From 678f32b3874e1216a45321205f657251c9c254bd Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Wed, 14 May 2025 10:52:50 -0500 Subject: [PATCH 01/19] Moved RunHelper to FWCore/Sources and renamed This will allow other sources to reuse the code --- .../Sources/interface/InputSourceRunHelper.h | 30 +++---- .../Sources/src/InputSourceRunHelper.cc | 78 ++++++++++--------- IOPool/Input/src/EmbeddedRootSource.cc | 4 +- IOPool/Input/src/EmbeddedRootSource.h | 6 +- IOPool/Input/src/PoolSource.cc | 6 +- IOPool/Input/src/PoolSource.h | 6 +- IOPool/Input/src/RepeatingCachedRootSource.cc | 7 +- IOPool/Input/src/RootFile.cc | 3 +- IOPool/Input/src/RootFile.h | 6 +- 9 files changed, 76 insertions(+), 70 deletions(-) rename IOPool/Input/src/RunHelper.h => FWCore/Sources/interface/InputSourceRunHelper.h (81%) rename IOPool/Input/src/RunHelper.cc => FWCore/Sources/src/InputSourceRunHelper.cc (75%) diff --git a/IOPool/Input/src/RunHelper.h b/FWCore/Sources/interface/InputSourceRunHelper.h similarity index 81% rename from IOPool/Input/src/RunHelper.h rename to FWCore/Sources/interface/InputSourceRunHelper.h index acdd8c4473edd..52efb47fe69a7 100644 --- a/IOPool/Input/src/RunHelper.h +++ b/FWCore/Sources/interface/InputSourceRunHelper.h @@ -1,5 +1,5 @@ -#ifndef IOPool_Input_RunHelper_h -#define IOPool_Input_RunHelper_h +#ifndef IOPool_Input_InputSourceRunHelper_h +#define IOPool_Input_InputSourceRunHelper_h #include "DataFormats/Provenance/interface/RunLumiEventNumber.h" #include "FWCore/Framework/interface/Frameworkfwd.h" @@ -12,10 +12,10 @@ namespace edm { class IndexIntoFile; class ParameterSetDescription; - class RunHelperBase { + class InputSourceRunHelperBase { public: - explicit RunHelperBase() = default; - virtual ~RunHelperBase(); + explicit InputSourceRunHelperBase() = default; + virtual ~InputSourceRunHelperBase(); virtual InputSource::ItemType nextItemType(InputSource::ItemType const& previousItemType, InputSource::ItemType const& newItemType, @@ -38,16 +38,16 @@ namespace edm { static void fillDescription(ParameterSetDescription& desc); }; - class DefaultRunHelper : public RunHelperBase { + class DefaultInputSourceRunHelper : public InputSourceRunHelperBase { public: - explicit DefaultRunHelper() = default; - ~DefaultRunHelper() override; + explicit DefaultInputSourceRunHelper() = default; + ~DefaultInputSourceRunHelper() override; }; - class SetRunHelper : public RunHelperBase { + class SetInputSourceRunHelper : public InputSourceRunHelperBase { public: - explicit SetRunHelper(ParameterSet const& pset); - ~SetRunHelper() override; + explicit SetInputSourceRunHelper(ParameterSet const& pset); + ~SetInputSourceRunHelper() override; void setForcedRunOffset(RunNumber_t firstRun) override; @@ -62,7 +62,7 @@ namespace edm { bool firstTime_; }; - class SetRunForEachLumiHelper : public RunHelperBase { + class SetRunForEachLumiHelper : public InputSourceRunHelperBase { public: explicit SetRunForEachLumiHelper(ParameterSet const& pset); ~SetRunForEachLumiHelper() override; @@ -89,9 +89,9 @@ namespace edm { bool firstTime_; }; - class FirstLuminosityBlockForEachRunHelper : public RunHelperBase { + class FirstLuminosityBlockForEachInputSourceRunHelper : public InputSourceRunHelperBase { public: - explicit FirstLuminosityBlockForEachRunHelper(ParameterSet const& pset); + explicit FirstLuminosityBlockForEachInputSourceRunHelper(ParameterSet const& pset); InputSource::ItemType nextItemType(InputSource::ItemType const& previousItemType, InputSource::ItemType const& newIemType, @@ -114,7 +114,7 @@ namespace edm { RunNumber_t lastUsedRunNumber_; bool fakeNewRun_; }; - std::unique_ptr makeRunHelper(ParameterSet const& pset); + std::unique_ptr makeInputSourceRunHelper(ParameterSet const& pset); } // namespace edm #endif diff --git a/IOPool/Input/src/RunHelper.cc b/FWCore/Sources/src/InputSourceRunHelper.cc similarity index 75% rename from IOPool/Input/src/RunHelper.cc rename to FWCore/Sources/src/InputSourceRunHelper.cc index cf1cfbeb76087..0407f6dce0c2e 100644 --- a/IOPool/Input/src/RunHelper.cc +++ b/FWCore/Sources/src/InputSourceRunHelper.cc @@ -1,6 +1,6 @@ /*---------------------------------------------------------------------- ----------------------------------------------------------------------*/ -#include "RunHelper.h" +#include "FWCore/Sources/interface/InputSourceRunHelper.h" #include "FWCore/ParameterSet/interface/ParameterSetDescription.h" #include "FWCore/Utilities/interface/EDMException.h" @@ -9,11 +9,11 @@ namespace edm { - std::unique_ptr makeRunHelper(ParameterSet const& pset) { + std::unique_ptr makeInputSourceRunHelper(ParameterSet const& pset) { if (pset.exists("setRunNumber")) { RunNumber_t run = pset.getUntrackedParameter("setRunNumber"); if (run != 0U) { - return std::make_unique(pset); + return std::make_unique(pset); } } if (pset.exists("setRunNumberForEachLumi")) { @@ -24,33 +24,34 @@ namespace edm { } if (pset.exists("firstLuminosityBlockForEachRun")) { if (not pset.getUntrackedParameter>("firstLuminosityBlockForEachRun").empty()) { - return std::make_unique(pset); + return std::make_unique(pset); } } - return std::make_unique(); + return std::make_unique(); } - RunHelperBase::~RunHelperBase() {} + InputSourceRunHelperBase::~InputSourceRunHelperBase() {} - void RunHelperBase::checkLumiConsistency(LuminosityBlockNumber_t lumi, LuminosityBlockNumber_t originalLumi) const { + void InputSourceRunHelperBase::checkLumiConsistency(LuminosityBlockNumber_t lumi, + LuminosityBlockNumber_t originalLumi) const { assert(lumi == originalLumi); } - void RunHelperBase::checkRunConsistency(RunNumber_t run, RunNumber_t originalRun) const { + void InputSourceRunHelperBase::checkRunConsistency(RunNumber_t run, RunNumber_t originalRun) const { assert(run == originalRun); } - DefaultRunHelper::~DefaultRunHelper() {} + DefaultInputSourceRunHelper::~DefaultInputSourceRunHelper() {} - SetRunHelper::SetRunHelper(ParameterSet const& pset) - : RunHelperBase(), + SetInputSourceRunHelper::SetInputSourceRunHelper(ParameterSet const& pset) + : InputSourceRunHelperBase(), setRun_(pset.getUntrackedParameter("setRunNumber")), forcedRunOffset_(0), firstTime_(true) {} - SetRunHelper::~SetRunHelper() {} + SetInputSourceRunHelper::~SetInputSourceRunHelper() {} - void SetRunHelper::setForcedRunOffset(RunNumber_t firstRun) { + void SetInputSourceRunHelper::setForcedRunOffset(RunNumber_t firstRun) { if (firstTime_ && setRun_ != 0) { forcedRunOffset_ = setRun_ - firstRun; if (forcedRunOffset_ < 0) { @@ -63,21 +64,21 @@ namespace edm { firstTime_ = false; } - void SetRunHelper::overrideRunNumber(RunID& id) { + void SetInputSourceRunHelper::overrideRunNumber(RunID& id) { id = RunID(id.run() + forcedRunOffset_); if (id < RunID::firstValidRun()) id = RunID::firstValidRun(); } - void SetRunHelper::overrideRunNumber(LuminosityBlockID& id) { + void SetInputSourceRunHelper::overrideRunNumber(LuminosityBlockID& id) { id = LuminosityBlockID(id.run() + forcedRunOffset_, id.luminosityBlock()); if (RunID(id.run()) < RunID::firstValidRun()) id = LuminosityBlockID(RunID::firstValidRun().run(), id.luminosityBlock()); } - void SetRunHelper::overrideRunNumber(EventID& id, bool isRealData) { + void SetInputSourceRunHelper::overrideRunNumber(EventID& id, bool isRealData) { if (isRealData) { - throw Exception(errors::Configuration, "SetRunHelper::overrideRunNumber()") + throw Exception(errors::Configuration, "SetInputSourceRunHelper::overrideRunNumber()") << "The 'setRunNumber' parameter of PoolSource cannot be used with real data.\n"; } id = EventID(id.run() + forcedRunOffset_, id.luminosityBlock(), id.event()); @@ -87,12 +88,12 @@ namespace edm { } } - void SetRunHelper::checkRunConsistency(RunNumber_t run, RunNumber_t originalRun) const { + void SetInputSourceRunHelper::checkRunConsistency(RunNumber_t run, RunNumber_t originalRun) const { assert(run == originalRun + forcedRunOffset_); } SetRunForEachLumiHelper::SetRunForEachLumiHelper(ParameterSet const& pset) - : RunHelperBase(), + : InputSourceRunHelperBase(), setRunNumberForEachLumi_(pset.getUntrackedParameter>("setRunNumberForEachLumi")), indexOfNextRunNumber_(0), realRunNumber_(0), @@ -163,17 +164,19 @@ namespace edm { assert(run == runNumberToUseForThisLumi()); } - FirstLuminosityBlockForEachRunHelper::FirstLuminosityBlockForEachRunHelper(ParameterSet const& pset) + FirstLuminosityBlockForEachInputSourceRunHelper::FirstLuminosityBlockForEachInputSourceRunHelper( + ParameterSet const& pset) : lumiToRun_(pset.getUntrackedParameter>("firstLuminosityBlockForEachRun")), realRunNumber_{0}, lastUsedRunNumber_{0}, fakeNewRun_{false} {} - InputSource::ItemType FirstLuminosityBlockForEachRunHelper::nextItemType(InputSource::ItemType const& previousItemType, - InputSource::ItemType const& newItemType, - RunNumber_t, - LuminosityBlockNumber_t iLumi, - EventNumber_t) { + InputSource::ItemType FirstLuminosityBlockForEachInputSourceRunHelper::nextItemType( + InputSource::ItemType const& previousItemType, + InputSource::ItemType const& newItemType, + RunNumber_t, + LuminosityBlockNumber_t iLumi, + EventNumber_t) { if (newItemType == InputSource::ItemType::IsLumi && previousItemType != InputSource::ItemType::IsRun) { auto run = findRunFromLumi(iLumi); if (run == 0) { @@ -189,9 +192,11 @@ namespace edm { return newItemType; } - RunNumber_t FirstLuminosityBlockForEachRunHelper::runNumberToUseForThisLumi() const { return lastUsedRunNumber_; } + RunNumber_t FirstLuminosityBlockForEachInputSourceRunHelper::runNumberToUseForThisLumi() const { + return lastUsedRunNumber_; + } - void FirstLuminosityBlockForEachRunHelper::checkForNewRun(RunNumber_t run, LuminosityBlockNumber_t iLumi) { + void FirstLuminosityBlockForEachInputSourceRunHelper::checkForNewRun(RunNumber_t run, LuminosityBlockNumber_t iLumi) { if (realRunNumber_ != 0 && run != realRunNumber_) { throw Exception(errors::MismatchedInputFiles, "PoolSource::checkForNewRun") << " Parameter 'firstLuminosityBlockForEachRun' can only process a single run.\n" @@ -202,25 +207,28 @@ namespace edm { fakeNewRun_ = false; } - void FirstLuminosityBlockForEachRunHelper::overrideRunNumber(RunID& id) { id = RunID(runNumberToUseForThisLumi()); } + void FirstLuminosityBlockForEachInputSourceRunHelper::overrideRunNumber(RunID& id) { + id = RunID(runNumberToUseForThisLumi()); + } - void FirstLuminosityBlockForEachRunHelper::overrideRunNumber(LuminosityBlockID& id) { + void FirstLuminosityBlockForEachInputSourceRunHelper::overrideRunNumber(LuminosityBlockID& id) { id = LuminosityBlockID(runNumberToUseForThisLumi(), id.luminosityBlock()); } - void FirstLuminosityBlockForEachRunHelper::overrideRunNumber(EventID& id, bool isRealData) { + void FirstLuminosityBlockForEachInputSourceRunHelper::overrideRunNumber(EventID& id, bool isRealData) { if (isRealData) { - throw Exception(errors::Configuration, "FirstLuminosityBlockForEachRunHelper::overrideRunNumber()") + throw Exception(errors::Configuration, "FirstLuminosityBlockForEachInputSourceRunHelper::overrideRunNumber()") << "The 'firstLuminosityBlockForEachRun' parameter of PoolSource cannot be used with real data.\n"; } id = EventID(runNumberToUseForThisLumi(), id.luminosityBlock(), id.event()); } - void FirstLuminosityBlockForEachRunHelper::checkRunConsistency(RunNumber_t run, RunNumber_t originalRun) const { + void FirstLuminosityBlockForEachInputSourceRunHelper::checkRunConsistency(RunNumber_t run, + RunNumber_t originalRun) const { assert(run == runNumberToUseForThisLumi()); } - RunNumber_t FirstLuminosityBlockForEachRunHelper::findRunFromLumi(LuminosityBlockNumber_t iLumi) const { + RunNumber_t FirstLuminosityBlockForEachInputSourceRunHelper::findRunFromLumi(LuminosityBlockNumber_t iLumi) const { RunNumber_t run = 0; for (auto const& lumiID : lumiToRun_) { if (lumiID.luminosityBlock() > iLumi) { @@ -229,7 +237,7 @@ namespace edm { run = lumiID.run(); } if (0 == run) { - throw Exception(errors::Configuration, "FirstLuminosityBlockForEachRunHelper::findRunFromLumi()") + throw Exception(errors::Configuration, "FirstLuminosityBlockForEachInputSourceRunHelper::findRunFromLumi()") << "The 'firstLuminosityBlockForEachRun' parameter does not have a matching Run number for LuminosityBlock " "number: " << iLumi << ".\n"; @@ -237,7 +245,7 @@ namespace edm { return run; } - void RunHelperBase::fillDescription(ParameterSetDescription& desc) { + void InputSourceRunHelperBase::fillDescription(ParameterSetDescription& desc) { desc.addOptionalNode(ParameterDescription("setRunNumber", 0U, false) xor ParameterDescription>( "setRunNumberForEachLumi", std::vector(), false) xor diff --git a/IOPool/Input/src/EmbeddedRootSource.cc b/IOPool/Input/src/EmbeddedRootSource.cc index f71854d29ad2f..2baef5e174948 100644 --- a/IOPool/Input/src/EmbeddedRootSource.cc +++ b/IOPool/Input/src/EmbeddedRootSource.cc @@ -2,11 +2,11 @@ ----------------------------------------------------------------------*/ #include "EmbeddedRootSource.h" #include "InputFile.h" -#include "RunHelper.h" #include "RootEmbeddedFileSequence.h" #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" #include "FWCore/ParameterSet/interface/ParameterSetDescription.h" #include "FWCore/Sources/interface/VectorInputSourceDescription.h" +#include "FWCore/Sources/interface/InputSourceRunHelper.h" namespace edm { @@ -29,7 +29,7 @@ namespace edm { bypassVersionCheck_(pset.getUntrackedParameter("bypassVersionCheck", false)), treeMaxVirtualSize_(pset.getUntrackedParameter("treeMaxVirtualSize", -1)), productSelectorRules_(pset, "inputCommands", "InputSource"), - runHelper_(new DefaultRunHelper()), + runHelper_(new DefaultInputSourceRunHelper()), catalog_(pset.getUntrackedParameter >("fileNames"), pset.getUntrackedParameter("overrideCatalog", std::string())), // Note: fileSequence_ needs to be initialized last, because it uses data members diff --git a/IOPool/Input/src/EmbeddedRootSource.h b/IOPool/Input/src/EmbeddedRootSource.h index 6456cf9fd701e..ce839648dafa8 100644 --- a/IOPool/Input/src/EmbeddedRootSource.h +++ b/IOPool/Input/src/EmbeddedRootSource.h @@ -27,7 +27,7 @@ namespace edm { class ConfigurationDescriptions; class FileCatalogItem; - class RunHelperBase; + class InputSourceRunHelperBase; class RootEmbeddedFileSequence; struct VectorInputSourceDescription; @@ -44,7 +44,7 @@ namespace edm { unsigned int nStreams() const { return nStreams_; } int treeMaxVirtualSize() const { return treeMaxVirtualSize_; } ProductSelectorRules const& productSelectorRules() const { return productSelectorRules_; } - RunHelperBase* runHelper() { return runHelper_.get(); } + InputSourceRunHelperBase* runHelper() { return runHelper_.get(); } static void fillDescriptions(ConfigurationDescriptions& descriptions); @@ -67,7 +67,7 @@ namespace edm { bool bypassVersionCheck_; int const treeMaxVirtualSize_; ProductSelectorRules productSelectorRules_; - std::unique_ptr runHelper_; + std::unique_ptr runHelper_; InputFileCatalog catalog_; edm::propagate_const> fileSequence_; diff --git a/IOPool/Input/src/PoolSource.cc b/IOPool/Input/src/PoolSource.cc index 6b3eb2b2db0d5..6753b329aa0cb 100644 --- a/IOPool/Input/src/PoolSource.cc +++ b/IOPool/Input/src/PoolSource.cc @@ -4,7 +4,6 @@ #include "InputFile.h" #include "RootPrimaryFileSequence.h" #include "RootSecondaryFileSequence.h" -#include "RunHelper.h" #include "DataFormats/Common/interface/ThinnedAssociation.h" #include "DataFormats/Provenance/interface/ProductDescription.h" #include "DataFormats/Provenance/interface/IndexIntoFile.h" @@ -19,6 +18,7 @@ #include "FWCore/Framework/interface/SharedResourcesAcquirer.h" #include "FWCore/Framework/interface/RunPrincipal.h" #include "FWCore/Framework/interface/ProductResolversFactory.h" +#include "FWCore/Sources/interface/InputSourceRunHelper.h" #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" #include "FWCore/ParameterSet/interface/ParameterSetDescription.h" #include "FWCore/Utilities/interface/EDMException.h" @@ -83,7 +83,7 @@ namespace edm { dropDescendants_(pset.getUntrackedParameter("dropDescendantsOfDroppedBranches")), labelRawDataLikeMC_(pset.getUntrackedParameter("labelRawDataLikeMC")), delayReadingEventProducts_(pset.getUntrackedParameter("delayReadingEventProducts")), - runHelper_(makeRunHelper(pset)), + runHelper_(makeInputSourceRunHelper(pset)), resourceSharedWithDelayedReaderPtr_(), // Note: primaryFileSequence_ and secondaryFileSequence_ need to be initialized last, because they use data members // initialized previously in their own initialization. @@ -329,7 +329,7 @@ namespace edm { ProductSelectorRules::fillDescription(desc, "inputCommands"); InputSource::fillDescription(desc); RootPrimaryFileSequence::fillDescription(desc); - RunHelperBase::fillDescription(desc); + InputSourceRunHelperBase::fillDescription(desc); descriptions.add("source", desc); } diff --git a/IOPool/Input/src/PoolSource.h b/IOPool/Input/src/PoolSource.h index 271049f375286..9f1279fadbebf 100644 --- a/IOPool/Input/src/PoolSource.h +++ b/IOPool/Input/src/PoolSource.h @@ -27,7 +27,7 @@ namespace edm { class FileCatalogItem; class RootPrimaryFileSequence; class RootSecondaryFileSequence; - class RunHelperBase; + class InputSourceRunHelperBase; class PoolSource : public InputSource { public: @@ -45,7 +45,7 @@ namespace edm { unsigned int nStreams() const { return nStreams_; } int treeMaxVirtualSize() const { return treeMaxVirtualSize_; } ProductSelectorRules const& productSelectorRules() const { return productSelectorRules_; } - RunHelperBase* runHelper() { return runHelper_.get(); } + InputSourceRunHelperBase* runHelper() { return runHelper_.get(); } static void fillDescriptions(ConfigurationDescriptions& descriptions); @@ -91,7 +91,7 @@ namespace edm { bool labelRawDataLikeMC_; bool delayReadingEventProducts_; - edm::propagate_const> runHelper_; + edm::propagate_const> runHelper_; std::unique_ptr resourceSharedWithDelayedReaderPtr_; // We do not use propagate_const because the acquirer is itself mutable. std::shared_ptr mutexSharedWithDelayedReader_; diff --git a/IOPool/Input/src/RepeatingCachedRootSource.cc b/IOPool/Input/src/RepeatingCachedRootSource.cc index edf1c765a8d46..0391319cdbff8 100644 --- a/IOPool/Input/src/RepeatingCachedRootSource.cc +++ b/IOPool/Input/src/RepeatingCachedRootSource.cc @@ -38,16 +38,15 @@ #include "FWCore/Utilities/interface/Exception.h" #include "FWCore/Utilities/interface/do_nothing_deleter.h" #include "FWCore/Sources/interface/EventSkipperByID.h" +#include "FWCore/Sources/interface/InputSourceRunHelper.h" #include "FWCore/Framework/interface/InputSourceMacros.h" -#include "RunHelper.h" #include "RootFile.h" #include "InputFile.h" #include "DuplicateChecker.h" namespace edm { - class RunHelperBase; class RepeatingCachedRootSource : public InputSource { public: @@ -142,7 +141,7 @@ namespace edm { RootServiceChecker rootServiceChecker_; ProductSelectorRules selectorRules_; - edm::propagate_const> runHelper_; + edm::propagate_const> runHelper_; std::unique_ptr rootFile_; std::vector orderedProcessHistoryIDs_; std::vector>> cachedWrappers_; @@ -176,7 +175,7 @@ using namespace edm; RepeatingCachedRootSource::RepeatingCachedRootSource(ParameterSet const& pset, InputSourceDescription const& desc) : InputSource(pset, desc), selectorRules_(pset, "inputCommands", "InputSource"), - runHelper_(std::make_unique()), + runHelper_(std::make_unique()), cachedWrappers_(pset.getUntrackedParameter("repeatNEvents")), eventAuxs_(cachedWrappers_.size()), provRetriever_(0), diff --git a/IOPool/Input/src/RootFile.cc b/IOPool/Input/src/RootFile.cc index f363f4e47fd11..3ae777a547901 100644 --- a/IOPool/Input/src/RootFile.cc +++ b/IOPool/Input/src/RootFile.cc @@ -5,8 +5,7 @@ #include "DuplicateChecker.h" #include "InputFile.h" #include "ProvenanceAdaptor.h" -#include "RunHelper.h" -#include "RootDelayedReaderBase.h" +#include "FWCore/Sources/interface/InputSourceRunHelper.h" #include "DataFormats/Common/interface/setIsMergeable.h" #include "DataFormats/Common/interface/ThinnedAssociation.h" diff --git a/IOPool/Input/src/RootFile.h b/IOPool/Input/src/RootFile.h index 7dc4a6233563e..cbdd37d4f92fa 100644 --- a/IOPool/Input/src/RootFile.h +++ b/IOPool/Input/src/RootFile.h @@ -57,7 +57,7 @@ namespace edm { class ProvenanceReaderBase; class ProvenanceAdaptor; class StoredMergeableRunProductMetadata; - class RunHelperBase; + class InputSourceRunHelperBase; class ThinnedAssociationsHelper; using EntryDescriptionMap = std::map; @@ -99,7 +99,7 @@ namespace edm { }; struct CrossFileInfo { - RunHelperBase* runHelper = nullptr; + InputSourceRunHelperBase* runHelper = nullptr; std::shared_ptr branchIDListHelper{}; ProcessBlockHelper* processBlockHelper = nullptr; std::shared_ptr thinnedAssociationsHelper{}; @@ -308,7 +308,7 @@ namespace edm { edm::propagate_const> fileThinnedAssociationsHelper_; edm::propagate_const> thinnedAssociationsHelper_; InputSource::ProcessingMode processingMode_; - edm::propagate_const runHelper_; + edm::propagate_const runHelper_; std::map newBranchToOldBranch_; edm::propagate_const eventHistoryTree_; // backward compatibility EventToProcessBlockIndexes eventToProcessBlockIndexes_; From fd9ee5f1d49531d5f815550288603dbf57f2a957 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Tue, 16 Sep 2025 12:26:48 -0500 Subject: [PATCH 02/19] All RNTupleTemp related files Names have changed but still using TTree for now. --- FWIO/RNTupleTempInput/BuildFile.xml | 15 + FWIO/RNTupleTempInput/src/DuplicateChecker.cc | 118 + FWIO/RNTupleTempInput/src/DuplicateChecker.h | 82 + .../src/EmbeddedRNTupleTempSource.cc | 102 + .../src/EmbeddedRNTupleTempSource.h | 79 + .../src/FixMissingStreamerInfos.cc | 74 + FWIO/RNTupleTempInput/src/InputFile.cc | 114 + FWIO/RNTupleTempInput/src/InputFile.h | 89 + FWIO/RNTupleTempInput/src/Module.cc | 9 + .../RNTupleTempInput/src/ProvenanceAdaptor.cc | 192 ++ FWIO/RNTupleTempInput/src/ProvenanceAdaptor.h | 58 + .../RNTupleTempInput/src/RNTupleTempSource.cc | 349 +++ FWIO/RNTupleTempInput/src/RNTupleTempSource.h | 103 + .../RNTupleTempInput/src/RootDelayedReader.cc | 90 + FWIO/RNTupleTempInput/src/RootDelayedReader.h | 75 + .../src/RootDelayedReaderBase.h | 70 + .../src/RootEmbeddedFileSequence.cc | 393 +++ .../src/RootEmbeddedFileSequence.h | 79 + FWIO/RNTupleTempInput/src/RootFile.cc | 2531 +++++++++++++++++ FWIO/RNTupleTempInput/src/RootFile.h | 331 +++ .../src/RootInputFileSequence.cc | 329 +++ .../src/RootInputFileSequence.h | 119 + .../src/RootPrimaryFileSequence.cc | 476 ++++ .../src/RootPrimaryFileSequence.h | 101 + .../src/RootPromptReadDelayedReader.cc | 109 + .../src/RootPromptReadDelayedReader.h | 85 + FWIO/RNTupleTempInput/src/RootRNTuple.cc | 627 ++++ FWIO/RNTupleTempInput/src/RootRNTuple.h | 265 ++ .../src/RootSecondaryFileSequence.cc | 108 + .../src/RootSecondaryFileSequence.h | 55 + FWIO/RNTupleTempInput/test/BuildFile.xml | 16 + FWIO/RNTupleTempInput/test/HeteroMerge_cfg.py | 17 + .../test/Pool2FileInputTest_cfg.py | 22 + .../test/PoolAliasTestStep1C_cfg.py | 37 + .../PoolAliasTestStep1_DifferentOrder_cfg.py | 43 + .../test/PoolAliasTestStep1_cfg.py | 43 + .../test/PoolAliasTestStep2A_cfg.py | 23 + .../test/PoolAliasTestStep2C_cfg.py | 17 + .../PoolAliasTestStep2_DifferentOrder_cfg.py | 31 + .../test/PoolAliasTestStep2_cfg.py | 31 + .../test/PoolEmptyTest2_cfg.py | 27 + .../test/PoolEmptyTest_cfg.py | 24 + .../RNTupleTempInput/test/PoolGUIDTest_cfg.py | 21 + .../test/PoolInputTest2_cfg.py | 23 + .../test/PoolInputTest3_cfg.py | 24 + .../test/PoolInputTest_cfg.py | 25 + .../test/PoolInputTest_noDelay_cfg.py | 21 + .../test/PoolInputTest_skipBadFiles_cfg.py | 35 + .../PoolInputTest_skip_with_failure_cfg.py | 25 + .../PoolNoParentDictionaryTestStep1_cfg.py | 24 + .../PoolNoParentDictionaryTestStep2_cfg.py | 17 + .../test/PrePool2FileInputTest_cfg.py | 31 + .../test/PrePoolInputTest2_cfg.py | 38 + .../test/PrePoolInputTest_cfg.py | 37 + .../test/RunPerLumiTest_cfg.py | 39 + FWIO/RNTupleTempInput/test/TestPoolInput.sh | 55 + .../test/TestPoolInputRunPerLumi.sh | 27 + .../firstLuminosityBlockForEachRunTest_cfg.py | 65 + ...sityBlockForEachRun_skipLumis2_Test_cfg.py | 54 + ...osityBlockForEachRun_skipLumis_Test_cfg.py | 55 + FWIO/RNTupleTempInput/test/preMerge2_cfg.py | 28 + FWIO/RNTupleTempInput/test/preMerge_cfg.py | 28 + .../noFallbackFile/local/storage.json | 10 + .../noFallbackFile/site-local-config.xml | 7 + .../useFallbackFile/local/storage.json | 18 + .../useFallbackFile/site-local-config.xml | 8 + .../test/testFileOpenErrorExitCode.sh | 40 + .../test/testForStreamerInfo.C | 111 + .../test/testNoParentDictionary.sh | 78 + .../testReducedProcessHistoryCreate_cfg.py | 54 + ...tReducedProcessHistoryHardwareResources.sh | 27 + .../test/testReducedProcessHistoryVersion.sh | 55 + .../test/testReducedProcessHistory_cfg.py | 117 + ...testRefProductIDMetadataConsistencyRoot.sh | 13 + ...oductIDMetadataConsistencyRootMerge_cfg.py | 19 + ...roductIDMetadataConsistencyRootTest_cfg.py | 12 + ...RefProductIDMetadataConsistencyRoot_cfg.py | 73 + .../test/test_fileOpenErrorExitCode_cfg.py | 13 + .../test/test_make_multi_lumi_cfg.py | 11 + .../test/test_make_overlapping_lumis_cfg.py | 28 + .../test/test_merge_two_files.py | 29 + .../test/test_read_overlapping_lumis_cfg.py | 31 + .../RunPerLumiTest.filtered.txt | 25 + FWIO/RNTupleTempOutput/BuildFile.xml | 17 + .../interface/RNTupleTempOutputModule.h | 243 ++ .../TimeoutRNTupleTempOutputModule.h | 46 + FWIO/RNTupleTempOutput/plugins/BuildFile.xml | 4 + FWIO/RNTupleTempOutput/plugins/Module.cc | 8 + .../src/RNTupleTempOutputModule.cc | 594 ++++ FWIO/RNTupleTempOutput/src/RootOutputFile.cc | 990 +++++++ FWIO/RNTupleTempOutput/src/RootOutputFile.h | 154 + .../src/RootOutputRNTuple.cc | 408 +++ .../RNTupleTempOutput/src/RootOutputRNTuple.h | 124 + .../src/TimeoutRNTupleTempOutputModule.cc | 55 + FWIO/RNTupleTempOutput/test/BuildFile.xml | 2 + .../test/PoolDropRead_cfg.py | 14 + .../test/PoolDropTest_cfg.py | 24 + .../test/PoolMissingRead_cfg.py | 14 + .../test/PoolMissingTest_cfg.py | 22 + .../test/PoolOutputAliasTestCheckResults.py | 8 + .../test/PoolOutputAliasTest_cfg.py | 19 + .../test/PoolOutputEmptyEventsTest_cfg.py | 26 + .../test/PoolOutputMergeWithEmptyFile_cfg.py | 13 + .../test/PoolOutputRead_cfg.py | 14 + .../test/PoolOutputTestOverrideGUID_cfg.py | 38 + .../test/PoolOutputTestUnscheduledRead_cfg.py | 52 + .../test/PoolOutputTestUnscheduled_cfg.py | 60 + .../test/PoolOutputTest_cfg.py | 31 + .../test/PoolTransientRead_cfg.py | 14 + .../test/PoolTransientTest_cfg.py | 25 + .../test/RNTupleTempOutputAliasTest.sh | 8 + FWIO/RNTupleTempOutput/test/TestProvA_cfg.py | 10 + FWIO/RNTupleTempOutput/test/TestProvB_cfg.py | 10 + FWIO/RNTupleTempOutput/test/TestProvC_cfg.py | 5 + .../test/TestRNTupleTempOutput.sh | 76 + 115 files changed, 12137 insertions(+) create mode 100644 FWIO/RNTupleTempInput/BuildFile.xml create mode 100644 FWIO/RNTupleTempInput/src/DuplicateChecker.cc create mode 100644 FWIO/RNTupleTempInput/src/DuplicateChecker.h create mode 100644 FWIO/RNTupleTempInput/src/EmbeddedRNTupleTempSource.cc create mode 100644 FWIO/RNTupleTempInput/src/EmbeddedRNTupleTempSource.h create mode 100644 FWIO/RNTupleTempInput/src/FixMissingStreamerInfos.cc create mode 100644 FWIO/RNTupleTempInput/src/InputFile.cc create mode 100644 FWIO/RNTupleTempInput/src/InputFile.h create mode 100644 FWIO/RNTupleTempInput/src/Module.cc create mode 100644 FWIO/RNTupleTempInput/src/ProvenanceAdaptor.cc create mode 100644 FWIO/RNTupleTempInput/src/ProvenanceAdaptor.h create mode 100644 FWIO/RNTupleTempInput/src/RNTupleTempSource.cc create mode 100644 FWIO/RNTupleTempInput/src/RNTupleTempSource.h create mode 100644 FWIO/RNTupleTempInput/src/RootDelayedReader.cc create mode 100644 FWIO/RNTupleTempInput/src/RootDelayedReader.h create mode 100644 FWIO/RNTupleTempInput/src/RootDelayedReaderBase.h create mode 100644 FWIO/RNTupleTempInput/src/RootEmbeddedFileSequence.cc create mode 100644 FWIO/RNTupleTempInput/src/RootEmbeddedFileSequence.h create mode 100644 FWIO/RNTupleTempInput/src/RootFile.cc create mode 100644 FWIO/RNTupleTempInput/src/RootFile.h create mode 100644 FWIO/RNTupleTempInput/src/RootInputFileSequence.cc create mode 100644 FWIO/RNTupleTempInput/src/RootInputFileSequence.h create mode 100644 FWIO/RNTupleTempInput/src/RootPrimaryFileSequence.cc create mode 100644 FWIO/RNTupleTempInput/src/RootPrimaryFileSequence.h create mode 100644 FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.cc create mode 100644 FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.h create mode 100644 FWIO/RNTupleTempInput/src/RootRNTuple.cc create mode 100644 FWIO/RNTupleTempInput/src/RootRNTuple.h create mode 100644 FWIO/RNTupleTempInput/src/RootSecondaryFileSequence.cc create mode 100644 FWIO/RNTupleTempInput/src/RootSecondaryFileSequence.h create mode 100644 FWIO/RNTupleTempInput/test/BuildFile.xml create mode 100644 FWIO/RNTupleTempInput/test/HeteroMerge_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/Pool2FileInputTest_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PoolAliasTestStep1C_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PoolAliasTestStep1_DifferentOrder_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PoolAliasTestStep1_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PoolAliasTestStep2A_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PoolAliasTestStep2C_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PoolAliasTestStep2_DifferentOrder_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PoolAliasTestStep2_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PoolEmptyTest2_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PoolEmptyTest_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PoolGUIDTest_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PoolInputTest2_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PoolInputTest3_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PoolInputTest_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PoolInputTest_noDelay_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PoolInputTest_skipBadFiles_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PoolInputTest_skip_with_failure_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PoolNoParentDictionaryTestStep1_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PoolNoParentDictionaryTestStep2_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PrePool2FileInputTest_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PrePoolInputTest2_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/PrePoolInputTest_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/RunPerLumiTest_cfg.py create mode 100755 FWIO/RNTupleTempInput/test/TestPoolInput.sh create mode 100755 FWIO/RNTupleTempInput/test/TestPoolInputRunPerLumi.sh create mode 100644 FWIO/RNTupleTempInput/test/firstLuminosityBlockForEachRunTest_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/firstLuminosityBlockForEachRun_skipLumis2_Test_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/firstLuminosityBlockForEachRun_skipLumis_Test_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/preMerge2_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/preMerge_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/sitelocalconfig/noFallbackFile/local/storage.json create mode 100644 FWIO/RNTupleTempInput/test/sitelocalconfig/noFallbackFile/site-local-config.xml create mode 100644 FWIO/RNTupleTempInput/test/sitelocalconfig/useFallbackFile/local/storage.json create mode 100644 FWIO/RNTupleTempInput/test/sitelocalconfig/useFallbackFile/site-local-config.xml create mode 100755 FWIO/RNTupleTempInput/test/testFileOpenErrorExitCode.sh create mode 100644 FWIO/RNTupleTempInput/test/testForStreamerInfo.C create mode 100755 FWIO/RNTupleTempInput/test/testNoParentDictionary.sh create mode 100644 FWIO/RNTupleTempInput/test/testReducedProcessHistoryCreate_cfg.py create mode 100755 FWIO/RNTupleTempInput/test/testReducedProcessHistoryHardwareResources.sh create mode 100755 FWIO/RNTupleTempInput/test/testReducedProcessHistoryVersion.sh create mode 100644 FWIO/RNTupleTempInput/test/testReducedProcessHistory_cfg.py create mode 100755 FWIO/RNTupleTempInput/test/testRefProductIDMetadataConsistencyRoot.sh create mode 100644 FWIO/RNTupleTempInput/test/testRefProductIDMetadataConsistencyRootMerge_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/testRefProductIDMetadataConsistencyRootTest_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/testRefProductIDMetadataConsistencyRoot_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/test_fileOpenErrorExitCode_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/test_make_multi_lumi_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/test_make_overlapping_lumis_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/test_merge_two_files.py create mode 100644 FWIO/RNTupleTempInput/test/test_read_overlapping_lumis_cfg.py create mode 100644 FWIO/RNTupleTempInput/test/unit_test_outputs/RunPerLumiTest.filtered.txt create mode 100644 FWIO/RNTupleTempOutput/BuildFile.xml create mode 100644 FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h create mode 100644 FWIO/RNTupleTempOutput/interface/TimeoutRNTupleTempOutputModule.h create mode 100644 FWIO/RNTupleTempOutput/plugins/BuildFile.xml create mode 100644 FWIO/RNTupleTempOutput/plugins/Module.cc create mode 100644 FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc create mode 100644 FWIO/RNTupleTempOutput/src/RootOutputFile.cc create mode 100644 FWIO/RNTupleTempOutput/src/RootOutputFile.h create mode 100644 FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc create mode 100644 FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h create mode 100644 FWIO/RNTupleTempOutput/src/TimeoutRNTupleTempOutputModule.cc create mode 100644 FWIO/RNTupleTempOutput/test/BuildFile.xml create mode 100644 FWIO/RNTupleTempOutput/test/PoolDropRead_cfg.py create mode 100644 FWIO/RNTupleTempOutput/test/PoolDropTest_cfg.py create mode 100644 FWIO/RNTupleTempOutput/test/PoolMissingRead_cfg.py create mode 100644 FWIO/RNTupleTempOutput/test/PoolMissingTest_cfg.py create mode 100644 FWIO/RNTupleTempOutput/test/PoolOutputAliasTestCheckResults.py create mode 100644 FWIO/RNTupleTempOutput/test/PoolOutputAliasTest_cfg.py create mode 100644 FWIO/RNTupleTempOutput/test/PoolOutputEmptyEventsTest_cfg.py create mode 100644 FWIO/RNTupleTempOutput/test/PoolOutputMergeWithEmptyFile_cfg.py create mode 100644 FWIO/RNTupleTempOutput/test/PoolOutputRead_cfg.py create mode 100644 FWIO/RNTupleTempOutput/test/PoolOutputTestOverrideGUID_cfg.py create mode 100644 FWIO/RNTupleTempOutput/test/PoolOutputTestUnscheduledRead_cfg.py create mode 100644 FWIO/RNTupleTempOutput/test/PoolOutputTestUnscheduled_cfg.py create mode 100644 FWIO/RNTupleTempOutput/test/PoolOutputTest_cfg.py create mode 100644 FWIO/RNTupleTempOutput/test/PoolTransientRead_cfg.py create mode 100644 FWIO/RNTupleTempOutput/test/PoolTransientTest_cfg.py create mode 100755 FWIO/RNTupleTempOutput/test/RNTupleTempOutputAliasTest.sh create mode 100644 FWIO/RNTupleTempOutput/test/TestProvA_cfg.py create mode 100644 FWIO/RNTupleTempOutput/test/TestProvB_cfg.py create mode 100644 FWIO/RNTupleTempOutput/test/TestProvC_cfg.py create mode 100755 FWIO/RNTupleTempOutput/test/TestRNTupleTempOutput.sh diff --git a/FWIO/RNTupleTempInput/BuildFile.xml b/FWIO/RNTupleTempInput/BuildFile.xml new file mode 100644 index 0000000000000..11f89571b4798 --- /dev/null +++ b/FWIO/RNTupleTempInput/BuildFile.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/FWIO/RNTupleTempInput/src/DuplicateChecker.cc b/FWIO/RNTupleTempInput/src/DuplicateChecker.cc new file mode 100644 index 0000000000000..49fbb2d788961 --- /dev/null +++ b/FWIO/RNTupleTempInput/src/DuplicateChecker.cc @@ -0,0 +1,118 @@ + +#include "FWIO/RNTupleTempInput/src/DuplicateChecker.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/Utilities/interface/Exception.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" + +#include +#include + +namespace edm::rntuple_temp { + + DuplicateChecker::DuplicateChecker(ParameterSet const& pset) + : dataType_(unknown), itIsKnownTheFileHasNoDuplicates_(false), disabled_(false) { + // The default value provided as the second argument to the getUntrackedParameter function call + // is not used when the ParameterSet has been validated and the parameters are not optional + // in the description. This is currently true when RNTupleTempSource is the primary input source. + // The modules that use RNTupleTempSource as a SecSource have not defined their fillDescriptions function + // yet, so the ParameterSet does not get validated yet. As soon as all the modules with a SecSource + // have defined descriptions, the defaults in the getUntrackedParameterSet function calls can + // and should be deleted from the code. + std::string duplicateCheckMode = + pset.getUntrackedParameter("duplicateCheckMode", std::string("checkAllFilesOpened")); + + if (duplicateCheckMode == std::string("noDuplicateCheck")) + duplicateCheckMode_ = noDuplicateCheck; + else if (duplicateCheckMode == std::string("checkEachFile")) + duplicateCheckMode_ = checkEachFile; + else if (duplicateCheckMode == std::string("checkEachRealDataFile")) + duplicateCheckMode_ = checkEachRealDataFile; + else if (duplicateCheckMode == std::string("checkAllFilesOpened")) + duplicateCheckMode_ = checkAllFilesOpened; + else { + throw cms::Exception("Configuration") + << "Illegal configuration parameter value passed to RNTupleTempSource for\n" + << "the \"duplicateCheckMode\" parameter, legal values are:\n" + << "\"noDuplicateCheck\", \"checkEachFile\", \"checkEachRealDataFile\", \"checkAllFilesOpened\"\n"; + } + } + + void DuplicateChecker::disable() { + disabled_ = true; + dataType_ = unknown; + relevantPreviousEvents_.clear(); + itIsKnownTheFileHasNoDuplicates_ = false; + } + + void DuplicateChecker::inputFileOpened(bool realData, + IndexIntoFile const& indexIntoFile, + std::vector > const& indexesIntoFiles, + std::vector >::size_type currentIndexIntoFile) { + dataType_ = realData ? isRealData : isSimulation; + if (checkDisabled()) + return; + + relevantPreviousEvents_.clear(); + itIsKnownTheFileHasNoDuplicates_ = false; + + if (duplicateCheckMode_ == checkAllFilesOpened) { + // Compares the current IndexIntoFile to all the previous ones and saves any duplicates. + // One unintended thing, it also saves the duplicate runs and lumis. + for (std::vector >::size_type i = 0; i < currentIndexIntoFile; ++i) { + if (indexesIntoFiles[i].get() != nullptr) { + indexIntoFile.set_intersection(*indexesIntoFiles[i], relevantPreviousEvents_); + } + } + } + if (relevantPreviousEvents_.empty()) { + if (!indexIntoFile.containsDuplicateEvents()) { + itIsKnownTheFileHasNoDuplicates_ = true; + } + } + } + + void DuplicateChecker::inputFileClosed() { + dataType_ = unknown; + relevantPreviousEvents_.clear(); + itIsKnownTheFileHasNoDuplicates_ = false; + } + + bool DuplicateChecker::isDuplicateAndCheckActive( + int index, RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event, std::string const& fileName) { + if (itIsKnownTheFileHasNoDuplicates_) + return false; + if (checkDisabled()) + return false; + + IndexIntoFile::IndexRunLumiEventKey newEvent(index, run, lumi, event); + bool duplicate = !relevantPreviousEvents_.insert(newEvent).second; + + if (duplicate) { + if (duplicateCheckMode_ == checkAllFilesOpened) { + LogWarning("DuplicateEvent") << "Duplicate Events found in entire set of input files.\n" + << "Both events were from run " << run << " and luminosity block " << lumi + << " with event number " << event << ".\n" + << "The duplicate was from file " << fileName << ".\n" + << "The duplicate will be skipped.\n"; + } else { + LogWarning("DuplicateEvent") << "Duplicate Events found in file " << fileName << ".\n" + << "Both events were from run " << run << " and luminosity block " << lumi + << " with event number " << event << ".\n" + << "The duplicate will be skipped.\n"; + } + return true; + } + return false; + } + + void DuplicateChecker::fillDescription(ParameterSetDescription& desc) { + std::string defaultString("checkAllFilesOpened"); + desc.addUntracked("duplicateCheckMode", defaultString) + ->setComment( + "'checkAllFilesOpened': check across all input files\n" + "'checkEachFile': check each input file independently\n" + "'checkEachRealDataFile': check each real data input file independently\n" + "'noDuplicateCheck': no duplicate checking\n"); + } +} // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempInput/src/DuplicateChecker.h b/FWIO/RNTupleTempInput/src/DuplicateChecker.h new file mode 100644 index 0000000000000..3ae8d40b99908 --- /dev/null +++ b/FWIO/RNTupleTempInput/src/DuplicateChecker.h @@ -0,0 +1,82 @@ +#ifndef DataFormats_Provenance_DuplicateChecker_h +#define DataFormats_Provenance_DuplicateChecker_h + +/*---------------------------------------------------------------------- + +FWIO/RNTupleTempInput/src/DuplicateChecker.h + +Used by RNTupleTempSource to detect events with +the same process history, run, lumi, and event number. +It is configurable whether it checks for duplicates +within the scope of each single input file or all input +files or does not check for duplicates at all. + +----------------------------------------------------------------------*/ + +#include "DataFormats/Provenance/interface/EventID.h" +#include "DataFormats/Provenance/interface/RunID.h" +#include "DataFormats/Provenance/interface/IndexIntoFile.h" + +#include +#include +#include +#include + +namespace edm { + class ParameterSet; + class ParameterSetDescription; +} // namespace edm + +namespace edm::rntuple_temp { + + class DuplicateChecker { + public: + DuplicateChecker(ParameterSet const& pset); + + void disable(); + + void inputFileOpened(bool realData, + IndexIntoFile const& indexIntoFile, + std::vector > const& indexesIntoFiles, + std::vector >::size_type currentIndexIntoFile); + + void inputFileClosed(); + + bool noDuplicatesInFile() const { return itIsKnownTheFileHasNoDuplicates_; } + + bool checkDisabled() const { + return duplicateCheckMode_ == noDuplicateCheck || + (duplicateCheckMode_ == checkEachRealDataFile && dataType_ == isSimulation) || disabled_; + } + + // Note that all references to the ProcessHistoryID in this class are to + // the "reduced" process history, including the index argument to this function. + bool isDuplicateAndCheckActive( + int index, RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event, std::string const& fileName); + + bool checkingAllFiles() const { return checkAllFilesOpened == duplicateCheckMode_; } + + static void fillDescription(ParameterSetDescription& desc); + + private: + enum DuplicateCheckMode { noDuplicateCheck, checkEachFile, checkEachRealDataFile, checkAllFilesOpened }; + + DuplicateCheckMode duplicateCheckMode_; + + enum DataType { isRealData, isSimulation, unknown }; + + DataType dataType_; + + // If checking the entire input for duplicates, then this holds + // events from previous files that duplicate events in the + // the current file. Plus it holds events that have been already + // processed in the current file. It is not used if there are + // no duplicates or duplicate checking has been disabled. + std::set relevantPreviousEvents_; + + bool itIsKnownTheFileHasNoDuplicates_; + + bool disabled_; + }; +} // namespace edm::rntuple_temp +#endif diff --git a/FWIO/RNTupleTempInput/src/EmbeddedRNTupleTempSource.cc b/FWIO/RNTupleTempInput/src/EmbeddedRNTupleTempSource.cc new file mode 100644 index 0000000000000..8de0ad820369a --- /dev/null +++ b/FWIO/RNTupleTempInput/src/EmbeddedRNTupleTempSource.cc @@ -0,0 +1,102 @@ +/*---------------------------------------------------------------------- +----------------------------------------------------------------------*/ +#include "EmbeddedRNTupleTempSource.h" +#include "InputFile.h" +#include "RootEmbeddedFileSequence.h" +#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" +#include "FWCore/Sources/interface/VectorInputSourceDescription.h" +#include "FWCore/Sources/interface/InputSourceRunHelper.h" + +namespace edm { + + class EventID; + class EventPrincipal; +} // namespace edm +namespace edm::rntuple_temp { + + EmbeddedRNTupleTempSource::EmbeddedRNTupleTempSource(ParameterSet const& pset, + VectorInputSourceDescription const& desc) + : VectorInputSource(pset, desc), + rootServiceChecker_(), + nStreams_(desc.allocations_->numberOfStreams()), + // The default value provided as the second argument to the getUntrackedParameter function call + // is not used when the ParameterSet has been validated and the parameters are not optional + // in the description. This is currently true when RNTupleTempSource is the primary input source. + // The modules that use RNTupleTempSource as a SecSource have not defined their fillDescriptions function + // yet, so the ParameterSet does not get validated yet. As soon as all the modules with a SecSource + // have defined descriptions, the defaults in the getUntrackedParameterSet function calls can + // and should be deleted from the code. + // + skipBadFiles_(pset.getUntrackedParameter("skipBadFiles", false)), + bypassVersionCheck_(pset.getUntrackedParameter("bypassVersionCheck", false)), + treeMaxVirtualSize_(pset.getUntrackedParameter("treeMaxVirtualSize", -1)), + productSelectorRules_(pset, "inputCommands", "InputSource"), + runHelper_(new DefaultInputSourceRunHelper()), + catalog_(pset.getUntrackedParameter >("fileNames"), + pset.getUntrackedParameter("overrideCatalog", std::string())), + // Note: fileSequence_ needs to be initialized last, because it uses data members + // initialized previously in its own initialization. + fileSequence_(new RootEmbeddedFileSequence(pset, *this, catalog_)) {} + + EmbeddedRNTupleTempSource::~EmbeddedRNTupleTempSource() {} + + void EmbeddedRNTupleTempSource::beginJob() {} + + void EmbeddedRNTupleTempSource::endJob() { + fileSequence_->endJob(); + InputFile::reportReadBranches(); + } + + void EmbeddedRNTupleTempSource::closeFile_() { fileSequence_->closeFile(); } + + bool EmbeddedRNTupleTempSource::readOneEvent(EventPrincipal& cache, + size_t& fileNameHash, + CLHEP::HepRandomEngine* engine, + EventID const* id, + bool recycleFiles) { + return fileSequence_->readOneEvent(cache, fileNameHash, engine, id, recycleFiles); + } + + void EmbeddedRNTupleTempSource::readOneSpecified(EventPrincipal& cache, + size_t& fileNameHash, + SecondaryEventIDAndFileInfo const& id) { + fileSequence_->readOneSpecified(cache, fileNameHash, id); + } + + void EmbeddedRNTupleTempSource::dropUnwantedBranches_(std::vector const& wantedBranches) { + std::vector rules; + rules.reserve(wantedBranches.size() + 1); + rules.emplace_back("drop *"); + for (std::string const& branch : wantedBranches) { + rules.push_back("keep " + branch + "_*"); + } + ParameterSet pset; + pset.addUntrackedParameter("inputCommands", rules); + productSelectorRules_ = ProductSelectorRules(pset, "inputCommands", "InputSource"); + } + + void EmbeddedRNTupleTempSource::fillDescriptions(ConfigurationDescriptions& descriptions) { + ParameterSetDescription desc; + + std::vector defaultStrings; + desc.setComment("Reads EDM/Root files for mixing."); + desc.addUntracked >("fileNames")->setComment("Names of files to be processed."); + desc.addUntracked("overrideCatalog", std::string()); + desc.addUntracked("skipBadFiles", false) + ->setComment( + "True: Ignore any missing or unopenable input file.\n" + "False: Throw exception if missing or unopenable input file."); + desc.addUntracked("bypassVersionCheck", false) + ->setComment( + "True: Bypass release version check.\n" + "False: Throw exception if reading file in a release prior to the release in which the file was written."); + desc.addUntracked("treeMaxVirtualSize", -1) + ->setComment("Size of ROOT TTree TBasket cache. Affects performance."); + + ProductSelectorRules::fillDescription(desc, "inputCommands"); + RootEmbeddedFileSequence::fillDescription(desc); + + descriptions.add("source", desc); + } +} // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempInput/src/EmbeddedRNTupleTempSource.h b/FWIO/RNTupleTempInput/src/EmbeddedRNTupleTempSource.h new file mode 100644 index 0000000000000..7e4bc176cdf7e --- /dev/null +++ b/FWIO/RNTupleTempInput/src/EmbeddedRNTupleTempSource.h @@ -0,0 +1,79 @@ +#ifndef FWIO_RNTupleTempInput_EmbeddedRNTupleTempSource_h +#define FWIO_RNTupleTempInput_EmbeddedRNTupleTempSource_h + +/*---------------------------------------------------------------------- + +EmbeddedRNTupleTempSource: This is an InputSource + +----------------------------------------------------------------------*/ + +#include "FWCore/Catalog/interface/InputFileCatalog.h" +#include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/Framework/interface/ProductSelectorRules.h" +#include "FWCore/Sources/interface/VectorInputSource.h" +#include "FWCore/Utilities/interface/propagate_const.h" +#include "IOPool/Common/interface/RootServiceChecker.h" + +#include +#include +#include +#include + +namespace CLHEP { + class HepRandomEngine; +} + +namespace edm { + + class ConfigurationDescriptions; + class FileCatalogItem; + class InputSourceRunHelperBase; + struct VectorInputSourceDescription; +} // namespace edm +namespace edm::rntuple_temp { + class RootEmbeddedFileSequence; + + class EmbeddedRNTupleTempSource : public VectorInputSource { + public: + explicit EmbeddedRNTupleTempSource(ParameterSet const& pset, VectorInputSourceDescription const& desc); + ~EmbeddedRNTupleTempSource() override; + using VectorInputSource::processHistoryRegistryForUpdate; + using VectorInputSource::productRegistryUpdate; + + // const accessors + bool skipBadFiles() const { return skipBadFiles_; } + bool bypassVersionCheck() const { return bypassVersionCheck_; } + unsigned int nStreams() const { return nStreams_; } + int treeMaxVirtualSize() const { return treeMaxVirtualSize_; } + ProductSelectorRules const& productSelectorRules() const { return productSelectorRules_; } + InputSourceRunHelperBase* runHelper() { return runHelper_.get(); } + + static void fillDescriptions(ConfigurationDescriptions& descriptions); + + private: + virtual void closeFile_(); + void beginJob() override; + void endJob() override; + bool readOneEvent(EventPrincipal& cache, + size_t& fileNameHash, + CLHEP::HepRandomEngine*, + EventID const* id, + bool recycleFiles) override; + void readOneSpecified(EventPrincipal& cache, size_t& fileNameHash, SecondaryEventIDAndFileInfo const& id) override; + void dropUnwantedBranches_(std::vector const& wantedBranches) override; + + RootServiceChecker rootServiceChecker_; + + unsigned int nStreams_; + bool skipBadFiles_; + bool bypassVersionCheck_; + int const treeMaxVirtualSize_; + ProductSelectorRules productSelectorRules_; + std::unique_ptr runHelper_; + + InputFileCatalog catalog_; + edm::propagate_const> fileSequence_; + + }; // class EmbeddedRNTupleTempSource +} // namespace edm::rntuple_temp +#endif diff --git a/FWIO/RNTupleTempInput/src/FixMissingStreamerInfos.cc b/FWIO/RNTupleTempInput/src/FixMissingStreamerInfos.cc new file mode 100644 index 0000000000000..d6e41df2e56ae --- /dev/null +++ b/FWIO/RNTupleTempInput/src/FixMissingStreamerInfos.cc @@ -0,0 +1,74 @@ +// -*- C++ -*- +// +// Package: Services +// Class : FixMissingStreamerInfos +// +// Implementation: + +/** \class edm::service::FixMissingStreamerInfos + +This service is used to open and close a ROOT file that contains +StreamerInfo objects causing them to be saved in memory. It is +used when reading a file written with a version of ROOT with a +bug that caused it to fail to write out StreamerInfo objects. +(see Issue 41246). + +CMSSW_13_0_0 had such a problem and files were written with +this problem. When using this service to read files written +with this release set the "fileInPath" parameter to the string +"IOPool/Input/data/fileContainingStreamerInfos_13_0_0.root". +This file is saved in the cms-data repository for IOPool/Input. +Note that it was difficult to identify all the problem classes +and we might have missed some. If there are additional problem +classes a new version of this file can be generated with script +IOPool/Input/scripts/makeFileContainingStreamerInfos.C. If the +problem ever recurs in ROOT with a different release, one could +use that script to generate a file containing StreamerInfos for +other releases. + + \author W. David Dagenhart, created 30 October, 2023 + +*/ + +#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" +#include "FWCore/ServiceRegistry/interface/ActivityRegistry.h" +#include "FWCore/ServiceRegistry/interface/ServiceMaker.h" +#include "FWCore/Utilities/interface/EDMException.h" +#include "FWCore/Utilities/interface/FileInPath.h" + +#include "TFile.h" + +namespace edm::rntuple_temp { + namespace service { + + class FixMissingStreamerInfos { + public: + FixMissingStreamerInfos(ParameterSet const&, ActivityRegistry&); + static void fillDescriptions(ConfigurationDescriptions&); + + private: + FileInPath fileInPath_; + }; + + FixMissingStreamerInfos::FixMissingStreamerInfos(ParameterSet const& pset, edm::ActivityRegistry&) + : fileInPath_(pset.getUntrackedParameter("fileInPath")) { + auto tFile = TFile::Open(fileInPath_.fullPath().c_str()); + if (!tFile || tFile->IsZombie()) { + throw cms::Exception("FixMissingStreamerInfo") + << "Failed opening file containing missing StreamerInfos: " << fileInPath_.fullPath(); + } + tFile->Close(); + } + + void FixMissingStreamerInfos::fillDescriptions(ConfigurationDescriptions& descriptions) { + ParameterSetDescription desc; + desc.addUntracked("fileInPath"); + descriptions.add("FixMissingStreamerInfos", desc); + } + } // namespace service +} // namespace edm::rntuple_temp + +using namespace edm::rntuple_temp::service; +DEFINE_FWK_SERVICE(FixMissingStreamerInfos); diff --git a/FWIO/RNTupleTempInput/src/InputFile.cc b/FWIO/RNTupleTempInput/src/InputFile.cc new file mode 100644 index 0000000000000..ed28ac4520710 --- /dev/null +++ b/FWIO/RNTupleTempInput/src/InputFile.cc @@ -0,0 +1,114 @@ +/*---------------------------------------------------------------------- +Holder for an input TFile. +----------------------------------------------------------------------*/ +#include "TList.h" +#include "TStreamerInfo.h" +#include "TClass.h" +#include "InputFile.h" + +#include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "FWCore/ServiceRegistry/interface/Service.h" +#include "FWCore/Utilities/interface/ExceptionPropagate.h" +#include "FWCore/Utilities/interface/TimeOfDay.h" + +#include +#include + +namespace edm::rntuple_temp { + InputFile::InputFile(char const* fileName, char const* msg, InputType inputType) + : file_(), fileName_(fileName), reportToken_(0), inputType_(inputType) { + logFileAction(msg, fileName); + { + // ROOT's context management implicitly assumes that a file is opened and + // closed on the same thread. To avoid the problem, we declare a local + // TContext object; when it goes out of scope, its destructor unregisters + // the context, guaranteeing the context is unregistered in the same thread + // it was registered in. Fixes issue #15524. + TDirectory::TContext contextEraser; + + file_ = std::unique_ptr(TFile::Open(fileName)); // propagate_const has no reset() function + } + std::exception_ptr e = edm::threadLocalException::getException(); + if (e != std::exception_ptr()) { + edm::threadLocalException::setException(std::exception_ptr()); + std::rethrow_exception(e); + } + if (!file_) { + throw edm::Exception(errors::FileOpenError) << "TFile::Open failed."; + } + if (file_->IsZombie()) { + throw edm::Exception(errors::FileOpenError) << "TFile::Open returned zombie."; + } + + logFileAction(" Successfully opened file ", fileName); + } + + InputFile::~InputFile() { Close(); } + + void InputFile::inputFileOpened(std::string const& logicalFileName, + std::string const& inputType, + std::string const& moduleName, + std::string const& label, + std::string const& fid, + std::vector const& branchNames) { + Service reportSvc; + reportToken_ = reportSvc->inputFileOpened( + fileName_, logicalFileName, std::string(), inputType, moduleName, label, fid, branchNames); + } + + void InputFile::eventReadFromFile() const { + Service reportSvc; + reportSvc->eventReadFromFile(inputType_, reportToken_); + } + + void InputFile::reportInputRunNumber(unsigned int run) const { + Service reportSvc; + reportSvc->reportInputRunNumber(run); + } + + void InputFile::reportInputLumiSection(unsigned int run, unsigned int lumi) const { + Service reportSvc; + reportSvc->reportInputLumiSection(run, lumi); + } + + void InputFile::reportSkippedFile(std::string const& fileName, std::string const& logicalFileName) { + Service reportSvc; + reportSvc->reportSkippedFile(fileName, logicalFileName); + } + + void InputFile::reportFallbackAttempt(std::string const& pfn, + std::string const& logicalFileName, + std::string const& errorMessage) { + Service reportSvc; + reportSvc->reportFallbackAttempt(pfn, logicalFileName, errorMessage); + } + + void InputFile::Close() { + if (file_->IsOpen()) { + file_->Close(); + try { + logFileAction(" Closed file ", fileName_.c_str()); + Service reportSvc; + reportSvc->inputFileClosed(inputType_, reportToken_); + } catch (std::exception const&) { + // If Close() called in a destructor after an exception throw, the services may no longer be active. + // Therefore, we catch any reasonable new exception. + } + } + } + + void InputFile::logFileAction(char const* msg, char const* fileName) const { + LogAbsolute("fileAction") << std::setprecision(0) << TimeOfDay() << msg << fileName; + FlushMessageLog(); + } + + void InputFile::reportReadBranches() { + Service reportSvc; + reportSvc->reportReadBranches(); + } + + void InputFile::reportReadBranch(InputType inputType, std::string const& branchName) { + Service reportSvc; + reportSvc->reportReadBranch(inputType, branchName); + } +} // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempInput/src/InputFile.h b/FWIO/RNTupleTempInput/src/InputFile.h new file mode 100644 index 0000000000000..758e4252cf813 --- /dev/null +++ b/FWIO/RNTupleTempInput/src/InputFile.h @@ -0,0 +1,89 @@ +#ifndef IOPool_Input_InputFile_h +#define IOPool_Input_InputFile_h + +/*---------------------------------------------------------------------- + +Holder for an input TFile. +----------------------------------------------------------------------*/ +#include "FWCore/MessageLogger/interface/JobReport.h" +#include "FWCore/Utilities/interface/InputType.h" +#include "FWCore/Utilities/interface/propagate_const.h" + +#include "TFile.h" +#include "TTree.h" +#include "TTreeCache.h" + +#include +#include +#include + +class TObject; + +namespace edm::rntuple_temp { + class InputFile { + public: + explicit InputFile(char const* fileName, char const* msg, InputType inputType); + ~InputFile(); + + InputFile(InputFile const&) = delete; // Disallow copying and moving + InputFile& operator=(InputFile const&) = delete; // Disallow copying and moving + + void Close(); + void inputFileOpened(std::string const& logicalFileName, + std::string const& inputType, + std::string const& moduleName, + std::string const& label, + std::string const& fid, + std::vector const& branchNames); + void eventReadFromFile() const; + void reportInputRunNumber(unsigned int run) const; + void reportInputLumiSection(unsigned int run, unsigned int lumi) const; + static void reportSkippedFile(std::string const& fileName, std::string const& logicalFileName); + static void reportFallbackAttempt(std::string const& pfn, + std::string const& logicalFileName, + std::string const& errorMessage); + // reportReadBranches is a per job report, rather than per file report. + // Nevertheless, it is defined here for convenience. + static void reportReadBranches(); + static void reportReadBranch(InputType inputType, std::string const& branchname); + + TObject* Get(char const* name) { return file_->Get(name); } + template + T* Get(char const* name) { return file_->Get(name); } + std::unique_ptr createCacheWithSize(TTree& iTree, unsigned int cacheSize) { + iTree.SetCacheSize(static_cast(cacheSize)); + std::unique_ptr newCache(dynamic_cast(file_->GetCacheRead(&iTree))); + file_->SetCacheRead(nullptr, &iTree, TFile::kDoNotDisconnect); + return newCache; + } + + class CacheGuard { + public: + CacheGuard(TFile* file, TObject* tree, TFileCacheRead* tfcr) : file_(file), tree_(tree) { + file_->SetCacheRead(tfcr, tree_, TFile::kDoNotDisconnect); + } + CacheGuard() = delete; + CacheGuard(CacheGuard const&) = delete; + CacheGuard& operator=(CacheGuard const&) = delete; + CacheGuard(CacheGuard&&) = delete; + CacheGuard& operator=(CacheGuard&&) = delete; + ~CacheGuard() { file_->SetCacheRead(nullptr, tree_, TFile::kDoNotDisconnect); } + + private: + TFile* file_; + TObject* tree_; + }; + [[nodiscard]] CacheGuard setCacheReadTemporarily(TFileCacheRead* tfcr, TObject* iTree) { + return CacheGuard(file_.get(), iTree, tfcr); + } + void clearCacheRead(TObject* iTree) { file_->SetCacheRead(nullptr, iTree, TFile::kDoNotDisconnect); } + void logFileAction(char const* msg, char const* fileName) const; + + private: + edm::propagate_const> file_; + std::string fileName_; + JobReport::Token reportToken_; + InputType inputType_; + }; +} // namespace edm::rntuple_temp +#endif diff --git a/FWIO/RNTupleTempInput/src/Module.cc b/FWIO/RNTupleTempInput/src/Module.cc new file mode 100644 index 0000000000000..5d210ecee99e1 --- /dev/null +++ b/FWIO/RNTupleTempInput/src/Module.cc @@ -0,0 +1,9 @@ +#include "FWCore/Framework/interface/InputSourceMacros.h" +#include "FWCore/Sources/interface/VectorInputSourceMacros.h" +#include "RNTupleTempSource.h" +#include "EmbeddedRNTupleTempSource.h" + +using edm::rntuple_temp::EmbeddedRNTupleTempSource; +using edm::rntuple_temp::RNTupleTempSource; +DEFINE_FWK_INPUT_SOURCE(RNTupleTempSource); +DEFINE_FWK_VECTOR_INPUT_SOURCE(EmbeddedRNTupleTempSource); diff --git a/FWIO/RNTupleTempInput/src/ProvenanceAdaptor.cc b/FWIO/RNTupleTempInput/src/ProvenanceAdaptor.cc new file mode 100644 index 0000000000000..50a22cde931dd --- /dev/null +++ b/FWIO/RNTupleTempInput/src/ProvenanceAdaptor.cc @@ -0,0 +1,192 @@ +/*---------------------------------------------------------------------- + +ProvenanceAdaptor.cc + +----------------------------------------------------------------------*/ +//------------------------------------------------------------ +// Class ProvenanceAdaptor: adapts old provenance (fileFormatVersion_.value() < 11) to new provenance. + +#include "ProvenanceAdaptor.h" + +#include +#include + +#include +#include +#include +#include +#include "DataFormats/Provenance/interface/BranchID.h" +#include "DataFormats/Provenance/interface/ProcessConfiguration.h" +#include "DataFormats/Provenance/interface/ProcessHistory.h" +#include "DataFormats/Provenance/interface/ProductRegistry.h" +#include "FWCore/Utilities/interface/Algorithms.h" +namespace edm::rntuple_temp { + + void ProvenanceAdaptor::fixProcessHistory(ProcessHistoryMap& pHistMap, ProcessHistoryVector& pHistVector) { + assert(pHistMap.empty() != pHistVector.empty()); + for (ProcessHistoryVector::const_iterator i = pHistVector.begin(), e = pHistVector.end(); i != e; ++i) { + pHistMap.insert(std::make_pair(i->id(), *i)); + } + pHistVector.clear(); + for (ProcessHistoryMap::const_iterator i = pHistMap.begin(), e = pHistMap.end(); i != e; ++i) { + ProcessHistory newHist; + ProcessHistoryID const& oldphID = i->first; + for (ProcessHistory::const_iterator it = i->second.begin(), et = i->second.end(); it != et; ++it) { + ParameterSetID const& newPsetID = convertID(it->parameterSetID()); + newHist.emplace_back(it->processName(), newPsetID, it->releaseVersion(), it->hardwareResourcesDescription()); + } + assert(newHist.size() == i->second.size()); + ProcessHistoryID newphID = newHist.id(); + pHistVector.push_back(std::move(newHist)); + if (newphID != oldphID) { + processHistoryIdConverter_.insert(std::make_pair(oldphID, newphID)); + } + } + assert(pHistVector.size() == pHistMap.size()); + } + + namespace { + typedef StringVector OneHistory; + typedef std::set Histories; + typedef std::pair Product; + typedef std::vector OrderedProducts; + struct Sorter { + explicit Sorter(Histories const& histories) : histories_(histories) {} + bool operator()(Product const& a, Product const& b) const; + Histories const histories_; + }; + bool Sorter::operator()(Product const& a, Product const& b) const { + assert(a != b); + if (a.first == b.first) + return false; + bool mayBeTrue = false; + for (Histories::const_iterator it = histories_.begin(), itEnd = histories_.end(); it != itEnd; ++it) { + OneHistory::const_iterator itA = find_in_all(*it, a.first); + if (itA == it->end()) + continue; + OneHistory::const_iterator itB = find_in_all(*it, b.first); + if (itB == it->end()) + continue; + assert(itA != itB); + if (itB < itA) { + // process b precedes process a; + return false; + } + // process a precedes process b; + mayBeTrue = true; + } + return mayBeTrue; + } + + void fillProcessConfiguration(ProcessHistoryVector const& pHistVec, ProcessConfigurationVector& procConfigVector) { + procConfigVector.clear(); + std::set pcset; + for (auto const& history : pHistVec) { + for (auto const& process : history) { + if (pcset.insert(process).second) { + procConfigVector.push_back(process); + } + } + } + } + + void fillListsAndIndexes(ProductRegistry& productRegistry, + ProcessHistoryMap const& pHistMap, + std::shared_ptr& branchIDLists, + std::vector& branchListIndexes) { + OrderedProducts orderedProducts; + std::set processNamesThatProduced; + ProductRegistry::ProductList& prodList = productRegistry.productListUpdator(); + for (auto& item : prodList) { + ProductDescription& prod = item.second; + if (prod.branchType() == InEvent) { + prod.init(); + processNamesThatProduced.insert(prod.processName()); + orderedProducts.emplace_back(prod.processName(), prod.branchID()); + } + } + assert(!orderedProducts.empty()); + Histories processHistories; + size_t max = 0; + for (ProcessHistoryMap::const_iterator it = pHistMap.begin(), itEnd = pHistMap.end(); it != itEnd; ++it) { + ProcessHistory const& pHist = it->second; + OneHistory processHistory; + for (ProcessHistory::const_iterator i = pHist.begin(), iEnd = pHist.end(); i != iEnd; ++i) { + if (processNamesThatProduced.find(i->processName()) != processNamesThatProduced.end()) { + processHistory.push_back(i->processName()); + } + } + max = (processHistory.size() > max ? processHistory.size() : max); + assert(max <= processNamesThatProduced.size()); + if (processHistory.size() > 1) { + processHistories.insert(processHistory); + } + } + stable_sort_all(orderedProducts, Sorter(processHistories)); + + auto pv = std::make_unique(); + auto p = std::make_unique(); + std::string processName; + BranchListIndex blix = 0; + for (OrderedProducts::const_iterator it = orderedProducts.begin(), itEnd = orderedProducts.end(); it != itEnd; + ++it) { + if (it->first != processName) { + if (!processName.empty()) { + branchListIndexes.push_back(blix); + ++blix; + pv->push_back(std::move(*p)); + p = std::make_unique(); + } + processName = it->first; + } + p->push_back(it->second.id()); + } + branchListIndexes.push_back(blix); + pv->push_back(std::move(*p)); + branchIDLists.reset(pv.release()); + } + } // namespace + + ProvenanceAdaptor::ProvenanceAdaptor(ProductRegistry& productRegistry, + ProcessHistoryMap& pHistMap, + ProcessHistoryVector& pHistVector, + ProcessConfigurationVector& procConfigVector, + ParameterSetIdConverter const& parameterSetIdConverter, + bool fullConversion) + : parameterSetIdConverter_(parameterSetIdConverter), + processHistoryIdConverter_(), + branchIDLists_(), + branchListIndexes_() { + fixProcessHistory(pHistMap, pHistVector); + fillProcessConfiguration(pHistVector, procConfigVector); + if (fullConversion) { + fillListsAndIndexes(productRegistry, pHistMap, branchIDLists_, branchListIndexes_); + } + } + + ProvenanceAdaptor::~ProvenanceAdaptor() {} + + ParameterSetID const& ProvenanceAdaptor::convertID(ParameterSetID const& oldID) const { + ParameterSetIdConverter::const_iterator it = parameterSetIdConverter_.find(oldID); + if (it == parameterSetIdConverter_.end()) { + return oldID; + } + return it->second; + } + + ProcessHistoryID const& ProvenanceAdaptor::convertID(ProcessHistoryID const& oldID) const { + ProcessHistoryIdConverter::const_iterator it = processHistoryIdConverter_.find(oldID); + if (it == processHistoryIdConverter_.end()) { + return oldID; + } + return it->second; + } + + std::shared_ptr ProvenanceAdaptor::releaseBranchIDLists() { + auto ptr = branchIDLists_; + branchIDLists_.reset(); + return ptr; + } + + void ProvenanceAdaptor::branchListIndexes(BranchListIndexes& indexes) const { indexes = branchListIndexes_; } +} // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempInput/src/ProvenanceAdaptor.h b/FWIO/RNTupleTempInput/src/ProvenanceAdaptor.h new file mode 100644 index 0000000000000..29dfb9e788cd0 --- /dev/null +++ b/FWIO/RNTupleTempInput/src/ProvenanceAdaptor.h @@ -0,0 +1,58 @@ +#ifndef IOPool_Input_ProvenanceAdaptor_h +#define IOPool_Input_ProvenanceAdaptor_h + +/*---------------------------------------------------------------------- + +ProvenanceAdaptor.h + +----------------------------------------------------------------------*/ +#include +#include +#include + +#include "DataFormats/Provenance/interface/BranchIDList.h" +#include "DataFormats/Provenance/interface/BranchListIndex.h" +#include "DataFormats/Provenance/interface/ParameterSetID.h" +#include "DataFormats/Provenance/interface/ProcessHistoryID.h" +#include "DataFormats/Provenance/interface/ProcessHistoryRegistry.h" +#include "DataFormats/Provenance/interface/ProvenanceFwd.h" +#include "FWCore/ParameterSet/interface/ParameterSetConverter.h" + +namespace edm::rntuple_temp { + + //------------------------------------------------------------ + // Class ProvenanceAdaptor: + + class ProvenanceAdaptor { + public: + typedef ParameterSetConverter::ParameterSetIdConverter ParameterSetIdConverter; + typedef std::map ProcessHistoryIdConverter; + ProvenanceAdaptor(ProductRegistry& productRegistry, + ProcessHistoryMap& pHistMap, + ProcessHistoryVector& pHistVector, + ProcessConfigurationVector& procConfigVector, + ParameterSetIdConverter const& parameterSetIdConverter, + bool fullConversion); + ~ProvenanceAdaptor(); + + ProvenanceAdaptor(ProvenanceAdaptor const&) = delete; // Disallow copying and moving + ProvenanceAdaptor& operator=(ProvenanceAdaptor const&) = delete; // Disallow copying and moving + + std::shared_ptr releaseBranchIDLists(); + + void branchListIndexes(BranchListIndexes& indexes) const; + + ParameterSetID const& convertID(ParameterSetID const& oldID) const; + + ProcessHistoryID const& convertID(ProcessHistoryID const& oldID) const; + + private: + void fixProcessHistory(ProcessHistoryMap& pHistMap, ProcessHistoryVector& pHistVector); + + ParameterSetIdConverter parameterSetIdConverter_; + ProcessHistoryIdConverter processHistoryIdConverter_; + std::shared_ptr branchIDLists_; + std::vector branchListIndexes_; + }; // class ProvenanceAdaptor +} // namespace edm::rntuple_temp +#endif diff --git a/FWIO/RNTupleTempInput/src/RNTupleTempSource.cc b/FWIO/RNTupleTempInput/src/RNTupleTempSource.cc new file mode 100644 index 0000000000000..47e0e2e9ab921 --- /dev/null +++ b/FWIO/RNTupleTempInput/src/RNTupleTempSource.cc @@ -0,0 +1,349 @@ +/*---------------------------------------------------------------------- +----------------------------------------------------------------------*/ +#include "RNTupleTempSource.h" +#include "InputFile.h" +#include "RootPrimaryFileSequence.h" +#include "RootSecondaryFileSequence.h" +#include "DataFormats/Common/interface/ThinnedAssociation.h" +#include "DataFormats/Provenance/interface/ProductDescription.h" +#include "DataFormats/Provenance/interface/IndexIntoFile.h" +#include "DataFormats/Provenance/interface/ProductRegistry.h" +#include "DataFormats/Provenance/interface/ThinnedAssociationsHelper.h" +#include "FWCore/Framework/interface/EventPrincipal.h" +#include "FWCore/Framework/interface/FileBlock.h" +#include "FWCore/Framework/interface/InputSourceDescription.h" +#include "FWCore/Framework/interface/LuminosityBlockPrincipal.h" +#include "FWCore/Framework/interface/PreallocationConfiguration.h" +#include "FWCore/Framework/interface/SharedResourcesRegistry.h" +#include "FWCore/Framework/interface/SharedResourcesAcquirer.h" +#include "FWCore/Framework/interface/RunPrincipal.h" +#include "FWCore/Framework/interface/ProductResolversFactory.h" +#include "FWCore/Sources/interface/InputSourceRunHelper.h" +#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" +#include "FWCore/Utilities/interface/EDMException.h" +#include "FWCore/Utilities/interface/Exception.h" +#include "FWCore/Utilities/interface/InputType.h" + +#include + +namespace edm { + + class BranchID; + class LuminosityBlockID; + class EventID; + class ThinnedAssociationsHelper; +} // namespace edm +namespace edm::rntuple_temp { + namespace { + void checkHistoryConsistency(Principal const& primary, Principal const& secondary) { + ProcessHistory const& ph1 = primary.processHistory(); + ProcessHistory const& ph2 = secondary.processHistory(); + if (ph1 != ph2 && !isAncestor(ph2, ph1)) { + throw Exception(errors::MismatchedInputFiles, "RNTupleTempSource::checkConsistency") + << "The secondary file is not an ancestor of the primary file\n"; + } + } + void checkConsistency(EventPrincipal const& primary, EventPrincipal const& secondary) { + if (!isSameEvent(primary, secondary)) { + throw Exception(errors::MismatchedInputFiles, "RNTupleTempSource::checkConsistency") + << primary.id() << " has inconsistent EventAuxiliary data in the primary and secondary file\n"; + } + } + void checkConsistency(LuminosityBlockAuxiliary const& primary, LuminosityBlockAuxiliary const& secondary) { + if (primary.id() != secondary.id()) { + throw Exception(errors::MismatchedInputFiles, "RNTupleTempSource::checkConsistency") + << primary.id() << " has inconsistent LuminosityBlockAuxiliary data in the primary and secondary file\n"; + } + } + void checkConsistency(RunAuxiliary const& primary, RunAuxiliary const& secondary) { + if (primary.id() != secondary.id()) { + throw Exception(errors::MismatchedInputFiles, "RNTupleTempSource::checkConsistency") + << primary.id() << " has inconsistent RunAuxiliary data in the primary and secondary file\n"; + } + } + } // namespace + + RNTupleTempSource::RNTupleTempSource(ParameterSet const& pset, InputSourceDescription const& desc) + : InputSource(pset, desc), + rootServiceChecker_(), + catalog_(pset.getUntrackedParameter >("fileNames"), + pset.getUntrackedParameter("overrideCatalog", std::string())), + secondaryCatalog_( + pset.getUntrackedParameter >("secondaryFileNames", std::vector()), + pset.getUntrackedParameter("overrideCatalog", std::string())), + secondaryRunPrincipal_(), + secondaryLumiPrincipal_(), + secondaryEventPrincipals_(), + branchIDsToReplace_(), + nStreams_(desc.allocations_->numberOfStreams()), + skipBadFiles_(pset.getUntrackedParameter("skipBadFiles")), + bypassVersionCheck_(pset.getUntrackedParameter("bypassVersionCheck")), + treeMaxVirtualSize_(pset.getUntrackedParameter("treeMaxVirtualSize")), + productSelectorRules_(pset, "inputCommands", "InputSource"), + dropDescendants_(pset.getUntrackedParameter("dropDescendantsOfDroppedBranches")), + labelRawDataLikeMC_(pset.getUntrackedParameter("labelRawDataLikeMC")), + delayReadingEventProducts_(pset.getUntrackedParameter("delayReadingEventProducts")), + runHelper_(makeInputSourceRunHelper(pset)), + resourceSharedWithDelayedReaderPtr_(), + // Note: primaryFileSequence_ and secondaryFileSequence_ need to be initialized last, because they use data members + // initialized previously in their own initialization. + primaryFileSequence_(new RootPrimaryFileSequence(pset, *this, catalog_)), + secondaryFileSequence_( + secondaryCatalog_.empty() ? nullptr : new RootSecondaryFileSequence(pset, *this, secondaryCatalog_)) { + auto resources = SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader(); + resourceSharedWithDelayedReaderPtr_ = std::make_unique(std::move(resources.first)); + mutexSharedWithDelayedReader_ = resources.second; + + if (secondaryCatalog_.empty() && pset.getUntrackedParameter("needSecondaryFileNames", false)) { + throw Exception(errors::Configuration, "RNTupleTempSource") << "'secondaryFileNames' must be specified\n"; + } + if (secondaryFileSequence_) { + secondaryEventPrincipals_.reserve(nStreams_); + for (unsigned int index = 0; index < nStreams_; ++index) { + secondaryEventPrincipals_.emplace_back(new EventPrincipal(secondaryFileSequence_->fileProductRegistry(), + edm::productResolversFactory::makePrimary, + secondaryFileSequence_->fileBranchIDListHelper(), + std::make_shared(), + processConfiguration(), + nullptr, + index)); + } + std::array, NumBranchTypes> idsToReplace; + ProductRegistry::ProductList const& secondary = secondaryFileSequence_->fileProductRegistry()->productList(); + ProductRegistry::ProductList const& primary = primaryFileSequence_->fileProductRegistry()->productList(); + std::set associationsFromSecondary; + //this is the registry used by the 'outside' world and only has the primary file information in it at present + ProductRegistry::ProductList& fullList = productRegistryUpdate().productListUpdator(); + for (auto const& item : secondary) { + if (item.second.present()) { + idsToReplace[item.second.branchType()].insert(item.second.branchID()); + if (item.second.branchType() == InEvent && item.second.unwrappedType() == typeid(ThinnedAssociation)) { + associationsFromSecondary.insert(item.second.branchID()); + } + //now make sure this is marked as not dropped else the product will not be 'get'table from the Event + auto itFound = fullList.find(item.first); + if (itFound != fullList.end()) { + // If the branch in primary file was dropped, need to initilize the dictionary information + if (itFound->second.dropped()) { + itFound->second.initFromDictionary(); + } + itFound->second.setDropped(false); + } + } + } + for (auto const& item : primary) { + if (item.second.present()) { + idsToReplace[item.second.branchType()].erase(item.second.branchID()); + associationsFromSecondary.erase(item.second.branchID()); + } + } + if (idsToReplace[InEvent].empty() && idsToReplace[InLumi].empty() && idsToReplace[InRun].empty()) { + secondaryFileSequence_ = nullptr; // propagate_const has no reset() function + } else { + for (int i = InEvent; i < NumBranchTypes; ++i) { + branchIDsToReplace_[i].reserve(idsToReplace[i].size()); + for (auto const& id : idsToReplace[i]) { + branchIDsToReplace_[i].push_back(id); + } + } + secondaryFileSequence_->initAssociationsFromSecondary(associationsFromSecondary); + } + } + } + + RNTupleTempSource::~RNTupleTempSource() {} + + void RNTupleTempSource::endJob() { + if (secondaryFileSequence_) + secondaryFileSequence_->endJob(); + primaryFileSequence_->endJob(); + InputFile::reportReadBranches(); + } + + std::shared_ptr RNTupleTempSource::readFile_() { + std::shared_ptr fb = primaryFileSequence_->readFile_(); + if (secondaryFileSequence_) { + fb->setNotFastClonable(FileBlock::HasSecondaryFileSequence); + } + return fb; + } + + void RNTupleTempSource::closeFile_() { primaryFileSequence_->closeFile(); } + + std::shared_ptr RNTupleTempSource::readRunAuxiliary_() { + return primaryFileSequence_->readRunAuxiliary_(); + } + + std::shared_ptr RNTupleTempSource::readLuminosityBlockAuxiliary_() { + return primaryFileSequence_->readLuminosityBlockAuxiliary_(); + } + + void RNTupleTempSource::fillProcessBlockHelper_() { primaryFileSequence_->fillProcessBlockHelper_(); } + + bool RNTupleTempSource::nextProcessBlock_(ProcessBlockPrincipal& processBlockPrincipal) { + return primaryFileSequence_->nextProcessBlock_(processBlockPrincipal); + } + + void RNTupleTempSource::readProcessBlock_(ProcessBlockPrincipal& processBlockPrincipal) { + primaryFileSequence_->readProcessBlock_(processBlockPrincipal); + } + + void RNTupleTempSource::readRun_(RunPrincipal& runPrincipal) { + bool shouldWeProcessRun = primaryFileSequence_->readRun_(runPrincipal); + if (secondaryFileSequence_ && shouldWeProcessRun && !branchIDsToReplace_[InRun].empty()) { + bool found = secondaryFileSequence_->skipToItem(runPrincipal.run(), 0U, 0U); + if (found) { + std::shared_ptr secondaryAuxiliary = secondaryFileSequence_->readRunAuxiliary_(); + checkConsistency(runPrincipal.aux(), *secondaryAuxiliary); + secondaryRunPrincipal_ = std::make_shared(secondaryFileSequence_->fileProductRegistry(), + edm::productResolversFactory::makePrimary, + processConfiguration(), + nullptr, + runPrincipal.index()); + secondaryRunPrincipal_->setAux(*secondaryAuxiliary); + secondaryFileSequence_->readRun_(*secondaryRunPrincipal_); + checkHistoryConsistency(runPrincipal, *secondaryRunPrincipal_); + runPrincipal.recombine(*secondaryRunPrincipal_, branchIDsToReplace_[InRun]); + } else { + throw Exception(errors::MismatchedInputFiles, "RNTupleTempSource::readRun_") + << " Run " << runPrincipal.run() << " is not found in the secondary input files\n"; + } + } + } + + void RNTupleTempSource::readLuminosityBlock_(LuminosityBlockPrincipal& lumiPrincipal) { + bool shouldWeProcessLumi = primaryFileSequence_->readLuminosityBlock_(lumiPrincipal); + if (secondaryFileSequence_ && shouldWeProcessLumi && !branchIDsToReplace_[InLumi].empty()) { + bool found = secondaryFileSequence_->skipToItem(lumiPrincipal.run(), lumiPrincipal.luminosityBlock(), 0U); + if (found) { + std::shared_ptr secondaryAuxiliary = + secondaryFileSequence_->readLuminosityBlockAuxiliary_(); + checkConsistency(lumiPrincipal.aux(), *secondaryAuxiliary); + secondaryLumiPrincipal_ = + std::make_shared(secondaryFileSequence_->fileProductRegistry(), + edm::productResolversFactory::makePrimary, + processConfiguration(), + nullptr, + lumiPrincipal.index()); + secondaryLumiPrincipal_->setAux(*secondaryAuxiliary); + secondaryFileSequence_->readLuminosityBlock_(*secondaryLumiPrincipal_); + checkHistoryConsistency(lumiPrincipal, *secondaryLumiPrincipal_); + lumiPrincipal.recombine(*secondaryLumiPrincipal_, branchIDsToReplace_[InLumi]); + } else { + throw Exception(errors::MismatchedInputFiles, "RNTupleTempSource::readLuminosityBlock_") + << " Run " << lumiPrincipal.run() << " LuminosityBlock " << lumiPrincipal.luminosityBlock() + << " is not found in the secondary input files\n"; + } + } + } + + void RNTupleTempSource::readEvent_(EventPrincipal& eventPrincipal) { + bool readAllProducts = not delayReadingEventProducts_; + bool readEventSucceeded = primaryFileSequence_->readEvent(eventPrincipal, readAllProducts); + assert(readEventSucceeded); + if (secondaryFileSequence_ && !branchIDsToReplace_[InEvent].empty()) { + bool found = secondaryFileSequence_->skipToItem( + eventPrincipal.run(), eventPrincipal.luminosityBlock(), eventPrincipal.id().event()); + if (found) { + EventPrincipal& secondaryEventPrincipal = *secondaryEventPrincipals_[eventPrincipal.streamID().value()]; + bool readEventSucceeded = secondaryFileSequence_->readEvent(secondaryEventPrincipal, readAllProducts); + checkConsistency(eventPrincipal, secondaryEventPrincipal); + checkHistoryConsistency(eventPrincipal, secondaryEventPrincipal); + assert(readEventSucceeded); + eventPrincipal.recombine(secondaryEventPrincipal, branchIDsToReplace_[InEvent]); + eventPrincipal.mergeProvenanceRetrievers(secondaryEventPrincipal); + secondaryEventPrincipal.clearPrincipal(); + } else { + throw Exception(errors::MismatchedInputFiles, "RNTupleTempSource::readEvent_") + << eventPrincipal.id() << " is not found in the secondary input files\n"; + } + } + if (readAllProducts) { + eventPrincipal.readAllFromSourceAndMergeImmediately(); + } + } + + bool RNTupleTempSource::readIt(EventID const& id, EventPrincipal& eventPrincipal, StreamContext& streamContext) { + bool found = primaryFileSequence_->skipToItem(id.run(), id.luminosityBlock(), id.event()); + if (!found) + return false; + EventSourceSentry sentry(*this, streamContext); + readEvent_(eventPrincipal); + return true; + } + + InputSource::ItemTypeInfo RNTupleTempSource::getNextItemType() { + RunNumber_t run = IndexIntoFile::invalidRun; + LuminosityBlockNumber_t lumi = IndexIntoFile::invalidLumi; + EventNumber_t event = IndexIntoFile::invalidEvent; + InputSource::ItemType itemType = primaryFileSequence_->getNextItemType(run, lumi, event); + if (secondaryFileSequence_ && (ItemType::IsSynchronize != state())) { + if (itemType == ItemType::IsRun || itemType == ItemType::IsLumi || itemType == ItemType::IsEvent) { + if (!secondaryFileSequence_->containedInCurrentFile(run, lumi, event)) { + return ItemType::IsSynchronize; + } + } + } + return runHelper_->nextItemType(state(), itemType, run, lumi, event); + } + + std::pair RNTupleTempSource::resourceSharedWithDelayedReader_() { + return std::make_pair(resourceSharedWithDelayedReaderPtr_.get(), mutexSharedWithDelayedReader_.get()); + } + + // Rewind to before the first event that was read. + void RNTupleTempSource::rewind_() { primaryFileSequence_->rewind_(); } + + // Advance "offset" events. Offset can be positive or negative (or zero). + void RNTupleTempSource::skip(int offset) { primaryFileSequence_->skipEvents(offset); } + + bool RNTupleTempSource::goToEvent_(EventID const& eventID) { return primaryFileSequence_->goToEvent(eventID); } + + void RNTupleTempSource::fillDescriptions(ConfigurationDescriptions& descriptions) { + ParameterSetDescription desc; + + std::vector defaultStrings; + desc.setComment("Reads EDM/Root files."); + desc.addUntracked >("fileNames")->setComment("Names of files to be processed."); + desc.addUntracked >("secondaryFileNames", defaultStrings) + ->setComment("Names of secondary files to be processed."); + desc.addUntracked("needSecondaryFileNames", false) + ->setComment("If True, 'secondaryFileNames' must be specified and be non-empty."); + desc.addUntracked("overrideCatalog", std::string()); + desc.addUntracked("skipBadFiles", false) + ->setComment( + "True: Ignore any missing or unopenable input file.\n" + "False: Throw exception if missing or unopenable input file."); + desc.addUntracked("bypassVersionCheck", false) + ->setComment( + "True: Bypass release version check.\n" + "False: Throw exception if reading file in a release prior to the release in which the file was written."); + desc.addUntracked("treeMaxVirtualSize", -1) + ->setComment("Size of ROOT TTree TBasket cache. Affects performance."); + desc.addUntracked("dropDescendantsOfDroppedBranches", true) + ->setComment("If True, also drop on input any descendent of any branch dropped on input."); + desc.addUntracked("labelRawDataLikeMC", true) + ->setComment("If True: replace module label for raw data to match MC. Also use 'LHC' as process."); + desc.addUntracked("delayReadingEventProducts", true) + ->setComment( + "If True: do not read a data product from the file until it is requested. If False: all event data " + "products are read upfront."); + ProductSelectorRules::fillDescription(desc, "inputCommands"); + InputSource::fillDescription(desc); + RootPrimaryFileSequence::fillDescription(desc); + InputSourceRunHelperBase::fillDescription(desc); + + descriptions.add("source", desc); + } + + bool RNTupleTempSource::randomAccess_() const { return true; } + + ProcessingController::ForwardState RNTupleTempSource::forwardState_() const { + return primaryFileSequence_->forwardState(); + } + + ProcessingController::ReverseState RNTupleTempSource::reverseState_() const { + return primaryFileSequence_->reverseState(); + } +} // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempInput/src/RNTupleTempSource.h b/FWIO/RNTupleTempInput/src/RNTupleTempSource.h new file mode 100644 index 0000000000000..e57cf5433a665 --- /dev/null +++ b/FWIO/RNTupleTempInput/src/RNTupleTempSource.h @@ -0,0 +1,103 @@ +#ifndef FWIO_RNTupleTempInput_RNTupleTempSource_h +#define FWIO_RNTupleTempInput_RNTupleTempSource_h + +/*---------------------------------------------------------------------- + +RNTupleTempSource: This is an InputSource + +----------------------------------------------------------------------*/ + +#include "DataFormats/Provenance/interface/BranchType.h" +#include "FWCore/Catalog/interface/InputFileCatalog.h" +#include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/Framework/interface/ProcessingController.h" +#include "FWCore/Framework/interface/ProductSelectorRules.h" +#include "FWCore/Framework/interface/InputSource.h" +#include "FWCore/Utilities/interface/propagate_const.h" +#include "IOPool/Common/interface/RootServiceChecker.h" + +#include +#include +#include +#include + +namespace edm { + class ConfigurationDescriptions; + class FileCatalogItem; + class InputSourceRunHelperBase; +} // namespace edm +namespace edm::rntuple_temp { + class RootPrimaryFileSequence; + class RootSecondaryFileSequence; + + class RNTupleTempSource : public InputSource { + public: + explicit RNTupleTempSource(ParameterSet const& pset, InputSourceDescription const& desc); + ~RNTupleTempSource() override; + using InputSource::processHistoryRegistryForUpdate; + using InputSource::productRegistryUpdate; + + // const accessors + bool skipBadFiles() const { return skipBadFiles_; } + bool dropDescendants() const { return dropDescendants_; } + bool bypassVersionCheck() const { return bypassVersionCheck_; } + bool labelRawDataLikeMC() const { return labelRawDataLikeMC_; } + bool delayReadingEventProducts() const { return delayReadingEventProducts_; } + unsigned int nStreams() const { return nStreams_; } + int treeMaxVirtualSize() const { return treeMaxVirtualSize_; } + ProductSelectorRules const& productSelectorRules() const { return productSelectorRules_; } + InputSourceRunHelperBase* runHelper() { return runHelper_.get(); } + + static void fillDescriptions(ConfigurationDescriptions& descriptions); + + protected: + ItemTypeInfo getNextItemType() override; + void readLuminosityBlock_(LuminosityBlockPrincipal& lumiPrincipal) override; + std::shared_ptr readLuminosityBlockAuxiliary_() override; + void readEvent_(EventPrincipal& eventPrincipal) override; + + private: + std::shared_ptr readRunAuxiliary_() override; + void readRun_(RunPrincipal& runPrincipal) override; + void fillProcessBlockHelper_() override; + bool nextProcessBlock_(ProcessBlockPrincipal&) override; + void readProcessBlock_(ProcessBlockPrincipal&) override; + std::shared_ptr readFile_() override; + void closeFile_() override; + void endJob() override; + bool readIt(EventID const& id, EventPrincipal& eventPrincipal, StreamContext& streamContext) override; + void skip(int offset) override; + bool goToEvent_(EventID const& eventID) override; + void rewind_() override; + bool randomAccess_() const override; + ProcessingController::ForwardState forwardState_() const override; + ProcessingController::ReverseState reverseState_() const override; + + std::pair resourceSharedWithDelayedReader_() override; + + RootServiceChecker rootServiceChecker_; + InputFileCatalog catalog_; + InputFileCatalog secondaryCatalog_; + edm::propagate_const> secondaryRunPrincipal_; + edm::propagate_const> secondaryLumiPrincipal_; + std::vector>> secondaryEventPrincipals_; + std::array, NumBranchTypes> branchIDsToReplace_; + + unsigned int nStreams_; + bool skipBadFiles_; + bool bypassVersionCheck_; + int const treeMaxVirtualSize_; + ProductSelectorRules productSelectorRules_; + bool dropDescendants_; + bool labelRawDataLikeMC_; + bool delayReadingEventProducts_; + + edm::propagate_const> runHelper_; + std::unique_ptr + resourceSharedWithDelayedReaderPtr_; // We do not use propagate_const because the acquirer is itself mutable. + std::shared_ptr mutexSharedWithDelayedReader_; + edm::propagate_const> primaryFileSequence_; + edm::propagate_const> secondaryFileSequence_; + }; // class RNTupleTempSource +} // namespace edm::rntuple_temp +#endif diff --git a/FWIO/RNTupleTempInput/src/RootDelayedReader.cc b/FWIO/RNTupleTempInput/src/RootDelayedReader.cc new file mode 100644 index 0000000000000..68406782e7539 --- /dev/null +++ b/FWIO/RNTupleTempInput/src/RootDelayedReader.cc @@ -0,0 +1,90 @@ +/*---------------------------------------------------------------------- +----------------------------------------------------------------------*/ + +#include "RootDelayedReader.h" +#include "InputFile.h" +#include "DataFormats/Common/interface/EDProductGetter.h" +#include "DataFormats/Common/interface/RefCoreStreamer.h" + +#include "FWCore/Framework/interface/SharedResourcesAcquirer.h" +#include "FWCore/Framework/interface/SharedResourcesRegistry.h" + +#include "IOPool/Common/interface/getWrapperBasePtr.h" + +#include "FWCore/Utilities/interface/EDMException.h" + +#include "TBranch.h" +#include "TClass.h" + +#include + +namespace edm::rntuple_temp { + + RootDelayedReader::RootDelayedReader(RootRNTuple const& tree, std::shared_ptr filePtr, InputType inputType) + : tree_(tree), filePtr_(filePtr), nextReader_(), inputType_(inputType) { + if (inputType == InputType::Primary) { + auto resources = SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader(); + resourceAcquirer_ = std::make_unique(std::move(resources.first)); + mutex_ = resources.second; + } + } + + RootDelayedReader::~RootDelayedReader() {} + + std::pair RootDelayedReader::sharedResources_() const { + return std::make_pair(resourceAcquirer_.get(), mutex_.get()); + } + + void RootDelayedReader::readAllProductsNow(EDProductGetter const* ep) {} + + std::shared_ptr RootDelayedReader::getProduct_(BranchID const& k, EDProductGetter const* ep) { + if (lastException_) { + try { + std::rethrow_exception(lastException_); + } catch (edm::Exception const& e) { + //avoid growing the context each time the exception is rethrown. + auto copy = e; + copy.addContext("Rethrowing an exception that happened on a different read request."); + throw copy; + } catch (cms::Exception& e) { + //If we do anything here to 'copy', we would lose the actual type of the exception. + e.addContext("Rethrowing an exception that happened on a different read request."); + throw; + } + } + auto branchInfo = getBranchInfo(k); + if (not branchInfo) { + if (nextReader_) { + return nextReader_->getProduct(k, ep); + } else { + return std::shared_ptr(); + } + } + TBranch* br = branchInfo->productBranch_; + if (br == nullptr) { + if (nextReader_) { + return nextReader_->getProduct(k, ep); + } else { + return std::shared_ptr(); + } + } + + RefCoreStreamerGuard guard(ep); + std::unique_ptr edp = branchInfo->newWrapper(); + void* edpPtr = edp.get(); + branchInfo->productBranch_->SetAddress(&edpPtr); + + try { + //Run, Lumi, and ProcessBlock only have 1 entry number, which is index 0 + tree_.getEntry(br, tree_.entryNumberForIndex(tree_.branchType() == InEvent ? ep->transitionIndex() : 0)); + } catch (...) { + lastException_ = std::current_exception(); + std::rethrow_exception(lastException_); + } + if (tree_.branchType() == InEvent) { + // CMS-THREADING For the primary input source calls to this function need to be serialized + InputFile::reportReadBranch(inputType_, std::string(br->GetName())); + } + return edp; + } +} // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempInput/src/RootDelayedReader.h b/FWIO/RNTupleTempInput/src/RootDelayedReader.h new file mode 100644 index 0000000000000..3f2ec7a57ae4a --- /dev/null +++ b/FWIO/RNTupleTempInput/src/RootDelayedReader.h @@ -0,0 +1,75 @@ +#ifndef IOPool_Input_RootDelayedReader_h +#define IOPool_Input_RootDelayedReader_h + +/*---------------------------------------------------------------------- + +RootDelayedReader.h // used by ROOT input sources + +----------------------------------------------------------------------*/ + +#include "DataFormats/Provenance/interface/BranchID.h" +#include "FWCore/Framework/interface/DelayedReader.h" +#include "FWCore/Utilities/interface/InputType.h" +#include "FWCore/Utilities/interface/propagate_const.h" +#include "FWCore/Utilities/interface/thread_safety_macros.h" +#include "RootRNTuple.h" +#include "RootDelayedReaderBase.h" + +#include +#include +#include +#include + +class TClass; +namespace edm { + class InputFile; + class SharedResourcesAcquirer; + class Exception; +} // namespace edm +namespace edm::rntuple_temp { + class RootRNTuple; + //------------------------------------------------------------ + // Class RootDelayedReader: + // + + class RootDelayedReader : public RootDelayedReaderBase { + public: + typedef rootrntuple::BranchInfo BranchInfo; + typedef rootrntuple::BranchMap BranchMap; + typedef rootrntuple::EntryNumber EntryNumber; + RootDelayedReader(RootRNTuple const& tree, std::shared_ptr filePtr, InputType inputType); + + ~RootDelayedReader() override; + + RootDelayedReader(RootDelayedReader const&) = delete; // Disallow copying and moving + RootDelayedReader& operator=(RootDelayedReader const&) = delete; // Disallow copying and moving + + void readAllProductsNow(EDProductGetter const* ep) override; + + private: + std::shared_ptr getProduct_(BranchID const& k, EDProductGetter const* ep) override; + void mergeReaders_(DelayedReader* other) override { nextReader_ = other; } + void reset_() override { nextReader_ = nullptr; } + std::pair sharedResources_() const override; + + BranchMap const& branches() const { return tree_.branches(); } + BranchInfo const* getBranchInfo(BranchID const& k) const { return branches().find(k); } + // NOTE: filePtr_ appears to be unused, but is needed to prevent + // the file containing the branch from being reclaimed. + RootRNTuple const& tree_; + edm::propagate_const> filePtr_; + edm::propagate_const nextReader_; + std::unique_ptr + resourceAcquirer_; // We do not use propagate_const because the acquirer is itself mutable. + std::shared_ptr mutex_; + InputType inputType_; + + //If a fatal exception happens we need to make a copy so we can + // rethrow that exception on other threads. This avoids TTree + // non-exception safety problems on later calls to TTree. + //All uses of the ROOT file are serialized + CMS_SA_ALLOW mutable std::exception_ptr lastException_; + }; // class RootDelayedReader + //------------------------------------------------------------ +} // namespace edm::rntuple_temp +#endif diff --git a/FWIO/RNTupleTempInput/src/RootDelayedReaderBase.h b/FWIO/RNTupleTempInput/src/RootDelayedReaderBase.h new file mode 100644 index 0000000000000..6d4607bd6c9db --- /dev/null +++ b/FWIO/RNTupleTempInput/src/RootDelayedReaderBase.h @@ -0,0 +1,70 @@ +#ifndef FWIO_RNTupleTempInput_RootDelayedReaderBase_h +#define FWIO_RNTupleTempInput_RootDelayedReaderBase_h + +/*---------------------------------------------------------------------- + +RootDelayedReaderBase.h // used by ROOT input sources + +----------------------------------------------------------------------*/ + +#include "DataFormats/Provenance/interface/BranchID.h" +#include "FWCore/Framework/interface/DelayedReader.h" +#include "FWCore/Utilities/interface/InputType.h" +#include "FWCore/Utilities/interface/propagate_const.h" +#include "FWCore/Utilities/interface/thread_safety_macros.h" +#include "RootRNTuple.h" + +#include +#include +#include +#include + +class TClass; +namespace edm { + class SharedResourcesAcquirer; + class Exception; +} // namespace edm +namespace edm::rntuple_temp { + class InputFile; + class RootRNTuple; + + //------------------------------------------------------------ + // Class RootDelayedReaderBase: pretends to support file reading. + // + + class RootDelayedReaderBase : public DelayedReader { + public: + RootDelayedReaderBase() = default; + + ~RootDelayedReaderBase() override = default; + + RootDelayedReaderBase(RootDelayedReaderBase const&) = delete; // Disallow copying and moving + RootDelayedReaderBase& operator=(RootDelayedReaderBase const&) = delete; // Disallow copying and moving + + signalslot::Signal const* preEventReadFromSourceSignal() + const final { + return preEventReadFromSourceSignal_; + } + signalslot::Signal const* postEventReadFromSourceSignal() + const final { + return postEventReadFromSourceSignal_; + } + + void setSignals( + signalslot::Signal const* preEventReadSource, + signalslot::Signal const* postEventReadSource) { + preEventReadFromSourceSignal_ = preEventReadSource; + postEventReadFromSourceSignal_ = postEventReadSource; + } + + virtual void readAllProductsNow(EDProductGetter const* ep) = 0; + + private: + signalslot::Signal const* preEventReadFromSourceSignal_ = + nullptr; + signalslot::Signal const* postEventReadFromSourceSignal_ = + nullptr; + }; // class RootDelayedReaderBase + //------------------------------------------------------------ +} // namespace edm::rntuple_temp +#endif diff --git a/FWIO/RNTupleTempInput/src/RootEmbeddedFileSequence.cc b/FWIO/RNTupleTempInput/src/RootEmbeddedFileSequence.cc new file mode 100644 index 0000000000000..12e4abcf2281c --- /dev/null +++ b/FWIO/RNTupleTempInput/src/RootEmbeddedFileSequence.cc @@ -0,0 +1,393 @@ +/*---------------------------------------------------------------------- +----------------------------------------------------------------------*/ +#include "EmbeddedRNTupleTempSource.h" +#include "InputFile.h" +#include "RootFile.h" +#include "RootEmbeddedFileSequence.h" +#include "RootRNTuple.h" + +#include "DataFormats/Provenance/interface/BranchID.h" +#include "DataFormats/Provenance/interface/BranchIDListHelper.h" +#include "DataFormats/Provenance/interface/ThinnedAssociationsHelper.h" +#include "FWCore/Catalog/interface/InputFileCatalog.h" +#include "FWCore/Catalog/interface/SiteLocalConfig.h" +#include "FWCore/Framework/interface/InputSource.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" +#include "FWCore/ServiceRegistry/interface/Service.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" + +#include "CLHEP/Random/RandFlat.h" + +#include +#include +#include + +namespace { + std::atomic badFilesSkipped_{0}; + auto operator"" _uz(unsigned long long i) -> std::size_t { return std::size_t{i}; } // uz will be in C++23 +} // namespace + +namespace edm { + class EventPrincipal; +} +namespace edm::rntuple_temp { + + RootEmbeddedFileSequence::RootEmbeddedFileSequence(ParameterSet const& pset, + EmbeddedRNTupleTempSource& input, + InputFileCatalog const& catalog) + : RootInputFileSequence(pset, catalog), + input_(input), + orderedProcessHistoryIDs_(), + sequential_(pset.getUntrackedParameter("sequential", false)), + sameLumiBlock_(pset.getUntrackedParameter("sameLumiBlock", false)), + fptr_(nullptr), + eventsRemainingInFile_(0), + // The default value provided as the second argument to the getUntrackedParameter function call + // is not used when the ParameterSet has been validated and the parameters are not optional + // in the description. This is currently true when RNTupleTempSource is the primary input source. + // The modules that use RNTupleTempSource as a SecSource have not defined their fillDescriptions function + // yet, so the ParameterSet does not get validated yet. As soon as all the modules with a SecSource + // have defined descriptions, the defaults in the getUntrackedParameterSet function calls can + // and should be deleted from the code. + initialNumberOfEventsToSkip_(pset.getUntrackedParameter("skipEvents", 0U)), + treeCacheSize_(pset.getUntrackedParameter("cacheSize", rootrntuple::defaultCacheSize)), + enablePrefetching_(false), + enforceGUIDInFileName_(pset.getUntrackedParameter("enforceGUIDInFileName", false)), + maxFileSkips_(pset.getUntrackedParameter("maxFileSkips", std::min(3_uz, numberOfFiles()))) { + if (noFiles()) { + throw Exception(errors::NoSecondaryFiles) + << "RootEmbeddedFileSequence no input files specified for secondary input source.\n"; + } + // + // The SiteLocalConfig controls the TTreeCache size and the prefetching settings. + Service pSLC; + if (pSLC.isAvailable()) { + if (treeCacheSize_ != 0U && pSLC->sourceTTreeCacheSize()) { + treeCacheSize_ = *(pSLC->sourceTTreeCacheSize()); + } + enablePrefetching_ = pSLC->enablePrefetching(); + } + + // Set the pointer to the function that reads an event. + if (sameLumiBlock_) { + if (sequential_) { + fptr_ = &RootEmbeddedFileSequence::readOneSequentialWithID; + } else { + fptr_ = &RootEmbeddedFileSequence::readOneRandomWithID; + } + } else { + if (sequential_) { + fptr_ = &RootEmbeddedFileSequence::readOneSequential; + } else { + fptr_ = &RootEmbeddedFileSequence::readOneRandom; + } + } + + // For the secondary input source we do not stage in. + if (sequential_) { + // We open the first file + if (!atFirstFile()) { + setAtFirstFile(); + initFile(false); + } + assert(rootFile()); + rootFile()->setAtEventEntry(IndexIntoFile::invalidEntry); + if (!sameLumiBlock_) { + skipEntries(initialNumberOfEventsToSkip_); + } + } else { + // We randomly choose the first file to open. + // We cannot use the random number service yet. + std::ifstream f("/dev/urandom"); + unsigned int seed; + f.read(reinterpret_cast(&seed), sizeof(seed)); + std::default_random_engine dre(seed); + std::uniform_int_distribution distribution(0, numberOfFiles() - 1); + while (!rootFile() && badFilesSkipped_ < maxFileSkips_) { + int offset = distribution(dre); + setAtFileSequenceNumber(offset); + initFile(input_.skipBadFiles()); + if (not rootFile()) { + ++badFilesSkipped_; + } + } + } + if (rootFile()) { + std::vector processOrder; + processingOrderMerge(input_.processHistoryRegistry(), processOrder); + input_.productRegistryUpdate().updateFromInput(rootFile()->productRegistry()->productList(), processOrder); + } else { + throw Exception(errors::FileOpenError) << "RootEmbeddedFileSequence::RootEmbeddedFileSequence(): " + << " input file retries exhausted.\n"; + } + } + + RootEmbeddedFileSequence::~RootEmbeddedFileSequence() {} + + void RootEmbeddedFileSequence::endJob() { closeFile(); } + + void RootEmbeddedFileSequence::closeFile_() { + // delete the RootFile object. + if (rootFile()) { + rootFile().reset(); + } + } + + void RootEmbeddedFileSequence::initFile_(bool skipBadFiles) { + initTheFile(skipBadFiles, false, nullptr, "mixingFiles", InputType::SecondarySource); + } + + RootEmbeddedFileSequence::RootFileSharedPtr RootEmbeddedFileSequence::makeRootFile( + std::shared_ptr filePtr) { + size_t currentIndexIntoFile = sequenceNumberOfFile(); + return std::make_shared(RootFile::FileOptions{.fileName = fileNames()[0], + .logicalFileName = logicalFileName(), + .filePtr = filePtr, + .bypassVersionCheck = input_.bypassVersionCheck(), + .enforceGUIDInFileName = enforceGUIDInFileName_}, + InputType::SecondarySource, + RootFile::ProcessingOptions{}, + RootFile::TTreeOptions{.treeCacheSize = treeCacheSize_, + .treeMaxVirtualSize = input_.treeMaxVirtualSize(), + .enablePrefetching = enablePrefetching_}, + RootFile::ProductChoices{.productSelectorRules = input_.productSelectorRules()}, + RootFile::CrossFileInfo{.runHelper = input_.runHelper(), + .indexesIntoFiles = indexesIntoFiles(), + .currentIndexIntoFile = currentIndexIntoFile}, + input_.nStreams(), + input_.processHistoryRegistryForUpdate(), + orderedProcessHistoryIDs_); + } + + void RootEmbeddedFileSequence::skipEntries(unsigned int offset) { + // offset is decremented by the number of events actually skipped. + bool completed = rootFile()->skipEntries(offset); + while (!completed) { + setAtNextFile(); + if (noMoreFiles()) { + setAtFirstFile(); + } + initFile(false); + assert(rootFile()); + rootFile()->setAtEventEntry(IndexIntoFile::invalidEntry); + completed = rootFile()->skipEntries(offset); + } + } + + bool RootEmbeddedFileSequence::readOneSequential( + EventPrincipal& cache, size_t& fileNameHash, CLHEP::HepRandomEngine*, EventID const*, bool recycleFiles) { + assert(rootFile()); + bool found = rootFile()->nextEventEntry(); + if (found) { + auto [found2, succeeded] = rootFile()->readCurrentEvent(cache); + found = found2; + } + if (!found) { + setAtNextFile(); + if (noMoreFiles()) { + if (recycleFiles) { + setAtFirstFile(); + } else { + return false; + } + } + initFile(false); + assert(rootFile()); + rootFile()->setAtEventEntry(IndexIntoFile::invalidEntry); + return readOneSequential(cache, fileNameHash, nullptr, nullptr, recycleFiles); + } + fileNameHash = lfnHash(); + return true; + } + + bool RootEmbeddedFileSequence::readOneSequentialWithID( + EventPrincipal& cache, size_t& fileNameHash, CLHEP::HepRandomEngine*, EventID const* idp, bool recycleFiles) { + assert(idp); + EventID const& id = *idp; + int offset = initialNumberOfEventsToSkip_; + initialNumberOfEventsToSkip_ = 0; + if (offset > 0) { + assert(rootFile()); + while (offset > 0) { + bool found = readOneSequentialWithID(cache, fileNameHash, nullptr, idp, recycleFiles); + if (!found) { + return false; + } + --offset; + } + } + assert(rootFile()); + if (noMoreFiles() || rootFile()->indexIntoFileIter().run() != id.run() || + rootFile()->indexIntoFileIter().lumi() != id.luminosityBlock()) { + bool found = skipToItem(id.run(), id.luminosityBlock(), 0, 0, false); + if (!found) { + return false; + } + } + assert(rootFile()); + bool found = rootFile()->setEntryAtNextEventInLumi(id.run(), id.luminosityBlock()); + if (found) { + auto [found2, succeeded] = rootFile()->readCurrentEvent(cache); + found = found2; + } + if (!found) { + found = skipToItemInNewFile(id.run(), id.luminosityBlock(), 0); + if (!found) { + return false; + } + return readOneSequentialWithID(cache, fileNameHash, nullptr, idp, recycleFiles); + } + fileNameHash = lfnHash(); + return true; + } + + void RootEmbeddedFileSequence::readOneSpecified(EventPrincipal& cache, + size_t& fileNameHash, + SecondaryEventIDAndFileInfo const& idx) { + EventID const& id = idx.eventID(); + bool found = skipToItem(id.run(), id.luminosityBlock(), id.event(), idx.fileNameHash()); + if (!found) { + throw Exception(errors::NotFound) << "RootEmbeddedFileSequence::readOneSpecified(): Secondary Input files" + << " do not contain specified event:\n" + << id << " in file id " << idx.fileNameHash() << "\n"; + } + assert(rootFile()); + auto [found2, succeeded] = rootFile()->readCurrentEvent(cache); + found = found2; + assert(found); + fileNameHash = idx.fileNameHash(); + if (fileNameHash == 0U) { + fileNameHash = lfnHash(); + } + } + + bool RootEmbeddedFileSequence::readOneRandom( + EventPrincipal& cache, size_t& fileNameHash, CLHEP::HepRandomEngine* engine, EventID const*, bool) { + assert(rootFile()); + assert(engine); + unsigned int currentSeqNumber = sequenceNumberOfFile(); + while (eventsRemainingInFile_ == 0) { + bool opened{false}; + while (!opened && badFilesSkipped_ < maxFileSkips_) { + unsigned int newSeqNumber = CLHEP::RandFlat::shootInt(engine, fileCatalogItems().size()); + setAtFileSequenceNumber(newSeqNumber); + if (newSeqNumber != currentSeqNumber) { + initFile(input_.skipBadFiles()); + currentSeqNumber = newSeqNumber; + } + if (rootFile()) { + eventsRemainingInFile_ = rootFile()->eventTree().entries(); + if (eventsRemainingInFile_ == 0) { + if (!input_.skipBadFiles()) { + throw Exception(errors::NotFound) << "RootEmbeddedFileSequence::readOneRandom(): Secondary Input file " + << fileNames()[0] << " contains no events.\n"; + } + LogWarning("RootEmbeddedFileSequence") << "RootEmbeddedFileSequence::readOneRandom(): Secondary Input file " + << fileNames()[0] << " contains no events and will be skipped.\n"; + ++badFilesSkipped_; + } else { + opened = true; + } + } else { + if (newSeqNumber != currentSeqNumber) { + ++badFilesSkipped_; + } + } + } + if (not opened) { + throw Exception(errors::FileOpenError) << "RootEmbeddedFileSequence::readOneRandom(): " + << " input file retries exhausted.\n"; + } + rootFile()->setAtEventEntry(CLHEP::RandFlat::shootInt(engine, eventsRemainingInFile_) - 1); + } + rootFile()->nextEventEntry(); + + auto [found, succeeded] = rootFile()->readCurrentEvent(cache); + if (!found) { + rootFile()->setAtEventEntry(0); + auto [found2, succeeded] = rootFile()->readCurrentEvent(cache); + assert(found2); + } + fileNameHash = lfnHash(); + --eventsRemainingInFile_; + return true; + } + + bool RootEmbeddedFileSequence::readOneRandomWithID(EventPrincipal& cache, + size_t& fileNameHash, + CLHEP::HepRandomEngine* engine, + EventID const* idp, + bool recycleFiles) { + assert(engine); + assert(idp); + EventID const& id = *idp; + if (noMoreFiles() || !rootFile() || rootFile()->indexIntoFileIter().run() != id.run() || + rootFile()->indexIntoFileIter().lumi() != id.luminosityBlock()) { + bool found = skipToItem(id.run(), id.luminosityBlock(), 0); + if (!found) { + return false; + } + int eventsInLumi = 0; + assert(rootFile()); + while (rootFile()->setEntryAtNextEventInLumi(id.run(), id.luminosityBlock())) + ++eventsInLumi; + found = skipToItem(id.run(), id.luminosityBlock(), 0); + assert(found); + int eventInLumi = CLHEP::RandFlat::shootInt(engine, eventsInLumi); + for (int i = 0; i < eventInLumi; ++i) { + bool foundEventInLumi = rootFile()->setEntryAtNextEventInLumi(id.run(), id.luminosityBlock()); + assert(foundEventInLumi); + } + } + assert(rootFile()); + bool found = rootFile()->setEntryAtNextEventInLumi(id.run(), id.luminosityBlock()); + if (found) { + auto [found2, succeeded] = rootFile()->readCurrentEvent(cache); + found = found2; + } + if (!found) { + found = rootFile()->setEntryAtItem(id.run(), id.luminosityBlock(), 0); + if (!found) { + return false; + } + return readOneRandomWithID(cache, fileNameHash, engine, idp, recycleFiles); + } + fileNameHash = lfnHash(); + return true; + } + + bool RootEmbeddedFileSequence::readOneEvent(EventPrincipal& cache, + size_t& fileNameHash, + CLHEP::HepRandomEngine* engine, + EventID const* id, + bool recycleFiles) { + assert(!sameLumiBlock_ || id != nullptr); + assert(sequential_ || engine != nullptr); + return (this->*fptr_)(cache, fileNameHash, engine, id, recycleFiles); + } + + void RootEmbeddedFileSequence::fillDescription(ParameterSetDescription& desc) { + desc.addUntracked("sequential", false) + ->setComment( + "True: loopEvents() reads events sequentially from beginning of first file.\n" + "False: loopEvents() first reads events beginning at random event. New files also chosen randomly"); + desc.addUntracked("sameLumiBlock", false) + ->setComment( + "True: loopEvents() reads events only in same lumi as the specified event.\n" + "False: loopEvents() reads events regardless of lumi."); + desc.addUntracked("skipEvents", 0U) + ->setComment( + "Skip the first 'skipEvents' events. Used only if 'sequential' is True and 'sameLumiBlock' is False"); + desc.addUntracked("maxFileSkips") + ->setComment( + "How many files to try if 'sequential' is False and 'skipBadFiles' is True.\n" + "Defaults to 3 (or # of files if smaller)."); + desc.addUntracked("cacheSize", rootrntuple::defaultCacheSize) + ->setComment("Size of ROOT TTree prefetch cache. Affects performance."); + desc.addUntracked("enforceGUIDInFileName", false) + ->setComment( + "True: file name part is required to be equal to the GUID of the file\n" + "False: file name can be anything"); + } +} // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempInput/src/RootEmbeddedFileSequence.h b/FWIO/RNTupleTempInput/src/RootEmbeddedFileSequence.h new file mode 100644 index 0000000000000..b36ba63142bde --- /dev/null +++ b/FWIO/RNTupleTempInput/src/RootEmbeddedFileSequence.h @@ -0,0 +1,79 @@ +#ifndef FWIO_RNTupleTempInput_RootEmbeddedFileSequence_h +#define FWIO_RNTupleTempInput_RootEmbeddedFileSequence_h + +/*---------------------------------------------------------------------- + +RootEmbeddedFileSequence: This is an InputSource + +----------------------------------------------------------------------*/ + +#include "RootInputFileSequence.h" +#include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/Sources/interface/VectorInputSource.h" +#include "DataFormats/Provenance/interface/ProcessHistoryID.h" + +#include +#include +#include + +namespace CLHEP { + class HepRandomEngine; +} + +namespace edm { + + class BranchID; + class FileCatalogItem; + class InputFileCatalog; + class ParameterSetDescription; +} // namespace edm +namespace edm::rntuple_temp { + class EmbeddedRNTupleTempSource; + class RootFile; + + class RootEmbeddedFileSequence : public RootInputFileSequence { + public: + explicit RootEmbeddedFileSequence(ParameterSet const& pset, + EmbeddedRNTupleTempSource& input, + InputFileCatalog const& catalog); + ~RootEmbeddedFileSequence() override; + + RootEmbeddedFileSequence(RootEmbeddedFileSequence const&) = delete; // Disallow copying and moving + RootEmbeddedFileSequence& operator=(RootEmbeddedFileSequence const&) = delete; // Disallow copying and moving + + void endJob(); + void skipEntries(unsigned int offset); + bool readOneEvent( + EventPrincipal& cache, size_t& fileNameHash, CLHEP::HepRandomEngine*, EventID const* id, bool recycleFiles); + bool readOneRandom(EventPrincipal& cache, size_t& fileNameHash, CLHEP::HepRandomEngine*, EventID const*, bool); + bool readOneRandomWithID( + EventPrincipal& cache, size_t& fileNameHash, CLHEP::HepRandomEngine*, EventID const* id, bool); + bool readOneSequential( + EventPrincipal& cache, size_t& fileNameHash, CLHEP::HepRandomEngine*, EventID const*, bool recycleFiles); + bool readOneSequentialWithID( + EventPrincipal& cache, size_t& fileNameHash, CLHEP::HepRandomEngine*, EventID const* id, bool); + void readOneSpecified(EventPrincipal& cache, size_t& fileNameHash, SecondaryEventIDAndFileInfo const& id); + + static void fillDescription(ParameterSetDescription& desc); + + private: + void closeFile_() override; + void initFile_(bool skipBadFiles) override; + RootFileSharedPtr makeRootFile(std::shared_ptr filePtr) override; + + EmbeddedRNTupleTempSource& input_; + + std::vector orderedProcessHistoryIDs_; + + bool sequential_; + bool sameLumiBlock_; + bool (RootEmbeddedFileSequence::*fptr_)(EventPrincipal&, size_t&, CLHEP::HepRandomEngine*, EventID const*, bool); + int eventsRemainingInFile_; + int initialNumberOfEventsToSkip_; + unsigned int treeCacheSize_; + bool enablePrefetching_; + bool enforceGUIDInFileName_; + unsigned int maxFileSkips_; + }; // class RootEmbeddedFileSequence +} // namespace edm::rntuple_temp +#endif diff --git a/FWIO/RNTupleTempInput/src/RootFile.cc b/FWIO/RNTupleTempInput/src/RootFile.cc new file mode 100644 index 0000000000000..00e27a0ef8906 --- /dev/null +++ b/FWIO/RNTupleTempInput/src/RootFile.cc @@ -0,0 +1,2531 @@ +/*---------------------------------------------------------------------- +----------------------------------------------------------------------*/ + +#include "RootFile.h" +#include "DuplicateChecker.h" +#include "InputFile.h" +#include "ProvenanceAdaptor.h" +#include "FWCore/Sources/interface/InputSourceRunHelper.h" +#include "RootDelayedReaderBase.h" + +#include "DataFormats/Common/interface/setIsMergeable.h" +#include "DataFormats/Common/interface/ThinnedAssociation.h" +#include "DataFormats/Provenance/interface/ProductDescription.h" +#include "DataFormats/Provenance/interface/BranchIDListHelper.h" +#include "DataFormats/Provenance/interface/BranchType.h" +#include "DataFormats/Provenance/interface/EventEntryInfo.h" +#include "DataFormats/Provenance/interface/ParameterSetBlob.h" +#include "DataFormats/Provenance/interface/ParentageRegistry.h" +#include "DataFormats/Provenance/interface/ProcessHistoryID.h" +#include "DataFormats/Provenance/interface/ProcessHistoryRegistry.h" +#include "DataFormats/Provenance/interface/ProductRegistry.h" +#include "DataFormats/Provenance/interface/processingOrderMerge.h" +#include "DataFormats/Provenance/interface/StoredMergeableRunProductMetadata.h" +#include "DataFormats/Provenance/interface/StoredProcessBlockHelper.h" +#include "DataFormats/Provenance/interface/StoredProductProvenance.h" +#include "DataFormats/Provenance/interface/ThinnedAssociationsHelper.h" +#include "DataFormats/Provenance/interface/RunID.h" +#include "FWCore/Common/interface/ProcessBlockHelper.h" +#include "FWCore/Framework/interface/FileBlock.h" +#include "FWCore/Framework/interface/EventPrincipal.h" +#include "FWCore/Framework/interface/ProductSelector.h" +#include "FWCore/Framework/interface/LuminosityBlockPrincipal.h" +#include "FWCore/Framework/interface/MergeableRunProductMetadata.h" +#include "FWCore/Framework/interface/ProcessBlockPrincipal.h" +#include "FWCore/Framework/interface/RunPrincipal.h" +#include "FWCore/Framework/interface/SharedResourcesAcquirer.h" +#include "FWCore/Framework/interface/SharedResourcesRegistry.h" +#include "FWCore/Framework/interface/DelayedReader.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/ParameterSet/interface/Registry.h" +#include "FWCore/Sources/interface/EventSkipperByID.h" +#include "FWCore/Sources/interface/DaqProvenanceHelper.h" +#include "FWCore/ServiceRegistry/interface/ServiceRegistry.h" +#include "FWCore/Utilities/interface/Algorithms.h" +#include "FWCore/Utilities/interface/do_nothing_deleter.h" +#include "FWCore/Utilities/interface/EDMException.h" +#include "FWCore/Utilities/interface/FriendlyName.h" +#include "FWCore/Utilities/interface/GlobalIdentifier.h" +#include "FWCore/Utilities/interface/ReleaseVersion.h" +#include "FWCore/Utilities/interface/stemFromPath.h" +#include "FWCore/Utilities/interface/thread_safety_macros.h" +#include "FWCore/Version/interface/GetReleaseVersion.h" +#include "IOPool/Common/interface/getWrapperBasePtr.h" + +#include "FWCore/Concurrency/interface/WaitingTaskHolder.h" +#include "FWCore/ServiceRegistry/interface/ModuleCallingContext.h" + +//used for backward compatibility +#include "DataFormats/Provenance/interface/EventAux.h" +#include "DataFormats/Provenance/interface/LuminosityBlockAux.h" +#include "DataFormats/Provenance/interface/RunAux.h" +#include "FWCore/ParameterSet/interface/ParameterSetConverter.h" + +#include "Rtypes.h" +#include "TClass.h" +#include "TString.h" +#include "TTree.h" +#include "TTreeCache.h" + +#include "ROOT/RNTuple.hxx" +#include "ROOT/RNTupleReader.hxx" + +#include +#include +#include +#include + +namespace edm::rntuple_temp { + + // Algorithm classes for making ProvenanceReader: + class MakeDummyProvenanceReader : public MakeProvenanceReader { + public: + std::unique_ptr makeReader(RootRNTuple& eventTree, + DaqProvenanceHelper const* daqProvenanceHelper) const override; + }; + class MakeOldProvenanceReader : public MakeProvenanceReader { + public: + MakeOldProvenanceReader(std::unique_ptr&& entryDescriptionMap) + : MakeProvenanceReader(), entryDescriptionMap_(std::move(entryDescriptionMap)) {} + std::unique_ptr makeReader(RootRNTuple& eventTree, + DaqProvenanceHelper const* daqProvenanceHelper) const override; + + private: + edm::propagate_const> entryDescriptionMap_; + }; + class MakeFullProvenanceReader : public MakeProvenanceReader { + public: + std::unique_ptr makeReader(RootRNTuple& eventTree, + DaqProvenanceHelper const* daqProvenanceHelper) const override; + }; + class MakeReducedProvenanceReader : public MakeProvenanceReader { + public: + MakeReducedProvenanceReader(std::vector const& parentageIDLookup) + : parentageIDLookup_(parentageIDLookup) {} + std::unique_ptr makeReader(RootRNTuple& eventTree, + DaqProvenanceHelper const* daqProvenanceHelper) const override; + + private: + std::vector const& parentageIDLookup_; + }; + + namespace { + void checkReleaseVersion(std::vector processHistoryVector, std::string const& fileName) { + std::string releaseVersion = getReleaseVersion(); + releaseversion::DecomposedReleaseVersion currentRelease(releaseVersion); + for (auto const& ph : processHistoryVector) { + for (auto const& pc : ph) { + if (releaseversion::isEarlierRelease(currentRelease, pc.releaseVersion())) { + throw Exception(errors::FormatIncompatibility) + << "The release you are using, " << getReleaseVersion() << " , predates\n" + << "a release (" << pc.releaseVersion() << ") used in writing the input file, " << fileName << ".\n" + << "Forward compatibility cannot be supported.\n"; + } + } + } + } + } // namespace + + // This is a helper class for IndexIntoFile. + class RootFileEventFinder : public IndexIntoFile::EventFinder { + public: + explicit RootFileEventFinder(RootRNTuple& eventTree) : eventTree_(eventTree) {} + ~RootFileEventFinder() override {} + + EventNumber_t getEventNumberOfEntry(rootrntuple::EntryNumber entry) const override { + rootrntuple::EntryNumber saveEntry = eventTree_.entryNumber(); + eventTree_.setEntryNumber(entry); + EventAuxiliary eventAux; + EventAuxiliary* pEvAux = &eventAux; + eventTree_.fillAux(pEvAux); + eventTree_.setEntryNumber(saveEntry); + return eventAux.event(); + } + + private: + RootRNTuple& eventTree_; + }; + + //--------------------------------------------------------------------- + RootFile::RootFile(FileOptions&& fileOptions, + InputType inputType, + ProcessingOptions&& processingOptions, + TTreeOptions&& ttreeOptions, + ProductChoices&& productChoices, + CrossFileInfo&& crossFileInfo, + unsigned int nStreams, + ProcessHistoryRegistry& processHistoryRegistry, + std::vector& orderedProcessHistoryIDs) + : file_(fileOptions.fileName), + logicalFile_(fileOptions.logicalFileName), + processHistoryRegistry_(&processHistoryRegistry), + filePtr_(fileOptions.filePtr), + eventSkipperByID_(processingOptions.eventSkipperByID), + fileFormatVersion_(), + fid_(), + indexIntoFileSharedPtr_(new IndexIntoFile), + indexIntoFile_(*indexIntoFileSharedPtr_), + indexIntoFileBegin_(indexIntoFile_.begin( + processingOptions.noRunLumiSort ? IndexIntoFile::entryOrder + : (processingOptions.noEventSort ? IndexIntoFile::firstAppearanceOrder + : IndexIntoFile::numericalOrder))), + indexIntoFileEnd_(indexIntoFileBegin_), + indexIntoFileIter_(indexIntoFileBegin_), + storedMergeableRunProductMetadata_((inputType == InputType::Primary) ? new StoredMergeableRunProductMetadata + : nullptr), + eventProcessHistoryIDs_(), + eventProcessHistoryIter_(eventProcessHistoryIDs_.begin()), + savedRunAuxiliary_(), + skipAnyEvents_(processingOptions.skipAnyEvents), + noRunLumiSort_(processingOptions.noRunLumiSort), + noEventSort_(processingOptions.noEventSort), + enforceGUIDInFileName_(fileOptions.enforceGUIDInFileName), + whyNotFastClonable_(0), + hasNewlyDroppedBranch_(), + branchListIndexesUnchanged_(false), + eventAuxCache_(), + eventTree_( + fileOptions.filePtr, InEvent, nStreams, ttreeOptions, rootrntuple::defaultLearningEntries, inputType), + lumiTree_(fileOptions.filePtr, + InLumi, + 1, + ttreeOptions.usingDefaultNonEventOptions(), + rootrntuple::defaultNonEventLearningEntries, + inputType), + runTree_(fileOptions.filePtr, + InRun, + 1, + ttreeOptions.usingDefaultNonEventOptions(), + rootrntuple::defaultNonEventLearningEntries, + inputType), + treePointers_(), + lastEventEntryNumberRead_(IndexIntoFile::invalidEntry), + productRegistry_(), + branchIDLists_(), + branchIDListHelper_(crossFileInfo.branchIDListHelper), + processBlockHelper_(crossFileInfo.processBlockHelper), + fileThinnedAssociationsHelper_(), + thinnedAssociationsHelper_(crossFileInfo.thinnedAssociationsHelper), + processingMode_(processingOptions.processingMode), + runHelper_(crossFileInfo.runHelper), + newBranchToOldBranch_(), + eventHistoryTree_(nullptr), + eventToProcessBlockIndexesBranch_( + inputType == InputType::Primary + ? eventTree_.tree()->GetBranch(poolNames::eventToProcessBlockIndexesBranchName().c_str()) + : nullptr), + history_(), + productDependencies_(new ProductDependencies), + duplicateChecker_(crossFileInfo.duplicateChecker), + provenanceAdaptor_(), + provenanceReaderMaker_(), + eventProductProvenanceRetrievers_(), + parentageIDLookup_(), + daqProvenanceHelper_(), + edProductClass_(TypeWithDict::byName("edm::WrapperBase").getClass()), + inputType_(inputType) { + hasNewlyDroppedBranch_.fill(false); + + treePointers_.resize(3); + treePointers_[InEvent] = &eventTree_; + treePointers_[InLumi] = &lumiTree_; + treePointers_[InRun] = &runTree_; + + // Read the metadata tree. + // We use a smart pointer so the tree will be deleted after use, and not kept for the life of the file. + std::unique_ptr metaDataTree(dynamic_cast(filePtr_->Get(poolNames::metaDataTreeName().c_str()))); + if (nullptr == metaDataTree.get()) { + throw Exception(errors::FileReadError) + << "Could not find tree " << poolNames::metaDataTreeName() << " in the input file.\n"; + } + + // To keep things simple, we just read in every possible branch that exists. + // We don't pay attention to which branches exist in which file format versions + + FileFormatVersion* fftPtr = &fileFormatVersion_; + if (metaDataTree->FindBranch(poolNames::fileFormatVersionBranchName().c_str()) != nullptr) { + TBranch* fft = metaDataTree->GetBranch(poolNames::fileFormatVersionBranchName().c_str()); + fft->SetAddress(&fftPtr); + rootrntuple::getEntry(fft, 0); + metaDataTree->SetBranchAddress(poolNames::fileFormatVersionBranchName().c_str(), &fftPtr); + } + + FileID* fidPtr = &fid_; + if (metaDataTree->FindBranch(poolNames::fileIdentifierBranchName().c_str()) != nullptr) { + metaDataTree->SetBranchAddress(poolNames::fileIdentifierBranchName().c_str(), &fidPtr); + } + + IndexIntoFile* iifPtr = &indexIntoFile_; + if (metaDataTree->FindBranch(poolNames::indexIntoFileBranchName().c_str()) != nullptr) { + metaDataTree->SetBranchAddress(poolNames::indexIntoFileBranchName().c_str(), &iifPtr); + } + + storedProcessBlockHelper_ = std::make_unique(); + StoredProcessBlockHelper& storedProcessBlockHelper = *storedProcessBlockHelper_; + StoredProcessBlockHelper* pStoredProcessBlockHelper = storedProcessBlockHelper_.get(); + if (inputType == InputType::Primary) { + if (metaDataTree->FindBranch(poolNames::processBlockHelperBranchName().c_str()) != nullptr) { + metaDataTree->SetBranchAddress(poolNames::processBlockHelperBranchName().c_str(), &pStoredProcessBlockHelper); + } + } + + StoredMergeableRunProductMetadata* smrc = nullptr; + if (inputType == InputType::Primary) { + smrc = &*storedMergeableRunProductMetadata_; + if (metaDataTree->FindBranch(poolNames::mergeableRunProductMetadataBranchName().c_str()) != nullptr) { + metaDataTree->SetBranchAddress(poolNames::mergeableRunProductMetadataBranchName().c_str(), &smrc); + } + } + + // Need to read to a temporary registry so we can do a translation of the BranchKeys. + // This preserves backward compatibility against friendly class name algorithm changes. + ProductRegistry inputProdDescReg; + ProductRegistry* ppReg = &inputProdDescReg; + metaDataTree->SetBranchAddress(poolNames::productDescriptionBranchName().c_str(), (&ppReg)); + + using PsetMap = std::map; + PsetMap psetMap; + { + auto rntuple = filePtr_->Get("ParameterSets"); + if(not rntuple) { + throw Exception(errors::FileReadError) + << "Could not find RNTuple 'ParameterSets' in the input file.\n"; + } + auto psets = ROOT::RNTupleReader::Open(*rntuple); + assert(psets.get()); + auto entry = psets->GetModel().CreateBareEntry(); + + std::pair idToBlob; + entry->BindRawPtr("IdToParameterSetsBlobs", &idToBlob); + + for (ROOT::NTupleSize_t i = 0; i < psets->GetNEntries(); ++i) { + psets->LoadEntry(i, *entry); + psetMap.insert(idToBlob); + } + } + + // backward compatibility + ProcessHistoryRegistry::collection_type pHistMap; + ProcessHistoryRegistry::collection_type* pHistMapPtr = &pHistMap; + if (metaDataTree->FindBranch(poolNames::processHistoryMapBranchName().c_str()) != nullptr) { + metaDataTree->SetBranchAddress(poolNames::processHistoryMapBranchName().c_str(), &pHistMapPtr); + } + + ProcessHistoryRegistry::vector_type pHistVector; + ProcessHistoryRegistry::vector_type* pHistVectorPtr = &pHistVector; + if (metaDataTree->FindBranch(poolNames::processHistoryBranchName().c_str()) != nullptr) { + metaDataTree->SetBranchAddress(poolNames::processHistoryBranchName().c_str(), &pHistVectorPtr); + } + + // backward compatibility + ProcessConfigurationVector processConfigurations; + ProcessConfigurationVector* procConfigVectorPtr = &processConfigurations; + if (metaDataTree->FindBranch(poolNames::processConfigurationBranchName().c_str()) != nullptr) { + metaDataTree->SetBranchAddress(poolNames::processConfigurationBranchName().c_str(), &procConfigVectorPtr); + } + + auto branchIDListsAPtr = std::make_unique(); + BranchIDLists* branchIDListsPtr = branchIDListsAPtr.get(); + if (metaDataTree->FindBranch(poolNames::branchIDListBranchName().c_str()) != nullptr) { + metaDataTree->SetBranchAddress(poolNames::branchIDListBranchName().c_str(), &branchIDListsPtr); + } + + ThinnedAssociationsHelper* thinnedAssociationsHelperPtr; // must remain in scope through getEntry() + if (inputType != InputType::SecondarySource) { + fileThinnedAssociationsHelper_ = + std::make_unique(); // propagate_const has no reset() function + thinnedAssociationsHelperPtr = fileThinnedAssociationsHelper_.get(); + if (metaDataTree->FindBranch(poolNames::thinnedAssociationsHelperBranchName().c_str()) != nullptr) { + metaDataTree->SetBranchAddress(poolNames::thinnedAssociationsHelperBranchName().c_str(), + &thinnedAssociationsHelperPtr); + } + } + + ProductDependencies* productDependenciesBuffer = productDependencies_.get(); + if (metaDataTree->FindBranch(poolNames::productDependenciesBranchName().c_str()) != nullptr) { + metaDataTree->SetBranchAddress(poolNames::productDependenciesBranchName().c_str(), &productDependenciesBuffer); + } + + // backward compatibility + std::vector* eventHistoryIDsPtr = &eventProcessHistoryIDs_; + if (metaDataTree->FindBranch(poolNames::eventHistoryBranchName().c_str()) != nullptr) { + metaDataTree->SetBranchAddress(poolNames::eventHistoryBranchName().c_str(), &eventHistoryIDsPtr); + } + + if (metaDataTree->FindBranch(poolNames::moduleDescriptionMapBranchName().c_str()) != nullptr) { + if (metaDataTree->GetBranch(poolNames::moduleDescriptionMapBranchName().c_str())->GetSplitLevel() != 0) { + metaDataTree->SetBranchStatus((poolNames::moduleDescriptionMapBranchName() + ".*").c_str(), false); + } else { + metaDataTree->SetBranchStatus(poolNames::moduleDescriptionMapBranchName().c_str(), false); + } + } + + // Here we read the metadata tree + rootrntuple::getEntry(metaDataTree.get(), 0); + + eventProcessHistoryIter_ = eventProcessHistoryIDs_.begin(); + + // Here we read the event history tree, if we have one. + readEventHistoryTree(); + + ParameterSetConverter::ParameterSetIdConverter psetIdConverter; + if (!fileFormatVersion().triggerPathsTracked()) { + ParameterSetConverter converter(psetMap, psetIdConverter, fileFormatVersion().parameterSetsByReference()); + } else { + // Merge into the parameter set registry. + pset::Registry& psetRegistry = *pset::Registry::instance(); + try { + for (auto const& psetEntry : psetMap) { + ParameterSet pset(psetEntry.second.pset()); + pset.setID(psetEntry.first); + // For thread safety, don't update global registries when a secondary source opens a file. + if (inputType != InputType::SecondarySource) { + psetRegistry.insertMapped(pset); + } + } + } catch (edm::Exception const& iExcept) { + if (iExcept.categoryCode() == edm::errors::Configuration) { + edm::Exception exception(edm::errors::FormatIncompatibility); + exception << iExcept.message(); + exception.addContext("Creating ParameterSets from file"); + throw exception; + } else { + throw; + } + } + } + std::shared_ptr mutableBranchIDLists; + if (!fileFormatVersion().splitProductIDs()) { + // Old provenance format input file. Create a provenance adaptor. + // propagate_const has no reset() function + provenanceAdaptor_ = std::make_unique( + inputProdDescReg, pHistMap, pHistVector, processConfigurations, psetIdConverter, true); + // Fill in the branchIDLists branch from the provenance adaptor + mutableBranchIDLists = provenanceAdaptor_->releaseBranchIDLists(); + } else { + if (!fileFormatVersion().triggerPathsTracked()) { + // New provenance format, but change in ParameterSet Format. Create a provenance adaptor. + // propagate_const has no reset() function + provenanceAdaptor_ = std::make_unique( + inputProdDescReg, pHistMap, pHistVector, processConfigurations, psetIdConverter, false); + } + // New provenance format input file. The branchIDLists branch was read directly from the input file. + if (metaDataTree->FindBranch(poolNames::branchIDListBranchName().c_str()) == nullptr) { + throw Exception(errors::EventCorruption) << "Failed to find branchIDLists branch in metaData tree.\n"; + } + mutableBranchIDLists.reset(branchIDListsAPtr.release()); + } + + if (fileFormatVersion().hasThinnedAssociations()) { + if (metaDataTree->FindBranch(poolNames::thinnedAssociationsHelperBranchName().c_str()) == nullptr) { + throw Exception(errors::EventCorruption) + << "Failed to find thinnedAssociationsHelper branch in metaData tree.\n"; + } + } + + if (!fileOptions.bypassVersionCheck) { + checkReleaseVersion(pHistVector, file()); + } + + if (productChoices.labelRawDataLikeMC) { + std::string const rawData("FEDRawDataCollection"); + std::string const source("source"); + ProductRegistry::ProductList& pList = inputProdDescReg.productListUpdator(); + BranchKey finder(rawData, source, "", ""); + ProductRegistry::ProductList::iterator it = pList.lower_bound(finder); + if (it != pList.end() && it->first.friendlyClassName() == rawData && it->first.moduleLabel() == source) { + // We found raw data with a module label of source. + // We need to change the module label and process name. + // Create helper. + it->second.init(); + // propagate_const has no reset() function + daqProvenanceHelper_ = std::make_unique(it->second.unwrappedTypeID()); + // Create the new branch description + ProductDescription const& newBD = daqProvenanceHelper_->productDescription(); + // Save info from the old and new branch descriptions + daqProvenanceHelper_->saveInfo(it->second, newBD); + // Map the new branch name to the old branch name. + newBranchToOldBranch_.insert(std::make_pair(newBD.branchName(), it->second.branchName())); + // Remove the old branch description from the product Registry. + pList.erase(it); + // Check that there was only one. + it = pList.lower_bound(finder); + assert(!(it != pList.end() && it->first.friendlyClassName() == rawData && it->first.moduleLabel() == source)); + // Insert the new branch description into the product registry. + inputProdDescReg.copyProduct(newBD); + // Fix up other per file metadata. + daqProvenanceHelper_->fixMetaData(processConfigurations, pHistVector); + daqProvenanceHelper_->fixMetaData(*mutableBranchIDLists); + daqProvenanceHelper_->fixMetaData(*productDependencies_); + } + branchIDLists_ = std::move(mutableBranchIDLists); + } + + for (auto const& history : pHistVector) { + processHistoryRegistry.registerProcessHistory(history); + } + + eventTree_.trainCache(BranchTypeToAuxiliaryBranchName(InEvent).c_str()); + + // Update the branch id info. This has to be done before validateFile since + // depending on the file format, the branchIDListHelper_ may have its fixBranchListIndexes call made + if (inputType == InputType::Primary) { + branchListIndexesUnchanged_ = branchIDListHelper_->updateFromInput(*branchIDLists_); + } + + validateFile(inputType, processingOptions.usingGoToEvent, orderedProcessHistoryIDs); + + // Here, we make the class that will make the ProvenanceReader + // It reads whatever trees it needs. + // propagate_const has no reset() function + provenanceReaderMaker_ = std::unique_ptr(makeProvenanceReaderMaker(inputType).release()); + + // Merge into the hashed registries. + if (eventSkipperByID_ && eventSkipperByID_->somethingToSkip()) { + whyNotFastClonable_ += FileBlock::EventsOrLumisSelectedByID; + } + + initializeDuplicateChecker(crossFileInfo.indexesIntoFiles, crossFileInfo.currentIndexIntoFile); + indexIntoFileIter_ = indexIntoFileBegin_ = indexIntoFile_.begin( + processingOptions.noRunLumiSort + ? IndexIntoFile::entryOrder + : (processingOptions.noEventSort ? IndexIntoFile::firstAppearanceOrder : IndexIntoFile::numericalOrder)); + indexIntoFileEnd_ = indexIntoFile_.end( + processingOptions.noRunLumiSort + ? IndexIntoFile::entryOrder + : (processingOptions.noEventSort ? IndexIntoFile::firstAppearanceOrder : IndexIntoFile::numericalOrder)); + runHelper_->setForcedRunOffset(indexIntoFileBegin_ == indexIntoFileEnd_ ? 1 : indexIntoFileBegin_.run()); + eventProcessHistoryIter_ = eventProcessHistoryIDs_.begin(); + + makeProcessBlockRootRNTuples(fileOptions.filePtr, ttreeOptions, inputType, storedProcessBlockHelper); + + setPresenceInProductRegistry(inputProdDescReg, storedProcessBlockHelper); + + auto newReg = std::make_unique(); + + // Do the translation from the old registry to the new one + { + ProductRegistry::ProductList const& prodList = inputProdDescReg.productList(); + for (auto const& product : prodList) { + ProductDescription const& prod = product.second; + std::string newFriendlyName = friendlyname::friendlyName(prod.className()); + if (newFriendlyName == prod.friendlyClassName()) { + newReg->copyProduct(prod); + } else { + if (fileFormatVersion().splitProductIDs()) { + throw Exception(errors::UnimplementedFeature) + << "Cannot change friendly class name algorithm without more development work\n" + << "to update BranchIDLists and ThinnedAssociationsHelper. Contact the framework group.\n"; + } + ProductDescription newBD(prod); + newBD.updateFriendlyClassName(); + newReg->copyProduct(newBD); + newBranchToOldBranch_.insert(std::make_pair(newBD.branchName(), prod.branchName())); + } + } + + dropOnInputAndReorder(*newReg, + productChoices.productSelectorRules, + productChoices.dropDescendantsOfDroppedProducts, + inputType, + storedProcessBlockHelper, + crossFileInfo.processBlockHelper); + + if (inputType == InputType::SecondaryFile) { + crossFileInfo.thinnedAssociationsHelper->updateFromSecondaryInput(*fileThinnedAssociationsHelper_, + *productChoices.associationsFromSecondary); + } else if (inputType == InputType::Primary) { + crossFileInfo.processBlockHelper->initializeFromPrimaryInput(storedProcessBlockHelper); + crossFileInfo.thinnedAssociationsHelper->updateFromPrimaryInput(*fileThinnedAssociationsHelper_); + } + + if (inputType == InputType::Primary) { + for (auto& product : newReg->productListUpdator()) { + setIsMergeable(product.second); + } + } + //inform system we want to use DelayedReader + for (auto& product : newReg->productListUpdator()) { + product.second.setOnDemand(true); + } + + for (auto& processBlockTree : processBlockTrees_) { + treePointers_.push_back(processBlockTree.get()); + } + auto processingOrder = inputProdDescReg.processOrder(); + processingOrderMerge(*processHistoryRegistry_, processingOrder); + newReg->setProcessOrder(processingOrder); + + // freeze the product registry + newReg->setFrozen(inputType != InputType::Primary); + productRegistry_.reset(newReg.release()); + } + + // Set up information from the product registry. + ProductRegistry::ProductList const& prodList = productRegistry()->productList(); + + { + std::vector nBranches(treePointers_.size(), 0); + for (auto const& product : prodList) { + if (product.second.branchType() == InProcess) { + std::vector const& processes = storedProcessBlockHelper.processesWithProcessBlockProducts(); + auto it = std::find(processes.begin(), processes.end(), product.second.processName()); + if (it != processes.end()) { + auto index = std::distance(processes.begin(), it); + ++nBranches[numberOfRunLumiEventProductTrees + index]; + } + } else { + ++nBranches[product.second.branchType()]; + } + } + + int i = 0; + for (auto& t : treePointers_) { + t->numberOfBranchesToAdd(nBranches[i]); + ++i; + } + } + for (auto const& product : prodList) { + ProductDescription const& prod = product.second; + if (prod.branchType() == InProcess) { + std::vector const& processes = storedProcessBlockHelper.processesWithProcessBlockProducts(); + auto it = std::find(processes.begin(), processes.end(), prod.processName()); + if (it != processes.end()) { + auto index = std::distance(processes.begin(), it); + treePointers_[numberOfRunLumiEventProductTrees + index]->addBranch(prod, + newBranchToOldBranch(prod.branchName())); + } + } else { + treePointers_[prod.branchType()]->addBranch(prod, newBranchToOldBranch(prod.branchName())); + } + } + + // Determine if this file is fast clonable. + setIfFastClonable(processingOptions.remainingEvents, processingOptions.remainingLumis); + + // We are done with our initial reading of EventAuxiliary. + indexIntoFile_.doneFileInitialization(); + + // Tell the event tree to begin training at the next read. + eventTree_.resetTraining(); + + // Train the run and lumi trees. + runTree_.trainCache("*"); + lumiTree_.trainCache("*"); + for (auto& processBlockTree : processBlockTrees_) { + processBlockTree->trainCache("*"); + } + } + + RootFile::~RootFile() {} + + void RootFile::readEntryDescriptionTree(EntryDescriptionMap& entryDescriptionMap, InputType inputType) { + // Called only for old format files. + // We use a smart pointer so the tree will be deleted after use, and not kept for the life of the file. + std::unique_ptr entryDescriptionTree( + dynamic_cast(filePtr_->Get(poolNames::entryDescriptionTreeName().c_str()))); + if (nullptr == entryDescriptionTree.get()) { + throw Exception(errors::FileReadError) + << "Could not find tree " << poolNames::entryDescriptionTreeName() << " in the input file.\n"; + } + + EntryDescriptionID idBuffer; + EntryDescriptionID* pidBuffer = &idBuffer; + entryDescriptionTree->SetBranchAddress(poolNames::entryDescriptionIDBranchName().c_str(), &pidBuffer); + + EventEntryDescription entryDescriptionBuffer; + EventEntryDescription* pEntryDescriptionBuffer = &entryDescriptionBuffer; + entryDescriptionTree->SetBranchAddress(poolNames::entryDescriptionBranchName().c_str(), &pEntryDescriptionBuffer); + + // Fill in the parentage registry. + ParentageRegistry& registry = *ParentageRegistry::instance(); + + for (Long64_t i = 0, numEntries = entryDescriptionTree->GetEntries(); i < numEntries; ++i) { + rootrntuple::getEntry(entryDescriptionTree.get(), i); + if (idBuffer != entryDescriptionBuffer.id()) { + throw Exception(errors::EventCorruption) << "Corruption of EntryDescription tree detected.\n"; + } + entryDescriptionMap.insert(std::make_pair(entryDescriptionBuffer.id(), entryDescriptionBuffer)); + Parentage parents; + parents.setParents(entryDescriptionBuffer.parents()); + if (daqProvenanceHelper_) { + ParentageID const oldID = parents.id(); + daqProvenanceHelper_->fixMetaData(parents.parentsForUpdate()); + ParentageID newID = parents.id(); + if (newID != oldID) { + daqProvenanceHelper_->setOldParentageIDToNew(oldID, newID); + } + } + // For thread safety, don't update global registries when a secondary source opens a file. + if (inputType != InputType::SecondarySource) { + registry.insertMapped(parents); + } + } + entryDescriptionTree->SetBranchAddress(poolNames::entryDescriptionIDBranchName().c_str(), nullptr); + entryDescriptionTree->SetBranchAddress(poolNames::entryDescriptionBranchName().c_str(), nullptr); + } + + void RootFile::readParentageTree(InputType inputType) { + // New format file + // We use a smart pointer so the tree will be deleted after use, and not kept for the life of the file. + std::unique_ptr parentageTree(dynamic_cast(filePtr_->Get(poolNames::parentageTreeName().c_str()))); + if (nullptr == parentageTree.get()) { + throw Exception(errors::FileReadError) + << "Could not find tree " << poolNames::parentageTreeName() << " in the input file.\n"; + } + + Parentage parents; + Parentage* pParentageBuffer = &parents; + parentageTree->SetBranchAddress(poolNames::parentageBranchName().c_str(), &pParentageBuffer); + + ParentageRegistry& registry = *ParentageRegistry::instance(); + + parentageIDLookup_.reserve(parentageTree->GetEntries()); + for (Long64_t i = 0, numEntries = parentageTree->GetEntries(); i < numEntries; ++i) { + rootrntuple::getEntry(parentageTree.get(), i); + if (daqProvenanceHelper_) { + ParentageID const oldID = parents.id(); + daqProvenanceHelper_->fixMetaData(parents.parentsForUpdate()); + ParentageID newID = parents.id(); + if (newID != oldID) { + daqProvenanceHelper_->setOldParentageIDToNew(oldID, newID); + } + } + // For thread safety, don't update global registries when a secondary source opens a file. + if (inputType != InputType::SecondarySource) { + registry.insertMapped(parents); + } + parentageIDLookup_.push_back(parents.id()); + } + parentageTree->SetBranchAddress(poolNames::parentageBranchName().c_str(), nullptr); + } + + void RootFile::setIfFastClonable(int remainingEvents, int remainingLumis) { + if (fileFormatVersion().noMetaDataTrees() and !fileFormatVersion().storedProductProvenanceUsed()) { + //we must avoid copying the old branch which stored the per product per event provenance + whyNotFastClonable_ += FileBlock::FileTooOld; + return; + } + if (!fileFormatVersion().splitProductIDs()) { + whyNotFastClonable_ += FileBlock::FileTooOld; + return; + } + if (processingMode_ != InputSource::RunsLumisAndEvents) { + whyNotFastClonable_ += FileBlock::NotProcessingEvents; + return; + } + // Find entry for first event in file + IndexIntoFile::IndexIntoFileItr it = indexIntoFileBegin_; + while (it != indexIntoFileEnd_ && it.getEntryType() != IndexIntoFile::kEvent) { + ++it; + } + if (it == indexIntoFileEnd_) { + whyNotFastClonable_ += FileBlock::NoEventsInFile; + return; + } + + // From here on, record all reasons we can't fast clone. + IndexIntoFile::SortOrder sortOrder = + (noRunLumiSort_ ? IndexIntoFile::entryOrder + : (noEventSort_ ? IndexIntoFile::firstAppearanceOrder : IndexIntoFile::numericalOrder)); + if (!indexIntoFile_.iterationWillBeInEntryOrder(sortOrder)) { + whyNotFastClonable_ += (noEventSort_ ? FileBlock::RunOrLumiNotContiguous : FileBlock::EventsToBeSorted); + } + if (skipAnyEvents_) { + whyNotFastClonable_ += FileBlock::InitialEventsSkipped; + } + if (remainingEvents >= 0 && eventTree_.entries() > remainingEvents) { + whyNotFastClonable_ += FileBlock::MaxEventsTooSmall; + } + if (remainingLumis >= 0 && lumiTree_.entries() > remainingLumis) { + whyNotFastClonable_ += FileBlock::MaxLumisTooSmall; + } + if (duplicateChecker_ && !duplicateChecker_->checkDisabled() && !duplicateChecker_->noDuplicatesInFile()) { + whyNotFastClonable_ += FileBlock::DuplicateEventsRemoved; + } + } + + std::shared_ptr RootFile::createFileBlock() { + std::vector processBlockTrees; + std::vector processesWithProcessBlockTrees; + processBlockTrees.reserve(processBlockTrees_.size()); + processesWithProcessBlockTrees.reserve(processBlockTrees_.size()); + for (auto& processBlockTree : processBlockTrees_) { + processBlockTrees.push_back(processBlockTree->tree()); + processesWithProcessBlockTrees.push_back(processBlockTree->processName()); + } + return std::make_shared(fileFormatVersion(), + eventTree_.tree(), + eventTree_.metaTree(), + lumiTree_.tree(), + lumiTree_.metaTree(), + runTree_.tree(), + runTree_.metaTree(), + std::move(processBlockTrees), + std::move(processesWithProcessBlockTrees), + whyNotFastClonable(), + hasNewlyDroppedBranch(), + file_, + branchListIndexesUnchanged(), + modifiedIDs(), + productDependencies()); + } + + void RootFile::updateFileBlock(FileBlock& fileBlock) { + std::vector processBlockTrees; + std::vector processesWithProcessBlockTrees; + processBlockTrees.reserve(processBlockTrees_.size()); + processesWithProcessBlockTrees.reserve(processBlockTrees_.size()); + for (auto& processBlockTree : processBlockTrees_) { + processBlockTrees.push_back(processBlockTree->tree()); + processesWithProcessBlockTrees.push_back(processBlockTree->processName()); + } + fileBlock.updateTTreePointers(eventTree_.tree(), + eventTree_.metaTree(), + lumiTree_.tree(), + lumiTree_.metaTree(), + runTree_.tree(), + runTree_.metaTree(), + std::move(processBlockTrees), + std::move(processesWithProcessBlockTrees)); + } + + std::string const& RootFile::newBranchToOldBranch(std::string const& newBranch) const { + std::map::const_iterator it = newBranchToOldBranch_.find(newBranch); + if (it != newBranchToOldBranch_.end()) { + return it->second; + } + return newBranch; + } + + IndexIntoFile::IndexIntoFileItr RootFile::indexIntoFileIter() const { return indexIntoFileIter_; } + + void RootFile::setPosition(IndexIntoFile::IndexIntoFileItr const& position) { + indexIntoFileIter_.copyPosition(position); + } + + void RootFile::initAssociationsFromSecondary(std::vector const& associationsFromSecondary) { + thinnedAssociationsHelper_->initAssociationsFromSecondary(associationsFromSecondary, + *fileThinnedAssociationsHelper_); + } + + bool RootFile::skipThisEntry() { + if (indexIntoFileIter_ == indexIntoFileEnd_) { + return false; + } + + if (eventSkipperByID_ && eventSkipperByID_->somethingToSkip()) { + // See first if the entire lumi or run is skipped, so we won't have to read the event Auxiliary in that case. + if (eventSkipperByID_->skipIt(indexIntoFileIter_.run(), indexIntoFileIter_.lumi(), 0U)) { + return true; + } + + // The Lumi is not skipped. If this is an event, see if the event is skipped. + if (indexIntoFileIter_.getEntryType() == IndexIntoFile::kEvent) { + auto eventAux = fillEventAuxiliary(indexIntoFileIter_.entry()); + if (eventSkipperByID_->skipIt(indexIntoFileIter_.run(), indexIntoFileIter_.lumi(), eventAux.id().event())) { + return true; + } + } + + // Skip runs with no lumis if either lumisToSkip or lumisToProcess have been set to select lumis + if (indexIntoFileIter_.getEntryType() == IndexIntoFile::kRun && eventSkipperByID_->skippingLumis()) { + // There are no lumis in this run, not even ones we will skip + if (indexIntoFileIter_.peekAheadAtLumi() == IndexIntoFile::invalidLumi) { + return true; + } + // If we get here there are lumis in the run, check to see if we are skipping all of them + do { + if (!eventSkipperByID_->skipIt(indexIntoFileIter_.run(), indexIntoFileIter_.peekAheadAtLumi(), 0U)) { + return false; + } + } while (indexIntoFileIter_.skipLumiInRun()); + return true; + } + } + return false; + } + + bool RootFile::isDuplicateEvent() { + assert(indexIntoFileIter_.getEntryType() == IndexIntoFile::kEvent); + if (duplicateChecker_.get() == nullptr) { + return false; + } + auto const eventAux = fillEventAuxiliary(indexIntoFileIter_.entry()); + return duplicateChecker_->isDuplicateAndCheckActive(indexIntoFileIter_.processHistoryIDIndex(), + indexIntoFileIter_.run(), + indexIntoFileIter_.lumi(), + eventAux.id().event(), + file_); + } + + bool RootFile::containsItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) const { + return indexIntoFile_.containsItem(run, lumi, event); + } + + IndexIntoFile::EntryType RootFile::getNextItemType(RunNumber_t& run, + LuminosityBlockNumber_t& lumi, + EventNumber_t& event) { + // First, account for consecutive skipped entries. + while (skipThisEntry()) { + if (indexIntoFileIter_.getEntryType() == IndexIntoFile::kRun) { + indexIntoFileIter_.advanceToNextRun(); + } else if (indexIntoFileIter_.getEntryType() == IndexIntoFile::kLumi) { + indexIntoFileIter_.advanceToNextLumiOrRun(); + } else { + ++indexIntoFileIter_; + } + } + // OK, we have an entry that is not skipped. + IndexIntoFile::EntryType entryType = indexIntoFileIter_.getEntryType(); + if (entryType == IndexIntoFile::kEnd) { + return IndexIntoFile::kEnd; + } + if (entryType == IndexIntoFile::kRun) { + run = indexIntoFileIter_.run(); + runHelper_->checkForNewRun(run, indexIntoFileIter_.peekAheadAtLumi()); + return IndexIntoFile::kRun; + } else if (processingMode_ == InputSource::Runs) { + indexIntoFileIter_.advanceToNextRun(); + return getNextItemType(run, lumi, event); + } + if (entryType == IndexIntoFile::kLumi) { + run = indexIntoFileIter_.run(); + lumi = indexIntoFileIter_.lumi(); + return IndexIntoFile::kLumi; + } else if (processingMode_ == InputSource::RunsAndLumis) { + indexIntoFileIter_.advanceToNextLumiOrRun(); + return getNextItemType(run, lumi, event); + } + if (isDuplicateEvent()) { + ++indexIntoFileIter_; + return getNextItemType(run, lumi, event); + } + run = indexIntoFileIter_.run(); + lumi = indexIntoFileIter_.lumi(); + auto eventAux = fillEventAuxiliary(indexIntoFileIter_.entry()); + event = eventAux.event(); + return IndexIntoFile::kEvent; + } + + bool RootFile::wasLastEventJustRead() const { + IndexIntoFile::IndexIntoFileItr itr(indexIntoFileIter_); + itr.advanceToEvent(); + return itr.getEntryType() == IndexIntoFile::kEnd; + } + + bool RootFile::wasFirstEventJustRead() const { + IndexIntoFile::IndexIntoFileItr itr(indexIntoFileIter_); + int phIndex; + RunNumber_t run; + LuminosityBlockNumber_t lumi; + IndexIntoFile::EntryNumber_t eventEntry; + itr.skipEventBackward(phIndex, run, lumi, eventEntry); + itr.skipEventBackward(phIndex, run, lumi, eventEntry); + return eventEntry == IndexIntoFile::invalidEntry; + } + + namespace { + struct RunItem { + RunItem(ProcessHistoryID const& phid, RunNumber_t const& run) : phid_(phid), run_(run) {} + ProcessHistoryID phid_; + RunNumber_t run_; + }; + struct RunItemSortByRun { + bool operator()(RunItem const& a, RunItem const& b) const { return a.run_ < b.run_; } + }; + struct RunItemSortByRunPhid { + bool operator()(RunItem const& a, RunItem const& b) const { + return a.run_ < b.run_ || (!(b.run_ < a.run_) && a.phid_ < b.phid_); + } + }; + struct LumiItem { + LumiItem(ProcessHistoryID const& phid, + RunNumber_t const& run, + LuminosityBlockNumber_t const& lumi, + IndexIntoFile::EntryNumber_t const& entry) + : phid_(phid), + run_(run), + lumi_(lumi), + firstEventEntry_(entry), + lastEventEntry_(entry == IndexIntoFile::invalidEntry ? IndexIntoFile::invalidEntry : entry + 1) {} + ProcessHistoryID phid_; + RunNumber_t run_; + LuminosityBlockNumber_t lumi_; + IndexIntoFile::EntryNumber_t firstEventEntry_; + IndexIntoFile::EntryNumber_t lastEventEntry_; + }; + struct LumiItemSortByRunLumi { + bool operator()(LumiItem const& a, LumiItem const& b) const { + return a.run_ < b.run_ || (!(b.run_ < a.run_) && a.lumi_ < b.lumi_); + } + }; + struct LumiItemSortByRunLumiPhid { + bool operator()(LumiItem const& a, LumiItem const& b) const { + if (a.run_ < b.run_) + return true; + if (b.run_ < a.run_) + return false; + if (a.lumi_ < b.lumi_) + return true; + if (b.lumi_ < a.lumi_) + return false; + return a.phid_ < b.phid_; + } + }; + } // namespace + + void RootFile::fillIndexIntoFile() { + // This function is for backward compatibility. + // If reading a current format file, indexIntoFile_ is read from the input + // file and should always be there. Note that the algorithm below will work + // sometimes but often fail with the new format introduced in release 3_8_0. + // If it ever becomes necessary to rebuild IndexIntoFile from the new format, + // probably a separate function should be written to deal with the task. + // This is possible just not implemented yet. + assert(!fileFormatVersion().hasIndexIntoFile()); + + typedef std::list LumiList; + LumiList lumis; // (declare 1) + + typedef std::set RunLumiSet; + RunLumiSet runLumiSet; // (declare 2) + + typedef std::list RunList; + RunList runs; // (declare 5) + + typedef std::set RunSet; + RunSet runSet; // (declare 4) + + typedef std::set RunItemSet; + RunItemSet runItemSet; // (declare 3) + + typedef std::map PHIDMap; + PHIDMap phidMap; + + RunNumber_t prevRun = 0; + LuminosityBlockNumber_t prevLumi = 0; + ProcessHistoryID prevPhid; + bool iFirst = true; + + indexIntoFile_.unsortedEventNumbers().clear(); // should already be empty, just being careful + indexIntoFile_.unsortedEventNumbers().reserve(eventTree_.entries()); + + // First, loop through the event tree. + EventSelectionIDVector eventSelectionIDs; + BranchListIndexes branchListIndexes; + while (eventTree_.next()) { + bool newRun = false; + bool newLumi = false; + auto evtAux = fillThisEventAuxiliary(); + fillEventHistory(evtAux, eventSelectionIDs, branchListIndexes); + + // Save the event numbers as we loop through the event auxiliary to avoid + // having to read through the event auxiliary again later. These event numbers + // are not actually used in this function, but could be needed elsewhere. + indexIntoFile_.unsortedEventNumbers().push_back(evtAux.event()); + + ProcessHistoryID reducedPHID = processHistoryRegistry_->reducedProcessHistoryID(evtAux.processHistoryID()); + + if (iFirst || prevPhid != reducedPHID || prevRun != evtAux.run()) { + iFirst = false; + newRun = newLumi = true; + } else if (prevLumi != evtAux.luminosityBlock()) { + newLumi = true; + } + prevPhid = reducedPHID; + prevRun = evtAux.run(); + prevLumi = evtAux.luminosityBlock(); + if (newLumi) { + lumis.emplace_back( + reducedPHID, evtAux.run(), evtAux.luminosityBlock(), eventTree_.entryNumber()); // (insert 1) + runLumiSet.insert(LuminosityBlockID(evtAux.run(), evtAux.luminosityBlock())); // (insert 2) + } else { + LumiItem& currentLumi = lumis.back(); + assert(currentLumi.lastEventEntry_ == eventTree_.entryNumber()); + ++currentLumi.lastEventEntry_; + } + if (newRun) { + // Insert run in list if it is not already there. + RunItem item(reducedPHID, evtAux.run()); + if (runItemSet.insert(item).second) { // (check 3, insert 3) + runs.push_back(std::move(item)); // (insert 5) + runSet.insert(evtAux.run()); // (insert 4) + phidMap.insert(std::make_pair(evtAux.run(), reducedPHID)); + } + } + } + // now clean up. + eventTree_.setEntryNumber(IndexIntoFile::invalidEntry); + lastEventEntryNumberRead_ = IndexIntoFile::invalidEntry; + + // Loop over run entries and fill information. + + typedef std::map RunMap; + RunMap runMap; // (declare 11) + + typedef std::vector RunVector; + RunVector emptyRuns; // (declare 12) + + if (runTree_.isValid()) { + while (runTree_.next()) { + // Note: adjacent duplicates will be skipped without an explicit check. + + std::shared_ptr runAux = fillRunAuxiliary(); + ProcessHistoryID reducedPHID = processHistoryRegistry_->reducedProcessHistoryID(runAux->processHistoryID()); + + if (runSet.insert(runAux->run()).second) { // (check 4, insert 4) + // This run was not associated with any events. + emptyRuns.emplace_back(reducedPHID, runAux->run()); // (insert 12) + } + runMap.insert(std::make_pair(runAux->run(), runTree_.entryNumber())); // (insert 11) + phidMap.insert(std::make_pair(runAux->run(), reducedPHID)); + } + // now clean up. + runTree_.setEntryNumber(IndexIntoFile::invalidEntry); + } + + // Insert the ordered empty runs into the run list. + RunItemSortByRun runItemSortByRun; + stable_sort_all(emptyRuns, runItemSortByRun); + + RunList::iterator itRuns = runs.begin(), endRuns = runs.end(); + for (auto const& emptyRun : emptyRuns) { + for (; itRuns != endRuns; ++itRuns) { + if (runItemSortByRun(emptyRun, *itRuns)) { + break; + } + } + runs.insert(itRuns, emptyRun); + } + + // Loop over luminosity block entries and fill information. + + typedef std::vector LumiVector; + LumiVector emptyLumis; // (declare 7) + + typedef std::map RunLumiMap; + RunLumiMap runLumiMap; // (declare 6) + + if (lumiTree_.isValid()) { + while (lumiTree_.next()) { + // Note: adjacent duplicates will be skipped without an explicit check. + std::shared_ptr lumiAux = fillLumiAuxiliary(); + LuminosityBlockID lumiID = LuminosityBlockID(lumiAux->run(), lumiAux->luminosityBlock()); + if (runLumiSet.insert(lumiID).second) { // (check 2, insert 2) + // This lumi was not associated with any events. + // Use the process history ID from the corresponding run. In cases of practical + // importance, this should be the correct process history ID, but it is possible + // to construct files where this is not the correct process history ID ... + PHIDMap::const_iterator iPhidMap = phidMap.find(lumiAux->run()); + assert(iPhidMap != phidMap.end()); + emptyLumis.emplace_back( + iPhidMap->second, lumiAux->run(), lumiAux->luminosityBlock(), IndexIntoFile::invalidEntry); // (insert 7) + } + runLumiMap.insert(std::make_pair(lumiID, lumiTree_.entryNumber())); + } + // now clean up. + lumiTree_.setEntryNumber(IndexIntoFile::invalidEntry); + } + + // Insert the ordered empty lumis into the lumi list. + LumiItemSortByRunLumi lumiItemSortByRunLumi; + stable_sort_all(emptyLumis, lumiItemSortByRunLumi); + + LumiList::iterator itLumis = lumis.begin(), endLumis = lumis.end(); + for (auto const& emptyLumi : emptyLumis) { + for (; itLumis != endLumis; ++itLumis) { + if (lumiItemSortByRunLumi(emptyLumi, *itLumis)) { + break; + } + } + lumis.insert(itLumis, emptyLumi); + } + + // Create a map of RunItems that gives the order of first appearance in the list. + // Also fill in the vector of process history IDs + typedef std::map RunCountMap; + RunCountMap runCountMap; // Declare (17) + std::vector& phids = indexIntoFile_.setProcessHistoryIDs(); + assert(phids.empty()); + std::vector& entries = indexIntoFile_.setRunOrLumiEntries(); + assert(entries.empty()); + int rcount = 0; + for (auto& run : runs) { + RunCountMap::const_iterator countMapItem = runCountMap.find(run); + if (countMapItem == runCountMap.end()) { + countMapItem = runCountMap.insert(std::make_pair(run, rcount)).first; // Insert (17) + assert(countMapItem != runCountMap.end()); + ++rcount; + } + std::vector::const_iterator phidItem = find_in_all(phids, run.phid_); + if (phidItem == phids.end()) { + phids.push_back(run.phid_); + phidItem = phids.end() - 1; + } + entries.emplace_back(countMapItem->second, // use (17) + IndexIntoFile::invalidEntry, + runMap[run.run_], // use (11) + phidItem - phids.begin(), + run.run_, + 0U, + IndexIntoFile::invalidEntry, + IndexIntoFile::invalidEntry); + } + + // Create a map of LumiItems that gives the order of first appearance in the list. + typedef std::map LumiCountMap; + LumiCountMap lumiCountMap; // Declare (19) + int lcount = 0; + for (auto& lumi : lumis) { + RunCountMap::const_iterator runCountMapItem = runCountMap.find(RunItem(lumi.phid_, lumi.run_)); + assert(runCountMapItem != runCountMap.end()); + LumiCountMap::const_iterator countMapItem = lumiCountMap.find(lumi); + if (countMapItem == lumiCountMap.end()) { + countMapItem = lumiCountMap.insert(std::make_pair(lumi, lcount)).first; // Insert (17) + assert(countMapItem != lumiCountMap.end()); + ++lcount; + } + std::vector::const_iterator phidItem = find_in_all(phids, lumi.phid_); + assert(phidItem != phids.end()); + entries.emplace_back(runCountMapItem->second, + countMapItem->second, + runLumiMap[LuminosityBlockID(lumi.run_, lumi.lumi_)], + phidItem - phids.begin(), + lumi.run_, + lumi.lumi_, + lumi.firstEventEntry_, + lumi.lastEventEntry_); + } + stable_sort_all(entries); + } + + void RootFile::validateFile(InputType inputType, + bool usingGoToEvent, + std::vector& orderedProcessHistoryIDs) { + // Validate the file. + if (!fid_.isValid()) { + fid_ = FileID(createGlobalIdentifier()); + } + if (!eventTree_.isValid()) { + throw Exception(errors::EventCorruption) << "'Events' tree is corrupted or not present\n" + << "in the input file.\n"; + } + if (enforceGUIDInFileName_) { + auto guidFromName = stemFromPath(file_); + if (guidFromName != fid_.fid()) { + throw edm::Exception(edm::errors::FileNameInconsistentWithGUID) + << "GUID " << guidFromName << " extracted from file name " << file_ + << " is inconsistent with the GUID read from the file " << fid_.fid(); + } + } + + if (fileFormatVersion().hasIndexIntoFile()) { + if (runTree().entries() > 0) { + assert(!indexIntoFile_.empty()); + } + if (!fileFormatVersion().useReducedProcessHistoryID()) { + if (daqProvenanceHelper_) { + std::vector& phidVec = indexIntoFile_.setProcessHistoryIDs(); + for (auto& phid : phidVec) { + phid = daqProvenanceHelper_->mapProcessHistoryID(phid); + } + } + indexIntoFile_.reduceProcessHistoryIDs(*processHistoryRegistry_); + } + } else { + assert(indexIntoFile_.empty()); + fillIndexIntoFile(); + } + + indexIntoFile_.fixIndexes(orderedProcessHistoryIDs); + indexIntoFile_.setNumberOfEvents(eventTree_.entries()); + indexIntoFile_.setEventFinder( + std::shared_ptr(std::make_shared(eventTree_))); + // We fill the event numbers explicitly if we need to find events in closed files, + // such as for secondary files (or secondary sources) or if duplicate checking across files. + bool needEventNumbers = false; + bool needIndexesForDuplicateChecker = + duplicateChecker_ && duplicateChecker_->checkingAllFiles() && !duplicateChecker_->checkDisabled(); + if (inputType != InputType::Primary || needIndexesForDuplicateChecker || usingGoToEvent) { + needEventNumbers = true; + } + bool needEventEntries = false; + if (inputType != InputType::Primary || !noEventSort_) { + // We need event entries for sorting or for secondary files or sources. + needEventEntries = true; + } + indexIntoFile_.fillEventNumbersOrEntries(needEventNumbers, needEventEntries); + } + + void RootFile::reportOpened(std::string const& inputType) { + // Report file opened. + std::string const label = "source"; + std::string moduleName = "RNTupleTempSource"; + filePtr_->inputFileOpened(logicalFile_, inputType, moduleName, label, fid_.fid(), eventTree_.branchNames()); + } + + void RootFile::close() { + // Just to play it safe, zero all pointers to objects in the InputFile to be closed. + eventHistoryTree_ = nullptr; + for (auto& treePointer : treePointers_) { + treePointer->close(); + treePointer = nullptr; + } + filePtr_->Close(); + filePtr_ = nullptr; // propagate_const has no reset() function + } + + EventAuxiliary const& RootFile::fillThisEventAuxiliary() { + if (lastEventEntryNumberRead_ == eventTree_.entryNumber()) { + // Already read. + return eventAuxCache_; + } + if (fileFormatVersion().newAuxiliary()) { + EventAuxiliary* pEvAux = &eventAuxCache_; + eventTree_.fillAux(pEvAux); + } else { + // for backward compatibility. + EventAux eventAux; + EventAux* pEvAux = &eventAux; + eventTree_.fillAux(pEvAux); + conversion(eventAux, eventAuxCache_); + } + lastEventEntryNumberRead_ = eventTree_.entryNumber(); + return eventAuxCache_; + } + + EventAuxiliary RootFile::fillEventAuxiliary(IndexIntoFile::EntryNumber_t entry) { + eventTree_.setEntryNumber(entry); + return fillThisEventAuxiliary(); + } + + void RootFile::fillEventToProcessBlockIndexes() { + TBranch* eventToProcessBlockIndexesBranch = get_underlying_safe(eventToProcessBlockIndexesBranch_); + if (eventToProcessBlockIndexesBranch == nullptr) { + if (processBlockHelper_.get() == nullptr) { + eventToProcessBlockIndexes_.setIndex(0); + } else { + eventToProcessBlockIndexes_.setIndex(processBlockHelper_->outerOffset()); + } + } else { + if (processBlockHelper_->cacheIndexVectorsPerFile().back() == 1u) { + eventToProcessBlockIndexes_.setIndex(processBlockHelper_->outerOffset()); + } else { + EventToProcessBlockIndexes* pEventToProcessBlockIndexes = &eventToProcessBlockIndexes_; + eventTree_.fillBranchEntry(eventToProcessBlockIndexesBranch, pEventToProcessBlockIndexes); + unsigned int updatedIndex = eventToProcessBlockIndexes_.index() + processBlockHelper_->outerOffset(); + eventToProcessBlockIndexes_.setIndex(updatedIndex); + } + } + } + + bool RootFile::fillEventHistory(EventAuxiliary& evtAux, + EventSelectionIDVector& eventSelectionIDs, + BranchListIndexes& branchListIndexes, + bool assertOnFailure) { + // We could consider doing delayed reading, but because we have to + // store this History object in a different tree than the event + // data tree, this is too hard to do in this first version. + if (fileFormatVersion().eventHistoryBranch()) { + // Lumi block number was not in EventID for the relevant releases. + EventID id(evtAux.id().run(), 0, evtAux.id().event()); + if (eventProcessHistoryIter_->eventID() != id) { + EventProcessHistoryID target(id, ProcessHistoryID()); + eventProcessHistoryIter_ = lower_bound_all(eventProcessHistoryIDs_, target); + assert(eventProcessHistoryIter_->eventID() == id); + } + evtAux.setProcessHistoryID(eventProcessHistoryIter_->processHistoryID()); + ++eventProcessHistoryIter_; + } else if (fileFormatVersion().eventHistoryTree()) { + // for backward compatibility. + History* pHistory = history_.get(); + TBranch* eventHistoryBranch = eventHistoryTree_->GetBranch(poolNames::eventHistoryBranchName().c_str()); + if (!eventHistoryBranch) { + throw Exception(errors::EventCorruption) << "Failed to find history branch in event history tree.\n"; + } + eventHistoryBranch->SetAddress(&pHistory); + rootrntuple::getEntry(eventHistoryTree_, eventTree_.entryNumber()); + eventHistoryBranch->SetAddress(nullptr); + evtAux.setProcessHistoryID(history_->processHistoryID()); + eventSelectionIDs.swap(history_->eventSelectionIDs()); + branchListIndexes.swap(history_->branchListIndexes()); + } else if (fileFormatVersion().noMetaDataTrees()) { + // Current format + EventSelectionIDVector* pESV = &eventSelectionIDs; + TBranch* eventSelectionIDBranch = eventTree_.tree()->GetBranch(poolNames::eventSelectionsBranchName().c_str()); + assert(eventSelectionIDBranch != nullptr); + eventTree_.fillBranchEntry(eventSelectionIDBranch, pESV); + BranchListIndexes* pBLI = &branchListIndexes; + TBranch* branchListIndexesBranch = eventTree_.tree()->GetBranch(poolNames::branchListIndexesBranchName().c_str()); + assert(branchListIndexesBranch != nullptr); + eventTree_.fillBranchEntry(branchListIndexesBranch, pBLI); + } + if (provenanceAdaptor_) { + evtAux.setProcessHistoryID(provenanceAdaptor_->convertID(evtAux.processHistoryID())); + for (auto& esID : eventSelectionIDs) { + esID = provenanceAdaptor_->convertID(esID); + } + } + if (daqProvenanceHelper_) { + evtAux.setProcessHistoryID(daqProvenanceHelper_->mapProcessHistoryID(evtAux.processHistoryID())); + } + if (!fileFormatVersion().splitProductIDs()) { + // old format. branchListIndexes_ must be filled in from the ProvenanceAdaptor. + provenanceAdaptor_->branchListIndexes(branchListIndexes); + } + if (branchIDListHelper_) { + return branchIDListHelper_->fixBranchListIndexes(branchListIndexes, assertOnFailure); + } + return true; + } + + std::shared_ptr RootFile::fillLumiAuxiliary() { + auto lumiAuxiliary = std::make_shared(); + if (fileFormatVersion().newAuxiliary()) { + LuminosityBlockAuxiliary* pLumiAux = lumiAuxiliary.get(); + lumiTree_.fillAux(pLumiAux); + } else { + LuminosityBlockAux lumiAux; + LuminosityBlockAux* pLumiAux = &lumiAux; + lumiTree_.fillAux(pLumiAux); + conversion(lumiAux, *lumiAuxiliary); + } + if (provenanceAdaptor_) { + lumiAuxiliary->setProcessHistoryID(provenanceAdaptor_->convertID(lumiAuxiliary->processHistoryID())); + } + if (daqProvenanceHelper_) { + lumiAuxiliary->setProcessHistoryID(daqProvenanceHelper_->mapProcessHistoryID(lumiAuxiliary->processHistoryID())); + } + if (lumiAuxiliary->luminosityBlock() == 0 && !fileFormatVersion().runsAndLumis()) { + lumiAuxiliary->id() = LuminosityBlockID(RunNumber_t(1), LuminosityBlockNumber_t(1)); + } + return lumiAuxiliary; + } + + std::shared_ptr RootFile::fillRunAuxiliary() { + auto runAuxiliary = std::make_shared(); + if (fileFormatVersion().newAuxiliary()) { + RunAuxiliary* pRunAux = runAuxiliary.get(); + runTree_.fillAux(pRunAux); + } else { + RunAux runAux; + RunAux* pRunAux = &runAux; + runTree_.fillAux(pRunAux); + conversion(runAux, *runAuxiliary); + } + if (provenanceAdaptor_) { + runAuxiliary->setProcessHistoryID(provenanceAdaptor_->convertID(runAuxiliary->processHistoryID())); + } + if (daqProvenanceHelper_) { + runAuxiliary->setProcessHistoryID(daqProvenanceHelper_->mapProcessHistoryID(runAuxiliary->processHistoryID())); + } + return runAuxiliary; + } + + bool RootFile::skipEvents(int& offset) { + while (offset > 0 && indexIntoFileIter_ != indexIntoFileEnd_) { + int phIndexOfSkippedEvent = IndexIntoFile::invalidIndex; + RunNumber_t runOfSkippedEvent = IndexIntoFile::invalidRun; + LuminosityBlockNumber_t lumiOfSkippedEvent = IndexIntoFile::invalidLumi; + IndexIntoFile::EntryNumber_t skippedEventEntry = IndexIntoFile::invalidEntry; + + indexIntoFileIter_.skipEventForward( + phIndexOfSkippedEvent, runOfSkippedEvent, lumiOfSkippedEvent, skippedEventEntry); + + // At the end of the file and there were no more events to skip + if (skippedEventEntry == IndexIntoFile::invalidEntry) + break; + + if (eventSkipperByID_ && eventSkipperByID_->somethingToSkip()) { + auto const evtAux = fillEventAuxiliary(skippedEventEntry); + if (eventSkipperByID_->skipIt(runOfSkippedEvent, lumiOfSkippedEvent, evtAux.id().event())) { + continue; + } + } + if (duplicateChecker_ && !duplicateChecker_->checkDisabled() && !duplicateChecker_->noDuplicatesInFile()) { + auto const evtAux = fillEventAuxiliary(skippedEventEntry); + if (duplicateChecker_->isDuplicateAndCheckActive( + phIndexOfSkippedEvent, runOfSkippedEvent, lumiOfSkippedEvent, evtAux.id().event(), file_)) { + continue; + } + } + --offset; + } + + while (offset < 0) { + if (duplicateChecker_) { + duplicateChecker_->disable(); + } + + int phIndexOfEvent = IndexIntoFile::invalidIndex; + RunNumber_t runOfEvent = IndexIntoFile::invalidRun; + LuminosityBlockNumber_t lumiOfEvent = IndexIntoFile::invalidLumi; + IndexIntoFile::EntryNumber_t eventEntry = IndexIntoFile::invalidEntry; + + indexIntoFileIter_.skipEventBackward(phIndexOfEvent, runOfEvent, lumiOfEvent, eventEntry); + + if (eventEntry == IndexIntoFile::invalidEntry) + break; + + if (eventSkipperByID_ && eventSkipperByID_->somethingToSkip()) { + auto const evtAux = fillEventAuxiliary(eventEntry); + if (eventSkipperByID_->skipIt(runOfEvent, lumiOfEvent, evtAux.id().event())) { + continue; + } + } + ++offset; + } + return (indexIntoFileIter_ == indexIntoFileEnd_); + } + + bool RootFile::goToEvent(EventID const& eventID) { + indexIntoFile_.fillEventNumbers(); + + if (duplicateChecker_) { + duplicateChecker_->disable(); + } + + IndexIntoFile::SortOrder sortOrder = IndexIntoFile::numericalOrder; + if (noEventSort_) + sortOrder = IndexIntoFile::firstAppearanceOrder; + if (noRunLumiSort_) { + sortOrder = IndexIntoFile::entryOrder; + } + + IndexIntoFile::IndexIntoFileItr iter = + indexIntoFile_.findPosition(sortOrder, eventID.run(), eventID.luminosityBlock(), eventID.event()); + + if (iter == indexIntoFile_.end(sortOrder)) { + return false; + } + indexIntoFileIter_ = iter; + return true; + } + + // readEvent() is responsible for creating, and setting up, the + // EventPrincipal. + // + // 1. create an EventPrincipal with a unique EventID + // 2. For each entry in the provenance, put in one ProductResolver, + // holding the Provenance for the corresponding EDProduct. + // 3. set up the the EventPrincipal to know about this ProductResolver. + // + // We do *not* create the EDProduct instance (the equivalent of reading + // the branch containing this EDProduct. That will be done by the Delayed Reader, + // when it is asked to do so. + // + bool RootFile::readEvent(EventPrincipal& principal, bool readAllProducts) { + assert(indexIntoFileIter_ != indexIntoFileEnd_); + assert(indexIntoFileIter_.getEntryType() == IndexIntoFile::kEvent); + // read the event + auto [found, succeeded] = readCurrentEvent(principal, false, readAllProducts); + auto const& evtAux = principal.aux(); + + runHelper_->checkRunConsistency(evtAux.run(), indexIntoFileIter_.run()); + runHelper_->checkLumiConsistency(evtAux.luminosityBlock(), indexIntoFileIter_.lumi()); + + ++indexIntoFileIter_; + return succeeded; + } + + // Reads event at the current entry in the event tree + std::tuple RootFile::readCurrentEvent(EventPrincipal& principal, + bool assertOnFailure, + bool readAllProducts) { + bool found = true; + bool succeeded = true; + if (!eventTree_.current()) { + found = false; + return {found, succeeded}; + } + auto evtAux = fillThisEventAuxiliary(); + if (!fileFormatVersion().lumiInEventID()) { + //ugly, but will disappear when the backward compatibility is done with schema evolution. + const_cast(evtAux.id()).setLuminosityBlockNumber(evtAux.oldLuminosityBlock()); + evtAux.resetObsoleteInfo(); + } + fillEventToProcessBlockIndexes(); + EventSelectionIDVector eventSelectionIDs; + BranchListIndexes branchListIndexes; + if (!fillEventHistory(evtAux, eventSelectionIDs, branchListIndexes, assertOnFailure)) { + succeeded = false; + } + runHelper_->overrideRunNumber(evtAux.id(), evtAux.isRealData()); + + // We're not done ... so prepare the EventPrincipal + eventTree_.insertEntryForIndex(principal.transitionIndex()); + auto provRetriever = makeProductProvenanceRetriever(principal.streamID().value()); + if (readAllProducts) { + eventTree_.rootDelayedReader()->readAllProductsNow(&principal); + provRetriever->unsafe_fillProvenance(); + } + auto history = processHistoryRegistry_->getMapped(evtAux.processHistoryID()); + principal.fillEventPrincipal(evtAux, + history, + std::move(eventSelectionIDs), + std::move(branchListIndexes), + eventToProcessBlockIndexes_, + *provRetriever, + eventTree_.resetAndGetRootDelayedReader()); + + // If this next assert shows up in performance profiling or significantly affects memory, then these three lines should be deleted. + // The IndexIntoFile should guarantee that it never fails. + ProcessHistoryID idToCheck = (daqProvenanceHelper_ && fileFormatVersion().useReducedProcessHistoryID() + ? *daqProvenanceHelper_->oldProcessHistoryID() + : evtAux.processHistoryID()); + ProcessHistoryID const& reducedPHID = processHistoryRegistry_->reducedProcessHistoryID(idToCheck); + assert(reducedPHID == indexIntoFile_.processHistoryID(indexIntoFileIter_.processHistoryIDIndex())); + + // report event read from file + filePtr_->eventReadFromFile(); + return {found, succeeded}; + } + + void RootFile::setAtEventEntry(IndexIntoFile::EntryNumber_t entry) { eventTree_.setEntryNumber(entry); } + + std::shared_ptr RootFile::readRunAuxiliary_() { + if (runHelper_->fakeNewRun()) { + auto runAuxiliary = std::make_shared(*savedRunAuxiliary()); + runHelper_->overrideRunNumber(runAuxiliary->id()); + return runAuxiliary; + } + assert(indexIntoFileIter_ != indexIntoFileEnd_); + assert(indexIntoFileIter_.getEntryType() == IndexIntoFile::kRun); + + // Begin code for backward compatibility before the existence of run trees. + if (!runTree_.isValid()) { + // prior to the support of run trees. + // RunAuxiliary did not contain a valid timestamp. Take it from the next event. + IndexIntoFile::EntryNumber_t eventEntry = indexIntoFileIter_.firstEventEntryThisRun(); + assert(eventEntry != IndexIntoFile::invalidEntry); + assert(eventTree_.current(eventEntry)); + auto const evtAux = fillEventAuxiliary(eventEntry); + + RunID run = RunID(indexIntoFileIter_.run()); + runHelper_->overrideRunNumber(run); + savedRunAuxiliary_ = std::make_shared(run.run(), evtAux.time(), Timestamp::invalidTimestamp()); + return savedRunAuxiliary(); + } + // End code for backward compatibility before the existence of run trees. + runTree_.setEntryNumber(indexIntoFileIter_.entry()); + std::shared_ptr runAuxiliary = fillRunAuxiliary(); + assert(runAuxiliary->run() == indexIntoFileIter_.run()); + runHelper_->overrideRunNumber(runAuxiliary->id()); + filePtr_->reportInputRunNumber(runAuxiliary->run()); + // If RunAuxiliary did not contain a valid begin timestamp, invalidate any end timestamp. + if (runAuxiliary->beginTime() == Timestamp::invalidTimestamp()) { + runAuxiliary->setEndTime(Timestamp::invalidTimestamp()); + } + + // If RunAuxiliary did not contain a valid timestamp, or if this an old format file from + // when the Run's ProcessHistory included only processes where products were added to the Run itself, + // we attempt to read the first event in the run to get appropriate info. + if (runAuxiliary->beginTime() == Timestamp::invalidTimestamp() || + !fileFormatVersion().processHistorySameWithinRun()) { + IndexIntoFile::EntryNumber_t eventEntry = indexIntoFileIter_.firstEventEntryThisRun(); + // If we have a valid event, use its information. + if (eventEntry != IndexIntoFile::invalidEntry) { + assert(eventTree_.current(eventEntry)); + auto evtAux = fillEventAuxiliary(eventEntry); + + // RunAuxiliary did not contain a valid timestamp. Take it from the next event in this run if there is one. + if (runAuxiliary->beginTime() == Timestamp::invalidTimestamp()) { + runAuxiliary->setBeginTime(evtAux.time()); + } + + // For backwards compatibility when the Run's ProcessHistory included only processes where products were added to the + // Run, and then the Run and Event auxiliaries could be different. Use the event ProcessHistoryID if there is one. It should + // almost always be correct by the current definition (processes included if any products are added. This makes the run, lumi, + // and event ProcessHistory's always be the same if no file merging occurs). + if (!fileFormatVersion().processHistorySameWithinRun()) { + EventSelectionIDVector eventSelectionIDs; + BranchListIndexes branchListIndexes; + fillEventHistory(evtAux, eventSelectionIDs, branchListIndexes); + runAuxiliary->setProcessHistoryID(evtAux.processHistoryID()); + } + } + } + savedRunAuxiliary_ = runAuxiliary; + return runAuxiliary; + } + + void RootFile::fillProcessBlockHelper_() { + assert(inputType_ == InputType::Primary); + std::vector nEntries; + nEntries.reserve(processBlockTrees_.size()); + for (auto const& processBlockTree : processBlockTrees_) { + nEntries.push_back(processBlockTree->entries()); + } + processBlockHelper_->fillFromPrimaryInput(*storedProcessBlockHelper_, nEntries); + storedProcessBlockHelper_ = + std::make_unique(); // propagate_const has no reset() function + } + + bool RootFile::initializeFirstProcessBlockEntry() { + if (processBlockTrees_[currentProcessBlockTree_]->entryNumber() == IndexIntoFile::invalidEntry) { + processBlockTrees_[currentProcessBlockTree_]->setEntryNumber(0); + assert(processBlockTrees_[currentProcessBlockTree_]->current()); + return true; + } + return false; + } + + bool RootFile::endOfProcessBlocksReached() const { return currentProcessBlockTree_ >= processBlockTrees_.size(); } + + bool RootFile::nextProcessBlock_(ProcessBlockPrincipal&) { + assert(inputType_ == InputType::Primary); + if (endOfProcessBlocksReached()) { + return false; + } + if (initializeFirstProcessBlockEntry()) { + return true; + } + // With the current design, the RootFile should always be + // set to a valid ProcessBlock entry in one of the TTrees + // if it not at the end. + assert(processBlockTrees_[currentProcessBlockTree_]->current()); + // Try for next entry in the same TTree + if (processBlockTrees_[currentProcessBlockTree_]->nextWithCache()) { + return true; + } + // Next ProcessBlock TTree + ++currentProcessBlockTree_; + if (endOfProcessBlocksReached()) { + return false; + } + // With current design there should always be at least one entry. + // Initialize for that entry. + processBlockTrees_[currentProcessBlockTree_]->setEntryNumber(0); + assert(processBlockTrees_[currentProcessBlockTree_]->current()); + return true; + } + + void RootFile::readProcessBlock_(ProcessBlockPrincipal& processBlockPrincipal) { + assert(inputType_ == InputType::Primary); + RootRNTuple* rootTree = processBlockTrees_[currentProcessBlockTree_].get(); + rootTree->insertEntryForIndex(0); + assert(!rootTree->processName().empty()); + processBlockPrincipal.fillProcessBlockPrincipal(rootTree->processName(), rootTree->resetAndGetRootDelayedReader()); + } + + bool RootFile::readRun_(RunPrincipal& runPrincipal) { + bool shouldProcessRun = indexIntoFileIter_.shouldProcessRun(); + + MergeableRunProductMetadata* mergeableRunProductMetadata = nullptr; + if (shouldProcessRun) { + if (inputType_ == InputType::Primary) { + mergeableRunProductMetadata = runPrincipal.mergeableRunProductMetadata(); + RootRNTuple::EntryNumber const& entryNumber = runTree_.entryNumber(); + assert(entryNumber >= 0); + mergeableRunProductMetadata->readRun( + entryNumber, *storedMergeableRunProductMetadata_, IndexIntoFileItrHolder(indexIntoFileIter_)); + } + } + + if (!runHelper_->fakeNewRun()) { + assert(indexIntoFileIter_ != indexIntoFileEnd_); + assert(indexIntoFileIter_.getEntryType() == IndexIntoFile::kRun); + ++indexIntoFileIter_; + } + // Begin code for backward compatibility before the existence of run trees. + if (!runTree_.isValid()) { + return shouldProcessRun; + } + // End code for backward compatibility before the existence of run trees. + if (shouldProcessRun) { + // NOTE: we use 0 for the index since do not do delayed reads for RunPrincipals + runTree_.insertEntryForIndex(0); + runPrincipal.fillRunPrincipal(*processHistoryRegistry_, runTree_.resetAndGetRootDelayedReader()); + // Read in all the products now. + runPrincipal.readAllFromSourceAndMergeImmediately(mergeableRunProductMetadata); + runPrincipal.setShouldWriteRun(RunPrincipal::kYes); + } else { + runPrincipal.fillRunPrincipal(*processHistoryRegistry_, nullptr); + if (runPrincipal.shouldWriteRun() != RunPrincipal::kYes) { + runPrincipal.setShouldWriteRun(RunPrincipal::kNo); + } + } + return shouldProcessRun; + } + + std::shared_ptr RootFile::readLuminosityBlockAuxiliary_() { + assert(indexIntoFileIter_ != indexIntoFileEnd_); + assert(indexIntoFileIter_.getEntryType() == IndexIntoFile::kLumi); + // Begin code for backward compatibility before the existence of lumi trees. + if (!lumiTree_.isValid()) { + IndexIntoFile::EntryNumber_t eventEntry = indexIntoFileIter_.firstEventEntryThisLumi(); + assert(eventEntry != IndexIntoFile::invalidEntry); + assert(eventTree_.current(eventEntry)); + auto const evtAux = fillEventAuxiliary(eventEntry); + + LuminosityBlockID lumi = LuminosityBlockID(indexIntoFileIter_.run(), indexIntoFileIter_.lumi()); + runHelper_->overrideRunNumber(lumi); + return std::make_shared( + lumi.run(), lumi.luminosityBlock(), evtAux.time(), Timestamp::invalidTimestamp()); + } + // End code for backward compatibility before the existence of lumi trees. + lumiTree_.setEntryNumber(indexIntoFileIter_.entry()); + std::shared_ptr lumiAuxiliary = fillLumiAuxiliary(); + assert(lumiAuxiliary->run() == indexIntoFileIter_.run()); + assert(lumiAuxiliary->luminosityBlock() == indexIntoFileIter_.lumi()); + runHelper_->overrideRunNumber(lumiAuxiliary->id()); + filePtr_->reportInputLumiSection(lumiAuxiliary->run(), lumiAuxiliary->luminosityBlock()); + if (lumiAuxiliary->beginTime() == Timestamp::invalidTimestamp()) { + IndexIntoFile::EntryNumber_t eventEntry = indexIntoFileIter_.firstEventEntryThisLumi(); + if (eventEntry != IndexIntoFile::invalidEntry) { + assert(eventTree_.current(eventEntry)); + auto const evtAux = fillEventAuxiliary(eventEntry); + + lumiAuxiliary->setBeginTime(evtAux.time()); + } + lumiAuxiliary->setEndTime(Timestamp::invalidTimestamp()); + } + if (!fileFormatVersion().processHistorySameWithinRun() && savedRunAuxiliary_) { + lumiAuxiliary->setProcessHistoryID(savedRunAuxiliary_->processHistoryID()); + } + return lumiAuxiliary; + } + + bool RootFile::readLuminosityBlock_(LuminosityBlockPrincipal& lumiPrincipal) { + bool shouldProcessLumi = indexIntoFileIter_.shouldProcessLumi(); + assert(indexIntoFileIter_ != indexIntoFileEnd_); + assert(indexIntoFileIter_.getEntryType() == IndexIntoFile::kLumi); + // Begin code for backward compatibility before the existence of lumi trees. + if (!lumiTree_.isValid()) { + ++indexIntoFileIter_; + return shouldProcessLumi; + } + // End code for backward compatibility before the existence of lumi trees. + if (shouldProcessLumi) { + lumiTree_.setEntryNumber(indexIntoFileIter_.entry()); + // NOTE: we use 0 for the index since do not do delayed reads for LuminosityBlockPrincipals + lumiTree_.insertEntryForIndex(0); + auto history = processHistoryRegistry_->getMapped(lumiPrincipal.aux().processHistoryID()); + lumiPrincipal.fillLuminosityBlockPrincipal(history, lumiTree_.resetAndGetRootDelayedReader()); + // Read in all the products now. + lumiPrincipal.readAllFromSourceAndMergeImmediately(); + lumiPrincipal.setShouldWriteLumi(LuminosityBlockPrincipal::kYes); + } else { + auto history = processHistoryRegistry_->getMapped(lumiPrincipal.aux().processHistoryID()); + lumiPrincipal.fillLuminosityBlockPrincipal(history, nullptr); + if (lumiPrincipal.shouldWriteLumi() != LuminosityBlockPrincipal::kYes) { + lumiPrincipal.setShouldWriteLumi(LuminosityBlockPrincipal::kNo); + } + } + ++indexIntoFileIter_; + return shouldProcessLumi; + } + + bool RootFile::setEntryAtEvent(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) { + indexIntoFileIter_ = indexIntoFile_.findEventPosition(run, lumi, event); + if (indexIntoFileIter_ == indexIntoFileEnd_) + return false; + eventTree_.setEntryNumber(indexIntoFileIter_.entry()); + return true; + } + + bool RootFile::setEntryAtLumi(RunNumber_t run, LuminosityBlockNumber_t lumi) { + indexIntoFileIter_ = indexIntoFile_.findLumiPosition(run, lumi); + if (indexIntoFileIter_ == indexIntoFileEnd_) + return false; + lumiTree_.setEntryNumber(indexIntoFileIter_.entry()); + return true; + } + + bool RootFile::setEntryAtRun(RunNumber_t run) { + indexIntoFileIter_ = indexIntoFile_.findRunPosition(run); + if (indexIntoFileIter_ == indexIntoFileEnd_) + return false; + runTree_.setEntryNumber(indexIntoFileIter_.entry()); + return true; + } + + bool RootFile::setEntryAtNextEventInLumi(RunNumber_t run, LuminosityBlockNumber_t lumi) { + if (indexIntoFileIter_.getEntryType() == IndexIntoFile::kEvent) { + ++indexIntoFileIter_; + } + indexIntoFileIter_.advanceToEvent(); + if (indexIntoFileIter_.getEntryType() != IndexIntoFile::kEvent) + return false; + if (run != indexIntoFileIter_.run()) + return false; + if (lumi != indexIntoFileIter_.lumi()) + return false; + //The following is used for its side effect of advancing the + // eventTree entry. + fillEventAuxiliary(indexIntoFileIter_.entry()); + return true; + } + + void RootFile::readEventHistoryTree() { + // Read in the event history tree, if we have one... + if (fileFormatVersion().eventHistoryTree()) { + history_ = std::make_unique(); // propagate_const has no reset() function + eventHistoryTree_ = dynamic_cast(filePtr_->Get(poolNames::eventHistoryTreeName().c_str())); + if (!eventHistoryTree_) { + throw Exception(errors::EventCorruption) << "Failed to find the event history tree.\n"; + } + } + } + + void RootFile::initializeDuplicateChecker( + std::vector> const& indexesIntoFiles, + std::vector>::size_type currentIndexIntoFile) { + if (duplicateChecker_ && !duplicateChecker_->checkDisabled()) { + if (eventTree_.next()) { + auto const evtAux = fillThisEventAuxiliary(); + + duplicateChecker_->inputFileOpened(evtAux.isRealData(), indexIntoFile_, indexesIntoFiles, currentIndexIntoFile); + } + eventTree_.setEntryNumber(IndexIntoFile::invalidEntry); + } + } + + void RootFile::setPresenceInProductRegistry(ProductRegistry& inputProdDescReg, + StoredProcessBlockHelper const& storedProcessBlockHelper) { + // Set product presence information in the product registry. + // "Presence" is a boolean that is true if and only if the TBranch exists + // in the TTree (except it will be false for ProcessBlock products in non-Primary + // input files). + ProductRegistry::ProductList& pList = inputProdDescReg.productListUpdator(); + for (auto& product : pList) { + ProductDescription& prod = product.second; + // Initialize ProductDescription from dictionary only if the + // branch is present. This allows a subsequent job to process + // data where a dictionary of a transient parent branch has been + // removed from the release after the file has been written. + prod.initBranchName(); + if (prod.branchType() == InProcess) { + std::vector const& processes = storedProcessBlockHelper.processesWithProcessBlockProducts(); + auto it = std::find(processes.begin(), processes.end(), prod.processName()); + if (it != processes.end()) { + auto index = std::distance(processes.begin(), it); + processBlockTrees_[index]->setPresence(prod, newBranchToOldBranch(prod.branchName())); + } else { + // Given current rules for saving ProductDescriptions, this case should only occur + // in non-Primary sequences. + prod.setDropped(true); + } + } else { + treePointers_[prod.branchType()]->setPresence(prod, newBranchToOldBranch(prod.branchName())); + } + if (prod.present()) { + prod.initFromDictionary(); + } + } + } + + void RootFile::markBranchToBeDropped(bool dropDescendants, + ProductDescription const& branch, + std::set& branchesToDrop, + std::map const& droppedToKeptAlias) const { + if (dropDescendants) { + productDependencies_->appendToDescendants(branch, branchesToDrop, droppedToKeptAlias); + } else { + branchesToDrop.insert(branch.branchID()); + } + } + + void RootFile::dropOnInputAndReorder(ProductRegistry& reg, + ProductSelectorRules const& rules, + bool dropDescendants, + InputType inputType, + StoredProcessBlockHelper& storedProcessBlockHelper, + ProcessBlockHelper const* processBlockHelper) { + ProductRegistry::ProductList& prodList = reg.productListUpdator(); + + // First fill in a map we will need to navigate to descendants + // in the case of EDAliases. + std::map droppedToKeptAlias; + for (auto const& product : prodList) { + ProductDescription const& prod = product.second; + if (prod.branchID() != prod.originalBranchID() && prod.present()) { + droppedToKeptAlias[prod.originalBranchID()] = prod.branchID(); + } + } + + // This object will select products based on the branchName and the + // keep and drop statements which are in the source configuration. + ProductSelector productSelector; + productSelector.initialize(rules, reg.allProductDescriptions()); + + // In this pass, fill in a set of branches to be dropped. + // Don't drop anything yet. + std::set branchesToDrop; + std::vector associationDescriptions; + for (auto const& product : prodList) { + ProductDescription const& prod = product.second; + if (inputType != InputType::Primary && prod.branchType() == InProcess) { + markBranchToBeDropped(dropDescendants, prod, branchesToDrop, droppedToKeptAlias); + } else if (prod.unwrappedType() == typeid(ThinnedAssociation) && prod.present()) { + // Special handling for ThinnedAssociations + if (inputType != InputType::SecondarySource) { + associationDescriptions.push_back(&prod); + } else { + markBranchToBeDropped(dropDescendants, prod, branchesToDrop, droppedToKeptAlias); + } + } else if (!productSelector.selected(prod)) { + markBranchToBeDropped(dropDescendants, prod, branchesToDrop, droppedToKeptAlias); + } + } + + if (inputType != InputType::SecondarySource) { + // Decide whether to keep the thinned associations and corresponding + // entries in the helper. For secondary source they are all dropped, + // but in other cases we look for thinned collections the associations + // redirect a Ref or Ptr to when dereferencing them. + + // Need a list of kept products in order to determine which thinned associations + // are kept. + std::set keptProductsInEvent; + for (auto const& product : prodList) { + ProductDescription const& prod = product.second; + if (branchesToDrop.find(prod.branchID()) == branchesToDrop.end() && prod.present() && + prod.branchType() == InEvent) { + keptProductsInEvent.insert(prod.branchID()); + } + } + + // Decide which ThinnedAssociations to keep and store the decision in keepAssociation + std::map keepAssociation; + fileThinnedAssociationsHelper_->selectAssociationProducts( + associationDescriptions, keptProductsInEvent, keepAssociation); + + for (auto association : associationDescriptions) { + if (!keepAssociation[association->branchID()]) { + markBranchToBeDropped(dropDescendants, *association, branchesToDrop, droppedToKeptAlias); + } + } + + // Also delete the dropped associations from the ThinnedAssociationsHelper + auto temp = std::make_unique(); + for (auto const& associationBranches : fileThinnedAssociationsHelper_->data()) { + auto iter = keepAssociation.find(associationBranches.association()); + if (iter != keepAssociation.end() && iter->second) { + temp->addAssociation(associationBranches); + } + } + // propagate_const has no reset() function + fileThinnedAssociationsHelper_ = std::unique_ptr(temp.release()); + } + + // On this pass, actually drop the branches. + std::set processesWithKeptProcessBlockProducts; + std::set::const_iterator branchesToDropEnd = branchesToDrop.end(); + for (ProductRegistry::ProductList::iterator it = prodList.begin(), itEnd = prodList.end(); it != itEnd;) { + ProductDescription const& prod = it->second; + bool drop = branchesToDrop.find(prod.branchID()) != branchesToDropEnd; + if (drop) { + if (!prod.dropped()) { + if (productSelector.selected(prod) && prod.unwrappedType() != typeid(ThinnedAssociation) && + prod.branchType() != InProcess) { + LogWarning("RootFile") << "Branch '" << prod.branchName() << "' is being dropped from the input\n" + << "of file '" << file_ << "' because it is dependent on a branch\n" + << "that was explicitly dropped.\n"; + } + if (prod.branchType() == InProcess) { + std::vector const& processes = storedProcessBlockHelper.processesWithProcessBlockProducts(); + auto it = std::find(processes.begin(), processes.end(), prod.processName()); + assert(it != processes.end()); + auto index = std::distance(processes.begin(), it); + processBlockTrees_[index]->dropBranch(newBranchToOldBranch(prod.branchName())); + } else { + treePointers_[prod.branchType()]->dropBranch(newBranchToOldBranch(prod.branchName())); + } + hasNewlyDroppedBranch_[prod.branchType()] = true; + } + ProductRegistry::ProductList::iterator icopy = it; + ++it; + prodList.erase(icopy); + } else { + if (prod.branchType() == InProcess && prod.present()) { + processesWithKeptProcessBlockProducts.insert(prod.processName()); + } + ++it; + } + } + + dropProcessesAndReorder(storedProcessBlockHelper, processesWithKeptProcessBlockProducts, processBlockHelper); + + // Drop on input mergeable run and lumi products, this needs to be invoked for secondary file input + if (inputType == InputType::SecondaryFile) { + TString tString; + for (ProductRegistry::ProductList::iterator it = prodList.begin(), itEnd = prodList.end(); it != itEnd;) { + ProductDescription const& prod = it->second; + if (prod.present() and prod.branchType() != InEvent and prod.branchType() != InProcess) { + TClass* cp = prod.wrappedType().getClass(); + void* p = cp->New(); + int offset = cp->GetBaseClassOffset(edProductClass_); + std::unique_ptr edp = getWrapperBasePtr(p, offset); + if (edp->isMergeable()) { + treePointers_[prod.branchType()]->dropBranch(newBranchToOldBranch(prod.branchName())); + ProductRegistry::ProductList::iterator icopy = it; + ++it; + prodList.erase(icopy); + } else { + ++it; + } + } else + ++it; + } + } + } + + void RootFile::dropProcessesAndReorder(StoredProcessBlockHelper& storedProcessBlockHelper, + std::set const& processesWithKeptProcessBlockProducts, + ProcessBlockHelper const* processBlockHelper) { + // Modify storedProcessBlockHelper and processBlockTrees_ + // This should account for dropOnInput and also make the + // order of process blocks in input files after the first + // be the same as the first. Processes with no ProcessBlock + // products should be removed. After this executes, + // the items in storedProcessBlockHelper + // and processBlockTrees should be in exact one to one + // correspondence and in the same order. For input files + // after the first, these items should be either the same + // as or a subset of the items in processBlockHelper and in + // the same order. + + if (processBlockTrees_.empty()) { + return; + } + + std::vector nEntries; + nEntries.reserve(processBlockTrees_.size()); + for (auto const& processBlockTree : processBlockTrees_) { + nEntries.push_back(processBlockTree->entries()); + } + + bool firstInputFile = !processBlockHelper->initializedFromInput(); + bool isModified = false; + std::vector finalIndexToStoredIndex; + + if (firstInputFile) { + isModified = processBlockHelper->firstFileDropProcessesAndReorderStored( + storedProcessBlockHelper, processesWithKeptProcessBlockProducts, nEntries, finalIndexToStoredIndex); + } else { + isModified = + processBlockHelper->dropProcessesAndReorderStored(storedProcessBlockHelper, + processesWithKeptProcessBlockProducts, + nEntries, + finalIndexToStoredIndex, + processBlockHelper->processesWithProcessBlockProducts()); + } + + // At this point, any modifications to storedProcessBlockHelper are done. + // Make consistent changes to processBlockTrees_ and this will cause + // unneeded RootRNTuples to be deleted. + if (isModified) { + std::vector>> newProcessBlockTrees; + unsigned int nFinalProducts = storedProcessBlockHelper.processesWithProcessBlockProducts().size(); + for (unsigned int j = 0; j < nFinalProducts; ++j) { + unsigned int iStored = finalIndexToStoredIndex[j]; + newProcessBlockTrees.push_back(std::move(processBlockTrees_[iStored])); + } + processBlockTrees_.swap(newProcessBlockTrees); + } + } + + void RootFile::setSignals( + signalslot::Signal const* preEventReadSource, + signalslot::Signal const* postEventReadSource) { + eventTree_.setSignals(preEventReadSource, postEventReadSource); + } + + void RootFile::makeProcessBlockRootRNTuples(std::shared_ptr filePtr, + TTreeOptions const& options, + InputType inputType, + StoredProcessBlockHelper const& storedProcessBlockHelper) { + // When this functions returns there will be exactly a 1-to-1 correspondence between the + // processes listed in storedProcessBlockHelper and the RootRNTuple objects created. processBlockTrees_ + // has pointers to the RootRNTuple's and will be filled in the same order. The RootRNTuple constructor + // will throw an exception if one of these TTree's is not in the file and this should be all of + // the ProcessBlock TTree's in the file. (later in the RootFile constructor, dropOnInput might + // remove some and also reordering may occur). + for (auto const& process : storedProcessBlockHelper.processesWithProcessBlockProducts()) { + processBlockTrees_.emplace_back(std::make_unique(filePtr, + InProcess, + process, + 1, + options.usingDefaultNonEventOptions(), + rootrntuple::defaultNonEventLearningEntries, + inputType)); + } + } + + std::unique_ptr RootFile::makeProvenanceReaderMaker(InputType inputType) { + if (fileFormatVersion_.storedProductProvenanceUsed()) { + readParentageTree(inputType); + return std::make_unique(parentageIDLookup_); + } else if (fileFormatVersion_.splitProductIDs()) { + readParentageTree(inputType); + return std::make_unique(); + } else if (fileFormatVersion_.perEventProductIDs()) { + auto entryDescriptionMap = std::make_unique(); + readEntryDescriptionTree(*entryDescriptionMap, inputType); + return std::make_unique(std::move(entryDescriptionMap)); + } else { + return std::make_unique(); + } + } + + std::shared_ptr RootFile::makeProductProvenanceRetriever(unsigned int iStreamID) { + if (eventProductProvenanceRetrievers_.size() <= iStreamID) { + eventProductProvenanceRetrievers_.resize(iStreamID + 1); + } + if (!eventProductProvenanceRetrievers_[iStreamID]) { + // propagate_const has no reset() function + eventProductProvenanceRetrievers_[iStreamID] = std::make_shared( + iStreamID, provenanceReaderMaker_->makeReader(eventTree_, daqProvenanceHelper_.get())); + } + eventProductProvenanceRetrievers_[iStreamID]->reset(); + return eventProductProvenanceRetriever(iStreamID); + } + + class ReducedProvenanceReader : public ProvenanceReaderBase { + public: + ReducedProvenanceReader(RootRNTuple* iRootRNTuple, + std::vector const& iParentageIDLookup, + DaqProvenanceHelper const* daqProvenanceHelper); + + std::set readProvenance(unsigned int) const override; + + void unsafe_fillProvenance(unsigned int) const override; + + private: + void readProvenanceAsync(WaitingTaskHolder task, + ModuleCallingContext const* moduleCallingContext, + unsigned int transitionIndex, + std::atomic*>& writeTo) const noexcept override; + + edm::propagate_const rootTree_; + edm::propagate_const provBranch_; + StoredProductProvenanceVector provVector_; + StoredProductProvenanceVector const* pProvVector_; + std::vector const& parentageIDLookup_; + DaqProvenanceHelper const* daqProvenanceHelper_; + std::shared_ptr mutex_; + SharedResourcesAcquirer acquirer_; + }; + + ReducedProvenanceReader::ReducedProvenanceReader(RootRNTuple* iRootRNTuple, + std::vector const& iParentageIDLookup, + DaqProvenanceHelper const* daqProvenanceHelper) + : ProvenanceReaderBase(), + rootTree_(iRootRNTuple), + pProvVector_(&provVector_), + parentageIDLookup_(iParentageIDLookup), + daqProvenanceHelper_(daqProvenanceHelper), + mutex_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().second), + acquirer_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().first) { + provBranch_ = + rootTree_->tree()->GetBranch(BranchTypeToProductProvenanceBranchName(rootTree_->branchType()).c_str()); + } + + namespace { + using SignalType = signalslot::Signal; + template + void readProvenanceAsyncImpl(R const* iThis, + SerialTaskQueueChain& chain, + WaitingTaskHolder task, + unsigned int transitionIndex, + std::atomic*>& writeTo, + ModuleCallingContext const* iContext, + SignalType const* pre, + SignalType const* post) { + if (nullptr == writeTo.load()) { + //need to be sure the task isn't run until after the read + WaitingTaskHolder taskHolder{task}; + auto pWriteTo = &writeTo; + + auto serviceToken = ServiceRegistry::instance().presentToken(); + + chain.push( + *taskHolder.group(), + [holder = std::move(taskHolder), + pWriteTo, + iThis, + transitionIndex, + iContext, + pre, + post, + serviceToken]() mutable { + if (nullptr == pWriteTo->load()) { + ServiceRegistry::Operate operate(serviceToken); + std::unique_ptr> prov; + try { + if (pre) { + pre->emit(*(iContext->getStreamContext()), *iContext); + } + prov = std::make_unique>(iThis->readProvenance(transitionIndex)); + if (post) { + post->emit(*(iContext->getStreamContext()), *iContext); + } + + } catch (...) { + if (post) { + post->emit(*(iContext->getStreamContext()), *iContext); + } + + holder.doneWaiting(std::current_exception()); + return; + } + const std::set* expected = nullptr; + + if (pWriteTo->compare_exchange_strong(expected, prov.get())) { + prov.release(); + } + } + holder.doneWaiting(std::exception_ptr()); + }); + } + } + } // namespace + + void ReducedProvenanceReader::readProvenanceAsync( + WaitingTaskHolder task, + ModuleCallingContext const* moduleCallingContext, + unsigned int transitionIndex, + std::atomic*>& writeTo) const noexcept { + readProvenanceAsyncImpl(this, + acquirer_.serialQueueChain(), + task, + transitionIndex, + writeTo, + moduleCallingContext, + rootTree_->rootDelayedReader()->preEventReadFromSourceSignal(), + rootTree_->rootDelayedReader()->postEventReadFromSourceSignal()); + } + + // + void ReducedProvenanceReader::unsafe_fillProvenance(unsigned int transitionIndex) const { + ReducedProvenanceReader* me = const_cast(this); + me->rootTree_->fillBranchEntry( + me->provBranch_, me->rootTree_->entryNumberForIndex(transitionIndex), me->pProvVector_); + } + + std::set ReducedProvenanceReader::readProvenance(unsigned int transitionIndex) const { + if (provVector_.empty()) { + std::lock_guard guard(*mutex_); + ReducedProvenanceReader* me = const_cast(this); + me->rootTree_->fillBranchEntry( + me->provBranch_, me->rootTree_->entryNumberForIndex(transitionIndex), me->pProvVector_); + } + std::set retValue; + if (daqProvenanceHelper_) { + for (auto const& prov : provVector_) { + BranchID bid(prov.branchID_); + retValue.emplace(daqProvenanceHelper_->mapBranchID(BranchID(prov.branchID_)), + daqProvenanceHelper_->mapParentageID(parentageIDLookup_[prov.parentageIDIndex_])); + } + } else { + for (auto const& prov : provVector_) { + if (prov.parentageIDIndex_ >= parentageIDLookup_.size()) { + throw edm::Exception(errors::LogicError) + << "ReducedProvenanceReader::ReadProvenance\n" + << "The parentage ID index value " << prov.parentageIDIndex_ + << " is out of bounds. The maximum value is " << parentageIDLookup_.size() - 1 << ".\n" + << "This should never happen.\n" + << "Please report this to the framework developers."; + } + retValue.emplace(BranchID(prov.branchID_), parentageIDLookup_[prov.parentageIDIndex_]); + } + } + //The results of this call are cached by the caller + const_cast(this)->provVector_.clear(); + + return retValue; + } + + class FullProvenanceReader : public ProvenanceReaderBase { + public: + explicit FullProvenanceReader(RootRNTuple* rootTree, DaqProvenanceHelper const* daqProvenanceHelper); + ~FullProvenanceReader() override {} + std::set readProvenance(unsigned int transitionIndex) const override; + + private: + void readProvenanceAsync(WaitingTaskHolder task, + ModuleCallingContext const* moduleCallingContext, + unsigned int transitionIndex, + std::atomic*>& writeTo) const noexcept override; + + RootRNTuple* rootTree_; + ProductProvenanceVector infoVector_; + //All access to a ROOT file is serialized + CMS_SA_ALLOW mutable ProductProvenanceVector* pInfoVector_; + DaqProvenanceHelper const* daqProvenanceHelper_; + std::shared_ptr mutex_; + SharedResourcesAcquirer acquirer_; + }; + + FullProvenanceReader::FullProvenanceReader(RootRNTuple* rootTree, DaqProvenanceHelper const* daqProvenanceHelper) + : ProvenanceReaderBase(), + rootTree_(rootTree), + infoVector_(), + pInfoVector_(&infoVector_), + daqProvenanceHelper_(daqProvenanceHelper), + mutex_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().second), + acquirer_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().first) {} + + void FullProvenanceReader::readProvenanceAsync( + WaitingTaskHolder task, + ModuleCallingContext const* moduleCallingContext, + unsigned int transitionIndex, + std::atomic*>& writeTo) const noexcept { + readProvenanceAsyncImpl(this, + acquirer_.serialQueueChain(), + task, + transitionIndex, + writeTo, + moduleCallingContext, + rootTree_->rootDelayedReader()->preEventReadFromSourceSignal(), + rootTree_->rootDelayedReader()->postEventReadFromSourceSignal()); + } + + std::set FullProvenanceReader::readProvenance(unsigned int transitionIndex) const { + { + std::lock_guard guard(*mutex_); + rootTree_->fillBranchEntryMeta( + rootTree_->branchEntryInfoBranch(), rootTree_->entryNumberForIndex(transitionIndex), pInfoVector_); + } + std::set retValue; + if (daqProvenanceHelper_) { + for (auto const& info : infoVector_) { + retValue.emplace(daqProvenanceHelper_->mapBranchID(info.branchID()), + daqProvenanceHelper_->mapParentageID(info.parentageID())); + } + } else { + for (auto const& info : infoVector_) { + retValue.emplace(info); + } + } + return retValue; + } + + class OldProvenanceReader : public ProvenanceReaderBase { + public: + explicit OldProvenanceReader(RootRNTuple* rootTree, + EntryDescriptionMap const& theMap, + DaqProvenanceHelper const* daqProvenanceHelper); + ~OldProvenanceReader() override {} + std::set readProvenance(unsigned int transitionIndex) const override; + + private: + void readProvenanceAsync(WaitingTaskHolder task, + ModuleCallingContext const* moduleCallingContext, + unsigned int transitionIndex, + std::atomic*>& writeTo) const noexcept override; + + edm::propagate_const rootTree_; + std::vector infoVector_; + //All access to ROOT file are serialized + CMS_SA_ALLOW mutable std::vector* pInfoVector_; + EntryDescriptionMap const& entryDescriptionMap_; + DaqProvenanceHelper const* daqProvenanceHelper_; + std::shared_ptr mutex_; + SharedResourcesAcquirer acquirer_; + }; + + OldProvenanceReader::OldProvenanceReader(RootRNTuple* rootTree, + EntryDescriptionMap const& theMap, + DaqProvenanceHelper const* daqProvenanceHelper) + : ProvenanceReaderBase(), + rootTree_(rootTree), + infoVector_(), + pInfoVector_(&infoVector_), + entryDescriptionMap_(theMap), + daqProvenanceHelper_(daqProvenanceHelper), + mutex_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().second), + acquirer_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().first) {} + + void OldProvenanceReader::readProvenanceAsync( + WaitingTaskHolder task, + ModuleCallingContext const* moduleCallingContext, + unsigned int transitionIndex, + std::atomic*>& writeTo) const noexcept { + readProvenanceAsyncImpl(this, + acquirer_.serialQueueChain(), + task, + transitionIndex, + writeTo, + moduleCallingContext, + rootTree_->rootDelayedReader()->preEventReadFromSourceSignal(), + rootTree_->rootDelayedReader()->postEventReadFromSourceSignal()); + } + + std::set OldProvenanceReader::readProvenance(unsigned int transitionIndex) const { + { + std::lock_guard guard(*mutex_); + rootTree_->branchEntryInfoBranch()->SetAddress(&pInfoVector_); + rootrntuple::getEntry(rootTree_->branchEntryInfoBranch(), rootTree_->entryNumberForIndex(transitionIndex)); + } + std::set retValue; + for (auto const& info : infoVector_) { + EntryDescriptionMap::const_iterator iter = entryDescriptionMap_.find(info.entryDescriptionID()); + assert(iter != entryDescriptionMap_.end()); + Parentage parentage(iter->second.parents()); + if (daqProvenanceHelper_) { + retValue.emplace(daqProvenanceHelper_->mapBranchID(info.branchID()), + daqProvenanceHelper_->mapParentageID(parentage.id())); + } else { + retValue.emplace(info.branchID(), parentage.id()); + } + } + return retValue; + } + + class DummyProvenanceReader : public ProvenanceReaderBase { + public: + DummyProvenanceReader(); + ~DummyProvenanceReader() override {} + + private: + std::set readProvenance(unsigned int) const override; + void readProvenanceAsync(WaitingTaskHolder task, + ModuleCallingContext const* moduleCallingContext, + unsigned int transitionIndex, + std::atomic*>& writeTo) const noexcept override; + }; + + DummyProvenanceReader::DummyProvenanceReader() : ProvenanceReaderBase() {} + + std::set DummyProvenanceReader::readProvenance(unsigned int) const { + // Not providing parentage!!! + return std::set{}; + } + void DummyProvenanceReader::readProvenanceAsync( + WaitingTaskHolder task, + ModuleCallingContext const* moduleCallingContext, + unsigned int transitionIndex, + std::atomic*>& writeTo) const noexcept { + if (nullptr == writeTo.load()) { + auto emptyProv = std::make_unique>(); + const std::set* expected = nullptr; + if (writeTo.compare_exchange_strong(expected, emptyProv.get())) { + emptyProv.release(); + } + } + } + + std::unique_ptr MakeDummyProvenanceReader::makeReader(RootRNTuple&, + DaqProvenanceHelper const*) const { + return std::make_unique(); + } + + std::unique_ptr MakeOldProvenanceReader::makeReader( + RootRNTuple& rootTree, DaqProvenanceHelper const* daqProvenanceHelper) const { + return std::make_unique(&rootTree, *entryDescriptionMap_, daqProvenanceHelper); + } + + std::unique_ptr MakeFullProvenanceReader::makeReader( + RootRNTuple& rootTree, DaqProvenanceHelper const* daqProvenanceHelper) const { + return std::make_unique(&rootTree, daqProvenanceHelper); + } + + std::unique_ptr MakeReducedProvenanceReader::makeReader( + RootRNTuple& rootTree, DaqProvenanceHelper const* daqProvenanceHelper) const { + return std::make_unique(&rootTree, parentageIDLookup_, daqProvenanceHelper); + } +} // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempInput/src/RootFile.h b/FWIO/RNTupleTempInput/src/RootFile.h new file mode 100644 index 0000000000000..e176cd08362e4 --- /dev/null +++ b/FWIO/RNTupleTempInput/src/RootFile.h @@ -0,0 +1,331 @@ +#ifndef FWIO_RNTupleTempInput_RootFile_h +#define FWIO_RNTupleTempInput_RootFile_h + +/*---------------------------------------------------------------------- + +RootFile.h // used by ROOT input sources + +----------------------------------------------------------------------*/ + +#include "RootRNTuple.h" +#include "DataFormats/Provenance/interface/ProductDependencies.h" +#include "DataFormats/Provenance/interface/BranchIDList.h" +#include "DataFormats/Provenance/interface/BranchListIndex.h" +#include "DataFormats/Provenance/interface/EntryDescriptionID.h" // backward compatibility +#include "DataFormats/Provenance/interface/EventAuxiliary.h" +#include "DataFormats/Provenance/interface/EventEntryDescription.h" // backward compatibility +#include "DataFormats/Provenance/interface/EventProcessHistoryID.h" // backward compatibility +#include "DataFormats/Provenance/interface/EventSelectionID.h" +#include "DataFormats/Provenance/interface/EventToProcessBlockIndexes.h" +#include "DataFormats/Provenance/interface/FileFormatVersion.h" +#include "DataFormats/Provenance/interface/FileID.h" +#include "DataFormats/Provenance/interface/History.h" +#include "DataFormats/Provenance/interface/IndexIntoFile.h" +#include "DataFormats/Provenance/interface/ParentageID.h" +#include "DataFormats/Provenance/interface/ProvenanceFwd.h" +#include "FWCore/Common/interface/FWCoreCommonFwd.h" +#include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/Framework/interface/InputSource.h" +#include "FWCore/Utilities/interface/InputType.h" +#include "FWCore/Utilities/interface/get_underlying_safe.h" +#include "FWCore/Utilities/interface/propagate_const.h" + +#include "TBranch.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace edm { + + //------------------------------------------------------------ + // Class RootFile: supports file reading. + + class BranchID; + class BranchIDListHelper; + class ProductProvenanceRetriever; + struct DaqProvenanceHelper; + class EventSkipperByID; + class ProcessHistoryRegistry; + class ProductSelectorRules; + class ProvenanceReaderBase; + class StoredMergeableRunProductMetadata; + class InputSourceRunHelperBase; + class ThinnedAssociationsHelper; + + namespace rntuple_temp { + class DuplicateChecker; + class ProvenanceAdaptor; + class InputFile; + + using EntryDescriptionMap = std::map; + + class MakeProvenanceReader { + public: + virtual std::unique_ptr makeReader( + RootRNTuple& eventTree, DaqProvenanceHelper const* daqProvenanceHelper) const = 0; + virtual ~MakeProvenanceReader() = default; + }; + + class RootFile { + public: + struct FileOptions { + std::string const& fileName; + std::string const& logicalFileName; + std::shared_ptr filePtr; + bool bypassVersionCheck; + bool enforceGUIDInFileName; + }; + + struct ProcessingOptions { + std::shared_ptr eventSkipperByID{}; + bool skipAnyEvents = false; + int remainingEvents = -1; + int remainingLumis = -1; + InputSource::ProcessingMode processingMode = InputSource::RunsLumisAndEvents; + bool noRunLumiSort = false; + bool noEventSort = false; + bool usingGoToEvent = false; + }; + + using TTreeOptions = RootRNTuple::Options; + struct ProductChoices { + ProductSelectorRules const& productSelectorRules; + std::vector const* associationsFromSecondary = nullptr; + bool dropDescendantsOfDroppedProducts = false; + bool labelRawDataLikeMC = false; + }; + + struct CrossFileInfo { + InputSourceRunHelperBase* runHelper = nullptr; + std::shared_ptr branchIDListHelper{}; + ProcessBlockHelper* processBlockHelper = nullptr; + std::shared_ptr thinnedAssociationsHelper{}; + std::shared_ptr duplicateChecker{}; + std::vector> const& indexesIntoFiles; //duplicate checking + std::vector>::size_type currentIndexIntoFile; + }; + + RootFile(FileOptions&& fileOptions, + InputType inputType, + ProcessingOptions&& processingOptions, + TTreeOptions&& ttreeOptions, + ProductChoices&& productChoices, + CrossFileInfo&& crossFileInfo, + unsigned int nStreams, + ProcessHistoryRegistry& processHistoryRegistry, + std::vector& orderedProcessHistoryIDs); + ~RootFile(); + + RootFile(RootFile const&) = delete; // Disallow copying and moving + RootFile& operator=(RootFile const&) = delete; // Disallow copying and moving + + void reportOpened(std::string const& inputType); + void close(); + std::tuple readCurrentEvent(EventPrincipal& cache, + bool assertOnFailure = true, + bool readAllProducts = false); + bool readEvent(EventPrincipal& cache, bool readAllProducts = false); + + std::shared_ptr readLuminosityBlockAuxiliary_(); + std::shared_ptr readRunAuxiliary_(); + std::shared_ptr readFakeRunAuxiliary_(); + + void fillProcessBlockHelper_(); + bool initializeFirstProcessBlockEntry(); + bool endOfProcessBlocksReached() const; + bool nextProcessBlock_(ProcessBlockPrincipal&); + void readProcessBlock_(ProcessBlockPrincipal&); + + bool readRun_(RunPrincipal& runPrincipal); + void readFakeRun_(RunPrincipal& runPrincipal); + bool readLuminosityBlock_(LuminosityBlockPrincipal& lumiPrincipal); + std::string const& file() const { return file_; } + std::shared_ptr productRegistry() const { return productRegistry_; } + // IndexIntoFile::EntryNumber_t const& entryNumber() const {return indexIntoFileIter().entry();} + // LuminosityBlockNumber_t const& luminosityBlockNumber() const {return indexIntoFileIter().lumi();} + // RunNumber_t const& runNumber() const {return indexIntoFileIter().run();} + RootRNTuple const& eventTree() const { return eventTree_; } + RootRNTuple const& lumiTree() const { return lumiTree_; } + RootRNTuple const& runTree() const { return runTree_; } + FileFormatVersion fileFormatVersion() const { return fileFormatVersion_; } + int whyNotFastClonable() const { return whyNotFastClonable_; } + std::array const& hasNewlyDroppedBranch() const { return hasNewlyDroppedBranch_; } + bool branchListIndexesUnchanged() const { return branchListIndexesUnchanged_; } + bool modifiedIDs() const { return daqProvenanceHelper_.get() != nullptr; } + std::shared_ptr createFileBlock(); + void updateFileBlock(FileBlock&); + + bool setEntryAtItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) { + return (event != 0) ? setEntryAtEvent(run, lumi, event) + : (lumi ? setEntryAtLumi(run, lumi) : setEntryAtRun(run)); + } + bool containsItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) const; + bool setEntryAtEvent(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event); + bool setEntryAtLumi(RunNumber_t run, LuminosityBlockNumber_t lumi); + bool setEntryAtRun(RunNumber_t run); + bool setEntryAtNextEventInLumi(RunNumber_t run, LuminosityBlockNumber_t lumi); + void setAtEventEntry(IndexIntoFile::EntryNumber_t entry); + + void rewind() { + indexIntoFileIter_ = indexIntoFileBegin_; + eventTree_.rewind(); + lumiTree_.rewind(); + runTree_.rewind(); + currentProcessBlockTree_ = 0; + for (auto& processBlockTree : processBlockTrees_) { + processBlockTree->rewindToInvalid(); + } + } + void setToLastEntry() { indexIntoFileIter_ = indexIntoFileEnd_; } + + bool skipEntries(unsigned int& offset) { return eventTree_.skipEntries(offset); } + bool skipEvents(int& offset); + bool goToEvent(EventID const& eventID); + bool nextEventEntry() { return eventTree_.nextWithCache(); } + IndexIntoFile::EntryType getNextItemType(RunNumber_t& run, LuminosityBlockNumber_t& lumi, EventNumber_t& event); + std::shared_ptr branchIDListHelper() const { + return get_underlying_safe(branchIDListHelper_); + } + std::shared_ptr& branchIDListHelper() { return get_underlying_safe(branchIDListHelper_); } + std::shared_ptr indexIntoFileSharedPtr() const { + return get_underlying_safe(indexIntoFileSharedPtr_); + } + std::shared_ptr& indexIntoFileSharedPtr() { return get_underlying_safe(indexIntoFileSharedPtr_); } + bool wasLastEventJustRead() const; + bool wasFirstEventJustRead() const; + IndexIntoFile::IndexIntoFileItr indexIntoFileIter() const; + void setPosition(IndexIntoFile::IndexIntoFileItr const& position); + void initAssociationsFromSecondary(std::vector const&); + + void setSignals( + signalslot::Signal const* preEventReadSource, + signalslot::Signal const* postEventReadSource); + + private: + void makeProcessBlockRootRNTuples(std::shared_ptr filePtr, + TTreeOptions const& ttreeOptions, + InputType inputType, + StoredProcessBlockHelper const& storedProcessBlockHelper); + bool skipThisEntry(); + void setIfFastClonable(int remainingEvents, int remainingLumis); + void validateFile(InputType inputType, + bool usingGoToEvent, + std::vector& orderedProcessHistoryIDs); + void fillIndexIntoFile(); + EventAuxiliary fillEventAuxiliary(IndexIntoFile::EntryNumber_t entry); + EventAuxiliary const& fillThisEventAuxiliary(); + void fillEventToProcessBlockIndexes(); + bool fillEventHistory(EventAuxiliary& evtAux, + EventSelectionIDVector& eventSelectionIDs, + BranchListIndexes& branchListIndexes, + bool assertOnFailure = true); + std::shared_ptr fillLumiAuxiliary(); + std::shared_ptr fillRunAuxiliary(); + std::string const& newBranchToOldBranch(std::string const& newBranch) const; + void setPresenceInProductRegistry(ProductRegistry&, StoredProcessBlockHelper const&); + void markBranchToBeDropped(bool dropDescendants, + ProductDescription const& branch, + std::set& branchesToDrop, + std::map const& droppedToKeptAlias) const; + void dropOnInputAndReorder(ProductRegistry&, + ProductSelectorRules const&, + bool dropDescendants, + InputType, + StoredProcessBlockHelper&, + ProcessBlockHelper const*); + void dropProcessesAndReorder(StoredProcessBlockHelper&, + std::set const& processesWithKeptProcessBlockProducts, + ProcessBlockHelper const*); + + void readParentageTree(InputType inputType); + void readEntryDescriptionTree(EntryDescriptionMap& entryDescriptionMap, + InputType inputType); // backward compatibility + void readEventHistoryTree(); + bool isDuplicateEvent(); + + void initializeDuplicateChecker(std::vector> const& indexesIntoFiles, + std::vector>::size_type currentIndexIntoFile); + + std::unique_ptr makeProvenanceReaderMaker(InputType inputType); + std::shared_ptr makeProductProvenanceRetriever(unsigned int iStreamIndex); + + std::shared_ptr savedRunAuxiliary() const { return get_underlying_safe(savedRunAuxiliary_); } + std::shared_ptr& savedRunAuxiliary() { return get_underlying_safe(savedRunAuxiliary_); } + + std::shared_ptr productDependencies() const { + return get_underlying_safe(productDependencies_); + } + std::shared_ptr& productDependencies() { return get_underlying_safe(productDependencies_); } + + std::shared_ptr eventProductProvenanceRetriever(size_t index) const { + return get_underlying_safe(eventProductProvenanceRetrievers_[index]); + } + std::shared_ptr& eventProductProvenanceRetriever(size_t index) { + return get_underlying_safe(eventProductProvenanceRetrievers_[index]); + } + + std::string const file_; + std::string const logicalFile_; + edm::propagate_const processHistoryRegistry_; // We don't own this + edm::propagate_const> filePtr_; + edm::propagate_const> eventSkipperByID_; + FileFormatVersion fileFormatVersion_; + FileID fid_; + edm::propagate_const> indexIntoFileSharedPtr_; + IndexIntoFile& indexIntoFile_; + IndexIntoFile::IndexIntoFileItr indexIntoFileBegin_; + IndexIntoFile::IndexIntoFileItr indexIntoFileEnd_; + IndexIntoFile::IndexIntoFileItr indexIntoFileIter_; + edm::propagate_const> storedMergeableRunProductMetadata_; + std::vector eventProcessHistoryIDs_; // backward compatibility + std::vector::const_iterator eventProcessHistoryIter_; // backward compatibility + edm::propagate_const> savedRunAuxiliary_; + bool skipAnyEvents_; + bool noRunLumiSort_; + bool noEventSort_; + bool enforceGUIDInFileName_; + int whyNotFastClonable_; + std::array hasNewlyDroppedBranch_; + bool branchListIndexesUnchanged_; + EventAuxiliary eventAuxCache_; //Should only be used by fillThisEventAuxiliary() + RootRNTuple eventTree_; + RootRNTuple lumiTree_; + RootRNTuple runTree_; + std::vector>> processBlockTrees_; + unsigned int currentProcessBlockTree_ = 0; + std::vector> treePointers_; + //Should only be used by fillThisEventAuxiliary() + IndexIntoFile::EntryNumber_t lastEventEntryNumberRead_; + std::shared_ptr productRegistry_; + std::shared_ptr branchIDLists_; + edm::propagate_const> branchIDListHelper_; + edm::propagate_const processBlockHelper_; + edm::propagate_const> storedProcessBlockHelper_; + edm::propagate_const> fileThinnedAssociationsHelper_; + edm::propagate_const> thinnedAssociationsHelper_; + InputSource::ProcessingMode processingMode_; + edm::propagate_const runHelper_; + std::map newBranchToOldBranch_; + edm::propagate_const eventHistoryTree_; // backward compatibility + EventToProcessBlockIndexes eventToProcessBlockIndexes_; + edm::propagate_const eventToProcessBlockIndexesBranch_; + edm::propagate_const> history_; // backward compatibility + edm::propagate_const> productDependencies_; + edm::propagate_const> duplicateChecker_; + edm::propagate_const> provenanceAdaptor_; // backward comatibility + edm::propagate_const> provenanceReaderMaker_; + std::vector>> eventProductProvenanceRetrievers_; + std::vector parentageIDLookup_; + edm::propagate_const> daqProvenanceHelper_; + edm::propagate_const edProductClass_; + InputType inputType_; + }; // class RootFile + } // namespace rntuple_temp + +} // namespace edm +#endif diff --git a/FWIO/RNTupleTempInput/src/RootInputFileSequence.cc b/FWIO/RNTupleTempInput/src/RootInputFileSequence.cc new file mode 100644 index 0000000000000..c3b57d161bc0a --- /dev/null +++ b/FWIO/RNTupleTempInput/src/RootInputFileSequence.cc @@ -0,0 +1,329 @@ +/*---------------------------------------------------------------------- +----------------------------------------------------------------------*/ +#include "RootFile.h" +#include "RootInputFileSequence.h" + +#include "DataFormats/Provenance/interface/BranchID.h" +#include "DataFormats/Provenance/interface/IndexIntoFile.h" +#include "DataFormats/Provenance/interface/ProductRegistry.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" +#include "Utilities/StorageFactory/interface/StorageFactory.h" +#include "Utilities/StorageFactory/interface/StatisticsSenderService.h" +#include "FWCore/ServiceRegistry/interface/Service.h" + +#include "TSystem.h" + +namespace edm { + class BranchIDListHelper; + class EventPrincipal; + class LuminosityBlockPrincipal; + class RunPrincipal; +} // namespace edm +namespace edm::rntuple_temp { + + RootInputFileSequence::RootInputFileSequence(ParameterSet const& pset, InputFileCatalog const& catalog) + : catalog_(catalog), + lfn_("unknown"), + lfnHash_(0U), + usedFallback_(false), + findFileForSpecifiedID_(nullptr), + fileIterBegin_(fileCatalogItems().begin()), + fileIterEnd_(fileCatalogItems().end()), + fileIter_(fileIterEnd_), + fileIterLastOpened_(fileIterEnd_), + rootFile_(), + indexesIntoFiles_(fileCatalogItems().size()) {} + + std::vector const& RootInputFileSequence::fileCatalogItems() const { + return catalog_.fileCatalogItems(); + } + + std::shared_ptr RootInputFileSequence::fileProductRegistry() const { + assert(rootFile()); + return rootFile()->productRegistry(); + } + + std::shared_ptr RootInputFileSequence::fileBranchIDListHelper() const { + assert(rootFile()); + return rootFile()->branchIDListHelper(); + } + + RootInputFileSequence::~RootInputFileSequence() {} + + std::shared_ptr RootInputFileSequence::readRunAuxiliary_() { + assert(rootFile()); + return rootFile()->readRunAuxiliary_(); + } + + std::shared_ptr RootInputFileSequence::readLuminosityBlockAuxiliary_() { + assert(rootFile()); + return rootFile()->readLuminosityBlockAuxiliary_(); + } + + bool RootInputFileSequence::readRun_(RunPrincipal& runPrincipal) { + assert(rootFile()); + return rootFile()->readRun_(runPrincipal); + } + + void RootInputFileSequence::fillProcessBlockHelper_() { + assert(rootFile()); + return rootFile()->fillProcessBlockHelper_(); + } + + bool RootInputFileSequence::nextProcessBlock_(ProcessBlockPrincipal& processBlockPrincipal) { + assert(rootFile()); + return rootFile()->nextProcessBlock_(processBlockPrincipal); + } + + void RootInputFileSequence::readProcessBlock_(ProcessBlockPrincipal& processBlockPrincipal) { + assert(rootFile()); + rootFile()->readProcessBlock_(processBlockPrincipal); + } + + bool RootInputFileSequence::readLuminosityBlock_(LuminosityBlockPrincipal& lumiPrincipal) { + assert(rootFile()); + return rootFile()->readLuminosityBlock_(lumiPrincipal); + } + + // readEvent() is responsible for setting up the EventPrincipal. + // + // 1. fill an EventPrincipal with a unique EventID + // 2. For each entry in the provenance, put in one ProductResolver, + // holding the Provenance for the corresponding EDProduct. + // 3. set up the caches in the EventPrincipal to know about this + // ProductResolver. + // + // We do *not* create the EDProduct instance (the equivalent of reading + // the branch containing this EDProduct. That will be done by the Delayed Reader, + // when it is asked to do so. + // + + bool RootInputFileSequence::readEvent(EventPrincipal& eventPrincipal, bool readAllProducts) { + assert(rootFile()); + return rootFile()->readEvent(eventPrincipal, readAllProducts); + } + + bool RootInputFileSequence::containedInCurrentFile(RunNumber_t run, + LuminosityBlockNumber_t lumi, + EventNumber_t event) const { + if (!rootFile()) + return false; + return rootFile()->containsItem(run, lumi, event); + } + + bool RootInputFileSequence::skipToItemInNewFile(RunNumber_t run, + LuminosityBlockNumber_t lumi, + EventNumber_t event, + size_t fileNameHash) { + // Look for item in files not yet opened. We have a hash of the logical file name + assert(fileNameHash != 0U); + // If the lookup table is not yet filled in, fill it. + if (!findFileForSpecifiedID_) { + // We use a multimap because there may be hash collisions (Two different LFNs could have the same hash). + // We map the hash of the LFN to the index into the list of files. + findFileForSpecifiedID_ = + std::make_unique>(); // propagate_const has no reset() function + auto hasher = std::hash(); + for (auto fileIter = fileIterBegin_; fileIter != fileIterEnd_; ++fileIter) { + findFileForSpecifiedID_->insert(std::make_pair(hasher(fileIter->logicalFileName()), fileIter - fileIterBegin_)); + } + } + // Look up the logical file name in the table + auto range = findFileForSpecifiedID_->equal_range(fileNameHash); + for (auto iter = range.first; iter != range.second; ++iter) { + // Don't look in files previously opened, because those have already been searched. + if (!indexesIntoFiles_[iter->second]) { + setAtFileSequenceNumber(iter->second); + initFile_(false); + assert(rootFile()); + bool found = rootFile()->setEntryAtItem(run, lumi, event); + if (found) { + return true; + } + } + } + // Not found + return false; + } + + bool RootInputFileSequence::skipToItemInNewFile(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) { + // Look for item in files not yet opened. We do not have a valid hash of the logical file name. + for (auto it = indexesIntoFiles_.begin(), itEnd = indexesIntoFiles_.end(); it != itEnd; ++it) { + if (!*it) { + // File not yet opened. + setAtFileSequenceNumber(it - indexesIntoFiles_.begin()); + initFile_(false); + assert(rootFile()); + bool found = rootFile()->setEntryAtItem(run, lumi, event); + if (found) { + return true; + } + } + } + // Not found + return false; + } + + bool RootInputFileSequence::skipToItem( + RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event, size_t fileNameHash, bool currentFileFirst) { + // Attempt to find item in currently open input file. + bool found = currentFileFirst && rootFile() && rootFile()->setEntryAtItem(run, lumi, event); + if (!found) { + // If only one input file, give up now, to save time. + if (currentFileFirst && rootFile() && indexesIntoFiles_.size() == 1) { + return false; + } + // Look for item (run/lumi/event) in files previously opened without reopening unnecessary files. + for (auto it = indexesIntoFiles_.begin(), itEnd = indexesIntoFiles_.end(); it != itEnd; ++it) { + if (*it && (*it)->containsItem(run, lumi, event)) { + // We found it. Close the currently open file, and open the correct one. + std::vector::const_iterator currentIter = fileIter_; + setAtFileSequenceNumber(it - indexesIntoFiles_.begin()); + if (fileIter_ != currentIter) { + initFile(false); + } + // Now get the item from the correct file. + assert(rootFile()); + found = rootFile()->setEntryAtItem(run, lumi, event); + assert(found); + return true; + } + } + return (fileNameHash != 0U && skipToItemInNewFile(run, lumi, event, fileNameHash)) || + skipToItemInNewFile(run, lumi, event); + } + return true; + } + + //Initiate the file using multiple data catalogs + void RootInputFileSequence::initTheFile( + bool skipBadFiles, bool deleteIndexIntoFile, InputSource* input, char const* inputTypeName, InputType inputType) { + // We are really going to close the open file. + + if (fileIterLastOpened_ != fileIterEnd_) { + size_t currentIndexIntoFile = fileIterLastOpened_ - fileIterBegin_; + if (deleteIndexIntoFile) { + indexesIntoFiles_[currentIndexIntoFile].reset(); + } else { + if (indexesIntoFiles_[currentIndexIntoFile]) + indexesIntoFiles_[currentIndexIntoFile]->inputFileClosed(); + } + fileIterLastOpened_ = fileIterEnd_; + } + closeFile(); + + if (noMoreFiles()) { + // No files specified + return; + } + + // Check if the logical file name was found. + if (fileNames()[0].empty()) { + // LFN not found in catalog. + InputFile::reportSkippedFile(fileNames()[0], logicalFileName()); + if (!skipBadFiles) { + throw cms::Exception("LogicalFileNameNotFound", "RootFileSequenceBase::initTheFile()\n") + << "Logical file name '" << logicalFileName() << "' was not found in the file catalog.\n" + << "If you wanted a local file, you forgot the 'file:' prefix\n" + << "before the file name in your configuration file.\n"; + } + LogWarning("") << "Input logical file: " << logicalFileName() + << " was not found in the catalog, and will be skipped.\n"; + return; + } + + lfn_ = logicalFileName().empty() ? fileNames()[0] : logicalFileName(); + lfnHash_ = std::hash()(lfn_); + usedFallback_ = false; + + std::shared_ptr filePtr; + std::list originalInfo; + + std::vector const& fNames = fileNames(); + + //this tries to open the file using multiple PFNs corresponding to different data catalogs + { + std::list exInfo; + std::list additionalMessage; + std::unique_ptr sentry( + input ? std::make_unique(*input, lfn_) : nullptr); + edm::Service service; + if (service.isAvailable()) { + service->openingFile(lfn(), inputType, -1); + } + for (std::vector::const_iterator it = fNames.begin(); it != fNames.end(); ++it) { + try { + usedFallback_ = (it != fNames.begin()); + std::unique_ptr name(gSystem->ExpandPathName(it->c_str())); + filePtr = std::make_shared(name.get(), " Initiating request to open file ", inputType); + break; + } catch (cms::Exception const& e) { + if (!skipBadFiles && std::next(it) == fNames.end()) { + InputFile::reportSkippedFile((*it), logicalFileName()); + errors::ErrorCodes errorCode = usedFallback_ ? errors::FallbackFileOpenError : errors::FileOpenError; + Exception ex(errorCode, "", e); + ex.addContext("Calling RootInputFileSequence::initTheFile()"); + std::ostringstream out; + out << "Input file " << (*it) << " could not be opened."; + ex.addAdditionalInfo(out.str()); + //report previous exceptions when use other names to open file + for (auto const& s : exInfo) + ex.addAdditionalInfo(s); + //report more information of the earlier file open failures in a log message + if (not additionalMessage.empty()) { + edm::LogWarning l("RootInputFileSequence"); + for (auto const& msg : additionalMessage) { + l << msg << "\n"; + } + } + throw ex; + } else { + exInfo.push_back("Calling RootInputFileSequence::initTheFile(): fail to open the file with name " + (*it)); + additionalMessage.push_back(fmt::format( + "Input file {} could not be opened, and fallback was attempted.\nAdditional information:", *it)); + char c = 'a'; + for (auto const& ai : e.additionalInfo()) { + additionalMessage.push_back(fmt::format(" [{}] {}", c, ai)); + ++c; + } + } + } + } + } + if (filePtr) { + size_t currentIndexIntoFile = fileIter_ - fileIterBegin_; + rootFile_ = makeRootFile(filePtr); + assert(rootFile_); + if (input) { + rootFile_->setSignals(&(input->preEventReadFromSourceSignal_), &(input->postEventReadFromSourceSignal_)); + } + fileIterLastOpened_ = fileIter_; + setIndexIntoFile(currentIndexIntoFile); + rootFile_->reportOpened(inputTypeName); + } else { + std::string fName = !fNames.empty() ? fNames[0] : ""; + InputFile::reportSkippedFile(fName, logicalFileName()); //0 cause exception? + if (!skipBadFiles) { + throw Exception(errors::FileOpenError) << "RootFileSequenceBase::initTheFile(): Input file " << fName + << " was not found or could not be opened.\n"; + } + LogWarning("RootInputFileSequence") + << "Input file: " << fName << " was not found or could not be opened, and will be skipped.\n"; + } + } + + void RootInputFileSequence::closeFile() { + edm::Service service; + if (rootFile() and service.isAvailable()) { + service->closedFile(lfn(), usedFallback()); + } + closeFile_(); + } + + void RootInputFileSequence::setIndexIntoFile(size_t index) { + indexesIntoFiles_[index] = rootFile()->indexIntoFileSharedPtr(); + } + +} // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempInput/src/RootInputFileSequence.h b/FWIO/RNTupleTempInput/src/RootInputFileSequence.h new file mode 100644 index 0000000000000..b45ef31297257 --- /dev/null +++ b/FWIO/RNTupleTempInput/src/RootInputFileSequence.h @@ -0,0 +1,119 @@ +#ifndef FWIO_RNTupleTempInput_RootInputFileSequence_h +#define FWIO_RNTupleTempInput_RootInputFileSequence_h + +/*---------------------------------------------------------------------- + +RootInputFileSequence: This is an InputSource. initTheFile tries to open +a file using a list of PFN names constructed from multiple data catalogs +in site-local-config.xml. These are accessed via FileCatalogItem iterator +fileIter_. + +----------------------------------------------------------------------*/ + +#include "InputFile.h" +#include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/Catalog/interface/InputFileCatalog.h" +#include "FWCore/Utilities/interface/InputType.h" +#include "FWCore/Utilities/interface/get_underlying_safe.h" + +#include +#include +#include +#include + +namespace edm { + class FileCatalogItem; + class IndexIntoFile; + class InputFileCatalog; + class ParameterSetDescription; +} // namespace edm +namespace edm::rntuple_temp { + class RootFile; + + class RootInputFileSequence { + public: + explicit RootInputFileSequence(ParameterSet const& pset, InputFileCatalog const& catalog); + virtual ~RootInputFileSequence(); + + RootInputFileSequence(RootInputFileSequence const&) = delete; // Disallow copying and moving + RootInputFileSequence& operator=(RootInputFileSequence const&) = delete; // Disallow copying and moving + + bool containedInCurrentFile(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) const; + bool readEvent(EventPrincipal& cache, bool readAllProducts = false); + std::shared_ptr readLuminosityBlockAuxiliary_(); + bool readLuminosityBlock_(LuminosityBlockPrincipal& lumiPrincipal); + std::shared_ptr readRunAuxiliary_(); + bool readRun_(RunPrincipal& runPrincipal); + void fillProcessBlockHelper_(); + bool nextProcessBlock_(ProcessBlockPrincipal&); + void readProcessBlock_(ProcessBlockPrincipal&); + bool skipToItem(RunNumber_t run, + LuminosityBlockNumber_t lumi, + EventNumber_t event, + size_t fileNameHash = 0U, + bool currentFileFirst = true); + std::shared_ptr fileProductRegistry() const; + std::shared_ptr fileBranchIDListHelper() const; + + void closeFile(); + + protected: + typedef std::shared_ptr RootFileSharedPtr; + void initFile(bool skipBadFiles) { initFile_(skipBadFiles); } + void initTheFile(bool skipBadFiles, + bool deleteIndexIntoFile, + InputSource* input, + char const* inputTypeName, + InputType inputType); + + bool skipToItemInNewFile(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event); + bool skipToItemInNewFile(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event, size_t fileNameHash); + + bool atFirstFile() const { return fileIter_ == fileIterBegin_; } + bool atLastFile() const { return fileIter_ + 1 == fileIterEnd_; } + bool noMoreFiles() const { return fileIter_ == fileIterEnd_; } + bool noFiles() const { return fileIterBegin_ == fileIterEnd_; } + size_t sequenceNumberOfFile() const { return fileIter_ - fileIterBegin_; } + size_t numberOfFiles() const { return fileIterEnd_ - fileIterBegin_; } + + void setAtFirstFile() { fileIter_ = fileIterBegin_; } + void setAtFileSequenceNumber(size_t offset) { fileIter_ = fileIterBegin_ + offset; } + void setNoMoreFiles() { fileIter_ = fileIterEnd_; } + void setAtNextFile() { ++fileIter_; } + void setAtPreviousFile() { --fileIter_; } + + std::vector const& fileNames() const { return fileIter_->fileNames(); } + + std::string const& logicalFileName() const { return fileIter_->logicalFileName(); } + std::string const& lfn() const { return lfn_; } + std::vector const& fileCatalogItems() const; + + std::vector> const& indexesIntoFiles() const { return indexesIntoFiles_; } + void setIndexIntoFile(size_t index); + size_t lfnHash() const { return lfnHash_; } + bool usedFallback() const { return usedFallback_; } + + std::shared_ptr rootFile() const { return get_underlying_safe(rootFile_); } + std::shared_ptr& rootFile() { return get_underlying_safe(rootFile_); } + + private: + InputFileCatalog const& catalog_; + std::string lfn_; + size_t lfnHash_; + bool usedFallback_; + edm::propagate_const>> findFileForSpecifiedID_; + std::vector::const_iterator const fileIterBegin_; + std::vector::const_iterator const fileIterEnd_; + std::vector::const_iterator fileIter_; + std::vector::const_iterator fileIterLastOpened_; + edm::propagate_const rootFile_; + std::vector> indexesIntoFiles_; + + private: + virtual RootFileSharedPtr makeRootFile(std::shared_ptr filePtr) = 0; + virtual void initFile_(bool skipBadFiles) = 0; + virtual void closeFile_() = 0; + + }; // class RootInputFileSequence +} // namespace edm::rntuple_temp +#endif diff --git a/FWIO/RNTupleTempInput/src/RootPrimaryFileSequence.cc b/FWIO/RNTupleTempInput/src/RootPrimaryFileSequence.cc new file mode 100644 index 0000000000000..dcebd1bf67655 --- /dev/null +++ b/FWIO/RNTupleTempInput/src/RootPrimaryFileSequence.cc @@ -0,0 +1,476 @@ +/*---------------------------------------------------------------------- +----------------------------------------------------------------------*/ +#include "DuplicateChecker.h" +#include "InputFile.h" +#include "RNTupleTempSource.h" +#include "RootFile.h" +#include "RootPrimaryFileSequence.h" +#include "RootRNTuple.h" + +#include "DataFormats/Provenance/interface/BranchID.h" +#include "DataFormats/Provenance/interface/ProductRegistry.h" +#include "DataFormats/Provenance/interface/ProcessHistoryRegistry.h" +#include "FWCore/Catalog/interface/InputFileCatalog.h" +#include "FWCore/Catalog/interface/SiteLocalConfig.h" +#include "FWCore/Framework/interface/FileBlock.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" +#include "FWCore/ServiceRegistry/interface/Service.h" +#include "Utilities/StorageFactory/interface/StorageFactory.h" + +namespace edm::rntuple_temp { + RootPrimaryFileSequence::RootPrimaryFileSequence(ParameterSet const& pset, + RNTupleTempSource& input, + InputFileCatalog const& catalog) + : RootInputFileSequence(pset, catalog), + input_(input), + firstFile_(true), + branchesMustMatch_(ProductDescription::Permissive), + orderedProcessHistoryIDs_(), + eventSkipperByID_(EventSkipperByID::create(pset).release()), + initialNumberOfEventsToSkip_(pset.getUntrackedParameter("skipEvents")), + noRunLumiSort_(pset.getUntrackedParameter("noRunLumiSort")), + noEventSort_(noRunLumiSort_ ? true : pset.getUntrackedParameter("noEventSort")), + treeCacheSize_(noEventSort_ ? pset.getUntrackedParameter("cacheSize") : 0U), + duplicateChecker_(new DuplicateChecker(pset)), + usingGoToEvent_(false), + enablePrefetching_(false), + enforceGUIDInFileName_(pset.getUntrackedParameter("enforceGUIDInFileName")) { + if (noRunLumiSort_ && (remainingEvents() >= 0 || remainingLuminosityBlocks() >= 0)) { + // There would need to be some Framework development work to allow stopping + // early with noRunLumiSort set true related to closing lumis and runs that + // were supposed to be continued but were not... We cannot have events written + // to output with no run or lumi written to output. + throw Exception(errors::Configuration, + "Illegal to configure noRunLumiSort and limit the number of events or luminosityBlocks"); + } + // The SiteLocalConfig controls the TTreeCache size and the prefetching settings. + Service pSLC; + if (pSLC.isAvailable()) { + if (treeCacheSize_ != 0U && pSLC->sourceTTreeCacheSize()) { + treeCacheSize_ = *(pSLC->sourceTTreeCacheSize()); + } + enablePrefetching_ = pSLC->enablePrefetching(); + } + + std::string branchesMustMatch = + pset.getUntrackedParameter("branchesMustMatch", std::string("permissive")); + if (branchesMustMatch == std::string("strict")) + branchesMustMatch_ = ProductDescription::Strict; + + // Prestage the files + for (setAtFirstFile(); !noMoreFiles(); setAtNextFile()) { + storage::StorageFactory::get()->stagein(fileNames()[0]); + } + // Open the first file. + for (setAtFirstFile(); !noMoreFiles(); setAtNextFile()) { + initFile(input_.skipBadFiles()); + if (rootFile()) + break; + } + if (rootFile()) { + std::vector processOrder; + processingOrderMerge(input_.processHistoryRegistry(), processOrder); + + input_.productRegistryUpdate().updateFromInput(rootFile()->productRegistry()->productList(), processOrder); + if (initialNumberOfEventsToSkip_ != 0) { + skipEventsAtBeginning(initialNumberOfEventsToSkip_); + } + } + } + + RootPrimaryFileSequence::~RootPrimaryFileSequence() {} + + void RootPrimaryFileSequence::endJob() { closeFile(); } + + std::shared_ptr RootPrimaryFileSequence::readFile_() { + std::shared_ptr fileBlock; + if (firstFile_) { + firstFile_ = false; + // Usually the first input file will already be open + if (!rootFile()) { + initFile(input_.skipBadFiles()); + } + } else if (goToEventInNewFile_) { + goToEventInNewFile_ = false; + setAtFileSequenceNumber(goToFileSequenceOffset_); + initFile(false); + assert(rootFile()); + bool found = rootFile()->goToEvent(goToEventID_); + assert(found); + } else if (skipIntoNewFile_) { + skipIntoNewFile_ = false; + setAtFileSequenceNumber(skipToFileSequenceNumber_); + initFile(false); + assert(rootFile()); + if (skipToOffsetInFinalFile_ < 0) { + rootFile()->setToLastEntry(); + } + bool atEnd = rootFile()->skipEvents(skipToOffsetInFinalFile_); + assert(!atEnd && skipToOffsetInFinalFile_ == 0); + } else { + if (!nextFile()) { + // handle case with last file bad and + // skipBadFiles true + fb_ = fileBlock; + return fileBlock; + } + } + if (!rootFile()) { + fileBlock = std::make_shared(); + fb_ = fileBlock; + return fileBlock; + } + fileBlock = rootFile()->createFileBlock(); + fb_ = fileBlock; + return fileBlock; + } + + void RootPrimaryFileSequence::closeFile_() { + // close the currently open file, if any, and delete the RootFile object. + if (rootFile()) { + auto sentry = std::make_unique(input_, lfn()); + rootFile()->close(); + if (duplicateChecker_) + duplicateChecker_->inputFileClosed(); + rootFile().reset(); + } + } + + void RootPrimaryFileSequence::initFile_(bool skipBadFiles) { + // If we are not duplicate checking across files and we are not using random access to find events, + // then we can delete the IndexIntoFile for the file we are closing. + // If we can't delete all of it, then we can delete the parts we do not need. + bool deleteIndexIntoFile = !usingGoToEvent_ && !(duplicateChecker_ && duplicateChecker_->checkingAllFiles() && + !duplicateChecker_->checkDisabled()); + initTheFile(skipBadFiles, deleteIndexIntoFile, &input_, "primaryFiles", InputType::Primary); + } + + RootPrimaryFileSequence::RootFileSharedPtr RootPrimaryFileSequence::makeRootFile(std::shared_ptr filePtr) { + size_t currentIndexIntoFile = sequenceNumberOfFile(); + return std::make_shared( + RootFile::FileOptions{.fileName = fileNames()[0], + .logicalFileName = logicalFileName(), + .filePtr = filePtr, + .bypassVersionCheck = input_.bypassVersionCheck(), + .enforceGUIDInFileName = enforceGUIDInFileName_}, + InputType::Primary, + RootFile::ProcessingOptions{.eventSkipperByID = eventSkipperByID(), + .skipAnyEvents = initialNumberOfEventsToSkip_ != 0, + .remainingEvents = remainingEvents(), + .remainingLumis = remainingLuminosityBlocks(), + .processingMode = input_.processingMode(), + .noRunLumiSort = noRunLumiSort_, + .noEventSort = noEventSort_, + .usingGoToEvent = usingGoToEvent_}, + RootFile::TTreeOptions{.treeCacheSize = treeCacheSize_, + .treeMaxVirtualSize = input_.treeMaxVirtualSize(), + .enablePrefetching = enablePrefetching_, + .promptReading = not input_.delayReadingEventProducts()}, + RootFile::ProductChoices{.productSelectorRules = input_.productSelectorRules(), + .associationsFromSecondary = nullptr, // associationsFromSecondary + .dropDescendantsOfDroppedProducts = input_.dropDescendants(), + .labelRawDataLikeMC = input_.labelRawDataLikeMC()}, + RootFile::CrossFileInfo{.runHelper = input_.runHelper(), + .branchIDListHelper = input_.branchIDListHelper(), + .processBlockHelper = input_.processBlockHelper().get(), + .thinnedAssociationsHelper = input_.thinnedAssociationsHelper(), + .duplicateChecker = duplicateChecker(), + .indexesIntoFiles = indexesIntoFiles(), + .currentIndexIntoFile = currentIndexIntoFile}, + input_.nStreams(), + input_.processHistoryRegistryForUpdate(), + orderedProcessHistoryIDs_); + } + + bool RootPrimaryFileSequence::nextFile() { + do { + if (!noMoreFiles()) + setAtNextFile(); + if (noMoreFiles()) { + return false; + } + + initFile(input_.skipBadFiles()); + if (rootFile()) { + break; + } + // If we are not skipping bad files and the file + // open failed, then initFile should have thrown + assert(input_.skipBadFiles()); + } while (true); + + // make sure the new product registry is compatible with the main one + std::string mergeInfo = + input_.productRegistryUpdate().merge(*rootFile()->productRegistry(), fileNames()[0], branchesMustMatch_); + if (!mergeInfo.empty()) { + throw Exception(errors::MismatchedInputFiles, "RootPrimaryFileSequence::nextFile()") << mergeInfo; + } + return true; + } + + bool RootPrimaryFileSequence::previousFile() { + if (atFirstFile()) { + return false; + } + setAtPreviousFile(); + + initFile(false); + + if (rootFile()) { + // make sure the new product registry is compatible to the main one + std::string mergeInfo = + input_.productRegistryUpdate().merge(*rootFile()->productRegistry(), fileNames()[0], branchesMustMatch_); + if (!mergeInfo.empty()) { + throw Exception(errors::MismatchedInputFiles, "RootPrimaryFileSequence::previousEvent()") << mergeInfo; + } + } + if (rootFile()) + rootFile()->setToLastEntry(); + return true; + } + + InputSource::ItemTypeInfo RootPrimaryFileSequence::getNextItemType(RunNumber_t& run, + LuminosityBlockNumber_t& lumi, + EventNumber_t& event) { + if (noMoreFiles() || skipToStop_) { + skipToStop_ = false; + return InputSource::ItemType::IsStop; + } + if (firstFile_ || goToEventInNewFile_ || skipIntoNewFile_) { + return InputSource::ItemType::IsFile; + } + if (rootFile()) { + IndexIntoFile::EntryType entryType = rootFile()->getNextItemType(run, lumi, event); + if (entryType == IndexIntoFile::kEvent) { + return InputSource::ItemType::IsEvent; + } else if (entryType == IndexIntoFile::kLumi) { + return InputSource::ItemType::IsLumi; + } else if (entryType == IndexIntoFile::kRun) { + return InputSource::ItemType::IsRun; + } + assert(entryType == IndexIntoFile::kEnd); + } + if (atLastFile()) { + return InputSource::ItemType::IsStop; + } + return InputSource::ItemType::IsFile; + } + + // Rewind to before the first event that was read. + void RootPrimaryFileSequence::rewind_() { + if (!atFirstFile()) { + closeFile(); + setAtFirstFile(); + } + if (!rootFile()) { + initFile(false); + } + rewindFile(); + firstFile_ = true; + goToEventInNewFile_ = false; + skipIntoNewFile_ = false; + skipToStop_ = false; + if (rootFile()) { + if (initialNumberOfEventsToSkip_ != 0) { + skipEventsAtBeginning(initialNumberOfEventsToSkip_); + } + } + } + + // Rewind to the beginning of the current file + void RootPrimaryFileSequence::rewindFile() { + if (rootFile()) + rootFile()->rewind(); + } + + // Advance "offset" events. Offset will be positive. + void RootPrimaryFileSequence::skipEventsAtBeginning(int offset) { + assert(rootFile()); + assert(offset >= 0); + while (offset != 0) { + bool atEnd = rootFile()->skipEvents(offset); + if ((offset > 0 || atEnd) && !nextFile()) { + return; + } + } + } + + // Advance "offset" events. Offset can be positive or negative (or zero). + void RootPrimaryFileSequence::skipEvents(int offset) { + assert(rootFile()); + + bool atEnd = rootFile()->skipEvents(offset); + if (!atEnd && offset == 0) { + // successfully completed skip within current file + return; + } + + // Return, if without closing the current file we know the skip cannot be completed + skipToStop_ = false; + if (offset > 0 || atEnd) { + if (atLastFile() || noMoreFiles()) { + skipToStop_ = true; + return; + } + } + if (offset < 0 && atFirstFile()) { + skipToStop_ = true; + return; + } + + // Save the current file and position so that we can restore them + size_t const originalFileSequenceNumber = sequenceNumberOfFile(); + IndexIntoFile::IndexIntoFileItr originalPosition = rootFile()->indexIntoFileIter(); + + if ((offset > 0 || atEnd) && !nextFile()) { + skipToStop_ = true; // Can only get here if skipBadFiles is true + } + if (offset < 0 && !previousFile()) { + skipToStop_ = true; // Can't actually get here + } + + if (!skipToStop_) { + while (offset != 0) { + skipToOffsetInFinalFile_ = offset; + bool atEnd = rootFile()->skipEvents(offset); + if ((offset > 0 || atEnd) && !nextFile()) { + skipToStop_ = true; + break; + } + if (offset < 0 && !previousFile()) { + skipToStop_ = true; + break; + } + } + if (!skipToStop_) { + skipIntoNewFile_ = true; + } + } + skipToFileSequenceNumber_ = sequenceNumberOfFile(); + + // Restore the original file and position + setAtFileSequenceNumber(originalFileSequenceNumber); + initFile(false); + assert(rootFile()); + rootFile()->setPosition(originalPosition); + rootFile()->updateFileBlock(*fb_); + } + + bool RootPrimaryFileSequence::goToEvent(EventID const& eventID) { + usingGoToEvent_ = true; + if (rootFile()) { + if (rootFile()->goToEvent(eventID)) { + return true; + } + // If only one input file, give up now, to save time. + if (rootFile() && indexesIntoFiles().size() == 1) { + return false; + } + // Look for item (run/lumi/event) in files previously opened without reopening unnecessary files. + for (auto it = indexesIntoFiles().begin(), itEnd = indexesIntoFiles().end(); it != itEnd; ++it) { + if (*it && (*it)->containsItem(eventID.run(), eventID.luminosityBlock(), eventID.event())) { + goToEventInNewFile_ = true; + goToFileSequenceOffset_ = it - indexesIntoFiles().begin(); + goToEventID_ = eventID; + return true; + } + } + + // Save the current file and position so that we can restore them + bool closedOriginalFile = false; + size_t const originalFileSequenceNumber = sequenceNumberOfFile(); + IndexIntoFile::IndexIntoFileItr originalPosition = rootFile()->indexIntoFileIter(); + + // Look for item in files not yet opened. + bool foundIt = false; + for (auto it = indexesIntoFiles().begin(), itEnd = indexesIntoFiles().end(); it != itEnd; ++it) { + if (!*it) { + setAtFileSequenceNumber(it - indexesIntoFiles().begin()); + initFile(false); + assert(rootFile()); + closedOriginalFile = true; + if ((*it)->containsItem(eventID.run(), eventID.luminosityBlock(), eventID.event())) { + foundIt = true; + goToEventInNewFile_ = true; + goToFileSequenceOffset_ = it - indexesIntoFiles().begin(); + goToEventID_ = eventID; + } + } + } + if (closedOriginalFile) { + setAtFileSequenceNumber(originalFileSequenceNumber); + initFile(false); + assert(rootFile()); + rootFile()->setPosition(originalPosition); + rootFile()->updateFileBlock(*fb_); + } + return foundIt; + } + return false; + } + + int RootPrimaryFileSequence::remainingEvents() const { return input_.remainingEvents(); } + + int RootPrimaryFileSequence::remainingLuminosityBlocks() const { return input_.remainingLuminosityBlocks(); } + + void RootPrimaryFileSequence::fillDescription(ParameterSetDescription& desc) { + desc.addUntracked("skipEvents", 0U) + ->setComment("Skip the first 'skipEvents' events that otherwise would have been processed."); + desc.addUntracked("noEventSort", true) + ->setComment( + "True: Process runs, lumis and events in the order they appear in the file (but see notes 1 and 2).\n" + "False: Process runs, lumis and events in each file in numerical order (run#, lumi#, event#) (but see note " + "3).\n" + "Note 1: Events within the same lumi will always be processed contiguously.\n" + "Note 2: Lumis within the same run will always be processed contiguously.\n" + "Note 3: Any sorting occurs independently in each input file (no sorting across input files)."); + desc.addUntracked("noRunLumiSort", false) + ->setComment( + "True: Process runs, lumis and events in the order they appear in the file.\n" + "False: Follow settings based on 'noEventSort' setting."); + desc.addUntracked("cacheSize", rootrntuple::defaultCacheSize) + ->setComment("Size of ROOT TTree prefetch cache. Affects performance."); + std::string defaultString("permissive"); + desc.addUntracked("branchesMustMatch", defaultString) + ->setComment( + "'strict': Branches in each input file must match those in the first file.\n" + "'permissive': Branches in each input file may be any subset of those in the first file."); + desc.addUntracked("enforceGUIDInFileName", false) + ->setComment( + "True: file name part is required to be equal to the GUID of the file\n" + "False: file name can be anything"); + + EventSkipperByID::fillDescription(desc); + DuplicateChecker::fillDescription(desc); + } + + ProcessingController::ForwardState RootPrimaryFileSequence::forwardState() const { + if (rootFile()) { + if (!rootFile()->wasLastEventJustRead()) { + return ProcessingController::kEventsAheadInFile; + } + if (noMoreFiles() || atLastFile()) { + return ProcessingController::kAtLastEvent; + } else { + return ProcessingController::kNextFileExists; + } + } + return ProcessingController::kUnknownForward; + } + + ProcessingController::ReverseState RootPrimaryFileSequence::reverseState() const { + if (rootFile()) { + if (!rootFile()->wasFirstEventJustRead()) { + return ProcessingController::kEventsBackwardsInFile; + } + if (!atFirstFile()) { + return ProcessingController::kPreviousFileExists; + } + return ProcessingController::kAtFirstEvent; + } + return ProcessingController::kUnknownReverse; + } + +} // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempInput/src/RootPrimaryFileSequence.h b/FWIO/RNTupleTempInput/src/RootPrimaryFileSequence.h new file mode 100644 index 0000000000000..ad153f19d8e63 --- /dev/null +++ b/FWIO/RNTupleTempInput/src/RootPrimaryFileSequence.h @@ -0,0 +1,101 @@ +#ifndef FWIO_RNTupleTempInput_RootPrimaryFileSequence_h +#define FWIO_RNTupleTempInput_RootPrimaryFileSequence_h + +/*---------------------------------------------------------------------- + +RootPrimaryFileSequence: This is an InputSource + +----------------------------------------------------------------------*/ + +#include "RootInputFileSequence.h" +#include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/Framework/interface/InputSource.h" +#include "FWCore/Framework/interface/ProductSelectorRules.h" +#include "FWCore/Framework/interface/ProcessingController.h" +#include "FWCore/Sources/interface/EventSkipperByID.h" +#include "FWCore/Utilities/interface/get_underlying_safe.h" +#include "DataFormats/Provenance/interface/ProductDescription.h" +#include "DataFormats/Provenance/interface/ProcessHistoryID.h" + +#include +#include +#include + +namespace edm { + + class BranchID; + class FileCatalogItem; + class InputFileCatalog; + class ParameterSetDescription; + namespace rntuple_temp { + class DuplicateChecker; + class RNTupleTempSource; + class RootFile; + + class RootPrimaryFileSequence : public RootInputFileSequence { + public: + explicit RootPrimaryFileSequence(ParameterSet const& pset, + RNTupleTempSource& input, + InputFileCatalog const& catalog); + ~RootPrimaryFileSequence() override; + + RootPrimaryFileSequence(RootPrimaryFileSequence const&) = delete; // Disallow copying and moving + RootPrimaryFileSequence& operator=(RootPrimaryFileSequence const&) = delete; // Disallow copying and moving + + std::shared_ptr readFile_(); + void endJob(); + InputSource::ItemTypeInfo getNextItemType(RunNumber_t& run, LuminosityBlockNumber_t& lumi, EventNumber_t& event); + void skipEventsAtBeginning(int offset); + void skipEvents(int offset); + bool goToEvent(EventID const& eventID); + void rewind_(); + static void fillDescription(ParameterSetDescription& desc); + ProcessingController::ForwardState forwardState() const; + ProcessingController::ReverseState reverseState() const; + + private: + void initFile_(bool skipBadFiles) override; + RootFileSharedPtr makeRootFile(std::shared_ptr filePtr) override; + bool nextFile(); + bool previousFile(); + void rewindFile(); + void closeFile_() override; + + int remainingEvents() const; + int remainingLuminosityBlocks() const; + + RNTupleTempSource& input_; + bool firstFile_; + ProductDescription::MatchMode branchesMustMatch_; + std::vector orderedProcessHistoryIDs_; + + std::shared_ptr eventSkipperByID() const { + return get_underlying_safe(eventSkipperByID_); + } + std::shared_ptr& eventSkipperByID() { return get_underlying_safe(eventSkipperByID_); } + std::shared_ptr duplicateChecker() const { + return get_underlying_safe(duplicateChecker_); + } + std::shared_ptr& duplicateChecker() { return get_underlying_safe(duplicateChecker_); } + + edm::propagate_const> fb_; + edm::propagate_const> eventSkipperByID_; + int initialNumberOfEventsToSkip_; + bool noRunLumiSort_; + bool noEventSort_; + unsigned int treeCacheSize_; + edm::propagate_const> duplicateChecker_; + bool usingGoToEvent_; + bool goToEventInNewFile_ = false; + size_t goToFileSequenceOffset_ = 0; + EventID goToEventID_; + bool skipToStop_ = false; + bool skipIntoNewFile_ = false; + size_t skipToFileSequenceNumber_ = 0; + int skipToOffsetInFinalFile_ = 0; + bool enablePrefetching_; + bool enforceGUIDInFileName_; + }; // class RootPrimaryFileSequence + } // namespace rntuple_temp +} // namespace edm +#endif diff --git a/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.cc b/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.cc new file mode 100644 index 0000000000000..6476433e188db --- /dev/null +++ b/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.cc @@ -0,0 +1,109 @@ +/*---------------------------------------------------------------------- +----------------------------------------------------------------------*/ + +#include "RootPromptReadDelayedReader.h" +#include "InputFile.h" +#include "DataFormats/Common/interface/EDProductGetter.h" +#include "DataFormats/Common/interface/RefCoreStreamer.h" + +#include "FWCore/Framework/interface/SharedResourcesAcquirer.h" +#include "FWCore/Framework/interface/SharedResourcesRegistry.h" + +#include "IOPool/Common/interface/getWrapperBasePtr.h" + +#include "FWCore/Utilities/interface/EDMException.h" + +#include "TBranch.h" +#include "TClass.h" +#include "TTree.h" + +#include + +namespace edm::rntuple_temp { + + RootPromptReadDelayedReader::RootPromptReadDelayedReader(RootRNTuple const& tree, + std::shared_ptr filePtr, + InputType inputType, + unsigned int iNIndexes) + : cacheMaps_(iNIndexes), tree_(tree), filePtr_(filePtr), nextReader_(), inputType_(inputType) { + if (inputType == InputType::Primary) { + auto resources = SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader(); + resourceAcquirer_ = std::make_unique(std::move(resources.first)); + mutex_ = resources.second; + } + } + + RootPromptReadDelayedReader::~RootPromptReadDelayedReader() {} + + std::pair RootPromptReadDelayedReader::sharedResources_() const { + return std::make_pair(resourceAcquirer_.get(), mutex_.get()); + } + + namespace { + unsigned int indexFor(RootRNTuple const& tree, EDProductGetter const* ep) { + return tree.branchType() == InEvent ? ep->transitionIndex() : 0; + } + } // namespace + std::shared_ptr RootPromptReadDelayedReader::getProduct_(BranchID const& k, EDProductGetter const* ep) { + if (lastException_) { + try { + std::rethrow_exception(lastException_); + } catch (edm::Exception const& e) { + //avoid growing the context each time the exception is rethrown. + auto copy = e; + copy.addContext("Rethrowing an exception that happened on a different read request."); + throw copy; + } catch (cms::Exception& e) { + //If we do anything here to 'copy', we would lose the actual type of the exception. + e.addContext("Rethrowing an exception that happened on a different read request."); + throw; + } + } + auto& cacheMap = cacheMaps_[indexFor(tree_, ep)]; + auto itFound = cacheMap.find(k.id()); + if (itFound != cacheMap.end()) { + auto& cache = itFound->second; + if (cache.wrapperBase_) { + if (tree_.branchType() == InEvent) { + // CMS-THREADING For the primary input source calls to this function need to be serialized + InputFile::reportReadBranch(inputType_, + std::string(tree_.branches().find(itFound->first)->productBranch_->GetName())); + } + return std::shared_ptr(std::move(cache.wrapperBase_)); + } + } + if (nextReader_) { + return nextReader_->getProduct(k, ep); + } + return std::shared_ptr(); + } + + void RootPromptReadDelayedReader::readAllProductsNow(EDProductGetter const* ep) { + // first set all the addresses + auto& cacheMap = cacheMaps_[indexFor(tree_, ep)]; + if (cacheMap.empty()) { + for (auto& cacheMap : cacheMaps_) { + cacheMap.reserve(tree_.branches().size()); + for (auto const& branch : tree_.branches()) { + cacheMap.emplace(branch.first, Cache{}); + } + } + } + for (auto& it : cacheMap) { + auto branchInfo = getBranchInfo(it.first); + if (branchInfo == nullptr || branchInfo->productBranch_ == nullptr) { + continue; // Skip if branch info or product branch is not available + } + auto& cache = it.second; + cache.wrapperBase_ = branchInfo->newWrapper(); + cache.wrapperBasePtr_ = cache.wrapperBase_.get(); + branchInfo->productBranch_->SetAddress(&cache.wrapperBasePtr_); + } + + { + // ROOT might use multiple threads while reading the entries + MultiThreadRefCoreStreamerGuard epGuard(ep); + tree_.getEntryForAllBranches(); + } + } +} // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.h b/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.h new file mode 100644 index 0000000000000..cadde91e1f407 --- /dev/null +++ b/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.h @@ -0,0 +1,85 @@ +#ifndef IOPool_Input_RootPromptReadDelayedReader_h +#define IOPool_Input_RootPromptReadDelayedReader_h + +/*---------------------------------------------------------------------- + +RootPromptReadDelayedReader.h // used by ROOT input sources + +----------------------------------------------------------------------*/ + +#include "DataFormats/Provenance/interface/BranchID.h" +#include "FWCore/Utilities/interface/InputType.h" +#include "FWCore/Utilities/interface/propagate_const.h" +#include "FWCore/Utilities/interface/thread_safety_macros.h" +#include "RootRNTuple.h" +#include "RootDelayedReaderBase.h" + +#include +#include +#include +#include + +class TClass; +namespace edm { + class SharedResourcesAcquirer; + class Exception; +} // namespace edm +namespace edm::rntuple_temp { + class InputFile; + class RootRNTuple; + //------------------------------------------------------------ + // Class RootPromptReadDelayedReader: pretends to support file reading. + // + + class RootPromptReadDelayedReader : public RootDelayedReaderBase { + public: + typedef rootrntuple::BranchInfo BranchInfo; + typedef rootrntuple::BranchMap BranchMap; + typedef rootrntuple::EntryNumber EntryNumber; + RootPromptReadDelayedReader(RootRNTuple const& tree, + std::shared_ptr filePtr, + InputType inputType, + unsigned int iNIndexes); + + ~RootPromptReadDelayedReader() override; + + RootPromptReadDelayedReader(RootPromptReadDelayedReader const&) = delete; + RootPromptReadDelayedReader& operator=(RootPromptReadDelayedReader const&) = delete; + RootPromptReadDelayedReader(RootPromptReadDelayedReader&&) = delete; + RootPromptReadDelayedReader& operator=(RootPromptReadDelayedReader&&) = delete; + + struct Cache { + std::unique_ptr wrapperBase_; + edm::WrapperBase* wrapperBasePtr_ = nullptr; // TBranch::SetAddress() needs a long live pointer to reference. + }; + + void readAllProductsNow(EDProductGetter const* ep) override; + + private: + std::shared_ptr getProduct_(BranchID const& k, EDProductGetter const* ep) override; + void mergeReaders_(DelayedReader* other) override { nextReader_ = other; } + void reset_() override { nextReader_ = nullptr; } + std::pair sharedResources_() const override; + + BranchMap const& branches() const { return tree_.branches(); } + BranchInfo const* getBranchInfo(unsigned int k) const { return branches().find(k); } + std::vector> cacheMaps_; + // NOTE: filePtr_ appears to be unused, but is needed to prevent + // the file containing the branch from being reclaimed. + RootRNTuple const& tree_; + edm::propagate_const> filePtr_; + edm::propagate_const nextReader_; + // We do not use propagate_const because the acquirer is itself mutable. + std::unique_ptr resourceAcquirer_; + std::shared_ptr mutex_; + InputType inputType_; + + //If a fatal exception happens we need to make a copy so we can + // rethrow that exception on other threads. This avoids TTree + // non-exception safety problems on later calls to TTree. + //All uses of the ROOT file are serialized + CMS_SA_ALLOW mutable std::exception_ptr lastException_; + }; // class RootPromptReadDelayedReader + //------------------------------------------------------------ +} // namespace edm::rntuple_temp +#endif diff --git a/FWIO/RNTupleTempInput/src/RootRNTuple.cc b/FWIO/RNTupleTempInput/src/RootRNTuple.cc new file mode 100644 index 0000000000000..0e0a5e1ae75ca --- /dev/null +++ b/FWIO/RNTupleTempInput/src/RootRNTuple.cc @@ -0,0 +1,627 @@ +#include "DataFormats/Provenance/interface/BranchType.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "FWCore/Utilities/interface/EDMException.h" +#include "FWCore/Utilities/interface/Exception.h" +#include "IOPool/Common/interface/getWrapperBasePtr.h" + +#include "InputFile.h" +#include "RootRNTuple.h" +#include "RootDelayedReader.h" +#include "RootPromptReadDelayedReader.h" + +#include "TTree.h" +#include "TTreeCache.h" +#include "TLeaf.h" + +#include "oneapi/tbb/task_arena.h" +#include + +namespace edm::rntuple_temp { + namespace { + TBranch* getAuxiliaryBranch(TTree* tree, BranchType const& branchType) { + TBranch* branch = tree->GetBranch(BranchTypeToAuxiliaryBranchName(branchType).c_str()); + if (branch == nullptr) { + branch = tree->GetBranch(BranchTypeToAuxBranchName(branchType).c_str()); + } + return branch; + } + TBranch* getProductProvenanceBranch(TTree* tree, BranchType const& branchType) { + TBranch* branch = tree->GetBranch(BranchTypeToBranchEntryInfoBranchName(branchType).c_str()); + return branch; + } + + std::unique_ptr makeRootDelayedReader(RootRNTuple const& tree, + std::shared_ptr filePtr, + InputType inputType, + unsigned int nIndexes, + bool promptRead) { + if (promptRead) { + return std::make_unique(tree, filePtr, inputType, nIndexes); + } + return std::make_unique(tree, filePtr, inputType); + } + } // namespace + + // Used for all RootRNTuples + // All the other constructors delegate to this one + RootRNTuple::RootRNTuple(std::shared_ptr filePtr, + BranchType const& branchType, + unsigned int nIndexes, + unsigned int learningEntries, + bool enablePrefetching, + bool promptRead, + InputType inputType) + : filePtr_(filePtr), + branchType_(branchType), + entryNumberForIndex_(std::make_unique>(nIndexes, IndexIntoFile::invalidEntry)), + learningEntries_(learningEntries), + enablePrefetching_(enablePrefetching), + enableTriggerCache_(branchType_ == InEvent), + promptRead_(promptRead), + rootDelayedReader_(makeRootDelayedReader(*this, filePtr, inputType, nIndexes, promptRead)) {} + + // Used for Event/Lumi/Run RootRNTuples + RootRNTuple::RootRNTuple(std::shared_ptr filePtr, + BranchType const& branchType, + unsigned int nIndexes, + Options const& options, + unsigned int learningEntries, + InputType inputType) + : RootRNTuple( + filePtr, branchType, nIndexes, learningEntries, options.enablePrefetching, options.promptReading, inputType) { + init(BranchTypeToProductTreeName(branchType), options.treeMaxVirtualSize, options.treeCacheSize); + metaTree_ = dynamic_cast(filePtr_->Get(BranchTypeToMetaDataTreeName(branchType).c_str())); + auxBranch_ = getAuxiliaryBranch(tree_, branchType_); + branchEntryInfoBranch_ = + metaTree_ ? getProductProvenanceBranch(metaTree_, branchType_) : getProductProvenanceBranch(tree_, branchType_); + infoTree_ = + dynamic_cast(filePtr->Get(BranchTypeToInfoTreeName(branchType).c_str())); // backward compatibility + } + + // Used for ProcessBlock RootRNTuples + RootRNTuple::RootRNTuple(std::shared_ptr filePtr, + BranchType const& branchType, + std::string const& processName, + unsigned int nIndexes, + Options const& options, + unsigned int learningEntries, + InputType inputType) + : RootRNTuple( + filePtr, branchType, nIndexes, learningEntries, options.enablePrefetching, options.promptReading, inputType) { + processName_ = processName; + init(BranchTypeToProductTreeName(branchType, processName), options.treeMaxVirtualSize, options.treeCacheSize); + } + + void RootRNTuple::init(std::string const& productTreeName, unsigned int maxVirtualSize, unsigned int cacheSize) { + if (filePtr_.get() != nullptr) { + tree_ = dynamic_cast(filePtr_->Get(productTreeName.c_str())); + } + if (not tree_) { + throw cms::Exception("WrongFileFormat") + << "The ROOT file does not contain a TTree named " << productTreeName + << "\n This is either not an edm ROOT file or is one that has been corrupted."; + } + entries_ = tree_->GetEntries(); + + // On merged files in older releases of ROOT, the autoFlush setting is always negative; we must guess. + // TODO: On newer merged files, we should be able to get this from the cluster iterator. + long treeAutoFlush = tree_->GetAutoFlush(); + if (treeAutoFlush < 0) { + // The "+1" is here to avoid divide-by-zero in degenerate cases. + Long64_t averageEventSizeBytes = tree_->GetZipBytes() / (tree_->GetEntries() + 1) + 1; + treeAutoFlush_ = cacheSize / averageEventSizeBytes + 1; + } else { + treeAutoFlush_ = treeAutoFlush; + } + if (treeAutoFlush_ < learningEntries_) { + learningEntries_ = treeAutoFlush_; + } + setTreeMaxVirtualSize(maxVirtualSize); + setCacheSize(cacheSize); + if (branchType_ == InEvent) { + Int_t branchCount = tree_->GetListOfBranches()->GetEntriesFast(); + trainedSet_.reserve(branchCount); + triggerSet_.reserve(branchCount); + } + } + + RootRNTuple::~RootRNTuple() {} + + RootRNTuple::EntryNumber const& RootRNTuple::entryNumberForIndex(unsigned int index) const { + assert(index < entryNumberForIndex_->size()); + return (*entryNumberForIndex_)[index]; + } + + void RootRNTuple::insertEntryForIndex(unsigned int index) { + assert(index < entryNumberForIndex_->size()); + (*entryNumberForIndex_)[index] = entryNumber(); + } + + bool RootRNTuple::isValid() const { + // ProcessBlock + if (branchType_ == InProcess) { + return tree_ != nullptr; + } + // Run/Lumi/Event + if (metaTree_ == nullptr || metaTree_->GetNbranches() == 0) { + return tree_ != nullptr && auxBranch_ != nullptr; + } + // Backward compatibility for Run/Lumi/Event + if (tree_ != nullptr && auxBranch_ != nullptr && metaTree_ != nullptr) { // backward compatibility + if (branchEntryInfoBranch_ != nullptr || infoTree_ != nullptr) + return true; // backward compatibility + return (entries_ == metaTree_->GetEntries() && + tree_->GetNbranches() <= metaTree_->GetNbranches() + 1); // backward compatibility + } // backward compatibility + return false; + } + + DelayedReader* RootRNTuple::resetAndGetRootDelayedReader() const { + rootDelayedReader_->reset(); + return rootDelayedReader_.get(); + } + + RootDelayedReaderBase* RootRNTuple::rootDelayedReader() const { return rootDelayedReader_.get(); } + + void RootRNTuple::setPresence(ProductDescription& prod, std::string const& oldBranchName) { + assert(isValid()); + if (tree_->GetBranch(oldBranchName.c_str()) == nullptr) { + prod.setDropped(true); + } + } + + void rootrntuple::BranchInfo::setBranch(TBranch* branch, TClass const* wrapperBaseTClass) { + productBranch_ = branch; + if (branch) { + classCache_ = TClass::GetClass(productDescription_.wrappedName().c_str()); + offsetToWrapperBase_ = classCache_->GetBaseClassOffset(wrapperBaseTClass); + } + } + std::unique_ptr rootrntuple::BranchInfo::newWrapper() const { + assert(nullptr != classCache_); + void* p = classCache_->New(); + return getWrapperBasePtr(p, offsetToWrapperBase_); + } + + void RootRNTuple::addBranch(ProductDescription const& prod, std::string const& oldBranchName) { + assert(isValid()); + static TClass const* const wrapperBaseTClass = TClass::GetClass("edm::WrapperBase"); + //use the translated branch name + TBranch* branch = tree_->GetBranch(oldBranchName.c_str()); + rootrntuple::BranchInfo info = rootrntuple::BranchInfo(prod); + info.productBranch_ = nullptr; + if (prod.present()) { + info.setBranch(branch, wrapperBaseTClass); + //we want the new branch name for the JobReport + branchNames_.push_back(prod.branchName()); + } + branches_.insert(prod.branchID(), info); + } + + void RootRNTuple::dropBranch(std::string const& oldBranchName) { + //use the translated branch name + TBranch* branch = tree_->GetBranch(oldBranchName.c_str()); + if (branch != nullptr) { + TObjArray* leaves = tree_->GetListOfLeaves(); + int entries = leaves->GetEntries(); + for (int i = 0; i < entries; ++i) { + TLeaf* leaf = (TLeaf*)(*leaves)[i]; + if (leaf == nullptr) + continue; + TBranch* br = leaf->GetBranch(); + if (br == nullptr) + continue; + if (br->GetMother() == branch) { + leaves->Remove(leaf); + } + } + leaves->Compress(); + tree_->GetListOfBranches()->Remove(branch); + tree_->GetListOfBranches()->Compress(); + delete branch; + } + } + + rootrntuple::BranchMap const& RootRNTuple::branches() const { return branches_; } + + std::shared_ptr RootRNTuple::createCacheWithSize(unsigned int cacheSize) const { + return filePtr_->createCacheWithSize(*tree_, cacheSize); + } + + void RootRNTuple::setCacheSize(unsigned int cacheSize) { + cacheSize_ = cacheSize; + treeCache_ = createCacheWithSize(cacheSize); + if (treeCache_) + treeCache_->SetEnablePrefetching(enablePrefetching_); + rawTreeCache_.reset(); + } + + void RootRNTuple::setTreeMaxVirtualSize(int treeMaxVirtualSize) { + if (treeMaxVirtualSize >= 0) + tree_->SetMaxVirtualSize(static_cast(treeMaxVirtualSize)); + } + + bool RootRNTuple::nextWithCache() { + bool returnValue = ++entryNumber_ < entries_; + if (returnValue) { + setEntryNumber(entryNumber_); + } + return returnValue; + } + + void RootRNTuple::setEntryNumber(EntryNumber theEntryNumber) { + { + auto guard = filePtr_->setCacheReadTemporarily(treeCache_.get(), tree_); + + // Detect a backward skip. If the skip is sufficiently large, we roll the dice and reset the treeCache. + // This will cause some amount of over-reading: we pre-fetch all the events in some prior cluster. + // However, because reading one event in the cluster is supposed to be equivalent to reading all events in the cluster, + // we're not incurring additional over-reading - we're just doing it more efficiently. + // NOTE: Constructor guarantees treeAutoFlush_ is positive, even if TTree->GetAutoFlush() is negative. + if (theEntryNumber < entryNumber_ and theEntryNumber >= 0) { + //We started reading the file near the end, now we need to correct for the learning length + if (switchOverEntry_ > tree_->GetEntries()) { + switchOverEntry_ = switchOverEntry_ - tree_->GetEntries(); + if (rawTreeCache_) { + rawTreeCache_->SetEntryRange(theEntryNumber, switchOverEntry_); + rawTreeCache_->FillBuffer(); + } + } + if (performedSwitchOver_ and triggerTreeCache_) { + //We are using the triggerTreeCache_ not the rawTriggerTreeCache_. + //The triggerTreeCache was originally told to start from an entry further in the file. + triggerTreeCache_->SetEntryRange(theEntryNumber, tree_->GetEntries()); + } else if (rawTriggerTreeCache_) { + //move the switch point to the end of the cluster holding theEntryNumber + rawTriggerSwitchOverEntry_ = -1; + TTree::TClusterIterator clusterIter = tree_->GetClusterIterator(theEntryNumber); + while ((rawTriggerSwitchOverEntry_ < theEntryNumber) || (rawTriggerSwitchOverEntry_ <= 0)) { + rawTriggerSwitchOverEntry_ = clusterIter(); + } + rawTriggerTreeCache_->SetEntryRange(theEntryNumber, rawTriggerSwitchOverEntry_); + } + } + if ((theEntryNumber < static_cast(entryNumber_ - treeAutoFlush_)) && (treeCache_) && + (!treeCache_->IsLearning()) && (entries_ > 0) && (switchOverEntry_ >= 0)) { + treeCache_->SetEntryRange(theEntryNumber, entries_); + treeCache_->FillBuffer(); + } + + entryNumber_ = theEntryNumber; + tree_->LoadTree(entryNumber_); + //want guard to end here + } + if (treeCache_ && trainNow_ && entryNumber_ >= 0) { + startTraining(); + trainNow_ = false; + trainedSet_.clear(); + triggerSet_.clear(); + rawTriggerSwitchOverEntry_ = -1; + } + if (not promptRead_ && treeCache_ && treeCache_->IsLearning() && switchOverEntry_ >= 0 && + entryNumber_ >= switchOverEntry_) { + stopTraining(); + } + } + + // The actual implementation is done below; it's split in this strange + // manner in order to keep a by-definition-rare code path out of the instruction cache. + inline TTreeCache* RootRNTuple::checkTriggerCache(TBranch* branch, EntryNumber entryNumber) const { + if (!treeCache_->IsAsyncReading() && enableTriggerCache_ && (trainedSet_.find(branch) == trainedSet_.end())) { + return checkTriggerCacheImpl(branch, entryNumber); + } else { + return nullptr; + } + } + + // See comments in the header. If this function is called, we already know + // the trigger cache is active and it was a cache miss for the regular cache. + TTreeCache* RootRNTuple::checkTriggerCacheImpl(TBranch* branch, EntryNumber entryNumber) const { + // This branch is not going to be in the cache. + // Assume this is a "trigger pattern". + // Always make sure the branch is added to the trigger set. + if (triggerSet_.find(branch) == triggerSet_.end()) { + triggerSet_.insert(branch); + if (triggerTreeCache_.get()) { + triggerTreeCache_->AddBranch(branch, kTRUE); + } + } + + if (rawTriggerSwitchOverEntry_ < 0) { + // The trigger has never fired before. Take everything not in the + // trainedSet and load it from disk + + // Calculate the end of the next cluster; triggers in the next cluster + // will use the triggerCache, not the rawTriggerCache. + // + // Guarantee that rawTriggerSwitchOverEntry_ is positive (non-zero) after completion + // of this if-block. + TTree::TClusterIterator clusterIter = tree_->GetClusterIterator(entryNumber); + while ((rawTriggerSwitchOverEntry_ < entryNumber) || (rawTriggerSwitchOverEntry_ <= 0)) { + rawTriggerSwitchOverEntry_ = clusterIter(); + } + + // ROOT will automatically expand the cache to fit one cluster; hence, we use + // 5 MB as the cache size below + rawTriggerTreeCache_ = createCacheWithSize(5 * 1024 * 1024); + if (rawTriggerTreeCache_) + rawTriggerTreeCache_->SetEnablePrefetching(false); + TObjArray* branches = tree_->GetListOfBranches(); + int branchCount = branches->GetEntriesFast(); + + // Train the rawTriggerCache to have everything not in the regular cache. + rawTriggerTreeCache_->SetLearnEntries(0); + rawTriggerTreeCache_->SetEntryRange(entryNumber, rawTriggerSwitchOverEntry_); + for (int i = 0; i < branchCount; i++) { + TBranch* tmp_branch = (TBranch*)branches->UncheckedAt(i); + if (trainedSet_.find(tmp_branch) != trainedSet_.end()) { + continue; + } + rawTriggerTreeCache_->AddBranch(tmp_branch, kTRUE); + } + performedSwitchOver_ = false; + rawTriggerTreeCache_->StopLearningPhase(); + + return rawTriggerTreeCache_.get(); + } else if (!performedSwitchOver_ and entryNumber_ < rawTriggerSwitchOverEntry_) { + // The raw trigger has fired and it contents are valid. + return rawTriggerTreeCache_.get(); + } else if (rawTriggerSwitchOverEntry_ > 0) { + // The raw trigger has fired, but we are out of the cache. Use the + // triggerCache instead. + if (!performedSwitchOver_) { + rawTriggerTreeCache_.reset(); + performedSwitchOver_ = true; + + // Train the triggerCache + triggerTreeCache_ = createCacheWithSize(5 * 1024 * 1024); + triggerTreeCache_->SetEnablePrefetching(false); + triggerTreeCache_->SetLearnEntries(0); + triggerTreeCache_->SetEntryRange(entryNumber, tree_->GetEntries()); + for (std::unordered_set::const_iterator it = triggerSet_.begin(), itEnd = triggerSet_.end(); + it != itEnd; + it++) { + triggerTreeCache_->AddBranch(*it, kTRUE); + } + triggerTreeCache_->StopLearningPhase(); + } + return triggerTreeCache_.get(); + } + + // By construction, this case should be impossible. + assert(false); + return nullptr; + } + + inline TTreeCache* RootRNTuple::selectCache(TBranch* branch, EntryNumber entryNumber) const { + TTreeCache* triggerCache = nullptr; + if (promptRead_) { + return rawTreeCache_.get(); + } + if (!treeCache_) { + return nullptr; + } else if (treeCache_->IsLearning() && rawTreeCache_) { + treeCache_->AddBranch(branch, kTRUE); + trainedSet_.insert(branch); + return rawTreeCache_.get(); + } else if ((triggerCache = checkTriggerCache(branch, entryNumber))) { + // A NULL return value from checkTriggerCache indicates the trigger cache case + // does not apply, and we should continue below. + return triggerCache; + } else { + // The "normal" TTreeCache case. + return treeCache_.get(); + } + } + TTreeCache* RootRNTuple::getAuxCache(TBranch* auxBranch) const { + if (not auxCache_ and cacheSize_ > 0) { + auxCache_ = createCacheWithSize(1 * 1024 * 1024); + if (auxCache_) { + auxCache_->SetEnablePrefetching(enablePrefetching_); + auxCache_->SetLearnEntries(0); + auxCache_->StartLearningPhase(); + auxCache_->SetEntryRange(0, tree_->GetEntries()); + auxCache_->AddBranch(auxBranch->GetName(), kTRUE); + auxCache_->StopLearningPhase(); + } + } + return auxCache_.get(); + } + + void RootRNTuple::getEntryForAllBranches() const { + oneapi::tbb::this_task_arena::isolate([&]() { + auto guard = filePtr_->setCacheReadTemporarily(treeCache_.get(), tree_); + tree_->GetEntry(entryNumber_); + }); + } + + void RootRNTuple::getEntry(TBranch* branch, EntryNumber entryNumber) const { + getEntryUsingCache(branch, entryNumber, selectCache(branch, entryNumber)); + } + + inline void RootRNTuple::getEntryUsingCache(TBranch* branch, EntryNumber entryNumber, TTreeCache* cache) const { + LogTrace("IOTrace").format( + "RootRNTuple::getEntryUsingCache() begin for branch {} entry {}", branch->GetName(), entryNumber); + try { + auto guard = filePtr_->setCacheReadTemporarily(cache, tree_); + branch->GetEntry(entryNumber); + } catch (cms::Exception const& e) { + // We make sure the treeCache_ is detached from the file, + // so that ROOT does not also delete it. + Exception t(errors::FileReadError, "", e); + t.addContext(std::string("Reading branch ") + branch->GetName()); + throw t; + } catch (std::exception const& e) { + Exception t(errors::FileReadError); + t << e.what(); + t.addContext(std::string("Reading branch ") + branch->GetName()); + throw t; + } catch (...) { + Exception t(errors::FileReadError); + t << "An exception of unknown type was thrown."; + t.addContext(std::string("Reading branch ") + branch->GetName()); + throw t; + } + LogTrace("IOTrace").format( + "RootRNTuple::getEntryUsingCache() end for branch {} entry {}", branch->GetName(), entryNumber); + } + + bool RootRNTuple::skipEntries(unsigned int& offset) { + entryNumber_ += offset; + bool retval = (entryNumber_ < entries_); + if (retval) { + offset = 0; + } else { + // Not enough entries in the file to skip. + // The +1 is needed because entryNumber_ is -1 at the initialization of the tree, not 0. + long long overshoot = entryNumber_ + 1 - entries_; + entryNumber_ = entries_; + offset = overshoot; + } + return retval; + } + + void RootRNTuple::startTraining() { + if (cacheSize_ == 0) { + return; + } + assert(treeCache_); + assert(branchType_ == InEvent); + assert(!rawTreeCache_); + treeCache_->SetLearnEntries(learningEntries_); + rawTreeCache_ = createCacheWithSize(cacheSize_); + rawTreeCache_->SetEnablePrefetching(false); + rawTreeCache_->SetLearnEntries(0); + if (promptRead_) { + switchOverEntry_ = entries_; + } else { + switchOverEntry_ = entryNumber_ + learningEntries_; + } + auto rawStart = entryNumber_; + auto rawEnd = switchOverEntry_; + auto treeStart = switchOverEntry_; + if (switchOverEntry_ >= tree_->GetEntries()) { + treeStart = switchOverEntry_ - tree_->GetEntries(); + rawEnd = tree_->GetEntries(); + } + rawTreeCache_->StartLearningPhase(); + rawTreeCache_->SetEntryRange(rawStart, rawEnd); + rawTreeCache_->AddBranch("*", kTRUE); + rawTreeCache_->StopLearningPhase(); + + treeCache_->StartLearningPhase(); + treeCache_->SetEntryRange(treeStart, tree_->GetEntries()); + // Make sure that 'branchListIndexes' branch exist in input file + if (filePtr_->Get(poolNames::branchListIndexesBranchName().c_str()) != nullptr) { + treeCache_->AddBranch(poolNames::branchListIndexesBranchName().c_str(), kTRUE); + } + treeCache_->AddBranch(BranchTypeToAuxiliaryBranchName(branchType_).c_str(), kTRUE); + trainedSet_.clear(); + triggerSet_.clear(); + assert(treeCache_->GetTree() == tree_); + } + + void RootRNTuple::stopTraining() { + auto guard = filePtr_->setCacheReadTemporarily(treeCache_.get(), tree_); + treeCache_->StopLearningPhase(); + rawTreeCache_.reset(); + } + + void RootRNTuple::close() { + // The TFile is about to be closed, and destructed. + // Just to play it safe, zero all pointers to quantities that are owned by the TFile. + auxBranch_ = branchEntryInfoBranch_ = nullptr; + tree_ = metaTree_ = infoTree_ = nullptr; + // We own the treeCache_. + // We make sure the treeCache_ is detached from the file, + // so that ROOT does not also delete it. + filePtr_->clearCacheRead(tree_); + // We *must* delete the TTreeCache here because the TFilePrefetch object + // references the TFile. If TFile is closed, before the TTreeCache is + // deleted, the TFilePrefetch may continue to do TFile operations, causing + // deadlocks or exceptions. + treeCache_.reset(); + rawTreeCache_.reset(); + triggerTreeCache_.reset(); + rawTriggerTreeCache_.reset(); + auxCache_.reset(); + // We give up our shared ownership of the TFile itself. + filePtr_.reset(); + } + + void RootRNTuple::trainCache(char const* branchNames) { + if (cacheSize_ == 0) { + return; + } + tree_->LoadTree(0); + assert(treeCache_); + { + auto guard = filePtr_->setCacheReadTemporarily(treeCache_.get(), tree_); + treeCache_->StartLearningPhase(); + treeCache_->SetEntryRange(0, tree_->GetEntries()); + treeCache_->AddBranch(branchNames, kTRUE); + treeCache_->StopLearningPhase(); + assert(treeCache_->GetTree() == tree_); + // We own the treeCache_. + // We make sure the treeCache_ is detached from the file, + // so that ROOT does not also delete it. + + //want guard to end here + } + + if (branchType_ == InEvent) { + // Must also manually add things to the trained set. + TObjArray* branches = tree_->GetListOfBranches(); + int branchCount = branches->GetEntriesFast(); + for (int i = 0; i < branchCount; i++) { + TBranch* branch = (TBranch*)branches->UncheckedAt(i); + if ((branchNames[0] == '*') || (strcmp(branchNames, branch->GetName()) == 0)) { + trainedSet_.insert(branch); + } + } + } + } + + void RootRNTuple::setSignals( + signalslot::Signal const* preEventReadSource, + signalslot::Signal const* postEventReadSource) { + rootDelayedReader_->setSignals(preEventReadSource, postEventReadSource); + } + + namespace rootrntuple { + Int_t getEntry(TBranch* branch, EntryNumber entryNumber) { + Int_t n = 0; + try { + n = branch->GetEntry(entryNumber); + } catch (cms::Exception const& e) { + throw Exception(errors::FileReadError, "", e); + } + return n; + } + + Int_t getEntry(TTree* tree, EntryNumber entryNumber) { + Int_t n = 0; + try { + n = tree->GetEntry(entryNumber); + } catch (cms::Exception const& e) { + throw Exception(errors::FileReadError, "", e); + } + return n; + } + + std::unique_ptr trainCache(TTree* tree, + InputFile& file, + unsigned int cacheSize, + char const* branchNames) { + tree->LoadTree(0); + std::unique_ptr treeCache = file.createCacheWithSize(*tree, cacheSize); + if (nullptr != treeCache.get()) { + treeCache->StartLearningPhase(); + treeCache->SetEntryRange(0, tree->GetEntries()); + treeCache->AddBranch(branchNames, kTRUE); + treeCache->StopLearningPhase(); + } + return treeCache; + } + } // namespace rootrntuple +} // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempInput/src/RootRNTuple.h b/FWIO/RNTupleTempInput/src/RootRNTuple.h new file mode 100644 index 0000000000000..8dc6e8b85ea76 --- /dev/null +++ b/FWIO/RNTupleTempInput/src/RootRNTuple.h @@ -0,0 +1,265 @@ +#ifndef FWIO_RNTupleTempInput_RootRNTuple_h +#define FWIO_RNTupleTempInput_RootRNTuple_h + +/*---------------------------------------------------------------------- + +RootRNTuple.h // used by ROOT input sources + +----------------------------------------------------------------------*/ + +#include "DataFormats/Provenance/interface/ProductDescription.h" +#include "DataFormats/Provenance/interface/BranchID.h" +#include "DataFormats/Provenance/interface/IndexIntoFile.h" +#include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/ServiceRegistry/interface/ServiceRegistryfwd.h" +#include "FWCore/Utilities/interface/BranchType.h" +#include "FWCore/Utilities/interface/InputType.h" +#include "FWCore/Utilities/interface/Signal.h" +#include "FWCore/Utilities/interface/thread_safety_macros.h" + +#include "Rtypes.h" +#include "TBranch.h" + +#include +#include +#include +#include +#include + +class TClass; +class TTree; +class TTreeCache; + +namespace edm::rntuple_temp { + class InputFile; + class RootDelayedReaderBase; + + namespace rootrntuple { + unsigned int const defaultCacheSize = 20U * 1024 * 1024; + unsigned int const defaultNonEventCacheSize = 1U * 1024 * 1024; + unsigned int const defaultLearningEntries = 20U; + unsigned int const defaultNonEventLearningEntries = 1U; + using EntryNumber = IndexIntoFile::EntryNumber_t; + struct BranchInfo { + BranchInfo(ProductDescription const& prod) + : productDescription_(prod), productBranch_(nullptr), classCache_(nullptr), offsetToWrapperBase_(0) {} + ProductDescription const productDescription_; + void setBranch(TBranch* branch, TClass const* wrapperBaseTClass); + std::unique_ptr newWrapper() const; + TBranch* productBranch_; + + private: + //All access to a ROOT file is serialized + TClass* classCache_; + Int_t offsetToWrapperBase_; + }; + + class BranchMap { + public: + using Map = std::unordered_map; + + void reserve(Map::size_type iSize) { map_.reserve(iSize); } + void insert(edm::BranchID const& iKey, BranchInfo const& iInfo) { map_.emplace(iKey.id(), iInfo); } + BranchInfo const* find(BranchID const& iKey) const { return find(iKey.id()); } + BranchInfo const* find(unsigned int iKey) const { + auto itFound = map_.find(iKey); + if (itFound == map_.end()) { + return nullptr; + } + return &itFound->second; + } + + using const_iterator = Map::const_iterator; + const_iterator begin() const { return map_.cbegin(); } + const_iterator end() const { return map_.cend(); } + Map::size_type size() const { return map_.size(); } + + private: + Map map_; + }; + + Int_t getEntry(TBranch* branch, EntryNumber entryNumber); + Int_t getEntry(TTree* tree, EntryNumber entryNumber); + std::unique_ptr trainCache(TTree* tree, + InputFile& file, + unsigned int cacheSize, + char const* branchNames); + } // namespace rootrntuple + + class RootRNTuple { + public: + using BranchMap = rootrntuple::BranchMap; + using EntryNumber = rootrntuple::EntryNumber; + struct Options { + unsigned int treeCacheSize = 0U; + int treeMaxVirtualSize; + bool enablePrefetching; + bool promptReading = false; + + Options usingDefaultNonEventOptions() const { + return {rootrntuple::defaultNonEventCacheSize, treeMaxVirtualSize, enablePrefetching, false}; + } + }; + + RootRNTuple(std::shared_ptr filePtr, + BranchType const& branchType, + unsigned int nIndexes, + Options const& options, + unsigned int learningEntries, + InputType inputType); + + RootRNTuple(std::shared_ptr filePtr, + BranchType const& branchType, + std::string const& processName, + unsigned int nIndexes, + Options const& options, + unsigned int learningEntries, + InputType inputType); + + void init(std::string const& productTreeName, unsigned int maxVirtualSize, unsigned int cacheSize); + + ~RootRNTuple(); + + RootRNTuple(RootRNTuple const&) = delete; // Disallow copying and moving + RootRNTuple& operator=(RootRNTuple const&) = delete; // Disallow copying and moving + + bool isValid() const; + void numberOfBranchesToAdd(BranchMap::Map::size_type iSize) { branches_.reserve(iSize); } + void addBranch(ProductDescription const& prod, std::string const& oldBranchName); + void dropBranch(std::string const& oldBranchName); + void getEntry(TBranch* branch, EntryNumber entry) const; + void getEntryForAllBranches() const; + void setPresence(ProductDescription& prod, std::string const& oldBranchName); + + bool next() { return ++entryNumber_ < entries_; } + bool nextWithCache(); + bool current() const { return entryNumber_ < entries_ && entryNumber_ >= 0; } + bool current(EntryNumber entry) const { return entry < entries_ && entry >= 0; } + void rewind() { entryNumber_ = 0; } + void rewindToInvalid() { entryNumber_ = IndexIntoFile::invalidEntry; } + void close(); + bool skipEntries(unsigned int& offset); + EntryNumber const& entryNumber() const { return entryNumber_; } + EntryNumber const& entryNumberForIndex(unsigned int index) const; + EntryNumber const& entries() const { return entries_; } + void setEntryNumber(EntryNumber theEntryNumber); + void insertEntryForIndex(unsigned int index); + std::vector const& branchNames() const { return branchNames_; } + RootDelayedReaderBase* rootDelayedReader() const; + DelayedReader* resetAndGetRootDelayedReader() const; + template + void fillAux(T*& pAux) { + auxBranch_->SetAddress(&pAux); + auto cache = getAuxCache(auxBranch_); + getEntryUsingCache(auxBranch_, entryNumber_, cache); + auxBranch_->SetAddress(nullptr); + } + + template + void fillBranchEntry(TBranch* branch, T*& pbuf) { + branch->SetAddress(&pbuf); + getEntry(branch, entryNumber_); + branch->SetAddress(nullptr); + } + + template + void fillBranchEntryMeta(TBranch* branch, EntryNumber entryNumber, T*& pbuf) { + if (metaTree_ != nullptr) { + // Metadata was in separate tree. Not cached. + branch->SetAddress(&pbuf); + rootrntuple::getEntry(branch, entryNumber); + } else { + fillBranchEntry(branch, entryNumber, pbuf); + } + } + + template + void fillBranchEntry(TBranch* branch, EntryNumber entryNumber, T*& pbuf) { + branch->SetAddress(&pbuf); + getEntry(branch, entryNumber); + } + + TTree const* tree() const { return tree_; } + TTree* tree() { return tree_; } + TTree const* metaTree() const { return metaTree_; } + TTree* metaTree() { return metaTree_; } + BranchMap const& branches() const; + + //For backwards compatibility + TBranch* branchEntryInfoBranch() const { return branchEntryInfoBranch_; } + + inline TTreeCache* checkTriggerCache(TBranch* branch, EntryNumber entryNumber) const; + TTreeCache* checkTriggerCacheImpl(TBranch* branch, EntryNumber entryNumber) const; + inline TTreeCache* selectCache(TBranch* branch, EntryNumber entryNumber) const; + void trainCache(char const* branchNames); + void resetTraining() { trainNow_ = true; } + + BranchType branchType() const { return branchType_; } + std::string const& processName() const { return processName_; } + + void setSignals( + signalslot::Signal const* preEventReadSource, + signalslot::Signal const* postEventReadSource); + + private: + RootRNTuple(std::shared_ptr filePtr, + BranchType const& branchType, + unsigned int nIndexes, + unsigned int learningEntries, + bool enablePrefetching, + bool promptRead, + InputType inputType); + + std::shared_ptr createCacheWithSize(unsigned int cacheSize) const; + void setCacheSize(unsigned int cacheSize); + void setTreeMaxVirtualSize(int treeMaxVirtualSize); + void startTraining(); + void stopTraining(); + void getEntryUsingCache(TBranch* branch, EntryNumber entry, TTreeCache*) const; + TTreeCache* getAuxCache(TBranch* auxBranch) const; + + std::shared_ptr filePtr_; + // We use bare pointers for pointers to some ROOT entities. + // Root owns them and uses bare pointers internally. + // Therefore,using smart pointers here will do no good. + TTree* tree_ = nullptr; + TTree* metaTree_ = nullptr; + BranchType branchType_; + std::string processName_; + TBranch* auxBranch_ = nullptr; + // We use a smart pointer to own the TTreeCache. + // Unfortunately, ROOT owns it when attached to a TFile, but not after it is detached. + // So, we make sure to it is detached before closing the TFile so there is no double delete. + std::shared_ptr treeCache_; + std::shared_ptr rawTreeCache_; + CMS_SA_ALLOW mutable std::shared_ptr auxCache_; + //All access to a ROOT file is serialized + CMS_SA_ALLOW mutable std::shared_ptr triggerTreeCache_; + CMS_SA_ALLOW mutable std::shared_ptr rawTriggerTreeCache_; + CMS_SA_ALLOW mutable std::unordered_set trainedSet_; + CMS_SA_ALLOW mutable std::unordered_set triggerSet_; + EntryNumber entries_ = 0; + EntryNumber entryNumber_ = IndexIntoFile::invalidEntry; + std::unique_ptr > entryNumberForIndex_; + std::vector branchNames_; + BranchMap branches_; + bool trainNow_ = false; + EntryNumber switchOverEntry_ = -1; + CMS_SA_ALLOW mutable EntryNumber rawTriggerSwitchOverEntry_ = -1; + CMS_SA_ALLOW mutable bool performedSwitchOver_ = false; + unsigned int learningEntries_; + unsigned int cacheSize_ = 0; + unsigned long treeAutoFlush_ = 0; + // Enable asynchronous I/O in ROOT (done in a separate thread). Only takes + // effect on the primary treeCache_; all other caches have this explicitly disabled. + bool enablePrefetching_; + bool enableTriggerCache_; + bool promptRead_; + std::unique_ptr rootDelayedReader_; + + TBranch* branchEntryInfoBranch_ = nullptr; //backwards compatibility + // below for backward compatibility + TTree* infoTree_ = nullptr; // backward compatibility + }; +} // namespace edm::rntuple_temp +#endif diff --git a/FWIO/RNTupleTempInput/src/RootSecondaryFileSequence.cc b/FWIO/RNTupleTempInput/src/RootSecondaryFileSequence.cc new file mode 100644 index 0000000000000..998226770f384 --- /dev/null +++ b/FWIO/RNTupleTempInput/src/RootSecondaryFileSequence.cc @@ -0,0 +1,108 @@ +/*---------------------------------------------------------------------- +----------------------------------------------------------------------*/ +#include "DuplicateChecker.h" +#include "RNTupleTempSource.h" +#include "InputFile.h" +#include "RootFile.h" +#include "RootSecondaryFileSequence.h" +#include "RootRNTuple.h" + +#include "DataFormats/Provenance/interface/BranchID.h" +#include "DataFormats/Provenance/interface/ProductRegistry.h" +#include "DataFormats/Provenance/interface/ProcessHistoryRegistry.h" +#include "FWCore/Catalog/interface/InputFileCatalog.h" +#include "FWCore/Catalog/interface/SiteLocalConfig.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" +#include "FWCore/ServiceRegistry/interface/Service.h" +#include "Utilities/StorageFactory/interface/StorageFactory.h" + +namespace edm::rntuple_temp { + RootSecondaryFileSequence::RootSecondaryFileSequence(ParameterSet const& pset, + RNTupleTempSource& input, + InputFileCatalog const& catalog) + : RootInputFileSequence(pset, catalog), + input_(input), + orderedProcessHistoryIDs_(), + enablePrefetching_(false), + enforceGUIDInFileName_(pset.getUntrackedParameter("enforceGUIDInFileName")) { + // The SiteLocalConfig controls the TTreeCache size and the prefetching settings. + Service pSLC; + if (pSLC.isAvailable()) { + enablePrefetching_ = pSLC->enablePrefetching(); + } + + // Prestage the files + //NOTE: we do not want to stage in all secondary files since we can be given a list of + // thousands of files and prestaging all those files can cause a site to fail. + // So, we stage in the first secondary file only. + setAtFirstFile(); + storage::StorageFactory::get()->stagein(fileNames()[0]); + + // Open the first file. + for (setAtFirstFile(); !noMoreFiles(); setAtNextFile()) { + initFile(input_.skipBadFiles()); + if (rootFile()) + break; + } + if (rootFile()) { + std::vector processOrder; + processingOrderMerge(input_.processHistoryRegistry(), processOrder); + input_.productRegistryUpdate().updateFromInput(rootFile()->productRegistry()->productList(), processOrder); + } + } + + RootSecondaryFileSequence::~RootSecondaryFileSequence() {} + + void RootSecondaryFileSequence::endJob() { closeFile(); } + + void RootSecondaryFileSequence::closeFile_() { + // close the currently open file, if any, and delete the RootFile object. + if (rootFile()) { + rootFile()->close(); + rootFile().reset(); + } + } + + void RootSecondaryFileSequence::initFile_(bool skipBadFiles) { + initTheFile(skipBadFiles, false, nullptr, "secondaryFiles", InputType::SecondaryFile); + } + + RootSecondaryFileSequence::RootFileSharedPtr RootSecondaryFileSequence::makeRootFile( + std::shared_ptr filePtr) { + size_t currentIndexIntoFile = sequenceNumberOfFile(); + return std::make_shared( + RootFile::FileOptions{.fileName = fileNames()[0], + .logicalFileName = logicalFileName(), + .filePtr = filePtr, + .bypassVersionCheck = input_.bypassVersionCheck(), + .enforceGUIDInFileName = enforceGUIDInFileName_}, + InputType::SecondaryFile, + RootFile::ProcessingOptions{ + .processingMode = input_.processingMode(), + }, + RootFile::TTreeOptions{.treeMaxVirtualSize = input_.treeMaxVirtualSize(), + .enablePrefetching = enablePrefetching_, + .promptReading = not input_.delayReadingEventProducts()}, + RootFile::ProductChoices{.productSelectorRules = input_.productSelectorRules(), + .associationsFromSecondary = &associationsFromSecondary_, + .dropDescendantsOfDroppedProducts = input_.dropDescendants(), + .labelRawDataLikeMC = input_.labelRawDataLikeMC()}, + RootFile::CrossFileInfo{.runHelper = input_.runHelper(), + .branchIDListHelper = input_.branchIDListHelper(), + .thinnedAssociationsHelper = input_.thinnedAssociationsHelper(), + .indexesIntoFiles = indexesIntoFiles(), + .currentIndexIntoFile = currentIndexIntoFile}, + input_.nStreams(), + input_.processHistoryRegistryForUpdate(), + orderedProcessHistoryIDs_); + } + + void RootSecondaryFileSequence::initAssociationsFromSecondary(std::set const& associationsFromSecondary) { + for (auto const& branchID : associationsFromSecondary) { + associationsFromSecondary_.push_back(branchID); + } + rootFile()->initAssociationsFromSecondary(associationsFromSecondary_); + } +} // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempInput/src/RootSecondaryFileSequence.h b/FWIO/RNTupleTempInput/src/RootSecondaryFileSequence.h new file mode 100644 index 0000000000000..222e7e1b0ee60 --- /dev/null +++ b/FWIO/RNTupleTempInput/src/RootSecondaryFileSequence.h @@ -0,0 +1,55 @@ +#ifndef IOPool_Input_RootSecondaryFileSequence_h +#define IOPool_Input_RootSecondaryFileSequence_h + +/*---------------------------------------------------------------------- + +RootSecondaryFileSequence: This is an InputSource + +----------------------------------------------------------------------*/ + +#include "RootInputFileSequence.h" +#include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/Framework/interface/ProductSelectorRules.h" +#include "DataFormats/Provenance/interface/ProcessHistoryID.h" + +#include +#include +#include +#include + +namespace edm { + + class BranchID; + class FileCatalogItem; + class InputFileCatalog; + class RNTupleTempSource; + class RootFile; +} // namespace edm +namespace edm::rntuple_temp { + + class RootSecondaryFileSequence : public RootInputFileSequence { + public: + explicit RootSecondaryFileSequence(ParameterSet const& pset, + RNTupleTempSource& input, + InputFileCatalog const& catalog); + ~RootSecondaryFileSequence() override; + + RootSecondaryFileSequence(RootSecondaryFileSequence const&) = delete; // Disallow copying and moving + RootSecondaryFileSequence& operator=(RootSecondaryFileSequence const&) = delete; // Disallow copying and moving + + void endJob(); + void initAssociationsFromSecondary(std::set const&); + + private: + void closeFile_() override; + void initFile_(bool skipBadFiles) override; + RootFileSharedPtr makeRootFile(std::shared_ptr filePtr) override; + + RNTupleTempSource& input_; + std::vector associationsFromSecondary_; + std::vector orderedProcessHistoryIDs_; + bool enablePrefetching_; + bool enforceGUIDInFileName_; + }; // class RootSecondaryFileSequence +} // namespace edm::rntuple_temp +#endif diff --git a/FWIO/RNTupleTempInput/test/BuildFile.xml b/FWIO/RNTupleTempInput/test/BuildFile.xml new file mode 100644 index 0000000000000..c60f57eb138c7 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/BuildFile.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + diff --git a/FWIO/RNTupleTempInput/test/HeteroMerge_cfg.py b/FWIO/RNTupleTempInput/test/HeteroMerge_cfg.py new file mode 100644 index 0000000000000..5ce4c89e87416 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/HeteroMerge_cfg.py @@ -0,0 +1,17 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( +'file:small.root', +'file:big.root' +) +) + + + diff --git a/FWIO/RNTupleTempInput/test/Pool2FileInputTest_cfg.py b/FWIO/RNTupleTempInput/test/Pool2FileInputTest_cfg.py new file mode 100644 index 0000000000000..1ccec0f841c17 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/Pool2FileInputTest_cfg.py @@ -0,0 +1,22 @@ +# The following comments couldn't be translated into the new config version: + +# Test storing OtherThing as well +# Configuration file for PrePoolInputTest + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTBOTHFILES") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.OtherThing = cms.EDProducer("OtherThingProducer") + +process.source = cms.Source("RNTupleTempSource", + secondaryFileNames = cms.untracked.vstring("file:PoolInputOther.root"), + fileNames = cms.untracked.vstring("file:PoolInput2FileTest.root") + ) + + +process.p = cms.Path(process.OtherThing) + + + diff --git a/FWIO/RNTupleTempInput/test/PoolAliasTestStep1C_cfg.py b/FWIO/RNTupleTempInput/test/PoolAliasTestStep1C_cfg.py new file mode 100644 index 0000000000000..5d953314141f0 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PoolAliasTestStep1C_cfg.py @@ -0,0 +1,37 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTPROD") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(11) +) +process.Thing = cms.EDProducer("ThingProducer") + +process.output = cms.OutputModule("RNTupleTempOutputModule", + outputCommands = cms.untracked.vstring("keep *", "drop *_Thing_*_TESTPROD"), + fileName = cms.untracked.string('aliastest_step1c.root') +) + +process.source = cms.Source("EmptySource", + firstLuminosityBlock = cms.untracked.uint32(6), + numberEventsInLuminosityBlock = cms.untracked.uint32(3), + firstRun = cms.untracked.uint32(561), + numberEventsInRun = cms.untracked.uint32(7) +) + + +process.OtherThing = cms.EDProducer("OtherThingProducer", + thingTag = cms.InputTag("Thing") + ) + +process.AltThing = cms.EDAlias( + Thing = cms.VPSet( + cms.PSet(type = cms.string('edmtestThings'), + fromProductInstance = cms.string('*'), + toProductInstance = cms.string('*')) + ) +) + +process.p = cms.Path(process.Thing*process.OtherThing) +process.ep = cms.EndPath(process.output) diff --git a/FWIO/RNTupleTempInput/test/PoolAliasTestStep1_DifferentOrder_cfg.py b/FWIO/RNTupleTempInput/test/PoolAliasTestStep1_DifferentOrder_cfg.py new file mode 100644 index 0000000000000..bdae1797021f9 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PoolAliasTestStep1_DifferentOrder_cfg.py @@ -0,0 +1,43 @@ +# The following comments couldn't be translated into the new config version: + +# Configuration file for PrePoolInputTest + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTPROD") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(11) +) +process.Thing = cms.EDProducer("ThingProducer") + +process.output = cms.OutputModule("RNTupleTempOutputModule", + outputCommands = cms.untracked.vstring("keep *", "drop *_Thing_*_TESTPROD"), + fileName = cms.untracked.string('aliastest_step1_diffOrder.root') +) + +process.source = cms.Source("EmptySource", + firstLuminosityBlock = cms.untracked.uint32(6), + numberEventsInLuminosityBlock = cms.untracked.uint32(3), + firstRun = cms.untracked.uint32(561), + numberEventsInRun = cms.untracked.uint32(7) +) + +process.OtherThing = cms.EDProducer("OtherThingProducer", + thingTag = cms.InputTag("ZThing") + ) + +#This changes the order of the original and alias branch names and branchIDs +process.ZThing = cms.EDAlias( + Thing = cms.VPSet( + cms.PSet(type = cms.string('edmtestThings'), + fromProductInstance = cms.string('*'), + toProductInstance = cms.string('*')) + ) +) + + +process.p = cms.Path(process.Thing*process.OtherThing) +process.ep = cms.EndPath(process.output) + diff --git a/FWIO/RNTupleTempInput/test/PoolAliasTestStep1_cfg.py b/FWIO/RNTupleTempInput/test/PoolAliasTestStep1_cfg.py new file mode 100644 index 0000000000000..1f020d905c8eb --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PoolAliasTestStep1_cfg.py @@ -0,0 +1,43 @@ +# The following comments couldn't be translated into the new config version: + +# Configuration file for PrePoolInputTest + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTPROD") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(11) +) +process.Thing = cms.EDProducer("ThingProducer") + +process.output = cms.OutputModule("RNTupleTempOutputModule", + outputCommands = cms.untracked.vstring("keep *", "drop *_Thing_*_TESTPROD"), + fileName = cms.untracked.string('step1.root') +) + +process.source = cms.Source("EmptySource", + firstLuminosityBlock = cms.untracked.uint32(6), + numberEventsInLuminosityBlock = cms.untracked.uint32(3), + firstRun = cms.untracked.uint32(561), + numberEventsInRun = cms.untracked.uint32(7) +) + + +process.OtherThing = cms.EDProducer("OtherThingProducer", + thingTag = cms.InputTag("AltThing") + ) + +process.AltThing = cms.EDAlias( + Thing = cms.VPSet( + cms.PSet(type = cms.string('edmtestThings'), + fromProductInstance = cms.string('*'), + toProductInstance = cms.string('*')) + ) +) + + +process.p = cms.Path(process.Thing*process.OtherThing) +process.ep = cms.EndPath(process.output) + diff --git a/FWIO/RNTupleTempInput/test/PoolAliasTestStep2A_cfg.py b/FWIO/RNTupleTempInput/test/PoolAliasTestStep2A_cfg.py new file mode 100644 index 0000000000000..e6a6c153f3f3c --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PoolAliasTestStep2A_cfg.py @@ -0,0 +1,23 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTANALYSIS") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) + +process.Analysis = cms.EDAnalyzer("OtherThingAnalyzer") + +process.source = cms.Source("RNTupleTempSource", + setRunNumber = cms.untracked.uint32(621), + fileNames = cms.untracked.vstring('file:step1.root') +) + +process.OtherThing = cms.EDProducer("OtherThingProducer", + thingTag = cms.InputTag("AltThing") +) + +process.p = cms.Path(process.OtherThing*process.Analysis) + + diff --git a/FWIO/RNTupleTempInput/test/PoolAliasTestStep2C_cfg.py b/FWIO/RNTupleTempInput/test/PoolAliasTestStep2C_cfg.py new file mode 100644 index 0000000000000..2971a767da84a --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PoolAliasTestStep2C_cfg.py @@ -0,0 +1,17 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTANALYSIS") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) + +process.Analysis = cms.EDAnalyzer("OtherThingAnalyzer") + +process.source = cms.Source("RNTupleTempSource", + setRunNumber = cms.untracked.uint32(621), + fileNames = cms.untracked.vstring('file:aliastest_step1c.root') +) + +process.p = cms.Path(process.Analysis) diff --git a/FWIO/RNTupleTempInput/test/PoolAliasTestStep2_DifferentOrder_cfg.py b/FWIO/RNTupleTempInput/test/PoolAliasTestStep2_DifferentOrder_cfg.py new file mode 100644 index 0000000000000..92db17c80eebe --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PoolAliasTestStep2_DifferentOrder_cfg.py @@ -0,0 +1,31 @@ +# The following comments couldn't be translated into the new config version: + +# Configuration file for PoolInputTest + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTANALYSIS") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) + +process.Analysis = cms.EDAnalyzer("OtherThingAnalyzer") + +process.source = cms.Source("RNTupleTempSource", + setRunNumber = cms.untracked.uint32(621), + fileNames = cms.untracked.vstring('file:aliastest_step1_diffOrder.root') +) + +process.p = cms.Path(process.Analysis) + +#Also test creating a Ref from the alias branch +process.OtherThing2 = cms.EDProducer("OtherThingProducer", + thingTag = cms.InputTag("ZThing") + ) + +process.Analysis2 = cms.EDAnalyzer("OtherThingAnalyzer", + other=cms.untracked.InputTag("OtherThing2","testUserTag")) + +process.p2 = cms.Path(process.OtherThing2+process.Analysis2) diff --git a/FWIO/RNTupleTempInput/test/PoolAliasTestStep2_cfg.py b/FWIO/RNTupleTempInput/test/PoolAliasTestStep2_cfg.py new file mode 100644 index 0000000000000..ee1ca4653861a --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PoolAliasTestStep2_cfg.py @@ -0,0 +1,31 @@ +# The following comments couldn't be translated into the new config version: + +# Configuration file for PoolInputTest + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTANALYSIS") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) + +process.Analysis = cms.EDAnalyzer("OtherThingAnalyzer") + +process.source = cms.Source("RNTupleTempSource", + setRunNumber = cms.untracked.uint32(621), + fileNames = cms.untracked.vstring('file:step1.root') +) + +process.p = cms.Path(process.Analysis) + +#Also test creating a Ref from the alias branch +process.OtherThing2 = cms.EDProducer("OtherThingProducer", + thingTag = cms.InputTag("AltThing") + ) + +process.Analysis2 = cms.EDAnalyzer("OtherThingAnalyzer", + other=cms.untracked.InputTag("OtherThing2","testUserTag")) + +process.p2 = cms.Path(process.OtherThing2+process.Analysis2) diff --git a/FWIO/RNTupleTempInput/test/PoolEmptyTest2_cfg.py b/FWIO/RNTupleTempInput/test/PoolEmptyTest2_cfg.py new file mode 100644 index 0000000000000..b762b7801a252 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PoolEmptyTest2_cfg.py @@ -0,0 +1,27 @@ +# The following comments couldn't be translated into the new config version: + +# Configuration file for PoolInputTest + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READEMPTY") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) +process.Thing = cms.EDProducer("ThingProducer") + +process.output = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('PoolEmptyTestOut.root') +) + +process.source = cms.Source("RNTupleTempSource", + skipBadFiles = cms.untracked.bool(True), + fileNames = cms.untracked.vstring('file:doesNotExist.root', 'file:PoolEmptyTest.root') +) + +process.p = cms.Path(process.Thing) +process.ep = cms.EndPath(process.output) + + diff --git a/FWIO/RNTupleTempInput/test/PoolEmptyTest_cfg.py b/FWIO/RNTupleTempInput/test/PoolEmptyTest_cfg.py new file mode 100644 index 0000000000000..d01e94512c668 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PoolEmptyTest_cfg.py @@ -0,0 +1,24 @@ +# Configuration file for PoolInputTest + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("WRITEEMPTY") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.Thing = cms.EDProducer("ThingProducer") + +process.output = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('PoolEmptyTest.root') +) + +# Select lumis that we know are not in the input file so +# that the output file is empty (contains no Runs, Lumis, +# or Events, but it does have metadata) +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:PoolInputTest.root'), + lumisToProcess = cms.untracked.VLuminosityBlockRange('1:1') +) + +process.p = cms.Path(process.Thing) +process.ep = cms.EndPath(process.output) + diff --git a/FWIO/RNTupleTempInput/test/PoolGUIDTest_cfg.py b/FWIO/RNTupleTempInput/test/PoolGUIDTest_cfg.py new file mode 100644 index 0000000000000..2492eda90efa1 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PoolGUIDTest_cfg.py @@ -0,0 +1,21 @@ +# Configuration file for PoolInputTest + +import FWCore.ParameterSet.Config as cms +import sys + +process = cms.Process("TESTRECO") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents.input = -1 +process.OtherThing = cms.EDProducer("OtherThingProducer") +process.Analysis = cms.EDAnalyzer("OtherThingAnalyzer") + +process.source = cms.Source("RNTupleTempSource", + setRunNumber = cms.untracked.uint32(621), + fileNames = cms.untracked.vstring(sys.argv[1]), + enforceGUIDInFileName = cms.untracked.bool(True) +) + +process.p = cms.Path(process.OtherThing*process.Analysis) + + diff --git a/FWIO/RNTupleTempInput/test/PoolInputTest2_cfg.py b/FWIO/RNTupleTempInput/test/PoolInputTest2_cfg.py new file mode 100644 index 0000000000000..dae9cc836907a --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PoolInputTest2_cfg.py @@ -0,0 +1,23 @@ +# The following comments couldn't be translated into the new config version: + +# Configuration file for PoolInputTest + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTRECO") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) +process.Analysis = cms.EDAnalyzer("OtherThingAnalyzer") + +process.source = cms.Source("RNTupleTempSource", + setRunNumber = cms.untracked.uint32(621), + fileNames = cms.untracked.vstring('file:PoolInputTest.root', + 'file:PoolInputOther.root') +) + +process.p = cms.Path(process.Analysis) + + diff --git a/FWIO/RNTupleTempInput/test/PoolInputTest3_cfg.py b/FWIO/RNTupleTempInput/test/PoolInputTest3_cfg.py new file mode 100644 index 0000000000000..3a36ca694fe2f --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PoolInputTest3_cfg.py @@ -0,0 +1,24 @@ +# The following comments couldn't be translated into the new config version: + +# Configuration file for PoolInputTest + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTRECO") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) +process.Analysis = cms.EDAnalyzer("OtherThingAnalyzer", + thingWasDropped = cms.untracked.bool(True) +) + +process.source = cms.Source("RNTupleTempSource", + setRunNumber = cms.untracked.uint32(621), + fileNames = cms.untracked.vstring('file:PoolInputDropTest.root') +) + +process.p = cms.Path(process.Analysis) + + diff --git a/FWIO/RNTupleTempInput/test/PoolInputTest_cfg.py b/FWIO/RNTupleTempInput/test/PoolInputTest_cfg.py new file mode 100644 index 0000000000000..3b6877de3a170 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PoolInputTest_cfg.py @@ -0,0 +1,25 @@ +# The following comments couldn't be translated into the new config version: + +# Configuration file for PoolInputTest + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTRECO") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) +process.OtherThing = cms.EDProducer("OtherThingProducer") + +process.Analysis = cms.EDAnalyzer("OtherThingAnalyzer") + +process.source = cms.Source("RNTupleTempSource", + setRunNumber = cms.untracked.uint32(621), + fileNames = cms.untracked.vstring('file:PoolInputTest.root', + 'file:PoolInputOther.root') +) + +process.p = cms.Path(process.OtherThing*process.Analysis) + + diff --git a/FWIO/RNTupleTempInput/test/PoolInputTest_noDelay_cfg.py b/FWIO/RNTupleTempInput/test/PoolInputTest_noDelay_cfg.py new file mode 100644 index 0000000000000..eec7dda05238a --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PoolInputTest_noDelay_cfg.py @@ -0,0 +1,21 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTRECO") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) +process.OtherThing = cms.EDProducer("OtherThingProducer") + +process.Analysis = cms.EDAnalyzer("OtherThingAnalyzer") + +process.source = cms.Source("RNTupleTempSource", + delayReadingEventProducts = cms.untracked.bool(False), + setRunNumber = cms.untracked.uint32(621), + fileNames = cms.untracked.vstring('file:PoolInputTest.root') +) + +process.p = cms.Path(process.OtherThing*process.Analysis) + +process.add_(cms.Service("Tracer")) diff --git a/FWIO/RNTupleTempInput/test/PoolInputTest_skipBadFiles_cfg.py b/FWIO/RNTupleTempInput/test/PoolInputTest_skipBadFiles_cfg.py new file mode 100644 index 0000000000000..82e700b965f1d --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PoolInputTest_skipBadFiles_cfg.py @@ -0,0 +1,35 @@ +# Test the skipBadFiles option of RNTupleTempSource +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTRECO") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) + +process.intProducer = cms.EDProducer("IntProducer", ivalue = cms.int32(3)) + +process.a1 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("intProducer") ), + expectedSum = cms.untracked.int32(54) +) + +process.source = cms.Source("RNTupleTempSource", + duplicateCheckMode = cms.untracked.string("noDuplicateCheck"), + skipBadFiles = cms.untracked.bool(True), + skipEvents = cms.untracked.uint32(15), #skips all events in first file + fileNames = cms.untracked.vstring('file:PoolInputTest.root', + 'file:this_file_doesnt_exist.root', + 'file:this_file_doesnt_exist.root', + 'file:PoolInputTest.root', + 'file:this_file_doesnt_exist.root', + 'file:this_file_doesnt_exist.root', + 'file:PoolInputTest.root', + 'file:this_file_doesnt_exist.root', + 'file:this_file_doesnt_exist.root') +) + +process.p = cms.Path(process.intProducer * process.a1) + + diff --git a/FWIO/RNTupleTempInput/test/PoolInputTest_skip_with_failure_cfg.py b/FWIO/RNTupleTempInput/test/PoolInputTest_skip_with_failure_cfg.py new file mode 100644 index 0000000000000..fc7b73c74008a --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PoolInputTest_skip_with_failure_cfg.py @@ -0,0 +1,25 @@ +# Configuration file for PoolInputTest + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTRECO") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) +process.OtherThing = cms.EDProducer("OtherThingProducer") + +process.Analysis = cms.EDAnalyzer("OtherThingAnalyzer") + +process.source = cms.Source("RNTupleTempSource", + skipBadFiles = cms.untracked.bool(True), + skipEvents = cms.untracked.uint32(15), #skips all events in first file + setRunNumber = cms.untracked.uint32(621), + fileNames = cms.untracked.vstring('file:PoolInputTest.root', + 'file:this_file_doesnt_exist.root') +) + +process.p = cms.Path(process.OtherThing*process.Analysis) + + diff --git a/FWIO/RNTupleTempInput/test/PoolNoParentDictionaryTestStep1_cfg.py b/FWIO/RNTupleTempInput/test/PoolNoParentDictionaryTestStep1_cfg.py new file mode 100644 index 0000000000000..a2e79ce431840 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PoolNoParentDictionaryTestStep1_cfg.py @@ -0,0 +1,24 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTPROD") +process.maxEvents.input = 11 + +process.source = cms.Source("EmptySource", + firstLuminosityBlock = cms.untracked.uint32(6), + numberEventsInLuminosityBlock = cms.untracked.uint32(3), + firstRun = cms.untracked.uint32(561), + numberEventsInRun = cms.untracked.uint32(7) +) + +process.parentIntProduct = cms.EDProducer("edmtest::TransientIntParentProducer", ivalue = cms.int32(1)) + +process.intProduct = cms.EDProducer("edmtest::IntProducerFromTransientParent", + src = cms.InputTag("parentIntProduct") +) + +process.output = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('noparentdict_step1.root') +) + +process.p = cms.Path(process.parentIntProduct+process.intProduct) +process.ep = cms.EndPath(process.output) diff --git a/FWIO/RNTupleTempInput/test/PoolNoParentDictionaryTestStep2_cfg.py b/FWIO/RNTupleTempInput/test/PoolNoParentDictionaryTestStep2_cfg.py new file mode 100644 index 0000000000000..5cb8c48939451 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PoolNoParentDictionaryTestStep2_cfg.py @@ -0,0 +1,17 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTANALYSIS") + +process.maxEvents.input = -1 + +process.source = cms.Source("RNTupleTempSource", + setRunNumber = cms.untracked.uint32(621), + fileNames = cms.untracked.vstring('file:noparentdict_step1.root') +) + +process.analysis = cms.EDAnalyzer("IntTestAnalyzer", + valueMustMatch = cms.untracked.int32(1), + moduleLabel = cms.untracked.InputTag("intProduct"), +) + +process.p = cms.Path(process.analysis) diff --git a/FWIO/RNTupleTempInput/test/PrePool2FileInputTest_cfg.py b/FWIO/RNTupleTempInput/test/PrePool2FileInputTest_cfg.py new file mode 100644 index 0000000000000..8bf8dc54c9d00 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PrePool2FileInputTest_cfg.py @@ -0,0 +1,31 @@ +# The following comments couldn't be translated into the new config version: + +# Test storing OtherThing as well +# Configuration file for PrePoolInputTest + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST2ND") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +#process.maxEvents = cms.untracked.PSet( +# input = cms.untracked.int32(11) +#) +#process.Thing = cms.EDProducer("ThingProducer") + +process.output = cms.OutputModule("RNTupleTempOutputModule", + outputCommands = cms.untracked.vstring('keep *', + 'drop *_Thing_*_*'), + fileName = cms.untracked.string('PoolInput2FileTest.root') +) + +process.OtherThing = cms.EDProducer("OtherThingProducer") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring("file:PoolInputOther.root") ) + + +process.p = cms.Path(process.OtherThing) +process.ep = cms.EndPath(process.output) + + diff --git a/FWIO/RNTupleTempInput/test/PrePoolInputTest2_cfg.py b/FWIO/RNTupleTempInput/test/PrePoolInputTest2_cfg.py new file mode 100644 index 0000000000000..995e5b4d2a287 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PrePoolInputTest2_cfg.py @@ -0,0 +1,38 @@ +# The following comments couldn't be translated into the new config version: + +# Test storing OtherThing as well +# Configuration file for PrePoolInputTest + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTPROD") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(11) +) +process.Thing = cms.EDProducer("ThingProducer") + +process.output = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('PoolInputTest.root') +) + +process.output2 = cms.OutputModule("RNTupleTempOutputModule", + outputCommands = cms.untracked.vstring('keep *', + 'drop *_Thing_*_*'), + fileName = cms.untracked.string('PoolInputDropTest.root') +) + +process.OtherThing = cms.EDProducer("OtherThingProducer") + +process.source = cms.Source("EmptySource", + firstLuminosityBlock = cms.untracked.uint32(6), + numberEventsInLuminosityBlock = cms.untracked.uint32(3), + firstRun = cms.untracked.uint32(561), + numberEventsInRun = cms.untracked.uint32(7) +) + +process.p = cms.Path(process.Thing*process.OtherThing) +process.ep = cms.EndPath(process.output*process.output2) + + diff --git a/FWIO/RNTupleTempInput/test/PrePoolInputTest_cfg.py b/FWIO/RNTupleTempInput/test/PrePoolInputTest_cfg.py new file mode 100644 index 0000000000000..b7989cc1a4017 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/PrePoolInputTest_cfg.py @@ -0,0 +1,37 @@ +# The following comments couldn't be translated into the new config version: + +# Configuration file for PrePoolInputTest + +import FWCore.ParameterSet.Config as cms +import sys + +useOtherThing = False +if len(sys.argv) > 7: + if sys.argv[7] == "useOtherThing": + useOtherThing = True + +process = cms.Process("TESTPROD") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents.input = int(sys.argv[2]) + +process.Thing = cms.EDProducer("ThingProducer") + +process.output = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string(sys.argv[1]) +) + +process.source = cms.Source("EmptySource", + firstRun = cms.untracked.uint32(int(sys.argv[3])), + numberEventsInRun = cms.untracked.uint32(int(sys.argv[4])), + firstLuminosityBlock = cms.untracked.uint32(int(sys.argv[5])), + numberEventsInLuminosityBlock = cms.untracked.uint32(int(sys.argv[6])) +) + +process.p = cms.Path(process.Thing) +if useOtherThing: + process.OtherThing = cms.EDProducer("OtherThingProducer") + process.p = cms.Path(process.Thing + process.OtherThing) +process.ep = cms.EndPath(process.output) + + diff --git a/FWIO/RNTupleTempInput/test/RunPerLumiTest_cfg.py b/FWIO/RNTupleTempInput/test/RunPerLumiTest_cfg.py new file mode 100644 index 0000000000000..cc7b2ed6defef --- /dev/null +++ b/FWIO/RNTupleTempInput/test/RunPerLumiTest_cfg.py @@ -0,0 +1,39 @@ +# The following comments couldn't be translated into the new config version: + +# Configuration file for PoolInputTest + +import FWCore.ParameterSet.Config as cms +from sys import argv + +process = cms.Process("TESTRECO") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents.input = int(argv[1]) + +runToLumi = [111,222,333,444,555] + +process.OtherThing = cms.EDProducer("OtherThingProducer") + +process.Analysis = cms.EDAnalyzer("OtherThingAnalyzer") + +process.source = cms.Source("RNTupleTempSource", + setRunNumberForEachLumi = cms.untracked.vuint32(*runToLumi), + fileNames = cms.untracked.vstring('file:RunPerLumiTest.root') +) + +process.output = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('OutputRunPerLumiTest.root') +) + +numberOfEventsInLumi = 5 +ids = cms.VEventID() +for l,r in enumerate(runToLumi): + for e in range(numberOfEventsInLumi): + ids.append(cms.EventID(r, l+1,l*5+e+1)) + +process.check = cms.EDAnalyzer("EventIDChecker", eventSequence = cms.untracked(ids)) + + +process.p = cms.Path(process.OtherThing*process.Analysis) + +process.e = cms.EndPath(process.check+process.output) diff --git a/FWIO/RNTupleTempInput/test/TestPoolInput.sh b/FWIO/RNTupleTempInput/test/TestPoolInput.sh new file mode 100755 index 0000000000000..4af94262f6b94 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/TestPoolInput.sh @@ -0,0 +1,55 @@ +#!/bin/sh -ex +# Pass in name and status +function die { echo $1: status $2 ; exit $2; } + +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} + +cmsRun -j PoolInputTest_jobreport.xml ${LOCAL_TEST_DIR}/PrePoolInputTest_cfg.py PoolInputTest.root 11 561 7 6 3 || die 'Failure using PrePoolInputTest_cfg.py' $? + +cmsRun -j PoolGuidTest_jobreport.xml ${LOCAL_TEST_DIR}/PoolGUIDTest_cfg.py file:PoolInputTest.root && die 'PoolGUIDTest_cfg.py PoolInputTest.root did not throw an exception' 1 +GUID_EXIT_CODE=$(edmFjrDump --exitCode PoolGuidTest_jobreport.xml) +if [ "x${GUID_EXIT_CODE}" != "x8034" ]; then + echo "Inconsistent GUID test reported exit code ${GUID_EXIT_CODE} which is different from the expected 8034" + exit 1 +fi +GUID_NAME=$(edmFjrDump --guid PoolInputTest_jobreport.xml).root +cp PoolInputTest.root ${GUID_NAME} +cmsRun ${LOCAL_TEST_DIR}/PoolGUIDTest_cfg.py file:${GUID_NAME} || die 'Failure using PoolGUIDTest_cfg.py ${GUID_NAME}' $? + + +cp PoolInputTest.root PoolInputOther.root + +cmsRun ${LOCAL_TEST_DIR}/PoolInputTest_cfg.py || die 'Failure using PoolInputTest_cfg.py' $? +cmsRun ${LOCAL_TEST_DIR}/PoolInputTest_noDelay_cfg.py >& PoolInputTest_noDelay_cfg.txt || die 'Failure using PoolInputTest_noDelay_cfg.py' $? +grep 'event delayed read from source' PoolInputTest_noDelay_cfg.txt && die 'Failure in PoolInputTest_noDelay_cfg.py, found delay reads from source' 1 +cmsRun ${LOCAL_TEST_DIR}/PoolInputTest_skip_with_failure_cfg.py || die 'Failure using PoolInputTest_skip_with_failure_cfg.py' $? +cmsRun ${LOCAL_TEST_DIR}/PoolInputTest_skipBadFiles_cfg.py || die 'Failure using PoolInputTest_skipBadFiles_cfg.py ' $? + +cmsRun ${LOCAL_TEST_DIR}/PrePool2FileInputTest_cfg.py || die 'Failure using PrePool2FileInputTest_cfg.py' $? +cmsRun ${LOCAL_TEST_DIR}/Pool2FileInputTest_cfg.py || die 'Failure using Pool2FileInputTest_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/PrePoolInputTest2_cfg.py || die 'Failure using PrePoolInputTest2_cfg.py' $? + +cp PoolInputTest.root PoolInputOther.root + +cmsRun ${LOCAL_TEST_DIR}/PoolInputTest2_cfg.py || die 'Failure using PoolInputTest2_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/PoolInputTest3_cfg.py || die 'Failure using PoolInputTest3_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/PoolEmptyTest_cfg.py || die 'Failure using PoolEmptyTest_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/PoolEmptyTest2_cfg.py || die 'Failure using PoolEmptyTest2_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/PoolAliasTestStep1_cfg.py || die 'Failure using PoolAliasTestStep1_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/PoolAliasTestStep2_cfg.py || die 'Failure using PoolAliasTestStep2_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/PoolAliasTestStep1_DifferentOrder_cfg.py || die 'Failure using PoolAliasTestStep1_DifferentOrder_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/PoolAliasTestStep2_DifferentOrder_cfg.py || die 'Failure using PoolAliasTestStep2_DifferentOrder_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/PoolAliasTestStep2A_cfg.py || die 'Failure using PoolAliasTestStep2A_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/PoolAliasTestStep1C_cfg.py || die 'Failure using PoolAliasTestStep2A_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/PoolAliasTestStep2C_cfg.py || die 'Failure using PoolAliasTestStep2A_cfg.py' $? diff --git a/FWIO/RNTupleTempInput/test/TestPoolInputRunPerLumi.sh b/FWIO/RNTupleTempInput/test/TestPoolInputRunPerLumi.sh new file mode 100755 index 0000000000000..66ac121e471f1 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/TestPoolInputRunPerLumi.sh @@ -0,0 +1,27 @@ +#!/bin/sh -ex +# Pass in name and status +function die { echo $1: status $2 ; exit $2; } + +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} + +cmsRun ${LOCAL_TEST_DIR}/PrePoolInputTest_cfg.py RunPerLumiTest.root 50 1 25 1 5 || die 'Failure using PrePoolInputTest_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/RunPerLumiTest_cfg.py 25 >& RunPerLumiTest.txt || die 'Failure using RunPerLumiTest_cfg.py' $? +grep 'record' RunPerLumiTest.txt | cut -d ' ' -f 4-11 > RunPerLumiTest.filtered.txt +diff ${LOCAL_TEST_DIR}/unit_test_outputs/RunPerLumiTest.filtered.txt RunPerLumiTest.filtered.txt || die 'incorrect output using RunPerLumiTest_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/RunPerLumiTest_cfg.py 50 >& tooManyLumis.txt && die 'RunPerLumiTest_cfg.py should have failed but did not' 1 +grep "MismatchedInputFiles" tooManyLumis.txt || die 'RunPerLumiTest_cfg.py should have failed but did not' $? + +cmsRun ${LOCAL_TEST_DIR}/firstLuminosityBlockForEachRunTest_cfg.py 'file:RunPerLumiTest.root' 25 1 25 1 5 || die 'Failure using firstLuminosityBlockForEachRunTest_cfg.py' $? +cmsRun ${LOCAL_TEST_DIR}/PrePoolInputTest_cfg.py firstLumiTest1.root 25 1 100 1 5 || die 'Failure using PrePoolInputTest_cfg.py' $? +cmsRun ${LOCAL_TEST_DIR}/PrePoolInputTest_cfg.py firstLumiTest2.root 25 1 100 6 5 || die 'Failure using PrePoolInputTest_cfg.py' $? +cmsRun ${LOCAL_TEST_DIR}/firstLuminosityBlockForEachRunTest_cfg.py 'file:firstLumiTest1.root,file:firstLumiTest2.root' 50 1 25 1 5 || die 'Failure using firstLuminosityBlockForEachRunTest_cfg.py with 2 files' $? +cmsRun ${LOCAL_TEST_DIR}/firstLuminosityBlockForEachRunTest_cfg.py 'file:firstLumiTest1.root,file:firstLumiTest2.root' 50 1 25 1 5 shareRun || die 'Failure using firstLuminosityBlockForEachRunTest_cfg.py with 2 files which share a run' $? +cmsRun ${LOCAL_TEST_DIR}/firstLuminosityBlockForEachRun_skipLumis_Test_cfg.py 'file:firstLumiTest1.root' 2 || die 'Failure using firstLuminosityBlockForEachRun_skipLumis_Test_cfg.py with first lumi 2' $? +cmsRun ${LOCAL_TEST_DIR}/firstLuminosityBlockForEachRun_skipLumis_Test_cfg.py 'file:firstLumiTest1.root' 4 || die 'Failure using firstLuminosityBlockForEachRun_skipLumis_Test_cfg.py with first lumi 4' $? +cmsRun ${LOCAL_TEST_DIR}/firstLuminosityBlockForEachRun_skipLumis_Test_cfg.py 'file:firstLumiTest1.root' 3 || die 'Failure using firstLuminosityBlockForEachRun_skipLumis_Test_cfg.py with first lumi 3' $? + +cmsRun ${LOCAL_TEST_DIR}/firstLuminosityBlockForEachRun_skipLumis2_Test_cfg.py 'file:firstLumiTest1.root' 2 || die 'Failure using firstLuminosityBlockForEachRun_skipLumis2_Test_cfg.py with first lumi 2' $? +cmsRun ${LOCAL_TEST_DIR}/firstLuminosityBlockForEachRun_skipLumis2_Test_cfg.py 'file:firstLumiTest1.root' 4 || die 'Failure using firstLuminosityBlockForEachRun_skipLumis2_Test_cfg.py with first lumi 4' $? +cmsRun ${LOCAL_TEST_DIR}/firstLuminosityBlockForEachRun_skipLumis2_Test_cfg.py 'file:firstLumiTest1.root' 3 || die 'Failure using firstLuminosityBlockForEachRun_skipLumis2_Test_cfg.py with first lumi 3' $? diff --git a/FWIO/RNTupleTempInput/test/firstLuminosityBlockForEachRunTest_cfg.py b/FWIO/RNTupleTempInput/test/firstLuminosityBlockForEachRunTest_cfg.py new file mode 100644 index 0000000000000..a167e8393c86b --- /dev/null +++ b/FWIO/RNTupleTempInput/test/firstLuminosityBlockForEachRunTest_cfg.py @@ -0,0 +1,65 @@ +# The following comments couldn't be translated into the new config version: + +# Configuration file for PoolInputTest + +import FWCore.ParameterSet.Config as cms +import sys + +process = cms.Process("TESTRECO") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(int(sys.argv[2])) +) + +firstRun = int(sys.argv[3]) +numberEventsInRun = int(sys.argv[4]) +firstLuminosityBlock = int(sys.argv[5]) +numberEventsInLuminosityBlock = int(sys.argv[6]) + +if len(sys.argv) > 7: + #have first run of second file continue the run + runToLumi = ((2,1),(10,3),(20,5), (30, 7), (40, 9) ) +else: + #have first run of second file be a new run + runToLumi = ((2,1),(10,3),(20,5), (30, 6), (40, 9) ) + +def findRunForLumi( lumi) : + lastRun = runToLumi[0][0] + for r,l in runToLumi: + if l > lumi: + break + lastRun = r + return lastRun + +ids = cms.VEventID() +numberOfEventsInLumi = 0 +numberOfEventsPerLumi = numberEventsInLuminosityBlock +lumi = 1 +event=0 +oldRun = 2 +numberOfFiles = len(sys.argv[1].split(",")) +numberOfEventsInFile = int(sys.argv[2])/numberOfFiles +for i in range(process.maxEvents.input.value()): + numberOfEventsInLumi +=1 + event += 1 + run = findRunForLumi(lumi) + if event > numberOfEventsInFile: + event = 1 + if numberOfEventsInLumi > numberOfEventsPerLumi: + numberOfEventsInLumi=1 + lumi += 1 + run = findRunForLumi(lumi) + if run != oldRun: + oldRun = run + ids.append(cms.EventID(run,lumi,event)) +process.check = cms.EDAnalyzer("EventIDChecker", eventSequence = cms.untracked(ids)) + + +process.source = cms.Source("RNTupleTempSource", + firstLuminosityBlockForEachRun = cms.untracked.VLuminosityBlockID(*[cms.LuminosityBlockID(x,y) for x,y in runToLumi]), + fileNames = cms.untracked.vstring(sys.argv[1].split(",")) +) + +process.e = cms.EndPath(process.check) + diff --git a/FWIO/RNTupleTempInput/test/firstLuminosityBlockForEachRun_skipLumis2_Test_cfg.py b/FWIO/RNTupleTempInput/test/firstLuminosityBlockForEachRun_skipLumis2_Test_cfg.py new file mode 100644 index 0000000000000..15b10b2de5ea1 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/firstLuminosityBlockForEachRun_skipLumis2_Test_cfg.py @@ -0,0 +1,54 @@ +# The following comments couldn't be translated into the new config version: + +# Configuration file for PoolInputTest + +import FWCore.ParameterSet.Config as cms +import sys + +process = cms.Process("TESTRECO") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents.input = -1 + +#have first run of second file be a new run +runToLumi = ((2,1),(10,3),(20,5)) + +def findRunForLumi( lumi) : + lastRun = runToLumi[0][0] + for r,l in runToLumi: + if l > lumi: + break + lastRun = r + return lastRun + +ids = cms.VEventID() +numberOfEventsInLumi = 0 +numberOfEventsPerLumi = 5 +lumi = int(sys.argv[2]) +event= numberOfEventsPerLumi*(lumi-1) +oldRun = 2 +numberOfFiles = 1 +numberOfEventsInFile = 15 +for i in range(numberOfEventsPerLumi*(6-lumi)): + numberOfEventsInLumi +=1 + event += 1 + run = findRunForLumi(lumi) +# if event > numberOfEventsInFile: +# event = 1 + if numberOfEventsInLumi > numberOfEventsPerLumi: + numberOfEventsInLumi=1 + lumi += 1 + run = findRunForLumi(lumi) + if run != oldRun: + oldRun = run + ids.append(cms.EventID(run,lumi,event)) +process.check = cms.EDAnalyzer("EventIDChecker", eventSequence = cms.untracked(ids)) + + +process.source = cms.Source("RNTupleTempSource", + lumisToProcess = cms.untracked.VLuminosityBlockRange('1:'+str(int(sys.argv[2]))+'-1:'+str(runToLumi[-1][1])), + firstLuminosityBlockForEachRun = cms.untracked.VLuminosityBlockID(*[cms.LuminosityBlockID(x,y) for x,y in runToLumi]), + fileNames = cms.untracked.vstring(sys.argv[1].split(",")) +) + +process.e = cms.EndPath(process.check) diff --git a/FWIO/RNTupleTempInput/test/firstLuminosityBlockForEachRun_skipLumis_Test_cfg.py b/FWIO/RNTupleTempInput/test/firstLuminosityBlockForEachRun_skipLumis_Test_cfg.py new file mode 100644 index 0000000000000..fade4b3bde54e --- /dev/null +++ b/FWIO/RNTupleTempInput/test/firstLuminosityBlockForEachRun_skipLumis_Test_cfg.py @@ -0,0 +1,55 @@ +# The following comments couldn't be translated into the new config version: + +# Configuration file for PoolInputTest + +import FWCore.ParameterSet.Config as cms +import sys + +process = cms.Process("TESTRECO") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents.input = -1 + +#have first run of second file be a new run +runToLumi = ((2,1),(10,3),(20,5)) + +def findRunForLumi( lumi) : + lastRun = runToLumi[0][0] + for r,l in runToLumi: + if l > lumi: + break + lastRun = r + return lastRun + +ids = cms.VEventID() +numberOfEventsInLumi = 0 +numberOfEventsPerLumi = 5 +lumi = int(sys.argv[2]) +event= numberOfEventsPerLumi*(lumi-1) +oldRun = 2 +numberOfFiles = 1 +numberOfEventsInFile = 15 +for i in range(numberOfEventsPerLumi*(6-lumi)): + numberOfEventsInLumi +=1 + event += 1 + run = findRunForLumi(lumi) +# if event > numberOfEventsInFile: +# event = 1 + if numberOfEventsInLumi > numberOfEventsPerLumi: + numberOfEventsInLumi=1 + lumi += 1 + run = findRunForLumi(lumi) + if run != oldRun: + oldRun = run + ids.append(cms.EventID(run,lumi,event)) +process.check = cms.EDAnalyzer("EventIDChecker", eventSequence = cms.untracked(ids)) + + +process.source = cms.Source("RNTupleTempSource", + firstLuminosityBlock = cms.untracked.uint32(int(sys.argv[2])), + firstLuminosityBlockForEachRun = cms.untracked.VLuminosityBlockID(*[cms.LuminosityBlockID(x,y) for x,y in runToLumi]), + fileNames = cms.untracked.vstring(sys.argv[1].split(",")) +) + +process.e = cms.EndPath(process.check) + diff --git a/FWIO/RNTupleTempInput/test/preMerge2_cfg.py b/FWIO/RNTupleTempInput/test/preMerge2_cfg.py new file mode 100644 index 0000000000000..2961206e73140 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/preMerge2_cfg.py @@ -0,0 +1,28 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTOUTPUT") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(1) +) +process.Thing = cms.EDProducer("ThingProducer") + +process.OtherThing = cms.EDProducer("OtherThingProducer", + useRefs = cms.untracked.bool(True) +) + +process.output = cms.OutputModule("RNTupleTempOutputModule", + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop edmtestThings_*__*' + ), + fileName = cms.untracked.string('file:big.root') +) + +process.source = cms.Source("EmptySource") + +process.p = cms.Path(process.Thing*process.OtherThing) +process.ep = cms.EndPath(process.output) + + diff --git a/FWIO/RNTupleTempInput/test/preMerge_cfg.py b/FWIO/RNTupleTempInput/test/preMerge_cfg.py new file mode 100644 index 0000000000000..eae25684e915d --- /dev/null +++ b/FWIO/RNTupleTempInput/test/preMerge_cfg.py @@ -0,0 +1,28 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTOUTPUT") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(1) +) +process.Thing = cms.EDProducer("ThingProducer") + +process.OtherThing = cms.EDProducer("OtherThingProducer", + useRefs = cms.untracked.bool(False) +) + +process.output = cms.OutputModule("RNTupleTempOutputModule", + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop edmtestThings_*__*' + ), + fileName = cms.untracked.string('file:small.root') +) + +process.source = cms.Source("EmptySource") + +process.p = cms.Path(process.Thing*process.OtherThing) +process.ep = cms.EndPath(process.output) + + diff --git a/FWIO/RNTupleTempInput/test/sitelocalconfig/noFallbackFile/local/storage.json b/FWIO/RNTupleTempInput/test/sitelocalconfig/noFallbackFile/local/storage.json new file mode 100644 index 0000000000000..dad76a157c5b6 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/sitelocalconfig/noFallbackFile/local/storage.json @@ -0,0 +1,10 @@ +[ + { "site": "DUMMY", + "volume": "DummyVolume", + "protocols": [ + { "protocol": "protocolThatDoesNotExist", + "prefix": "abc" + } + ] + } +] diff --git a/FWIO/RNTupleTempInput/test/sitelocalconfig/noFallbackFile/site-local-config.xml b/FWIO/RNTupleTempInput/test/sitelocalconfig/noFallbackFile/site-local-config.xml new file mode 100644 index 0000000000000..4cd0590b4c036 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/sitelocalconfig/noFallbackFile/site-local-config.xml @@ -0,0 +1,7 @@ + + + + + + + diff --git a/FWIO/RNTupleTempInput/test/sitelocalconfig/useFallbackFile/local/storage.json b/FWIO/RNTupleTempInput/test/sitelocalconfig/useFallbackFile/local/storage.json new file mode 100644 index 0000000000000..33a0c430c0751 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/sitelocalconfig/useFallbackFile/local/storage.json @@ -0,0 +1,18 @@ +[ + { "site": "DUMMY", + "volume": "DummyVolume1", + "protocols": [ + { "protocol": "protocolThatDoesNotExist1", + "prefix": "abc" + } + ] + }, + { "site": "DUMMY", + "volume": "DummyVolume2", + "protocols": [ + { "protocol": "protocolThatDoesNotExist2", + "prefix": "def" + } + ] + } +] diff --git a/FWIO/RNTupleTempInput/test/sitelocalconfig/useFallbackFile/site-local-config.xml b/FWIO/RNTupleTempInput/test/sitelocalconfig/useFallbackFile/site-local-config.xml new file mode 100644 index 0000000000000..538412e075877 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/sitelocalconfig/useFallbackFile/site-local-config.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/FWIO/RNTupleTempInput/test/testFileOpenErrorExitCode.sh b/FWIO/RNTupleTempInput/test/testFileOpenErrorExitCode.sh new file mode 100755 index 0000000000000..31e1c55a8823d --- /dev/null +++ b/FWIO/RNTupleTempInput/test/testFileOpenErrorExitCode.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Pass in name and status +function die { echo $1: status $2 ; exit $2; } + +mkdir -p SITECONF +mkdir -p SITECONF/local +mkdir -p SITECONF/local/JobConfig + +export SITECONFIG_PATH=${PWD}/SITECONF/local +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} + +cp ${LOCAL_TEST_DIR}/sitelocalconfig/noFallbackFile/site-local-config.xml ${SITECONFIG_PATH}/JobConfig/ +cp ${LOCAL_TEST_DIR}/sitelocalconfig/noFallbackFile/local/storage.json ${SITECONFIG_PATH}/ +F1=${LOCAL_TEST_DIR}/test_fileOpenErrorExitCode_cfg.py +cmsRun -j NoFallbackFile_jobreport.xml $F1 --input FileThatDoesNotExist.root && die "$F1 should have failed but didn't, exit code was 0" 1 + +CMSRUN_EXIT_CODE=$(edmFjrDump --exitCode NoFallbackFile_jobreport.xml) +echo "Exit code after first run of test_fileOpenErrorExitCode_cfg.py is ${CMSRUN_EXIT_CODE}" +if [ "x${CMSRUN_EXIT_CODE}" != "x8020" ]; then + echo "Unexpected cmsRun exit code after FileOpenError, exit code from jobReport ${CMSRUN_EXIT_CODE} which is different from the expected 8020" + exit 1 +fi + +cp ${LOCAL_TEST_DIR}/sitelocalconfig/useFallbackFile/site-local-config.xml ${SITECONFIG_PATH}/JobConfig/ +cp ${LOCAL_TEST_DIR}/sitelocalconfig/useFallbackFile/local/storage.json ${SITECONFIG_PATH}/ +cmsRun -j UseFallbackFile_jobreport.xml $F1 --input FileThatDoesNotExist.root > UseFallbackFile_output.log 2>&1 && die "$F1 should have failed after file fallback but didn't, exit code was 0" 1 +grep -q "Input file abc/store/FileThatDoesNotExist.root could not be opened, and fallback was attempted" UseFallbackFile_output.log +RET=$? +if [ "${RET}" != "0" ]; then + cat UseFallbackFile_output.log + die "UseFallbackFile_output.log did not contain the detailed error message of the first file open failure " $RET +fi + +CMSRUN_EXIT_CODE=$(edmFjrDump --exitCode UseFallbackFile_jobreport.xml) +echo "Exit code after second run of test_fileOpenErrorExitCode_cfg.py is ${CMSRUN_EXIT_CODE}" +if [ "x${CMSRUN_EXIT_CODE}" != "x8028" ]; then + echo "Unexpected cmsRun exit code after FallbackFileOpenError, exit code from jobReport ${CMSRUN_EXIT_CODE} which is different from the expected 8028" + exit 1 +fi diff --git a/FWIO/RNTupleTempInput/test/testForStreamerInfo.C b/FWIO/RNTupleTempInput/test/testForStreamerInfo.C new file mode 100644 index 0000000000000..caa4474333012 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/testForStreamerInfo.C @@ -0,0 +1,111 @@ +// Use something like the following to run this file: +// +// root.exe -b -l -q SchemaEvolutionTest.root 'IOPool/Input/test/testForStreamerInfo.C(gFile)' | sort -u +// +// In the output, lines with "Missing" indicate problems. + +// Note the script ignores classes whose name start with "T" +// to ignore ROOT classes. In the context this is used, none +// of the interesting classes start with "T" (although if used +// generally use I can imagine this could cause some confusion...). + +// This is a modified version of some temporary code Philippe Canal +// provided us while debugging a problem. + +#include "TFile.h" +#include "TStreamerInfo.h" +#include "TList.h" +#include "TVirtualCollectionProxy.h" +#include "TStreamerElement.h" +#include + +void check(TClass &cl, TList &streamerInfoList) { + std::string name(cl.GetName()); + if (name == "string") + return; + if (0 == name.compare(0, strlen("pair<"), "pair<")) + return; + std::cout << "check TClass: " << name << std::endl; + bool found = streamerInfoList.FindObject(cl.GetName()) != 0; + if (!found) + std::cout << "Missing: " << cl.GetName() << '\n'; +} + +void check(TVirtualCollectionProxy &proxy, TList &streamerInfoList) { + auto inner = proxy.GetValueClass(); + if (inner) { + auto subproxy = inner->GetCollectionProxy(); + if (subproxy) { + check(*subproxy, streamerInfoList); + } else { + check(*inner, streamerInfoList); + } + } +} + +void check(TStreamerElement &element, TList &streamerInfoList) { + auto cl = element.GetClass(); + if (cl == nullptr) { + return; + } + // Ignore all classes that start with a T with the intent + // to ignore all internal ROOT classes + // (In general this might ignore other interesting classes, + // but this is intended to be used in a specific test where the + // the interesting classes don't start with T). + if (*(cl->GetName()) == 'T') { + return; + } + if (cl->GetCollectionProxy()) { + check(*cl->GetCollectionProxy(), streamerInfoList); + } else { + check(*cl, streamerInfoList); + } +} + +// This is called once for each TStreamerInfo in +// streamerInfoList. The info is the first argument +// and the list of all the infos is the second argument. +void scan(TStreamerInfo &info, TList &streamerInfoList) { + auto cl = TClass::GetClass(info.GetName()); + // print error message and do skip the info if there + // is not a TClass available. + if (!cl) { + //std::cerr << "Error no TClass for " << info.GetName() << '\n'; + return; + } + auto proxy = cl->GetCollectionProxy(); + if (proxy) + check(*proxy, streamerInfoList); + for (auto e : TRangeDynCast(*info.GetElements())) { + if (!e) + continue; + check(*e, streamerInfoList); + } +} + +void scan(TList *streamerInfoList) { + if (!streamerInfoList) + return; + for (auto l : TRangeDynCast(*streamerInfoList)) { + if (!l) + continue; + // Ignore all classes that start with a T with the intent + // to ignore all internal ROOT classes + // (In general this might ignore other interesting classes, + // but this is intended to be used in a specific test where the + // the interesting classes don't start with T). + if (*(l->GetName()) == 'T') { + continue; + } + //std::cout << "Seeing: " << l->GetName() << " " << l->GetClassVersion() << '\n'; + scan(*l, *streamerInfoList); + } + delete streamerInfoList; +} + +void testForStreamerInfo(TFile *file) { + if (!file) + return; + scan(file->GetStreamerInfoList()); +} diff --git a/FWIO/RNTupleTempInput/test/testNoParentDictionary.sh b/FWIO/RNTupleTempInput/test/testNoParentDictionary.sh new file mode 100755 index 0000000000000..1650603ef7715 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/testNoParentDictionary.sh @@ -0,0 +1,78 @@ +#!/bin/bash -e + +function die { echo $1: status $2 ; exit $2; } + +SCRAM_TEST_NAME=TestIOPoolInputNoParentDictionary +rm -rf $SCRAM_TEST_NAME +mkdir $SCRAM_TEST_NAME +cd $SCRAM_TEST_NAME + +# Create a new CMSSW dev area and build modified DataFormats/TestObjects in it +NEW_CMSSW_BASE=$(/bin/pwd -P)/$CMSSW_VERSION +scram -a $SCRAM_ARCH project $CMSSW_VERSION +pushd $CMSSW_VERSION/src +mkdir DataFormats + +# Copy DataFormats/Common if it was modified locally +if [ -d ${CMSSW_BASE}/src/DataFormats/Common ]; then + cp -Lr ${CMSSW_BASE}/src/DataFormats/Common DataFormats/ +fi +if [ -d ${CMSSW_BASE}/src/DataFormats/Provenance ]; then + cp -Lr ${CMSSW_BASE}/src/DataFormats/Provenance DataFormats/ +fi +#DataFormats/Common and DataFormat/Provenance depend on Utilities +if [ -d ${CMSSW_BASE}/src/FWCore/Utilities ]; then + mkdir -p FWCore + cp -Lr ${CMSSW_BASE}/src/FWCore/Utilities FWCore/ +fi +#DataFormats/Common depends on MessageLogger +if [ -d ${CMSSW_BASE}/src/FWCore/MessageLogger ]; then + mkdir -p FWCore + cp -Lr ${CMSSW_BASE}/src/FWCore/MessageLogger FWCore/ +fi +#DataFormats/Provenance depends on Reflection +if [ -d ${CMSSW_BASE}/src/FWCore/Reflection ]; then + mkdir -p FWCore + cp -Lr ${CMSSW_BASE}/src/FWCore/Reflection FWCore/ +fi + +#DataFormats/TestObjects depends on FWCore/SOA +if [ -d ${CMSSW_BASE}/src/FWCore/SOA ]; then + mkdir -p FWCore + cp -Lr ${CMSSW_BASE}/src/FWCore/SOA FWCore/ +fi + +# Copy DataFormats/TestObjects code to be able to edit it to make ROOT header parsing to fail +for DIR in ${CMSSW_BASE} ${CMSSW_RELEASE_BASE} ${CMSSW_FULL_RELEASE_BASE} ; do + if [ -d ${DIR}/src/DataFormats/TestObjects ]; then + cp -Lr ${DIR}/src/DataFormats/TestObjects DataFormats/ + break + fi +done +if [ ! -e DataFormats/TestObjects ]; then + echo "Failed to copy DataFormats/TestObjects from local or release area" + exit 1; +fi + +# Enable TransientIntParentT dictionaries +cat DataFormats/TestObjects/test/BuildFile_extra.xml >> DataFormats/TestObjects/test/BuildFile.xml +#Set env and build in sub-shel +(eval $(scram run -sh) ; scram build -j $(nproc)) + +popd + +# Prepend NEW_CMSSW_BASE's lib/src paths in to LD_LIBRARY_PATH and ROOT_INCLUDE_PATH +export LD_LIBRARY_PATH=${NEW_CMSSW_BASE}/lib/${SCRAM_ARCH}:${LD_LIBRARY_PATH} +export ROOT_INCLUDE_PATH=${NEW_CMSSW_BASE}/src:${ROOT_INCLUDE_PATH} + +echo "Produce a file with TransientIntParentT<1> product" +cmsRun ${LOCALTOP}/src/IOPool/Input/test/PoolNoParentDictionaryTestStep1_cfg.py || die 'Failed cmsRun PoolNoParentDictionaryTestStep1_cfg.py' $? + + +# Then make attempt to load TransientIntParentT<1> to fail +echo "PREVENT_HEADER_PARSING" >> ${NEW_CMSSW_BASE}/src/DataFormats/TestObjects/interface/ToyProducts.h +rm ${NEW_CMSSW_BASE}/lib/${SCRAM_ARCH}/DataFormatsTestObjectsParent1_xr_rdict.pcm ${NEW_CMSSW_BASE}/lib/${SCRAM_ARCH}/libDataFormatsTestObjectsParent1.so +sed -i -e 's/libDataFormatsTestObjectsParent1.so/libDataFormatsTestObjectsParent2.so/' ${NEW_CMSSW_BASE}/lib/${SCRAM_ARCH}/DataFormatsTestObjectsParent1_xr.rootmap + +echo "Read the file without TransientIntParentT<1> dictionary" +cmsRun ${LOCALTOP}/src/IOPool/Input/test/PoolNoParentDictionaryTestStep2_cfg.py || die 'Failed cmsRun PoolNoParentDictionaryTestStep2_cfg.py' $? diff --git a/FWIO/RNTupleTempInput/test/testReducedProcessHistoryCreate_cfg.py b/FWIO/RNTupleTempInput/test/testReducedProcessHistoryCreate_cfg.py new file mode 100644 index 0000000000000..10b535788873b --- /dev/null +++ b/FWIO/RNTupleTempInput/test/testReducedProcessHistoryCreate_cfg.py @@ -0,0 +1,54 @@ +import FWCore.ParameterSet.Config as cms +import argparse + +parser = argparse.ArgumentParser(description='Create files for reduced ProcessHistory test') +parser.add_argument("--version", type=str, help="CMSSW version to be used in the ProcessHistory (default is unset") +parser.add_argument("--accelerators", type=str, nargs='+', help="Propagated to process.options.accelerators (default is unset)") +parser.add_argument("--firstEvent", default=1, type=int, help="Number of first event") +parser.add_argument("--lumi", default=1, type=int, help="LuminosityBlock number") +parser.add_argument("--output", type=str, help="Output file name") + +args = parser.parse_args() + +class ProcessAcceleratorTest(cms.ProcessAccelerator): + def __init__(self): + super(ProcessAcceleratorTest,self).__init__() + self._labels = ["test-one", "test-two"] + def labels(self): + return self._labels + def enabledLabels(self): + return self._labels + + +process = cms.Process("PROD") +if args.version: + process._specialOverrideReleaseVersionOnlyForTesting(args.version) +if args.accelerators: + process.add_(ProcessAcceleratorTest()) + process.options.accelerators = args.accelerators + +process.maxEvents.input = 10 + +from FWCore.Modules.modules import EmptySource +process.source = EmptySource( + firstEvent = args.firstEvent, + firstLuminosityBlock = args.lumi, +) + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.out = RNTupleTempOutputModule( + fileName = args.output +) + +from FWCore.Framework.modules import IntProducer +process.intProducer = IntProducer(ivalue = 42) + +from FWCore.Integration.modules import ThingWithMergeProducer +process.thingWithMergeProducer = ThingWithMergeProducer() + +process.t = cms.Task( + process.intProducer, + process.thingWithMergeProducer, +) +process.p = cms.Path(process.t) +process.ep = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempInput/test/testReducedProcessHistoryHardwareResources.sh b/FWIO/RNTupleTempInput/test/testReducedProcessHistoryHardwareResources.sh new file mode 100755 index 0000000000000..3a750962c2fda --- /dev/null +++ b/FWIO/RNTupleTempInput/test/testReducedProcessHistoryHardwareResources.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +function die { echo $1: status $2 ; exit $2; } +function runSuccess { + echo "cmsRun $@" + cmsRun $@ || die "cmsRun $*" $? + echo +} + + +# Check that changing hardware resources does not lead to new lumi or run +runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --accelerators test-one --firstEvent 1 --output test-one.root +runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --accelerators test-two --firstEvent 101 --output test-two.root + +#edmProvDump test-one.root | grep -q "accelerators: test-one" || die "Did not find test-one from test-one.root provenance" $? +#edmProvDump test-two.root | grep -q "accelerators: test-two" || die "Did not find test-two from test-two.root provenance" $? + +runSuccess ${SCRAM_TEST_PATH}/test_merge_two_files.py test-one.root test-two.root + +runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_files.root + +#can't handle RNTUple +#edmProvDump merged_files.root | grep -q "accelerators: test-one" || die "Did not find test-one from merged_files.root provenance" $? +#edmProvDump merged_files.root | grep -q "accelerators: test-two" || die "Did not find test-two from merged_files.root provenance" $? + + +exit 0 diff --git a/FWIO/RNTupleTempInput/test/testReducedProcessHistoryVersion.sh b/FWIO/RNTupleTempInput/test/testReducedProcessHistoryVersion.sh new file mode 100755 index 0000000000000..2b2ced3dcd98e --- /dev/null +++ b/FWIO/RNTupleTempInput/test/testReducedProcessHistoryVersion.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +function die { echo $1: status $2 ; exit $2; } +function runSuccess { + echo "cmsRun $@" + cmsRun $@ || die "cmsRun $*" $? + echo +} +function runFailure { + echo "cmsRun $@ (expected to fail)" + cmsRun $@ && die "cmsRun $*" 1 + echo +} + +VERSION_ARR=(${CMSSW_VERSION//_/ }) +VERSION1="${VERSION_ARR[0]}_${VERSION_ARR[1]}_${VERSION_ARR[2]}_0" +VERSION2="${VERSION_ARR[0]}_${VERSION_ARR[1]}_${VERSION_ARR[2]}_1" +VERSION3="${VERSION_ARR[0]}_${VERSION_ARR[1]}_$((${VERSION_ARR[2]}+1))_0" + +# Check that changing the patch version does not lead to new lumi or run +runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --version ${VERSION1} --firstEvent 1 --output version1.root +runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --version ${VERSION2} --firstEvent 101 --output version2.root + +#edmProvDump version1.root | grep -q "version: '${VERSION1}'" || die "Did not find ${VERSION1} from version.root provenance" $? +#edmProvDump version2.root | grep -q "version: '${VERSION2}'" || die "Did not find ${VERSION2} from version.root provenance" $? + +runSuccess ${SCRAM_TEST_PATH}/test_merge_two_files.py version1.root version2.root + +runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_files.root + + +# Check that changing the minor version leads to new lumi +runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --version ${VERSION3} --firstEvent 201 --output version3_lumi.root + +#edmProvDump version3_lumi.root | grep -q "version: '${VERSION3}'" || die "Did not find ${VERSION3} from version3_lumi.root provenance" $? + +runSuccess ${SCRAM_TEST_PATH}/test_merge_two_files.py version1.root version3_lumi.root --output merged_files3_lumi.root --bypassVersionCheck + +runFailure ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_files3_lumi.root --bypassVersionCheck + +runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_files3_lumi.root --bypassVersionCheck --expectNewLumi + + +# Check that changing the minor version leads to new run +runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --version ${VERSION3} --firstEvent 201 --lumi 2 --output version3_run.root + +#edmProvDump version3_run.root | grep -q "version: '${VERSION3}'" || die "Did not find ${VERSION3} from version3_lumi.root provenance" $? + +runSuccess ${SCRAM_TEST_PATH}/test_merge_two_files.py version1.root version3_run.root --output merged_files3_run.root --bypassVersionCheck + +runFailure ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_files3_run.root --bypassVersionCheck + +runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_files3_run.root --bypassVersionCheck --expectNewRun + +exit 0 diff --git a/FWIO/RNTupleTempInput/test/testReducedProcessHistory_cfg.py b/FWIO/RNTupleTempInput/test/testReducedProcessHistory_cfg.py new file mode 100644 index 0000000000000..d5126e9dea186 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/testReducedProcessHistory_cfg.py @@ -0,0 +1,117 @@ +import FWCore.ParameterSet.Config as cms +import argparse + +parser = argparse.ArgumentParser(description='Test reduced ProcessHistory') +parser.add_argument("--input", type=str, help="Input file") +parser.add_argument("--bypassVersionCheck", action="store_true", help="Bypass version check") +parser.add_argument("--expectNewLumi", action="store_true", help="Set this if a new lumi is expected between the original files") +parser.add_argument("--expectNewRun", action="store_true", help="Set this if a new run is expected between the original files") + +args = parser.parse_args() + +process = cms.Process("READ") + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = [f"file:{args.input}"], + bypassVersionCheck = args.bypassVersionCheck, +) + +from FWCore.Framework.modules import TestMergeResults, RunLumiEventAnalyzer +process.testmerge = TestMergeResults( + expectedBeginRunProd = [10001, 20004, 10003], + expectedEndRunProd = [100001, 200004, 100003], + expectedBeginLumiProd = [101, 204, 103], + expectedEndLumiProd = [1001, 2004, 1003], + + expectedBeginRunNew = [10001, 10002, 10003], + expectedEndRunNew = [100001, 100002, 100003], + expectedBeginLumiNew = [101, 102, 103], + expectedEndLumiNew = [1001, 1002, 1003], + expectedProcessHistoryInRuns = [ + 'PROD', + 'MERGETWOFILES', + 'READ' + ] +) +def setWithMergeAndCopyEntry(p, value): + p[1] = value + p[3:5] = p[0:2] +if args.expectNewLumi or args.expectNewRun: + setWithMergeAndCopyEntry(process.testmerge.expectedBeginRunProd, 10002) + setWithMergeAndCopyEntry(process.testmerge.expectedEndRunProd, 100002) + setWithMergeAndCopyEntry(process.testmerge.expectedBeginLumiProd, 102) + setWithMergeAndCopyEntry(process.testmerge.expectedEndLumiProd, 1002) + +process.test = RunLumiEventAnalyzer( + expectedRunLumiEvents = [ + 1, 0, 0, # beginRun + 1, 1, 0, # beginLumi + 1, 1, 1, + 1, 1, 2, + 1, 1, 3, + 1, 1, 4, + 1, 1, 5, + 1, 1, 6, + 1, 1, 7, + 1, 1, 8, + 1, 1, 9, + 1, 1, 10, + 1, 1, 101, + 1, 1, 102, + 1, 1, 103, + 1, 1, 104, + 1, 1, 105, + 1, 1, 106, + 1, 1, 107, + 1, 1, 108, + 1, 1, 109, + 1, 1, 110, + 1, 1, 0, # endLumi + 1, 0, 0, # endRun + ] +) +endFirstFileIndex = 3*(10+2) +if args.expectNewLumi: + process.test.expectedRunLumiEvents = process.test.expectedRunLumiEvents[:endFirstFileIndex] + [ + 1, 1, 0, # endLumi + 1, 0, 0, # endRun + 1, 0, 0, # beginRun + 1, 1, 0, # beginLumi + 1, 1, 201, + 1, 1, 202, + 1, 1, 203, + 1, 1, 204, + 1, 1, 205, + 1, 1, 206, + 1, 1, 207, + 1, 1, 208, + 1, 1, 209, + 1, 1, 210, + 1, 1, 0, # endLumi + 1, 0, 0, # endRun + ] +elif args.expectNewRun: + process.test.expectedRunLumiEvents = process.test.expectedRunLumiEvents[:endFirstFileIndex] + [ + 1, 1, 0, # endLumi + 1, 0, 0, # endRun + 1, 0, 0, # beginRun + 1, 2, 0, # beginLumi + 1, 2, 201, + 1, 2, 202, + 1, 2, 203, + 1, 2, 204, + 1, 2, 205, + 1, 2, 206, + 1, 2, 207, + 1, 2, 208, + 1, 2, 209, + 1, 2, 210, + 1, 2, 0, # endLumi + 1, 0, 0, # endRun + ] + +process.p = cms.Path( + process.testmerge + + process.test +) diff --git a/FWIO/RNTupleTempInput/test/testRefProductIDMetadataConsistencyRoot.sh b/FWIO/RNTupleTempInput/test/testRefProductIDMetadataConsistencyRoot.sh new file mode 100755 index 0000000000000..230a854dc9c0c --- /dev/null +++ b/FWIO/RNTupleTempInput/test/testRefProductIDMetadataConsistencyRoot.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +function die { echo Failure $1: status $2 ; exit $2 ; } +function runSuccess { + echo "cmsRun $@" + cmsRun $@ || die "cmsRun $*" $? + echo +} + +runSuccess ${SCRAM_TEST_PATH}/testRefProductIDMetadataConsistencyRoot_cfg.py +runSuccess ${SCRAM_TEST_PATH}/testRefProductIDMetadataConsistencyRoot_cfg.py --enableOther +runSuccess ${SCRAM_TEST_PATH}/testRefProductIDMetadataConsistencyRootMerge_cfg.py +runSuccess ${SCRAM_TEST_PATH}/testRefProductIDMetadataConsistencyRootTest_cfg.py diff --git a/FWIO/RNTupleTempInput/test/testRefProductIDMetadataConsistencyRootMerge_cfg.py b/FWIO/RNTupleTempInput/test/testRefProductIDMetadataConsistencyRootMerge_cfg.py new file mode 100644 index 0000000000000..93859030234bf --- /dev/null +++ b/FWIO/RNTupleTempInput/test/testRefProductIDMetadataConsistencyRootMerge_cfg.py @@ -0,0 +1,19 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring("file:refconsistency_1.root", + "file:refconsistency_10.root") +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string("refconsistency_merge.root") +) + +process.tester = cms.EDAnalyzer("OtherThingAnalyzer", + other = cms.untracked.InputTag("otherThing","testUserTag") +) + +process.o = cms.EndPath(process.out+process.tester) + diff --git a/FWIO/RNTupleTempInput/test/testRefProductIDMetadataConsistencyRootTest_cfg.py b/FWIO/RNTupleTempInput/test/testRefProductIDMetadataConsistencyRootTest_cfg.py new file mode 100644 index 0000000000000..ec00e05b44cc5 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/testRefProductIDMetadataConsistencyRootTest_cfg.py @@ -0,0 +1,12 @@ +import FWCore.ParameterSet.Config as cms +process = cms.Process("TEST") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring("file:refconsistency_merge.root") +) + +process.tester = cms.EDAnalyzer("OtherThingAnalyzer", + other = cms.untracked.InputTag("otherThing","testUserTag") +) + +process.e = cms.EndPath(process.tester) diff --git a/FWIO/RNTupleTempInput/test/testRefProductIDMetadataConsistencyRoot_cfg.py b/FWIO/RNTupleTempInput/test/testRefProductIDMetadataConsistencyRoot_cfg.py new file mode 100644 index 0000000000000..716b0fc5fe424 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/testRefProductIDMetadataConsistencyRoot_cfg.py @@ -0,0 +1,73 @@ +import FWCore.ParameterSet.Config as cms + +import argparse +import sys + +parser = argparse.ArgumentParser(prog=sys.argv[0], description='Test ModuleTypeResolver and Ref') +parser.add_argument("--enableOther", action="store_true", help="Enable other accelerator. Also sets Lumi to 2") +args = parser.parse_args() + +class ModuleTypeResolverTest: + def __init__(self, accelerators): + self._variants = [] + if "other" in accelerators: + self._variants.append("other") + if "cpu" in accelerators: + self._variants.append("cpu") + if len(self._variants) == 0: + raise cms.EDMException(cms.edm.errors.UnavailableAccelerator, "No 'cpu' or 'other' accelerator available") + + def plugin(self): + return "edm::test::ConfigurableTestTypeResolverMaker" + + def setModuleVariant(self, module): + if module.type_().startswith("generic::"): + if hasattr(module, "variant"): + if module.variant.value() not in self._variants: + raise cms.EDMException(cms.edm.errors.UnavailableAccelerator, "Module {} has the Test variant set explicitly to {}, but its accelerator is not available for the job".format(module.label_(), module.variant.value())) + else: + module.variant = cms.untracked.string(self._variants[0]) + +class ProcessAcceleratorTest(cms.ProcessAccelerator): + def __init__(self): + super(ProcessAcceleratorTest,self).__init__() + self._labels = ["other"] + self._enabled = [] + if args.enableOther: + self._enabled.append("other") + def labels(self): + return self._labels + def enabledLabels(self): + return self._enabled + def moduleTypeResolver(self, accelerators): + return ModuleTypeResolverTest(accelerators) + +process = cms.Process("PROD1") +process.add_(ProcessAcceleratorTest()) + +process.source = cms.Source("EmptySource", + firstEvent = cms.untracked.uint32(10 if args.enableOther else 1), +) +process.maxEvents.input = 3 + +process.intprod = cms.EDProducer("generic::IntTransformer", valueCpu = cms.int32(1), valueOther = cms.int32(2)) + +process.thing = cms.EDProducer("ThingProducer") + +process.otherThing = cms.EDProducer("OtherThingProducer", + thingTag=cms.InputTag("thing") +) + +process.t = cms.Task( + process.intprod, + process.thing, + process.otherThing +) +process.p = cms.Path(process.t) + +fname = "refconsistency_{}".format(process.source.firstEvent.value()) +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string(fname+".root") +) + +process.ep = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempInput/test/test_fileOpenErrorExitCode_cfg.py b/FWIO/RNTupleTempInput/test/test_fileOpenErrorExitCode_cfg.py new file mode 100644 index 0000000000000..7f88a6d7fb53e --- /dev/null +++ b/FWIO/RNTupleTempInput/test/test_fileOpenErrorExitCode_cfg.py @@ -0,0 +1,13 @@ +import FWCore.ParameterSet.Config as cms +import argparse +import sys + +parser = argparse.ArgumentParser(prog=sys.argv[0], description="Test FileOpenErrorExitCode") +parser.add_argument("--input", type=str, default=[], nargs="*", help="Optional list of input files") +args = parser.parse_args() + +process = cms.Process("TEST") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring("/store/"+x for x in args.input) +) diff --git a/FWIO/RNTupleTempInput/test/test_make_multi_lumi_cfg.py b/FWIO/RNTupleTempInput/test/test_make_multi_lumi_cfg.py new file mode 100644 index 0000000000000..8e848b340b022 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/test_make_multi_lumi_cfg.py @@ -0,0 +1,11 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("WRITE") + +process.source = cms.Source("EmptySource", numberEventsInLuminosityBlock = cms.untracked.uint32(4)) + +process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32(20)) + +process.out = cms.OutputModule("RNTupleTempOutputModule", fileName = cms.untracked.string("multi_lumi.root")) + +process.o = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempInput/test/test_make_overlapping_lumis_cfg.py b/FWIO/RNTupleTempInput/test/test_make_overlapping_lumis_cfg.py new file mode 100644 index 0000000000000..2fb930538546f --- /dev/null +++ b/FWIO/RNTupleTempInput/test/test_make_overlapping_lumis_cfg.py @@ -0,0 +1,28 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("WRITE") + +process.a = cms.EDProducer("IntProducer", + ivalue = cms.int32(10)) + +process.b = cms.EDProducer("BusyWaitIntProducer",ivalue = cms.int32(1), iterations = cms.uint32(100*1000)) + +process.f = cms.EDFilter("ModuloEventIDFilter", modulo = cms.uint32(3), offset = cms.uint32(2) ) + +process.p = cms.Path(process.f+process.b, cms.Task(process.a) ) + +process.out = cms.OutputModule("RNTupleTempOutputModule", fileName = cms.untracked.string("overlap.root"), + outputCommands = cms.untracked.vstring("drop *", "keep *_b_*_*","keep *_a_*_*") ) + +process.prnt = cms.OutputModule("AsciiOutputModule", + outputCommands = cms.untracked.vstring("drop *", "keep *_b_*_*","keep *_a_*_*") ) + +process.source = cms.Source("EmptySource", numberEventsInLuminosityBlock = cms.untracked.uint32(3)) +process.maxEvents = cms.untracked.PSet(input = cms.untracked.int32(10)) + +process.o = cms.EndPath(process.out) +process.o2 = cms.EndPath(process.prnt) + +process.options = cms.untracked.PSet( numberOfThreads = cms.untracked.uint32(3), + numberOfStreams = cms.untracked.uint32(0), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(2)) diff --git a/FWIO/RNTupleTempInput/test/test_merge_two_files.py b/FWIO/RNTupleTempInput/test/test_merge_two_files.py new file mode 100644 index 0000000000000..7155a5afc2952 --- /dev/null +++ b/FWIO/RNTupleTempInput/test/test_merge_two_files.py @@ -0,0 +1,29 @@ +import FWCore.ParameterSet.Config as cms +import argparse + +parser = argparse.ArgumentParser(description='Create files for reduced ProcessHistory test') +parser.add_argument("file1", type=str, help="First file to merge") +parser.add_argument("file2", type=str, help="Second file to merge") +parser.add_argument("--output", default="merged_files.root", help="Output file name") +parser.add_argument("--bypassVersionCheck", action="store_true", help="Bypass version check") + +args = parser.parse_args() + +process = cms.Process("MERGETWOFILES") + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = ["file:"+args.file1,"file:"+args.file2], + duplicateCheckMode = "noDuplicateCheck", + bypassVersionCheck = args.bypassVersionCheck, +) + +from FWCore.Integration.modules import ThingWithMergeProducer +process.thingWithMergeProducer = ThingWithMergeProducer() + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.out = RNTupleTempOutputModule(fileName = args.output) + +process.p = cms.Path(process.thingWithMergeProducer) + +process.t = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempInput/test/test_read_overlapping_lumis_cfg.py b/FWIO/RNTupleTempInput/test/test_read_overlapping_lumis_cfg.py new file mode 100644 index 0000000000000..9686375a797bf --- /dev/null +++ b/FWIO/RNTupleTempInput/test/test_read_overlapping_lumis_cfg.py @@ -0,0 +1,31 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READ") + +process.source = cms.Source("RNTupleTempSource", fileNames = cms.untracked.vstring("file:overlap.root")) +process.tst = cms.EDAnalyzer("RunLumiEventChecker", + eventSequence = cms.untracked.VEventID( + cms.EventID(1,0,0), + cms.EventID(1,1,0), + cms.EventID(1,1,1), + cms.EventID(1,1,2), + cms.EventID(1,1,3), + cms.EventID(1,1,0), + cms.EventID(1,2,0), + cms.EventID(1,2,4), + cms.EventID(1,2,5), + cms.EventID(1,2,6), + cms.EventID(1,2,0), + cms.EventID(1,3,0), + cms.EventID(1,3,7), + cms.EventID(1,3,8), + cms.EventID(1,3,9), + cms.EventID(1,3,0), + cms.EventID(1,4,0), + cms.EventID(1,4,10), + cms.EventID(1,4,0), + cms.EventID(1,0,0) + ), + unorderedEvents = cms.untracked.bool(True)) + +process.out = cms.EndPath(process.tst) diff --git a/FWIO/RNTupleTempInput/test/unit_test_outputs/RunPerLumiTest.filtered.txt b/FWIO/RNTupleTempInput/test/unit_test_outputs/RunPerLumiTest.filtered.txt new file mode 100644 index 0000000000000..01b2bdf46aeed --- /dev/null +++ b/FWIO/RNTupleTempInput/test/unit_test_outputs/RunPerLumiTest.filtered.txt @@ -0,0 +1,25 @@ +1st record. Run 111, Event 1, LumiSection 1 +2nd record. Run 111, Event 2, LumiSection 1 +3rd record. Run 111, Event 3, LumiSection 1 +4th record. Run 111, Event 4, LumiSection 1 +5th record. Run 111, Event 5, LumiSection 1 +6th record. Run 222, Event 6, LumiSection 2 +7th record. Run 222, Event 7, LumiSection 2 +8th record. Run 222, Event 8, LumiSection 2 +9th record. Run 222, Event 9, LumiSection 2 +10th record. Run 222, Event 10, LumiSection 2 +11th record. Run 333, Event 11, LumiSection 3 +12th record. Run 333, Event 12, LumiSection 3 +13th record. Run 333, Event 13, LumiSection 3 +14th record. Run 333, Event 14, LumiSection 3 +15th record. Run 333, Event 15, LumiSection 3 +16th record. Run 444, Event 16, LumiSection 4 +17th record. Run 444, Event 17, LumiSection 4 +18th record. Run 444, Event 18, LumiSection 4 +19th record. Run 444, Event 19, LumiSection 4 +20th record. Run 444, Event 20, LumiSection 4 +21st record. Run 555, Event 21, LumiSection 5 +22nd record. Run 555, Event 22, LumiSection 5 +23rd record. Run 555, Event 23, LumiSection 5 +24th record. Run 555, Event 24, LumiSection 5 +25th record. Run 555, Event 25, LumiSection 5 diff --git a/FWIO/RNTupleTempOutput/BuildFile.xml b/FWIO/RNTupleTempOutput/BuildFile.xml new file mode 100644 index 0000000000000..7bd594cf5f0d6 --- /dev/null +++ b/FWIO/RNTupleTempOutput/BuildFile.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h b/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h new file mode 100644 index 0000000000000..10d19608a766f --- /dev/null +++ b/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h @@ -0,0 +1,243 @@ +#ifndef FWIO_RNTupleTempOutput_RNTupleTempOutputModule_h +#define FWIO_RNTupleTempOutput_RNTupleTempOutputModule_h + +////////////////////////////////////////////////////////////////////// +// +// Class RNTupleTempOutputModule. Output module to POOL file +// +// Oringinal Author: Luca Lista +// Current Author: Bill Tanenbaum +// +////////////////////////////////////////////////////////////////////// + +#include +#include +#include +#include +#include +#include +#include + +#include "IOPool/Common/interface/RootServiceChecker.h" +#include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/Framework/interface/one/OutputModule.h" +#include "FWCore/Utilities/interface/BranchType.h" +#include "FWCore/Utilities/interface/propagate_const.h" +#include "DataFormats/Provenance/interface/ProductDependencies.h" +#include "DataFormats/Provenance/interface/BranchID.h" +#include "DataFormats/Provenance/interface/BranchType.h" +#include "DataFormats/Provenance/interface/ParentageID.h" +#include "DataFormats/Provenance/interface/ProductRegistry.h" + +class TTree; +namespace edm { + + class EDGetToken; + class ModuleCallingContext; + class ParameterSet; + class ConfigurationDescriptions; + class ProductProvenanceRetriever; +} // namespace edm +namespace edm::rntuple_temp { + class RootOutputFile; + + class RNTupleTempOutputModule : public one::OutputModule { + public: + enum DropMetaData { DropNone, DropDroppedPrior, DropPrior, DropAll }; + explicit RNTupleTempOutputModule(ParameterSet const& ps); + ~RNTupleTempOutputModule() override; + RNTupleTempOutputModule(RNTupleTempOutputModule const&) = delete; // Disallow copying and moving + RNTupleTempOutputModule& operator=(RNTupleTempOutputModule const&) = delete; // Disallow copying and moving + std::string const& fileName() const { return fileName_; } + std::string const& logicalFileName() const { return logicalFileName_; } + int compressionLevel() const { return compressionLevel_; } + std::string const& compressionAlgorithm() const { return compressionAlgorithm_; } + int basketSize() const { return basketSize_; } + int eventAuxiliaryBasketSize() const { return eventAuxBasketSize_; } + int eventAutoFlushSize() const { return eventAutoFlushSize_; } + int splitLevel() const { return splitLevel_; } + std::string const& basketOrder() const { return basketOrder_; } + int treeMaxVirtualSize() const { return treeMaxVirtualSize_; } + bool overrideInputFileSplitLevels() const { return overrideInputFileSplitLevels_; } + bool compactEventAuxiliary() const { return compactEventAuxiliary_; } + bool mergeJob() const { return mergeJob_; } + DropMetaData const& dropMetaData() const { return dropMetaData_; } + std::string const& catalog() const { return catalog_; } + std::string const& moduleLabel() const { return moduleLabel_; } + unsigned int maxFileSize() const { return maxFileSize_; } + int inputFileCount() const { return inputFileCount_; } + int whyNotFastClonable() const { return whyNotFastClonable_; } + + std::string const& currentFileName() const; + + static void fillDescription(ParameterSetDescription& desc); + static void fillDescriptions(ConfigurationDescriptions& descriptions); + + using OutputModule::selectorConfig; + + struct AuxItem { + AuxItem(); + ~AuxItem() {} + int basketSize_; + }; + using AuxItemArray = std::array; + AuxItemArray const& auxItems() const { return auxItems_; } + + struct OutputItem { + class Sorter { + public: + explicit Sorter(TTree* tree); + bool operator()(OutputItem const& lh, OutputItem const& rh) const; + + private: + std::shared_ptr> treeMap_; + }; + + explicit OutputItem(ProductDescription const* bd, EDGetToken const& token, int splitLevel, int basketSize); + + BranchID branchID() const { return productDescription_->branchID(); } + std::string const& branchName() const { return productDescription_->branchName(); } + + bool operator<(OutputItem const& rh) const { return *productDescription_ < *rh.productDescription_; } + + ProductDescription const* productDescription() const { return productDescription_; } + EDGetToken token() const { return token_; } + void const* const product() const { return product_; } + void const*& product() { return product_; } + void setProduct(void const* iProduct) { product_ = iProduct; } + int splitLevel() const { return splitLevel_; } + int basketSize() const { return basketSize_; } + + private: + ProductDescription const* productDescription_; + EDGetToken token_; + void const* product_; + int splitLevel_; + int basketSize_; + }; + + using OutputItemList = std::vector; + + struct SpecialSplitLevelForBranch { + SpecialSplitLevelForBranch(std::string const& iBranchName, int iSplitLevel) + : branch_(convert(iBranchName)), + splitLevel_(iSplitLevel < 1 ? 1 : iSplitLevel) //minimum is 1 + {} + bool match(std::string const& iBranchName) const; + std::regex convert(std::string const& iGlobBranchExpression) const; + + std::regex branch_; + int splitLevel_; + }; + + struct AliasForBranch { + AliasForBranch(std::string const& iBranchName, std::string const& iAlias) + : branch_{convert(iBranchName)}, alias_{iAlias} {} + + bool match(std::string const& iBranchName) const; + std::regex convert(std::string const& iGlobBranchExpression) const; + + std::regex branch_; + std::string alias_; + }; + + std::vector const& selectedOutputItemList() const { return selectedOutputItemList_; } + + std::vector& selectedOutputItemList() { return selectedOutputItemList_; } + + ProductDependencies const& productDependencies() const { return productDependencies_; } + + std::vector const& aliasForBranches() const { return aliasForBranches_; } + + protected: + ///allow inheriting classes to override but still be able to call this method in the overridden version + bool shouldWeCloseFile() const override; + void write(EventForOutput const& e) override; + + virtual std::pair physicalAndLogicalNameForNewFile(); + virtual void doExtrasAfterCloseFile(); + + private: + void preActionBeforeRunEventAsync(WaitingTaskHolder iTask, + ModuleCallingContext const& iModuleCallingContext, + Principal const& iPrincipal) const noexcept override; + + void openFile(FileBlock const& fb) override; + void respondToOpenInputFile(FileBlock const& fb) override; + void respondToCloseInputFile(FileBlock const& fb) override; + void writeLuminosityBlock(LuminosityBlockForOutput const&) override; + void writeRun(RunForOutput const&) override; + void writeProcessBlock(ProcessBlockForOutput const&) override; + bool isFileOpen() const override; + void reallyOpenFile(); + void reallyCloseFile() override; + void beginJob() override; + void initialRegistry(edm::ProductRegistry const& iReg) override; + + void setProcessesWithSelectedMergeableRunProducts(std::set const&) override; + + using BranchParents = std::map>; + void updateBranchParentsForOneBranch(ProductProvenanceRetriever const* provRetriever, BranchID const& branchID); + void updateBranchParents(EventForOutput const& e); + void fillDependencyGraph(); + + void startEndFile(); + void writeFileFormatVersion(); + void writeFileIdentifier(); + void writeIndexIntoFile(); + void writeStoredMergeableRunProductMetadata(); + void writeProcessHistoryRegistry(); + void writeParameterSetRegistry(); + void writeProductDescriptionRegistry(); + void writeParentageRegistry(); + void writeBranchIDListRegistry(); + void writeThinnedAssociationsHelper(); + void writeProductDependencies(); + void writeEventAuxiliary(); + void writeProcessBlockHelper(); + void finishEndFile(); + + void fillSelectedItemList(BranchType branchtype, + std::string const& processName, + TTree* theInputTree, + OutputItemList&); + void beginInputFile(FileBlock const& fb); + + RootServiceChecker rootServiceChecker_; + AuxItemArray auxItems_; + std::vector selectedOutputItemList_; + std::vector specialSplitLevelForBranches_; + std::vector aliasForBranches_; + std::unique_ptr reg_; + std::string const fileName_; + std::string const logicalFileName_; + std::string const catalog_; + unsigned int const maxFileSize_; + int const compressionLevel_; + std::string const compressionAlgorithm_; + int const basketSize_; + int const eventAuxBasketSize_; + int const eventAutoFlushSize_; + int const splitLevel_; + std::string basketOrder_; + int const treeMaxVirtualSize_; + int whyNotFastClonable_; + DropMetaData dropMetaData_; + std::string const moduleLabel_; + bool initializedFromInput_; + int outputFileCount_; + int inputFileCount_; + BranchParents branchParents_; + ProductDependencies productDependencies_; + std::vector producedBranches_; + bool overrideInputFileSplitLevels_; + bool compactEventAuxiliary_; + bool mergeJob_; + edm::propagate_const> rootOutputFile_; + std::string statusFileName_; + std::string overrideGUID_; + std::vector processesWithSelectedMergeableRunProducts_; + }; +} // namespace edm::rntuple_temp + +#endif diff --git a/FWIO/RNTupleTempOutput/interface/TimeoutRNTupleTempOutputModule.h b/FWIO/RNTupleTempOutput/interface/TimeoutRNTupleTempOutputModule.h new file mode 100644 index 0000000000000..954d4f9059fa5 --- /dev/null +++ b/FWIO/RNTupleTempOutput/interface/TimeoutRNTupleTempOutputModule.h @@ -0,0 +1,46 @@ +#ifndef FWIO_RNTupleTempOutput_TimeoutRNTupleTempOutputModule_h +#define FWIO_RNTupleTempOutput_TimeoutRNTupleTempOutputModule_h + +////////////////////////////////////////////////////////////////////// +// +// Class TimeoutRNTupleTempOutputModule. Output module to POOL file with file +// closure based on timeout. First file has only one event, second +// file is closed after 15 seconds if at least one event was processed. +// Then timeout is increased to 30 seconds and 60 seconds. After that +// all other files are closed with timeout of 60 seconds. +// +// Created by Dmytro.Kovalskyi@cern.ch +// +////////////////////////////////////////////////////////////////////// + +#include "FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h" + +namespace edm { + class ConfigurationDescriptions; + class ModuleCallingContext; + class ParameterSet; +} // namespace edm +namespace edm::rntuple_temp { + + class TimeoutRNTupleTempOutputModule : public RNTupleTempOutputModule { + public: + explicit TimeoutRNTupleTempOutputModule(ParameterSet const& ps); + ~TimeoutRNTupleTempOutputModule() override {} + TimeoutRNTupleTempOutputModule(TimeoutRNTupleTempOutputModule const&) = delete; // Disallow copying and moving + TimeoutRNTupleTempOutputModule& operator=(TimeoutRNTupleTempOutputModule const&) = + delete; // Disallow copying and moving + + static void fillDescriptions(ConfigurationDescriptions& descriptions); + + protected: + bool shouldWeCloseFile() const override; + void write(EventForOutput const& e) override; + + private: + mutable time_t m_lastEvent; + mutable unsigned int eventsWrittenInCurrentFile; + mutable int m_timeout; + }; +} // namespace edm::rntuple_temp + +#endif diff --git a/FWIO/RNTupleTempOutput/plugins/BuildFile.xml b/FWIO/RNTupleTempOutput/plugins/BuildFile.xml new file mode 100644 index 0000000000000..5bc5d93c949ce --- /dev/null +++ b/FWIO/RNTupleTempOutput/plugins/BuildFile.xml @@ -0,0 +1,4 @@ + + + + diff --git a/FWIO/RNTupleTempOutput/plugins/Module.cc b/FWIO/RNTupleTempOutput/plugins/Module.cc new file mode 100644 index 0000000000000..939d2cf8975c8 --- /dev/null +++ b/FWIO/RNTupleTempOutput/plugins/Module.cc @@ -0,0 +1,8 @@ +#include "FWCore/Framework/interface/MakerMacros.h" +#include "FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h" +#include "FWIO/RNTupleTempOutput/interface/TimeoutRNTupleTempOutputModule.h" + +using edm::rntuple_temp::RNTupleTempOutputModule; +using edm::rntuple_temp::TimeoutRNTupleTempOutputModule; +DEFINE_FWK_MODULE(RNTupleTempOutputModule); +DEFINE_FWK_MODULE(TimeoutRNTupleTempOutputModule); diff --git a/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc b/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc new file mode 100644 index 0000000000000..b23945983b8c5 --- /dev/null +++ b/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc @@ -0,0 +1,594 @@ +#include "FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h" + +#include "FWIO/RNTupleTempOutput/src/RootOutputFile.h" + +#include "FWCore/Framework/interface/EventForOutput.h" +#include "FWCore/Framework/interface/LuminosityBlockForOutput.h" +#include "FWCore/Framework/interface/RunForOutput.h" +#include "FWCore/Framework/interface/FileBlock.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" +#include "FWCore/ServiceRegistry/interface/Service.h" +#include "DataFormats/Provenance/interface/ProductDescription.h" +#include "DataFormats/Provenance/interface/Parentage.h" +#include "DataFormats/Provenance/interface/ParentageRegistry.h" +#include "DataFormats/Provenance/interface/ProductProvenance.h" +#include "FWCore/Framework/interface/ProductProvenanceRetriever.h" +#include "FWCore/Utilities/interface/Algorithms.h" +#include "FWCore/Utilities/interface/EDMException.h" +#include "FWCore/Utilities/interface/TimeOfDay.h" +#include "FWCore/Utilities/interface/WrappedClassName.h" + +#include "TTree.h" +#include "TBranchElement.h" +#include "TObjArray.h" +#include "RVersion.h" +#include "TDictAttributeMap.h" + +#include +#include +#include +#include "boost/algorithm/string.hpp" + +namespace edm::rntuple_temp { + RNTupleTempOutputModule::RNTupleTempOutputModule(ParameterSet const& pset) + : edm::one::OutputModuleBase::OutputModuleBase(pset), + one::OutputModule(pset), + rootServiceChecker_(), + auxItems_(), + selectedOutputItemList_(), + fileName_(pset.getUntrackedParameter("fileName")), + logicalFileName_(pset.getUntrackedParameter("logicalFileName")), + catalog_(pset.getUntrackedParameter("catalog")), + maxFileSize_(pset.getUntrackedParameter("maxSize")), + compressionLevel_(pset.getUntrackedParameter("compressionLevel")), + compressionAlgorithm_(pset.getUntrackedParameter("compressionAlgorithm")), + basketSize_(pset.getUntrackedParameter("basketSize")), + eventAuxBasketSize_(pset.getUntrackedParameter("eventAuxiliaryBasketSize")), + eventAutoFlushSize_(pset.getUntrackedParameter("eventAutoFlushCompressedSize")), + splitLevel_(std::min(pset.getUntrackedParameter("splitLevel") + 1, 99)), + basketOrder_(pset.getUntrackedParameter("sortBaskets")), + treeMaxVirtualSize_(pset.getUntrackedParameter("treeMaxVirtualSize")), + whyNotFastClonable_(pset.getUntrackedParameter("fastCloning") ? FileBlock::CanFastClone + : FileBlock::DisabledInConfigFile), + dropMetaData_(DropNone), + moduleLabel_(pset.getParameter("@module_label")), + initializedFromInput_(false), + outputFileCount_(0), + inputFileCount_(0), + branchParents_(), + productDependencies_(), + overrideInputFileSplitLevels_(pset.getUntrackedParameter("overrideInputFileSplitLevels")), + compactEventAuxiliary_(pset.getUntrackedParameter("compactEventAuxiliary")), + mergeJob_(pset.getUntrackedParameter("mergeJob")), + rootOutputFile_(), + statusFileName_(), + overrideGUID_(pset.getUntrackedParameter("overrideGUID")) { + if (pset.getUntrackedParameter("writeStatusFile")) { + std::ostringstream statusfilename; + statusfilename << moduleLabel_ << '_' << getpid(); + statusFileName_ = statusfilename.str(); + } + + std::string dropMetaData(pset.getUntrackedParameter("dropMetaData")); + if (dropMetaData.empty()) + dropMetaData_ = DropNone; + else if (dropMetaData == std::string("NONE")) + dropMetaData_ = DropNone; + else if (dropMetaData == std::string("DROPPED")) + dropMetaData_ = DropDroppedPrior; + else if (dropMetaData == std::string("PRIOR")) + dropMetaData_ = DropPrior; + else if (dropMetaData == std::string("ALL")) + dropMetaData_ = DropAll; + else { + throw edm::Exception(errors::Configuration, "Illegal dropMetaData parameter value: ") + << dropMetaData << ".\n" + << "Legal values are 'NONE', 'DROPPED', 'PRIOR', and 'ALL'.\n"; + } + + if (!wantAllEvents()) { + whyNotFastClonable_ += FileBlock::EventSelectionUsed; + } + + auto const& specialSplit{pset.getUntrackedParameterSetVector("overrideBranchesSplitLevel")}; + + specialSplitLevelForBranches_.reserve(specialSplit.size()); + for (auto const& s : specialSplit) { + specialSplitLevelForBranches_.emplace_back(s.getUntrackedParameter("branch"), + s.getUntrackedParameter("splitLevel")); + } + + auto const& branchAliases{pset.getUntrackedParameterSetVector("branchAliases")}; + aliasForBranches_.reserve(branchAliases.size()); + for (auto const& a : branchAliases) { + aliasForBranches_.emplace_back(a.getUntrackedParameter("branch"), + a.getUntrackedParameter("alias")); + } + // We don't use this next parameter, but we read it anyway because it is part + // of the configuration of this module. An external parser creates the + // configuration by reading this source code. + pset.getUntrackedParameterSet("dataset"); + } + + void RNTupleTempOutputModule::beginJob() {} + + void RNTupleTempOutputModule::initialRegistry(edm::ProductRegistry const& iReg) { + reg_ = std::make_unique(iReg.productList()); + } + + std::string const& RNTupleTempOutputModule::currentFileName() const { return rootOutputFile_->fileName(); } + + RNTupleTempOutputModule::AuxItem::AuxItem() : basketSize_(ProductDescription::invalidBasketSize) {} + + RNTupleTempOutputModule::OutputItem::OutputItem(ProductDescription const* bd, + EDGetToken const& token, + int splitLevel, + int basketSize) + : productDescription_(bd), token_(token), product_(nullptr), splitLevel_(splitLevel), basketSize_(basketSize) {} + + RNTupleTempOutputModule::OutputItem::Sorter::Sorter(TTree* tree) : treeMap_(new std::map) { + // Fill a map mapping branch names to an index specifying the order in the tree. + if (tree != nullptr) { + TObjArray* branches = tree->GetListOfBranches(); + for (int i = 0; i < branches->GetEntries(); ++i) { + TBranchElement* br = (TBranchElement*)branches->At(i); + treeMap_->insert(std::make_pair(std::string(br->GetName()), i)); + } + } + } + + bool RNTupleTempOutputModule::OutputItem::Sorter::operator()(OutputItem const& lh, OutputItem const& rh) const { + // Provides a comparison for sorting branches according to the index values in treeMap_. + // Branches not found are always put at the end (i.e. not found > found). + if (treeMap_->empty()) + return lh < rh; + std::string const& lstring = lh.productDescription_->branchName(); + std::string const& rstring = rh.productDescription_->branchName(); + std::map::const_iterator lit = treeMap_->find(lstring); + std::map::const_iterator rit = treeMap_->find(rstring); + bool lfound = (lit != treeMap_->end()); + bool rfound = (rit != treeMap_->end()); + if (lfound && rfound) { + return lit->second < rit->second; + } else if (lfound) { + return true; + } else if (rfound) { + return false; + } + return lh < rh; + } + + namespace { + std::regex convertBranchExpression(std::string const& iGlobBranchExpression) { + std::string tmp(iGlobBranchExpression); + boost::replace_all(tmp, "*", ".*"); + boost::replace_all(tmp, "?", "."); + return std::regex(tmp); + } + } // namespace + + inline bool RNTupleTempOutputModule::SpecialSplitLevelForBranch::match(std::string const& iBranchName) const { + return std::regex_match(iBranchName, branch_); + } + + std::regex RNTupleTempOutputModule::SpecialSplitLevelForBranch::convert( + std::string const& iGlobBranchExpression) const { + return convertBranchExpression(iGlobBranchExpression); + } + + bool RNTupleTempOutputModule::AliasForBranch::match(std::string const& iBranchName) const { + return std::regex_match(iBranchName, branch_); + } + + std::regex RNTupleTempOutputModule::AliasForBranch::convert(std::string const& iGlobBranchExpression) const { + return convertBranchExpression(iGlobBranchExpression); + } + + void RNTupleTempOutputModule::fillSelectedItemList(BranchType branchType, + std::string const& processName, + TTree* theInputTree, + OutputItemList& outputItemList) { + SelectedProducts const& keptVector = keptProducts()[branchType]; + + if (branchType != InProcess) { + AuxItem& auxItem = auxItems_[branchType]; + + auto basketSize = (InEvent == branchType) ? eventAuxBasketSize_ : basketSize_; + + // Fill AuxItem + if (theInputTree != nullptr && !overrideInputFileSplitLevels_) { + TBranch* auxBranch = theInputTree->GetBranch(BranchTypeToAuxiliaryBranchName(branchType).c_str()); + if (auxBranch) { + auxItem.basketSize_ = auxBranch->GetBasketSize(); + } else { + auxItem.basketSize_ = basketSize; + } + } else { + auxItem.basketSize_ = basketSize; + } + } + + // Fill outputItemList with an entry for each branch. + for (auto const& kept : keptVector) { + int splitLevel = ProductDescription::invalidSplitLevel; + int basketSize = ProductDescription::invalidBasketSize; + + ProductDescription const& prod = *kept.first; + if (branchType == InProcess && processName != prod.processName()) { + continue; + } + TBranch* theBranch = ((!prod.produced() && theInputTree != nullptr && !overrideInputFileSplitLevels_) + ? theInputTree->GetBranch(prod.branchName().c_str()) + : nullptr); + + if (theBranch != nullptr) { + splitLevel = theBranch->GetSplitLevel(); + basketSize = theBranch->GetBasketSize(); + } else { + auto wp = prod.wrappedType().getClass()->GetAttributeMap(); + auto wpSplitLevel = ProductDescription::invalidSplitLevel; + if (wp && wp->HasKey("splitLevel")) { + wpSplitLevel = strtol(wp->GetPropertyAsString("splitLevel"), nullptr, 0); + if (wpSplitLevel < 0) { + throw cms::Exception("IllegalSplitLevel") << "' An illegal ROOT split level of " << wpSplitLevel + << " is specified for class " << prod.wrappedName() << ".'\n"; + } + wpSplitLevel += 1; //Compensate for wrapper + } + splitLevel = (wpSplitLevel == ProductDescription::invalidSplitLevel ? splitLevel_ : wpSplitLevel); + for (auto const& b : specialSplitLevelForBranches_) { + if (b.match(prod.branchName())) { + splitLevel = b.splitLevel_; + } + } + auto wpBasketSize = ProductDescription::invalidBasketSize; + if (wp && wp->HasKey("basketSize")) { + wpBasketSize = strtol(wp->GetPropertyAsString("basketSize"), nullptr, 0); + if (wpBasketSize <= 0) { + throw cms::Exception("IllegalBasketSize") << "' An illegal ROOT basket size of " << wpBasketSize + << " is specified for class " << prod.wrappedName() << "'.\n"; + } + } + basketSize = (wpBasketSize == ProductDescription::invalidBasketSize ? basketSize_ : wpBasketSize); + } + outputItemList.emplace_back(&prod, kept.second, splitLevel, basketSize); + } + + // Sort outputItemList to allow fast copying. + // The branches in outputItemList must be in the same order as in the input tree, with all new branches at the end. + sort_all(outputItemList, OutputItem::Sorter(theInputTree)); + } + + void RNTupleTempOutputModule::beginInputFile(FileBlock const& fb) { + if (isFileOpen()) { + //Faster to read ChildrenBranches directly from input + // file than to build it every event + auto const& branchToChildMap = fb.productDependencies().childLookup(); + for (auto const& parentToChildren : branchToChildMap) { + for (auto const& child : parentToChildren.second) { + productDependencies_.insertChild(parentToChildren.first, child); + } + } + rootOutputFile_->beginInputFile(fb, remainingEvents()); + } + } + + void RNTupleTempOutputModule::openFile(FileBlock const& fb) { + if (!isFileOpen()) { + reallyOpenFile(); + beginInputFile(fb); + } + } + + void RNTupleTempOutputModule::respondToOpenInputFile(FileBlock const& fb) { + if (!initializedFromInput_) { + std::vector const& processesWithProcessBlockProducts = + outputProcessBlockHelper().processesWithProcessBlockProducts(); + unsigned int numberOfProcessesWithProcessBlockProducts = processesWithProcessBlockProducts.size(); + unsigned int numberOfTTrees = numberOfRunLumiEventProductTrees + numberOfProcessesWithProcessBlockProducts; + selectedOutputItemList_.resize(numberOfTTrees); + + for (unsigned int i = InEvent; i < NumBranchTypes; ++i) { + BranchType branchType = static_cast(i); + if (branchType != InProcess) { + std::string processName; + TTree* theInputTree = + (branchType == InEvent ? fb.tree() : (branchType == InLumi ? fb.lumiTree() : fb.runTree())); + OutputItemList& outputItemList = selectedOutputItemList_[branchType]; + fillSelectedItemList(branchType, processName, theInputTree, outputItemList); + } else { + // Handle output items in ProcessBlocks + for (unsigned int k = InProcess; k < numberOfTTrees; ++k) { + OutputItemList& outputItemList = selectedOutputItemList_[k]; + std::string const& processName = processesWithProcessBlockProducts[k - InProcess]; + TTree* theInputTree = fb.processBlockTree(processName); + fillSelectedItemList(branchType, processName, theInputTree, outputItemList); + } + } + } + initializedFromInput_ = true; + } + ++inputFileCount_; + beginInputFile(fb); + } + + void RNTupleTempOutputModule::respondToCloseInputFile(FileBlock const& fb) { + if (rootOutputFile_) + rootOutputFile_->respondToCloseInputFile(fb); + } + + void RNTupleTempOutputModule::setProcessesWithSelectedMergeableRunProducts(std::set const& processes) { + processesWithSelectedMergeableRunProducts_.assign(processes.begin(), processes.end()); + } + + RNTupleTempOutputModule::~RNTupleTempOutputModule() {} + + void RNTupleTempOutputModule::write(EventForOutput const& e) { + updateBranchParents(e); + rootOutputFile_->writeOne(e); + if (!statusFileName_.empty()) { + std::ofstream statusFile(statusFileName_.c_str()); + statusFile << e.id() << " time: " << std::setprecision(3) << TimeOfDay() << '\n'; + statusFile.close(); + } + } + + void RNTupleTempOutputModule::writeLuminosityBlock(LuminosityBlockForOutput const& lb) { + rootOutputFile_->writeLuminosityBlock(lb); + } + + void RNTupleTempOutputModule::writeRun(RunForOutput const& r) { + if (!reg_ or (reg_->size() < r.productRegistry().size())) { + reg_ = std::make_unique(r.productRegistry().productList()); + } + rootOutputFile_->writeRun(r); + } + + void RNTupleTempOutputModule::writeProcessBlock(ProcessBlockForOutput const& pb) { + rootOutputFile_->writeProcessBlock(pb); + } + + void RNTupleTempOutputModule::reallyCloseFile() { + writeEventAuxiliary(); + fillDependencyGraph(); + branchParents_.clear(); + startEndFile(); + writeFileFormatVersion(); + writeFileIdentifier(); + writeIndexIntoFile(); + writeStoredMergeableRunProductMetadata(); + writeProcessHistoryRegistry(); + writeParameterSetRegistry(); + writeProductDescriptionRegistry(); + writeParentageRegistry(); + writeBranchIDListRegistry(); + writeThinnedAssociationsHelper(); + writeProductDependencies(); //productDependencies used here + writeProcessBlockHelper(); + productDependencies_.clear(); + finishEndFile(); + + doExtrasAfterCloseFile(); + } + + // At some later date, we may move functionality from finishEndFile() to here. + void RNTupleTempOutputModule::startEndFile() {} + + void RNTupleTempOutputModule::writeFileFormatVersion() { rootOutputFile_->writeFileFormatVersion(); } + void RNTupleTempOutputModule::writeFileIdentifier() { rootOutputFile_->writeFileIdentifier(); } + void RNTupleTempOutputModule::writeIndexIntoFile() { rootOutputFile_->writeIndexIntoFile(); } + void RNTupleTempOutputModule::writeStoredMergeableRunProductMetadata() { + rootOutputFile_->writeStoredMergeableRunProductMetadata(); + } + void RNTupleTempOutputModule::writeProcessHistoryRegistry() { rootOutputFile_->writeProcessHistoryRegistry(); } + void RNTupleTempOutputModule::writeParameterSetRegistry() { rootOutputFile_->writeParameterSetRegistry(); } + void RNTupleTempOutputModule::writeProductDescriptionRegistry() { + assert(reg_); + rootOutputFile_->writeProductDescriptionRegistry(*reg_); + } + void RNTupleTempOutputModule::writeParentageRegistry() { rootOutputFile_->writeParentageRegistry(); } + void RNTupleTempOutputModule::writeBranchIDListRegistry() { rootOutputFile_->writeBranchIDListRegistry(); } + void RNTupleTempOutputModule::writeThinnedAssociationsHelper() { rootOutputFile_->writeThinnedAssociationsHelper(); } + void RNTupleTempOutputModule::writeProductDependencies() { rootOutputFile_->writeProductDependencies(); } + void RNTupleTempOutputModule::writeEventAuxiliary() { rootOutputFile_->writeEventAuxiliary(); } + void RNTupleTempOutputModule::writeProcessBlockHelper() { rootOutputFile_->writeProcessBlockHelper(); } + void RNTupleTempOutputModule::finishEndFile() { + rootOutputFile_->finishEndFile(); + rootOutputFile_ = nullptr; + } // propagate_const has no reset() function + void RNTupleTempOutputModule::doExtrasAfterCloseFile() {} + bool RNTupleTempOutputModule::isFileOpen() const { return rootOutputFile_.get() != nullptr; } + bool RNTupleTempOutputModule::shouldWeCloseFile() const { return rootOutputFile_->shouldWeCloseFile(); } + + std::pair RNTupleTempOutputModule::physicalAndLogicalNameForNewFile() { + if (inputFileCount_ == 0) { + throw edm::Exception(errors::LogicError) << "Attempt to open output file before input file. " + << "Please report this to the core framework developers.\n"; + } + std::string suffix(".root"); + std::string::size_type offset = fileName().rfind(suffix); + bool ext = (offset == fileName().size() - suffix.size()); + if (!ext) + suffix.clear(); + std::string fileBase(ext ? fileName().substr(0, offset) : fileName()); + std::ostringstream ofilename; + std::ostringstream lfilename; + ofilename << fileBase; + lfilename << logicalFileName(); + if (outputFileCount_) { + ofilename << std::setw(3) << std::setfill('0') << outputFileCount_; + if (!logicalFileName().empty()) { + lfilename << std::setw(3) << std::setfill('0') << outputFileCount_; + } + } + ofilename << suffix; + ++outputFileCount_; + + return std::make_pair(ofilename.str(), lfilename.str()); + } + + void RNTupleTempOutputModule::reallyOpenFile() { + auto names = physicalAndLogicalNameForNewFile(); + rootOutputFile_ = std::make_unique(this, + names.first, + names.second, + processesWithSelectedMergeableRunProducts_, + overrideGUID_); // propagate_const has no reset() function + // Override the GUID of the first file only, in order to avoid two + // output files from one Output Module to have identical GUID. + overrideGUID_.clear(); + } + + void RNTupleTempOutputModule::updateBranchParentsForOneBranch(ProductProvenanceRetriever const* provRetriever, + BranchID const& branchID) { + ProductProvenance const* provenance = provRetriever->branchIDToProvenanceForProducedOnly(branchID); + if (provenance != nullptr) { + BranchParents::iterator it = branchParents_.find(branchID); + if (it == branchParents_.end()) { + it = branchParents_.insert(std::make_pair(branchID, std::set())).first; + } + it->second.insert(provenance->parentageID()); + } + } + + void RNTupleTempOutputModule::updateBranchParents(EventForOutput const& e) { + ProductProvenanceRetriever const* provRetriever = e.productProvenanceRetrieverPtr(); + if (producedBranches_.empty()) { + for (auto const& prod : e.productRegistry().productList()) { + ProductDescription const& desc = prod.second; + if (desc.produced() && desc.branchType() == InEvent && !desc.isAlias()) { + producedBranches_.emplace_back(desc.branchID()); + } + } + } + for (auto const& bid : producedBranches_) { + updateBranchParentsForOneBranch(provRetriever, bid); + } + } + + void RNTupleTempOutputModule::preActionBeforeRunEventAsync(WaitingTaskHolder iTask, + ModuleCallingContext const& iModuleCallingContext, + Principal const& iPrincipal) const noexcept { + if (DropAll != dropMetaData_) { + auto const* ep = dynamic_cast(&iPrincipal); + if (ep) { + auto pr = ep->productProvenanceRetrieverPtr(); + if (pr) { + pr->readProvenanceAsync(iTask, &iModuleCallingContext); + } + } + } + } + + void RNTupleTempOutputModule::fillDependencyGraph() { + for (auto const& branchParent : branchParents_) { + BranchID const& child = branchParent.first; + std::set const& eIds = branchParent.second; + for (auto const& eId : eIds) { + Parentage entryDesc; + ParentageRegistry::instance()->getMapped(eId, entryDesc); + std::vector const& parents = entryDesc.parents(); + for (auto const& parent : parents) { + productDependencies_.insertChild(parent, child); + } + } + } + } + + void RNTupleTempOutputModule::fillDescription(ParameterSetDescription& desc) { + std::string defaultString; + + desc.setComment("Writes runs, lumis, and events into EDM/ROOT files."); + desc.addUntracked("fileName")->setComment("Name of output file."); + desc.addUntracked("logicalFileName", defaultString) + ->setComment("Passed to job report. Otherwise unused by module."); + desc.addUntracked("catalog", defaultString) + ->setComment("Passed to job report. Otherwise unused by module."); + desc.addUntracked("maxSize", 0x7f000000) + ->setComment( + "Maximum output file size, in kB.\n" + "If over maximum, new output file will be started at next input file transition."); + desc.addUntracked("compressionLevel", 4)->setComment("ROOT compression level of output file."); + desc.addUntracked("compressionAlgorithm", "ZSTD") + ->setComment( + "Algorithm used to compress data in the ROOT output file, allowed values are ZLIB, LZMA, LZ4, and ZSTD"); + desc.addUntracked("basketSize", 16384)->setComment("Default ROOT basket size in output file."); + desc.addUntracked("eventAuxiliaryBasketSize", 16384) + ->setComment("Default ROOT basket size in output file for EventAuxiliary branch."); + desc.addUntracked("eventAutoFlushCompressedSize", 20 * 1024 * 1024) + ->setComment( + "Set ROOT auto flush stored data size (in bytes) for event TTree. The value sets how large the compressed " + "buffer is allowed to get. The uncompressed buffer can be quite a bit larger than this depending on the " + "average compression ratio. The value of -1 just uses ROOT's default value. The value of 0 turns off this " + "feature. A value of -N changes the behavior to flush after every Nth event."); + desc.addUntracked("splitLevel", 99)->setComment("Default ROOT branch split level in output file."); + desc.addUntracked("sortBaskets", std::string("sortbasketsbyoffset")) + ->setComment( + "Legal values: 'sortbasketsbyoffset', 'sortbasketsbybranch', 'sortbasketsbyentry'.\n" + "Used by ROOT when fast copying. Affects performance."); + desc.addUntracked("treeMaxVirtualSize", -1) + ->setComment("Size of ROOT TTree TBasket cache. Affects performance."); + desc.addUntracked("fastCloning", true) + ->setComment( + "True: Allow fast copying, if possible.\n" + "False: Disable fast copying."); + desc.addUntracked("mergeJob", false) + ->setComment( + "If set to true and fast copying is disabled, copy input file compression and basket sizes to the output " + "file."); + desc.addUntracked("compactEventAuxiliary", false) + ->setComment( + "False: Write EventAuxiliary as we go like any other event metadata branch.\n" + "True: Optimize the file layout by deferring writing the EventAuxiliary branch until the output file is " + "closed."); + desc.addUntracked("overrideInputFileSplitLevels", false) + ->setComment( + "False: Use branch split levels and basket sizes from input file, if possible.\n" + "True: Always use specified or default split levels and basket sizes."); + desc.addUntracked("writeStatusFile", false) + ->setComment("Write a status file. Intended for use by workflow management."); + desc.addUntracked("dropMetaData", defaultString) + ->setComment( + "Determines handling of per product per event metadata. Options are:\n" + "'NONE': Keep all of it.\n" + "'DROPPED': Keep it for products produced in current process and all kept products. Drop it for dropped " + "products produced in prior processes.\n" + "'PRIOR': Keep it for products produced in current process. Drop it for products produced in prior " + "processes.\n" + "'ALL': Drop all of it."); + desc.addUntracked("overrideGUID", defaultString) + ->setComment( + "Allows to override the GUID of the file. Intended to be used only in Tier0 for re-creating files.\n" + "The GUID needs to be of the proper format. If a new output file is started (see maxSize), the GUID of\n" + "the first file only is overridden, i.e. the subsequent output files have different, generated GUID."); + { + ParameterSetDescription dataSet; + dataSet.setAllowAnything(); + desc.addUntracked("dataset", dataSet) + ->setComment("PSet is only used by Data Operations and not by this module."); + } + { + ParameterSetDescription specialSplit; + specialSplit.addUntracked("branch")->setComment( + "Name of branch needing a special split level. The name can contain wildcards '*' and '?'"); + specialSplit.addUntracked("splitLevel")->setComment("The special split level for the branch"); + desc.addVPSetUntracked("overrideBranchesSplitLevel", specialSplit, std::vector()); + } + { + ParameterSetDescription alias; + alias.addUntracked("branch")->setComment( + "Name of branch which will get alias. The name can contain wildcards '*' and '?'"); + alias.addUntracked("alias")->setComment("The alias to give to the TBranch"); + desc.addVPSetUntracked("branchAliases", alias, std::vector()); + } + OutputModule::fillDescription(desc); + } + + void RNTupleTempOutputModule::fillDescriptions(ConfigurationDescriptions& descriptions) { + ParameterSetDescription desc; + RNTupleTempOutputModule::fillDescription(desc); + descriptions.add("edmOutput", desc); + } +} // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempOutput/src/RootOutputFile.cc b/FWIO/RNTupleTempOutput/src/RootOutputFile.cc new file mode 100644 index 0000000000000..c4d14b4105e25 --- /dev/null +++ b/FWIO/RNTupleTempOutput/src/RootOutputFile.cc @@ -0,0 +1,990 @@ + +#include "FWIO/RNTupleTempOutput/src/RootOutputFile.h" + +#include "FWCore/Utilities/interface/GlobalIdentifier.h" + +#include "DataFormats/Provenance/interface/EventAuxiliary.h" +#include "DataFormats/Provenance/interface/ProductDescription.h" +#include "FWCore/Version/interface/GetFileFormatVersion.h" +#include "DataFormats/Provenance/interface/FileFormatVersion.h" +#include "FWCore/Utilities/interface/EDMException.h" +#include "FWCore/Utilities/interface/Algorithms.h" +#include "FWCore/Utilities/interface/Digest.h" +#include "FWCore/Common/interface/OutputProcessBlockHelper.h" +#include "FWCore/Framework/interface/FileBlock.h" +#include "FWCore/Framework/interface/EventForOutput.h" +#include "FWCore/Framework/interface/LuminosityBlockForOutput.h" +#include "FWCore/Framework/interface/MergeableRunProductMetadata.h" +#include "FWCore/Framework/interface/OccurrenceForOutput.h" +#include "FWCore/Framework/interface/ProcessBlockForOutput.h" +#include "FWCore/Framework/interface/RunForOutput.h" +#include "FWCore/MessageLogger/interface/JobReport.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "DataFormats/Common/interface/BasicHandle.h" +#include "DataFormats/Provenance/interface/ProductDependencies.h" +#include "DataFormats/Provenance/interface/BranchIDList.h" +#include "DataFormats/Provenance/interface/Parentage.h" +#include "DataFormats/Provenance/interface/ParentageRegistry.h" +#include "DataFormats/Provenance/interface/EventID.h" +#include "DataFormats/Provenance/interface/EventToProcessBlockIndexes.h" +#include "DataFormats/Provenance/interface/ParameterSetBlob.h" +#include "DataFormats/Provenance/interface/ParameterSetID.h" +#include "DataFormats/Provenance/interface/ProcessHistoryID.h" +#include "DataFormats/Provenance/interface/ProductRegistry.h" +#include "DataFormats/Provenance/interface/StoredProcessBlockHelper.h" +#include "DataFormats/Provenance/interface/ThinnedAssociationsHelper.h" +#include "DataFormats/Provenance/interface/ProductRegistry.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/ParameterSet/interface/Registry.h" +#include "FWCore/ServiceRegistry/interface/Service.h" +#include "FWCore/Utilities/interface/ExceptionPropagate.h" +#include "IOPool/Common/interface/getWrapperBasePtr.h" +#include "IOPool/Provenance/interface/CommonProvenanceFiller.h" + +#include "TTree.h" +#include "TFile.h" +#include "TClass.h" +#include "Rtypes.h" +#include "RVersion.h" + +#include "ROOT/RNTuple.hxx" +#include "ROOT/RNTupleWriter.hxx" + + +#include "Compression.h" + +#include +#include +#include + +namespace edm::rntuple_temp { + + namespace { + bool sorterForJobReportHash(ProductDescription const* lh, ProductDescription const* rh) { + return lh->fullClassName() < rh->fullClassName() ? true + : lh->fullClassName() > rh->fullClassName() ? false + : lh->moduleLabel() < rh->moduleLabel() ? true + : lh->moduleLabel() > rh->moduleLabel() ? false + : lh->productInstanceName() < rh->productInstanceName() ? true + : lh->productInstanceName() > rh->productInstanceName() ? false + : lh->processName() < rh->processName() ? true + : false; + } + + TFile* openTFile(char const* name, int compressionLevel) { + TFile* file = TFile::Open(name, "recreate", "", compressionLevel); + std::exception_ptr e = edm::threadLocalException::getException(); + if (e != std::exception_ptr()) { + edm::threadLocalException::setException(std::exception_ptr()); + std::rethrow_exception(e); + } + return file; + } + } // namespace + + RootOutputFile::RootOutputFile(RNTupleTempOutputModule* om, + std::string const& fileName, + std::string const& logicalFileName, + std::vector const& processesWithSelectedMergeableRunProducts, + std::string const& overrideGUID) + : file_(fileName), + logicalFile_(logicalFileName), + reportToken_(0), + om_(om), + whyNotFastClonable_(om_->whyNotFastClonable()), + canFastCloneAux_(false), + filePtr_(openTFile(file_.c_str(), om_->compressionLevel())), + fid_(), + eventEntryNumber_(0LL), + lumiEntryNumber_(0LL), + runEntryNumber_(0LL), + indexIntoFile_(), + storedMergeableRunProductMetadata_(processesWithSelectedMergeableRunProducts), + nEventsInLumi_(0), + metaDataTree_(nullptr), + parentageTree_(nullptr), + lumiAux_(), + runAux_(), + pEventAux_(nullptr), + pLumiAux_(&lumiAux_), + pRunAux_(&runAux_), + eventEntryInfoVector_(), + pEventEntryInfoVector_(&eventEntryInfoVector_), + pBranchListIndexes_(nullptr), + pEventSelectionIDs_(nullptr), + eventTree_(filePtr(), InEvent, om_->splitLevel(), om_->treeMaxVirtualSize()), + lumiTree_(filePtr(), InLumi, om_->splitLevel(), om_->treeMaxVirtualSize()), + runTree_(filePtr(), InRun, om_->splitLevel(), om_->treeMaxVirtualSize()), + dataTypeReported_(false), + processHistoryRegistry_(), + parentageIDs_(), + branchesWithStoredHistory_(), + wrapperBaseTClass_(TClass::GetClass("edm::WrapperBase")) { + std::vector const& processesWithProcessBlockProducts = + om_->outputProcessBlockHelper().processesWithProcessBlockProducts(); + for (auto const& processName : processesWithProcessBlockProducts) { + processBlockTrees_.emplace_back(std::make_unique( + filePtr(), InProcess, om_->splitLevel(), om_->treeMaxVirtualSize(), processName)); + } + + if (om_->compressionAlgorithm() == std::string("ZLIB")) { + filePtr_->SetCompressionAlgorithm(ROOT::RCompressionSetting::EAlgorithm::kZLIB); + } else if (om_->compressionAlgorithm() == std::string("LZMA")) { + filePtr_->SetCompressionAlgorithm(ROOT::RCompressionSetting::EAlgorithm::kLZMA); + } else if (om_->compressionAlgorithm() == std::string("ZSTD")) { + filePtr_->SetCompressionAlgorithm(ROOT::RCompressionSetting::EAlgorithm::kZSTD); + } else if (om_->compressionAlgorithm() == std::string("LZ4")) { + filePtr_->SetCompressionAlgorithm(ROOT::RCompressionSetting::EAlgorithm::kLZ4); + } else { + throw Exception(errors::Configuration) + << "RNTupleTempOutputModule configured with unknown compression algorithm '" << om_->compressionAlgorithm() + << "'\n" + << "Allowed compression algorithms are ZLIB, LZMA, LZ4, and ZSTD\n"; + } + if (-1 != om->eventAutoFlushSize()) { + eventTree_.setAutoFlush(-1 * om->eventAutoFlushSize()); + } + if (om_->compactEventAuxiliary()) { + eventTree_.addAuxiliary( + BranchTypeToAuxiliaryBranchName(InEvent), pEventAux_, om_->auxItems()[InEvent].basketSize_, false); + eventTree_.tree()->SetBranchStatus(BranchTypeToAuxiliaryBranchName(InEvent).c_str(), + false); // see writeEventAuxiliary + } else { + eventTree_.addAuxiliary( + BranchTypeToAuxiliaryBranchName(InEvent), pEventAux_, om_->auxItems()[InEvent].basketSize_); + } + + eventTree_.addAuxiliary(BranchTypeToProductProvenanceBranchName(InEvent), + pEventEntryInfoVector(), + om_->auxItems()[InEvent].basketSize_); + eventTree_.addAuxiliary( + poolNames::eventSelectionsBranchName(), pEventSelectionIDs_, om_->auxItems()[InEvent].basketSize_, false); + eventTree_.addAuxiliary( + poolNames::branchListIndexesBranchName(), pBranchListIndexes_, om_->auxItems()[InEvent].basketSize_); + + if (om_->outputProcessBlockHelper().productsFromInputKept()) { + eventTree_.addAuxiliary(poolNames::eventToProcessBlockIndexesBranchName(), + pEventToProcessBlockIndexes_, + om_->auxItems()[InEvent].basketSize_); + } + + lumiTree_.addAuxiliary( + BranchTypeToAuxiliaryBranchName(InLumi), pLumiAux_, om_->auxItems()[InLumi].basketSize_); + + runTree_.addAuxiliary( + BranchTypeToAuxiliaryBranchName(InRun), pRunAux_, om_->auxItems()[InRun].basketSize_); + + treePointers_.emplace_back(&eventTree_); + treePointers_.emplace_back(&lumiTree_); + treePointers_.emplace_back(&runTree_); + for (auto& processBlockTree : processBlockTrees_) { + treePointers_.emplace_back(processBlockTree.get()); + } + + for (unsigned int i = 0; i < treePointers_.size(); ++i) { + RootOutputRNTuple* theTree = treePointers_[i]; + for (auto& item : om_->selectedOutputItemList()[i]) { + item.setProduct(nullptr); + ProductDescription const& desc = *item.productDescription(); + theTree->addBranch(desc.branchName(), + desc.wrappedName(), + item.product(), + item.splitLevel(), + item.basketSize(), + item.productDescription()->produced()); + //make sure we always store product registry info for all branches we create + branchesWithStoredHistory_.insert(item.branchID()); + } + } + // Don't split metadata tree or event description tree + metaDataTree_ = RootOutputRNTuple::makeTTree(filePtr_.get(), poolNames::metaDataTreeName(), 0); + parentageTree_ = RootOutputRNTuple::makeTTree(filePtr_.get(), poolNames::parentageTreeName(), 0); + + if (overrideGUID.empty()) { + fid_ = FileID(createGlobalIdentifier()); + } else { + if (not isValidGlobalIdentifier(overrideGUID)) { + throw edm::Exception(errors::Configuration) + << "GUID to be used for output file is not valid (is '" << overrideGUID << "')"; + } + fid_ = FileID(overrideGUID); + } + + // For the Job Report, get a vector of branch names in the "Events" tree. + // Also create a hash of all the branch names in the "Events" tree + // in a deterministic order, except use the full class name instead of the friendly class name. + // To avoid extra string copies, we create a vector of pointers into the product registry, + // and use a custom comparison operator for sorting. + std::vector branchNames; + std::vector branches; + branchNames.reserve(om_->selectedOutputItemList()[InEvent].size()); + branches.reserve(om->selectedOutputItemList()[InEvent].size()); + for (auto const& item : om_->selectedOutputItemList()[InEvent]) { + branchNames.push_back(item.productDescription()->branchName()); + branches.push_back(item.productDescription()); + } + // Now sort the branches for the hash. + sort_all(branches, sorterForJobReportHash); + // Now, make a concatenated string. + std::ostringstream oss; + char const underscore = '_'; + for (auto const& branch : branches) { + ProductDescription const& bd = *branch; + oss << bd.fullClassName() << underscore << bd.moduleLabel() << underscore << bd.productInstanceName() + << underscore << bd.processName() << underscore; + } + std::string stringrep = oss.str(); + cms::Digest md5alg(stringrep); + + // Register the output file with the JobReport service + // and get back the token for it. + std::string moduleName = "RNTupleTempOutputModule"; + Service reportSvc; + reportToken_ = reportSvc->outputFileOpened(file_, + logicalFile_, // PFN and LFN + om_->catalog(), // catalog + moduleName, // module class name + om_->moduleLabel(), // module label + fid_.fid(), // file id (guid) + std::string(), // data type (not yet known, so string is empty). + md5alg.digest().toString(), // branch hash + branchNames); // branch names being written + } + + namespace { + void maybeIssueWarning(int whyNotFastClonable, std::string const& ifileName, std::string const& ofileName) { + // No message if fast cloning was deliberately disabled, or if there are no events to copy anyway. + if ((whyNotFastClonable & (FileBlock::DisabledInConfigFile | FileBlock::NoRootInputSource | + FileBlock::NotProcessingEvents | FileBlock::NoEventsInFile)) != 0) { + return; + } + + // There will be a message stating every reason that fast cloning was not possible. + // If at one or more of the reasons was because of something the user explicitly specified (e.g. event selection, skipping events), + // or if the input file was in an old format, the message will be informational. Otherwise, the message will be a warning. + bool isWarning = true; + std::ostringstream message; + message << "Fast copying of file " << ifileName << " to file " << ofileName << " is disabled because:\n"; + if ((whyNotFastClonable & FileBlock::HasSecondaryFileSequence) != 0) { + message << "a SecondaryFileSequence was specified.\n"; + whyNotFastClonable &= ~(FileBlock::HasSecondaryFileSequence); + isWarning = false; + } + if ((whyNotFastClonable & FileBlock::FileTooOld) != 0) { + message << "the input file is in an old format.\n"; + whyNotFastClonable &= ~(FileBlock::FileTooOld); + isWarning = false; + } + if ((whyNotFastClonable & FileBlock::EventsToBeSorted) != 0) { + message << "events need to be sorted.\n"; + whyNotFastClonable &= ~(FileBlock::EventsToBeSorted); + } + if ((whyNotFastClonable & FileBlock::RunOrLumiNotContiguous) != 0) { + message << "a run or a lumi is not contiguous in the input file.\n"; + whyNotFastClonable &= ~(FileBlock::RunOrLumiNotContiguous); + } + if ((whyNotFastClonable & FileBlock::EventsOrLumisSelectedByID) != 0) { + message << "events or lumis were selected or skipped by ID.\n"; + whyNotFastClonable &= ~(FileBlock::EventsOrLumisSelectedByID); + isWarning = false; + } + if ((whyNotFastClonable & FileBlock::InitialEventsSkipped) != 0) { + message << "initial events, lumis or runs were skipped.\n"; + whyNotFastClonable &= ~(FileBlock::InitialEventsSkipped); + isWarning = false; + } + if ((whyNotFastClonable & FileBlock::DuplicateEventsRemoved) != 0) { + message << "some events were skipped because of duplicate checking.\n"; + whyNotFastClonable &= ~(FileBlock::DuplicateEventsRemoved); + } + if ((whyNotFastClonable & FileBlock::MaxEventsTooSmall) != 0) { + message << "some events were not copied because of maxEvents limit.\n"; + whyNotFastClonable &= ~(FileBlock::MaxEventsTooSmall); + isWarning = false; + } + if ((whyNotFastClonable & FileBlock::MaxLumisTooSmall) != 0) { + message << "some events were not copied because of maxLumis limit.\n"; + whyNotFastClonable &= ~(FileBlock::MaxLumisTooSmall); + isWarning = false; + } + if ((whyNotFastClonable & FileBlock::ParallelProcesses) != 0) { + message << "parallel processing was specified.\n"; + whyNotFastClonable &= ~(FileBlock::ParallelProcesses); + isWarning = false; + } + if ((whyNotFastClonable & FileBlock::EventSelectionUsed) != 0) { + message << "an EventSelector was specified.\n"; + whyNotFastClonable &= ~(FileBlock::EventSelectionUsed); + isWarning = false; + } + if ((whyNotFastClonable & FileBlock::OutputMaxEventsTooSmall) != 0) { + message << "some events were not copied because of maxEvents output limit.\n"; + whyNotFastClonable &= ~(FileBlock::OutputMaxEventsTooSmall); + isWarning = false; + } + if ((whyNotFastClonable & FileBlock::SplitLevelMismatch) != 0) { + message << "the split level or basket size of a branch or branches was modified.\n"; + whyNotFastClonable &= ~(FileBlock::SplitLevelMismatch); + } + if ((whyNotFastClonable & FileBlock::BranchMismatch) != 0) { + message << "The format of a data product has changed.\n"; + whyNotFastClonable &= ~(FileBlock::BranchMismatch); + } + assert(whyNotFastClonable == FileBlock::CanFastClone); + if (isWarning) { + LogWarning("FastCloningDisabled") << message.str(); + } else { + LogInfo("FastCloningDisabled") << message.str(); + } + } + } // namespace + + void RootOutputFile::beginInputFile(FileBlock const& fb, int remainingEvents) { + // Reset per input file information + whyNotFastClonable_ = om_->whyNotFastClonable(); + canFastCloneAux_ = false; + + if (fb.tree() != nullptr) { + whyNotFastClonable_ |= fb.whyNotFastClonable(); + + if (remainingEvents >= 0 && remainingEvents < fb.tree()->GetEntries()) { + whyNotFastClonable_ |= FileBlock::OutputMaxEventsTooSmall; + } + + bool match = eventTree_.checkSplitLevelsAndBasketSizes(fb.tree()); + if (!match) { + if (om_->overrideInputFileSplitLevels()) { + // We may be fast copying. We must disable fast copying if the split levels + // or basket sizes do not match. + whyNotFastClonable_ |= FileBlock::SplitLevelMismatch; + } else { + // We are using the input split levels and basket sizes from the first input file + // for copied output branches. In this case, we throw an exception if any branches + // have different split levels or basket sizes in a subsequent input file. + // If the mismatch is in the first file, there is a bug somewhere, so we assert. + assert(om_->inputFileCount() > 1); + throw Exception(errors::MismatchedInputFiles, "RootOutputFile::beginInputFile()") + << "Merge failure because input file " << file_ << " has different ROOT split levels or basket sizes\n" + << "than previous files. To allow merging in spite of this, use the configuration parameter\n" + << "overrideInputFileSplitLevels=cms.untracked.bool(True)\n" + << "in every RNTupleTempOutputModule.\n"; + } + } + + // Since this check can be time consuming, we do it only if we would otherwise fast clone. + if (whyNotFastClonable_ == FileBlock::CanFastClone) { + if (!eventTree_.checkIfFastClonable(fb.tree())) { + whyNotFastClonable_ |= FileBlock::BranchMismatch; + } + } + + // reasons for whyNotFastClonable that are also inconsistent with a merge job + constexpr auto setSubBranchBasketConditions = + FileBlock::EventsOrLumisSelectedByID | FileBlock::InitialEventsSkipped | FileBlock::MaxEventsTooSmall | + FileBlock::MaxLumisTooSmall | FileBlock::EventSelectionUsed | FileBlock::OutputMaxEventsTooSmall | + FileBlock::SplitLevelMismatch | FileBlock::BranchMismatch; + + if (om_->inputFileCount() == 1) { + if (om_->mergeJob()) { + // for merge jobs always forward the compression mode + auto infile = fb.tree()->GetCurrentFile(); + if (infile != nullptr) { + filePtr_->SetCompressionSettings(infile->GetCompressionSettings()); + } + } + + // if we aren't fast cloning, and the reason why is consistent with a + // merge job or is only because of parallel processes, then forward all + // the sub-branch basket sizes + if (whyNotFastClonable_ != FileBlock::CanFastClone && + ((om_->mergeJob() && (whyNotFastClonable_ & setSubBranchBasketConditions) == 0) || + (whyNotFastClonable_ == FileBlock::ParallelProcesses))) { + eventTree_.setSubBranchBasketSizes(fb.tree()); + } + } + + // We now check if we can fast copy the auxiliary branches. + // We can do so only if we can otherwise fast copy, + // the input file has the current format (these branches are in the Events Tree), + // there are no newly dropped or produced products, + // no metadata has been dropped, + // ID's have not been modified, + // and the branch list indexes do not need modification. + + // Note: Fast copy of the EventProductProvenance branch is unsafe + // unless we can enforce that the parentage information for a fully copied + // output file will be the same as for the input file, with nothing dropped. + // This has never been enforced, and, withthe EDAlias feature, it may no longer + // work by accident. + // So, for now, we do not enable fast cloning of the non-product branches. + /* + canFastCloneAux_ = (whyNotFastClonable_ == FileBlock::CanFastClone) && + fb.fileFormatVersion().noMetaDataTrees() && + !om_->hasNewlyDroppedBranch()[InEvent] && + !fb.hasNewlyDroppedBranch()[InEvent] && + om_->dropMetaData() == RNTupleTempOutputModule::DropNone && + !reg->anyProductProduced() && + !fb.modifiedIDs() && + fb.branchListIndexesUnchanged(); + */ + + // Report the fast copying status. + Service reportSvc; + reportSvc->reportFastCopyingStatus(reportToken_, fb.fileName(), whyNotFastClonable_ == FileBlock::CanFastClone); + } else { + whyNotFastClonable_ |= FileBlock::NoRootInputSource; + } + + eventTree_.maybeFastCloneTree( + whyNotFastClonable_ == FileBlock::CanFastClone, canFastCloneAux_, fb.tree(), om_->basketOrder()); + + // Possibly issue warning or informational message if we haven't fast cloned. + if (fb.tree() != nullptr && whyNotFastClonable_ != FileBlock::CanFastClone) { + maybeIssueWarning(whyNotFastClonable_, fb.fileName(), file_); + } + + if (om_->compactEventAuxiliary() && + (whyNotFastClonable_ & (FileBlock::EventsOrLumisSelectedByID | FileBlock::InitialEventsSkipped | + FileBlock::EventSelectionUsed)) == 0) { + long long int reserve = remainingEvents; + if (fb.tree() != nullptr) { + reserve = reserve > 0 ? std::min(fb.tree()->GetEntries(), reserve) : fb.tree()->GetEntries(); + } + if (reserve > 0) { + compactEventAuxiliary_.reserve(compactEventAuxiliary_.size() + reserve); + } + } + } + + void RootOutputFile::respondToCloseInputFile(FileBlock const&) { + // We can't do setEntries() on the event tree if the EventAuxiliary branch is empty & disabled + if (not om_->compactEventAuxiliary()) { + eventTree_.setEntries(); + } + lumiTree_.setEntries(); + runTree_.setEntries(); + } + + bool RootOutputFile::shouldWeCloseFile() const { + unsigned int const oneK = 1024; + Long64_t size = filePtr_->GetSize() / oneK; + return (size >= om_->maxFileSize()); + } + + void RootOutputFile::writeOne(EventForOutput const& e) { + // Auxiliary branch + pEventAux_ = &e.eventAuxiliary(); + + // Because getting the data may cause an exception to be thrown we want to do that + // first before writing anything to the file about this event + // NOTE: pEventAux_, pBranchListIndexes_, pEventSelectionIDs_, and pEventEntryInfoVector_ + // must be set before calling fillBranches since they get written out in that routine. + assert(pEventAux_->processHistoryID() == e.processHistoryID()); + pBranchListIndexes_ = &e.branchListIndexes(); + pEventToProcessBlockIndexes_ = &e.eventToProcessBlockIndexes(); + + // Note: The EventSelectionIDVector should have a one to one correspondence with the processes in the process history. + // Therefore, a new entry should be added if and only if the current process has been added to the process history, + // which is done if and only if there is a produced product. + EventSelectionIDVector esids = e.eventSelectionIDs(); + if (e.productRegistry().anyProductProduced() || !om_->wantAllEvents()) { + esids.push_back(om_->selectorConfig()); + } + pEventSelectionIDs_ = &esids; + ProductProvenanceRetriever const* provRetriever = e.productProvenanceRetrieverPtr(); + assert(provRetriever); + unsigned int ttreeIndex = InEvent; + fillBranches(InEvent, e, ttreeIndex, pEventEntryInfoVector_, provRetriever); + + // Add the dataType to the job report if it hasn't already been done + if (!dataTypeReported_) { + Service reportSvc; + std::string dataType("MC"); + if (pEventAux_->isRealData()) + dataType = "Data"; + reportSvc->reportDataType(reportToken_, dataType); + dataTypeReported_ = true; + } + + // Store the process history. + processHistoryRegistry_.registerProcessHistory(e.processHistory()); + // Store the reduced ID in the IndexIntoFile + ProcessHistoryID reducedPHID = processHistoryRegistry_.reducedProcessHistoryID(e.processHistoryID()); + // Add event to index + indexIntoFile_.addEntry( + reducedPHID, pEventAux_->run(), pEventAux_->luminosityBlock(), pEventAux_->event(), eventEntryNumber_); + ++eventEntryNumber_; + + if (om_->compactEventAuxiliary()) { + compactEventAuxiliary_.push_back(*pEventAux_); + } + + // Report event written + Service reportSvc; + reportSvc->eventWrittenToFile(reportToken_, e.id().run(), e.id().event()); + ++nEventsInLumi_; + } + + void RootOutputFile::writeLuminosityBlock(LuminosityBlockForOutput const& lb) { + // Auxiliary branch + // NOTE: lumiAux_ must be filled before calling fillBranches since it gets written out in that routine. + lumiAux_ = lb.luminosityBlockAuxiliary(); + // Use the updated process historyID + lumiAux_.setProcessHistoryID(lb.processHistoryID()); + // Store the process history. + processHistoryRegistry_.registerProcessHistory(lb.processHistory()); + // Store the reduced ID in the IndexIntoFile + ProcessHistoryID reducedPHID = processHistoryRegistry_.reducedProcessHistoryID(lb.processHistoryID()); + // Add lumi to index. + indexIntoFile_.addEntry(reducedPHID, lumiAux_.run(), lumiAux_.luminosityBlock(), 0U, lumiEntryNumber_); + ++lumiEntryNumber_; + unsigned int ttreeIndex = InLumi; + fillBranches(InLumi, lb, ttreeIndex); + lumiTree_.optimizeBaskets(10ULL * 1024 * 1024); + + Service reportSvc; + reportSvc->reportLumiSection(reportToken_, lb.id().run(), lb.id().luminosityBlock(), nEventsInLumi_); + nEventsInLumi_ = 0; + } + + void RootOutputFile::writeRun(RunForOutput const& r) { + // Auxiliary branch + // NOTE: runAux_ must be filled before calling fillBranches since it gets written out in that routine. + runAux_ = r.runAuxiliary(); + // Use the updated process historyID + runAux_.setProcessHistoryID(r.processHistoryID()); + // Store the process history. + processHistoryRegistry_.registerProcessHistory(r.processHistory()); + // Store the reduced ID in the IndexIntoFile + ProcessHistoryID reducedPHID = processHistoryRegistry_.reducedProcessHistoryID(r.processHistoryID()); + // Add run to index. + indexIntoFile_.addEntry(reducedPHID, runAux_.run(), 0U, 0U, runEntryNumber_); + r.mergeableRunProductMetadata()->addEntryToStoredMetadata(storedMergeableRunProductMetadata_); + ++runEntryNumber_; + unsigned int ttreeIndex = InRun; + fillBranches(InRun, r, ttreeIndex); + runTree_.optimizeBaskets(10ULL * 1024 * 1024); + + Service reportSvc; + reportSvc->reportRunNumber(reportToken_, r.run()); + } + + void RootOutputFile::writeProcessBlock(ProcessBlockForOutput const& pb) { + std::string const& processName = pb.processName(); + std::vector const& processesWithProcessBlockProducts = + om_->outputProcessBlockHelper().processesWithProcessBlockProducts(); + std::vector::const_iterator it = + std::find(processesWithProcessBlockProducts.cbegin(), processesWithProcessBlockProducts.cend(), processName); + if (it == processesWithProcessBlockProducts.cend()) { + return; + } + unsigned int ttreeIndex = InProcess + std::distance(processesWithProcessBlockProducts.cbegin(), it); + fillBranches(InProcess, pb, ttreeIndex); + treePointers_[ttreeIndex]->optimizeBaskets(10ULL * 1024 * 1024); + } + + void RootOutputFile::writeParentageRegistry() { + Parentage const* desc(nullptr); + + if (!parentageTree_->Branch(poolNames::parentageBranchName().c_str(), &desc, om_->basketSize(), 0)) + throw Exception(errors::FatalRootError) << "Failed to create a branch for Parentages in the output file"; + + ParentageRegistry& ptReg = *ParentageRegistry::instance(); + + std::vector orderedIDs(parentageIDs_.size()); + for (auto const& parentageID : parentageIDs_) { + orderedIDs[parentageID.second] = parentageID.first; + } + //now put them into the TTree in the correct order + for (auto const& orderedID : orderedIDs) { + desc = ptReg.getMapped(orderedID); + //NOTE: some old format files have missing Parentage info + // so a null value of desc can't be fatal. + // Root will default construct an object in that case. + parentageTree_->Fill(); + } + } + + void RootOutputFile::writeFileFormatVersion() { + FileFormatVersion fileFormatVersion(getFileFormatVersion()); + FileFormatVersion* pFileFmtVsn = &fileFormatVersion; + TBranch* b = + metaDataTree_->Branch(poolNames::fileFormatVersionBranchName().c_str(), &pFileFmtVsn, om_->basketSize(), 0); + assert(b); + b->Fill(); + } + + void RootOutputFile::writeFileIdentifier() { + FileID* fidPtr = &fid_; + TBranch* b = metaDataTree_->Branch(poolNames::fileIdentifierBranchName().c_str(), &fidPtr, om_->basketSize(), 0); + assert(b); + b->Fill(); + } + + void RootOutputFile::writeIndexIntoFile() { + if (eventTree_.checkEntriesInReadBranches(eventEntryNumber_) == false) { + Exception ex(errors::OtherCMS); + ex << "The number of entries in at least one output TBranch whose entries\n" + "were copied from the input does not match the number of events\n" + "recorded in IndexIntoFile. This might (or might not) indicate a\n" + "problem related to fast copy."; + ex.addContext("Calling RootOutputFile::writeIndexIntoFile"); + throw ex; + } + indexIntoFile_.sortVector_Run_Or_Lumi_Entries(); + IndexIntoFile* iifPtr = &indexIntoFile_; + TBranch* b = metaDataTree_->Branch(poolNames::indexIntoFileBranchName().c_str(), &iifPtr, om_->basketSize(), 0); + assert(b); + b->Fill(); + } + + void RootOutputFile::writeStoredMergeableRunProductMetadata() { + storedMergeableRunProductMetadata_.optimizeBeforeWrite(); + StoredMergeableRunProductMetadata* ptr = &storedMergeableRunProductMetadata_; + TBranch* b = + metaDataTree_->Branch(poolNames::mergeableRunProductMetadataBranchName().c_str(), &ptr, om_->basketSize(), 0); + assert(b); + b->Fill(); + } + + void RootOutputFile::writeProcessHistoryRegistry() { + fillProcessHistoryBranch(metaDataTree_.get(), om_->basketSize(), processHistoryRegistry_); + } + + void RootOutputFile::writeBranchIDListRegistry() { + BranchIDLists const* p = om_->branchIDLists(); + TBranch* b = metaDataTree_->Branch(poolNames::branchIDListBranchName().c_str(), &p, om_->basketSize(), 0); + assert(b); + b->Fill(); + } + + void RootOutputFile::writeThinnedAssociationsHelper() { + ThinnedAssociationsHelper const* p = om_->thinnedAssociationsHelper(); + TBranch* b = + metaDataTree_->Branch(poolNames::thinnedAssociationsHelperBranchName().c_str(), &p, om_->basketSize(), 0); + assert(b); + b->Fill(); + } + + void RootOutputFile::writeParameterSetRegistry() { + auto model = ROOT::RNTupleModel::CreateBare(); + { + auto field = + ROOT::RFieldBase::Create("IdToParameterSetsBlobs", "std::pair,edm::ParameterSetBlob>").Unwrap(); + model->AddField(std::move(field)); + } + auto writeOptions = ROOT::RNTupleWriteOptions(); + //writeOptions.SetCompression(convert(iConfig.compressionAlgo), iConfig.compressionLevel); + auto parameterSets = ROOT::RNTupleWriter::Append(std::move(model), poolNames::parameterSetsTreeName(), *filePtr_, writeOptions); + + std::pair idToBlob; + + auto rentry = parameterSets->CreateEntry(); + rentry->BindRawPtr("IdToParameterSetsBlobs", static_cast(&idToBlob)); + + for (auto const& pset : *pset::Registry::instance()) { + idToBlob.first = pset.first; + idToBlob.second.pset() = pset.second.toString(); + + parameterSets->Fill(*rentry); + } + } + + void RootOutputFile::writeProductDescriptionRegistry(ProductRegistry const& iReg) { + // Make a local copy of the ProductRegistry, removing any transient or pruned products. + using ProductList = ProductRegistry::ProductList; + ProductRegistry pReg(iReg.productList()); + ProductList& pList = const_cast(pReg.productList()); + for (auto const& prod : pList) { + if (prod.second.branchID() != prod.second.originalBranchID()) { + if (branchesWithStoredHistory_.find(prod.second.branchID()) != branchesWithStoredHistory_.end()) { + branchesWithStoredHistory_.insert(prod.second.originalBranchID()); + } + } + } + std::set::iterator end = branchesWithStoredHistory_.end(); + for (ProductList::iterator it = pList.begin(); it != pList.end();) { + if (branchesWithStoredHistory_.find(it->second.branchID()) == end) { + // avoid invalidating iterator on deletion + ProductList::iterator itCopy = it; + ++it; + pList.erase(itCopy); + + } else { + ++it; + } + } + + ProductRegistry* ppReg = &pReg; + TBranch* b = metaDataTree_->Branch(poolNames::productDescriptionBranchName().c_str(), &ppReg, om_->basketSize(), 0); + assert(b); + b->Fill(); + } + void RootOutputFile::writeProductDependencies() { + ProductDependencies& pDeps = const_cast(om_->productDependencies()); + ProductDependencies* ppDeps = &pDeps; + TBranch* b = + metaDataTree_->Branch(poolNames::productDependenciesBranchName().c_str(), &ppDeps, om_->basketSize(), 0); + assert(b); + b->Fill(); + } + + // For duplicate removal and to determine if fast cloning is possible, the input + // module by default reads the entire EventAuxiliary branch when it opens the + // input files. If EventAuxiliary is written in the usual way, this results + // in many small reads scattered throughout the file, which can have very poor + // performance characteristics on some filesystems. As a workaround, we save + // EventAuxiliary and write it at the end of the file. + + void RootOutputFile::writeEventAuxiliary() { + constexpr std::size_t maxEaBasketSize = 4 * 1024 * 1024; + + if (om_->compactEventAuxiliary()) { + auto tree = eventTree_.tree(); + auto const& bname = BranchTypeToAuxiliaryBranchName(InEvent).c_str(); + + tree->SetBranchStatus(bname, true); + auto basketsize = + std::min(maxEaBasketSize, + compactEventAuxiliary_.size() * (sizeof(EventAuxiliary) + 26)); // 26 is an empirical fudge factor + tree->SetBasketSize(bname, basketsize); + auto b = tree->GetBranch(bname); + + assert(b); + + LogDebug("writeEventAuxiliary") << "EventAuxiliary ratio extras/GUIDs/all = " + << compactEventAuxiliary_.extrasSize() << "/" + << compactEventAuxiliary_.guidsSize() << "/" << compactEventAuxiliary_.size(); + + for (auto const& aux : compactEventAuxiliary_) { + const auto ea = aux.eventAuxiliary(); + pEventAux_ = &ea; + // Fill EventAuxiliary branch + b->Fill(); + } + eventTree_.setEntries(); + } + } + + void RootOutputFile::writeProcessBlockHelper() { + if (!om_->outputProcessBlockHelper().processesWithProcessBlockProducts().empty()) { + StoredProcessBlockHelper storedProcessBlockHelper( + om_->outputProcessBlockHelper().processesWithProcessBlockProducts()); + om_->outputProcessBlockHelper().fillCacheIndices(storedProcessBlockHelper); + + StoredProcessBlockHelper* pStoredProcessBlockHelper = &storedProcessBlockHelper; + TBranch* b = metaDataTree_->Branch( + poolNames::processBlockHelperBranchName().c_str(), &pStoredProcessBlockHelper, om_->basketSize(), 0); + assert(b); + b->Fill(); + } + } + + void RootOutputFile::finishEndFile() { + std::string_view status = "beginning"; + std::string_view value = ""; + try { + metaDataTree_->SetEntries(-1); + status = "writeTTree() for metadata"; + RootOutputRNTuple::writeTTree(metaDataTree_); + + status = "writeTTree() for parentage"; + RootOutputRNTuple::writeTTree(parentageTree_); + + // Create branch aliases for all the branches in the + // events/lumis/runs/processblock trees. The loop is over + // all types of data products. + status = "writeTree() for "; + for (unsigned int i = 0; i < treePointers_.size(); ++i) { + std::string processName; + BranchType branchType = InProcess; + if (i < InProcess) { + branchType = static_cast(i); + } else { + processName = om_->outputProcessBlockHelper().processesWithProcessBlockProducts()[i - InProcess]; + } + setBranchAliases(treePointers_[i]->tree(), om_->keptProducts()[branchType], processName); + value = treePointers_[i]->tree()->GetName(); + treePointers_[i]->writeTree(); + } + + // close the file -- mfp + // Just to play it safe, zero all pointers to objects in the TFile to be closed. + status = "closing TTrees"; + value = ""; + metaDataTree_ = parentageTree_ = nullptr; + for (auto& treePointer : treePointers_) { + treePointer->close(); + treePointer = nullptr; + } + status = "closing TFile"; + filePtr_->Close(); + filePtr_ = nullptr; // propagate_const has no reset() function + + // report that file has been closed + status = "reporting to JobReport"; + Service reportSvc; + reportSvc->outputFileClosed(reportToken_); + } catch (cms::Exception& e) { + e.addContext("Calling RootOutputFile::finishEndFile() while closing " + file_); + e.addAdditionalInfo("While calling " + std::string(status) + std::string(value)); + throw; + } + } + + void RootOutputFile::setBranchAliases(TTree* tree, + SelectedProducts const& branches, + std::string const& processName) const { + if (tree && tree->GetNbranches() != 0) { + auto const& aliasForBranches = om_->aliasForBranches(); + for (auto const& selection : branches) { + ProductDescription const& pd = *selection.first; + if (pd.branchType() == InProcess && processName != pd.processName()) { + continue; + } + std::string const& full = pd.branchName() + "obj"; + bool matched = false; + for (auto const& matcher : aliasForBranches) { + if (matcher.match(pd.branchName())) { + tree->SetAlias(matcher.alias_.c_str(), full.c_str()); + matched = true; + } + } + if (not matched and pd.branchAliases().empty()) { + std::string const& alias = (pd.productInstanceName().empty() ? pd.moduleLabel() : pd.productInstanceName()); + tree->SetAlias(alias.c_str(), full.c_str()); + } else { + for (auto const& alias : pd.branchAliases()) { + tree->SetAlias(alias.c_str(), full.c_str()); + } + } + } + } + } + + void RootOutputFile::insertAncestors(ProductProvenance const& iGetParents, + ProductProvenanceRetriever const* iMapper, + bool produced, + std::set const& iProducedIDs, + std::set& oToFill) { + assert(om_->dropMetaData() != RNTupleTempOutputModule::DropAll); + assert(produced || om_->dropMetaData() != RNTupleTempOutputModule::DropPrior); + if (om_->dropMetaData() == RNTupleTempOutputModule::DropDroppedPrior && !produced) + return; + std::vector const& parentIDs = iGetParents.parentage().parents(); + for (auto const& parentID : parentIDs) { + branchesWithStoredHistory_.insert(parentID); + ProductProvenance const* info = iMapper->branchIDToProvenance(parentID); + if (info) { + if (om_->dropMetaData() == RNTupleTempOutputModule::DropNone || + (iProducedIDs.end() != iProducedIDs.find(info->branchID()))) { + if (insertProductProvenance(*info, oToFill)) { + //haven't seen this one yet + insertAncestors(*info, iMapper, produced, iProducedIDs, oToFill); + } + } + } + } + } + + void RootOutputFile::fillBranches(BranchType const& branchType, + OccurrenceForOutput const& occurrence, + unsigned int ttreeIndex, + StoredProductProvenanceVector* productProvenanceVecPtr, + ProductProvenanceRetriever const* provRetriever) { + std::vector > dummies; + + OutputItemList& items = om_->selectedOutputItemList()[ttreeIndex]; + + bool const doProvenance = + (productProvenanceVecPtr != nullptr) && (om_->dropMetaData() != RNTupleTempOutputModule::DropAll); + bool const keepProvenanceForPrior = doProvenance && om_->dropMetaData() != RNTupleTempOutputModule::DropPrior; + + bool const fastCloning = (branchType == InEvent) && (whyNotFastClonable_ == FileBlock::CanFastClone); + std::set provenanceToKeep; + // + //If we are dropping some of the meta data we need to know + // which BranchIDs were produced in this process because + // we may be storing meta data for only those products + // We do this only for event products. + std::set producedBranches; + if (doProvenance && branchType == InEvent && om_->dropMetaData() != RNTupleTempOutputModule::DropNone) { + for (auto bd : occurrence.productRegistry().allProductDescriptions()) { + if (bd->produced() && bd->branchType() == InEvent) { + producedBranches.insert(bd->branchID()); + } + } + } + + // Loop over EDProduct branches, possibly fill the provenance, and write the branch. + for (auto& item : items) { + BranchID const& id = item.productDescription()->branchID(); + branchesWithStoredHistory_.insert(id); + + bool produced = item.productDescription()->produced(); + bool getProd = + (produced || !fastCloning || treePointers_[ttreeIndex]->uncloned(item.productDescription()->branchName())); + bool keepProvenance = doProvenance && (produced || keepProvenanceForPrior); + + WrapperBase const* product = nullptr; + ProductProvenance const* productProvenance = nullptr; + if (getProd) { + BasicHandle result = occurrence.getByToken(item.token(), item.productDescription()->unwrappedTypeID()); + product = result.wrapper(); + if (result.isValid() && keepProvenance) { + productProvenance = result.provenance()->productProvenance(); + } + if (product == nullptr) { + // No product with this ID is in the event. + // Add a null product. + TClass* cp = item.productDescription()->wrappedType().getClass(); + assert(cp != nullptr); + int offset = cp->GetBaseClassOffset(wrapperBaseTClass_); + void* p = cp->New(); + std::unique_ptr dummy = getWrapperBasePtr(p, offset); + product = dummy.get(); + dummies.emplace_back(std::move(dummy)); + } + item.setProduct(product); + } + if (keepProvenance && productProvenance == nullptr) { + productProvenance = provRetriever->branchIDToProvenance(item.productDescription()->originalBranchID()); + } + if (productProvenance) { + insertProductProvenance(*productProvenance, provenanceToKeep); + insertAncestors(*productProvenance, provRetriever, produced, producedBranches, provenanceToKeep); + } + } + + if (doProvenance) + productProvenanceVecPtr->assign(provenanceToKeep.begin(), provenanceToKeep.end()); + treePointers_[ttreeIndex]->fillTree(); + if (doProvenance) + productProvenanceVecPtr->clear(); + } + + bool RootOutputFile::insertProductProvenance(const edm::ProductProvenance& iProv, + std::set& oToInsert) { + StoredProductProvenance toStore; + toStore.branchID_ = iProv.branchID().id(); + std::set::iterator itFound = oToInsert.find(toStore); + if (itFound == oToInsert.end()) { + //get the index to the ParentageID or insert a new value if not already present + std::pair::iterator, bool> i = + parentageIDs_.insert(std::make_pair(iProv.parentageID(), static_cast(parentageIDs_.size()))); + toStore.parentageIDIndex_ = i.first->second; + if (toStore.parentageIDIndex_ >= parentageIDs_.size()) { + throw edm::Exception(errors::LogicError) + << "RootOutputFile::insertProductProvenance\n" + << "The parentage ID index value " << toStore.parentageIDIndex_ + << " is out of bounds. The maximum value is currently " << parentageIDs_.size() - 1 << ".\n" + << "This should never happen.\n" + << "Please report this to the framework developers."; + } + + oToInsert.insert(toStore); + return true; + } + return false; + } +} // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempOutput/src/RootOutputFile.h b/FWIO/RNTupleTempOutput/src/RootOutputFile.h new file mode 100644 index 0000000000000..ec7e5650a451b --- /dev/null +++ b/FWIO/RNTupleTempOutput/src/RootOutputFile.h @@ -0,0 +1,154 @@ +#ifndef IOPool_Output_RootOutputFile_h +#define IOPool_Output_RootOutputFile_h + +////////////////////////////////////////////////////////////////////// +// +// Class RootOutputFile +// +// Current Author: Bill Tanenbaum +// +////////////////////////////////////////////////////////////////////// + +#include +#include +#include +#include + +#include + +#include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/MessageLogger/interface/JobReport.h" +#include "FWCore/Utilities/interface/BranchType.h" +#include "FWCore/Utilities/interface/get_underlying_safe.h" +#include "FWCore/Utilities/interface/propagate_const.h" +#include "DataFormats/Provenance/interface/BranchListIndex.h" +#include "DataFormats/Provenance/interface/EventSelectionID.h" +#include "DataFormats/Provenance/interface/FileID.h" +#include "DataFormats/Provenance/interface/IndexIntoFile.h" +#include "DataFormats/Provenance/interface/LuminosityBlockAuxiliary.h" +#include "DataFormats/Provenance/interface/ParentageID.h" +#include "DataFormats/Provenance/interface/ProcessHistoryRegistry.h" +#include "DataFormats/Provenance/interface/ProductProvenance.h" +#include "DataFormats/Provenance/interface/StoredProductProvenance.h" +#include "DataFormats/Provenance/interface/StoredMergeableRunProductMetadata.h" +#include "DataFormats/Provenance/interface/RunAuxiliary.h" +#include "DataFormats/Provenance/interface/SelectedProducts.h" +#include "DataFormats/Provenance/interface/CompactEventAuxiliaryVector.h" +#include "FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h" +#include "FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h" + +class TTree; +class TFile; +class TClass; + +namespace edm { + class OccurrenceForOutput; +} +namespace edm::rntuple_temp { + class RNTupleTempOutputModule; + + class RootOutputFile { + public: + using OutputItem = RNTupleTempOutputModule::OutputItem; + using OutputItemList = RNTupleTempOutputModule::OutputItemList; + explicit RootOutputFile(RNTupleTempOutputModule* om, + std::string const& fileName, + std::string const& logicalFileName, + std::vector const& processesWithSelectedMergeableRunProducts, + std::string const& overrideGUID); + ~RootOutputFile() {} + void writeOne(EventForOutput const& e); + //void endFile(); + void writeLuminosityBlock(LuminosityBlockForOutput const& lb); + void writeRun(RunForOutput const& r); + void writeProcessBlock(ProcessBlockForOutput const&); + void writeFileFormatVersion(); + void writeFileIdentifier(); + void writeIndexIntoFile(); + void writeStoredMergeableRunProductMetadata(); + void writeProcessHistoryRegistry(); + void writeParameterSetRegistry(); + void writeProductDescriptionRegistry(ProductRegistry const&); + void writeParentageRegistry(); + void writeBranchIDListRegistry(); + void writeThinnedAssociationsHelper(); + void writeProductDependencies(); + void writeEventAuxiliary(); + void writeProcessBlockHelper(); + + void finishEndFile(); + void beginInputFile(FileBlock const& fb, int remainingEvents); + void respondToCloseInputFile(FileBlock const& fb); + bool shouldWeCloseFile() const; + + std::string const& fileName() const { return file_; } + + private: + void setBranchAliases(TTree* tree, SelectedProducts const& branches, std::string const& processName) const; + + void fillBranches(BranchType const& branchType, + OccurrenceForOutput const& occurrence, + unsigned int ttreeIndex, + StoredProductProvenanceVector* productProvenanceVecPtr = nullptr, + ProductProvenanceRetriever const* provRetriever = nullptr); + + void insertAncestors(ProductProvenance const& iGetParents, + ProductProvenanceRetriever const* iMapper, + bool produced, + std::set const& producedBranches, + std::set& oToFill); + + bool insertProductProvenance(const ProductProvenance&, std::set& oToInsert); + + std::shared_ptr filePtr() const { return get_underlying_safe(filePtr_); } + std::shared_ptr& filePtr() { return get_underlying_safe(filePtr_); } + StoredProductProvenanceVector const* pEventEntryInfoVector() const { + return get_underlying_safe(pEventEntryInfoVector_); + } + StoredProductProvenanceVector*& pEventEntryInfoVector() { return get_underlying_safe(pEventEntryInfoVector_); } + + //------------------------------- + // Member data + + std::string file_; + std::string logicalFile_; + JobReport::Token reportToken_; + edm::propagate_const om_; + int whyNotFastClonable_; + bool canFastCloneAux_; + edm::propagate_const> filePtr_; + FileID fid_; + IndexIntoFile::EntryNumber_t eventEntryNumber_; + IndexIntoFile::EntryNumber_t lumiEntryNumber_; + IndexIntoFile::EntryNumber_t runEntryNumber_; + IndexIntoFile indexIntoFile_; + StoredMergeableRunProductMetadata storedMergeableRunProductMetadata_; + unsigned long nEventsInLumi_; + edm::propagate_const metaDataTree_; + edm::propagate_const parentageTree_; + LuminosityBlockAuxiliary lumiAux_; + RunAuxiliary runAux_; + EventAuxiliary const* pEventAux_; + LuminosityBlockAuxiliary const* pLumiAux_; + RunAuxiliary const* pRunAux_; + StoredProductProvenanceVector eventEntryInfoVector_; + edm::propagate_const pEventEntryInfoVector_; + BranchListIndexes const* pBranchListIndexes_; + EventToProcessBlockIndexes const* pEventToProcessBlockIndexes_; + EventSelectionIDVector const* pEventSelectionIDs_; + RootOutputRNTuple eventTree_; + RootOutputRNTuple lumiTree_; + RootOutputRNTuple runTree_; + std::vector>> processBlockTrees_; + std::vector> treePointers_; + bool dataTypeReported_; + ProcessHistoryRegistry processHistoryRegistry_; + std::map parentageIDs_; + std::set branchesWithStoredHistory_; + edm::propagate_const wrapperBaseTClass_; + CompactEventAuxiliaryVector compactEventAuxiliary_; + }; + +} // namespace edm::rntuple_temp + +#endif diff --git a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc new file mode 100644 index 0000000000000..4cff2e979c5ac --- /dev/null +++ b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc @@ -0,0 +1,408 @@ + +#include "RootOutputRNTuple.h" + +#include "DataFormats/Common/interface/RefCoreStreamer.h" +#include "DataFormats/Provenance/interface/ProductDescription.h" +#include "FWCore/AbstractServices/interface/RootHandlers.h" +#include "FWCore/MessageLogger/interface/JobReport.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "FWCore/ServiceRegistry/interface/Service.h" +#include "FWCore/Utilities/interface/Algorithms.h" +#include "FWCore/Utilities/interface/EDMException.h" +#include "FWCore/Catalog/interface/SiteLocalConfig.h" +#include "FWCore/ServiceRegistry/interface/Service.h" + +#include "TBranch.h" +#include "TBranchElement.h" +#include "TCollection.h" +#include "TFile.h" +#include "TTreeCloner.h" +#include "Rtypes.h" +#include "RVersion.h" + +#include + +#include "oneapi/tbb/task_arena.h" + +namespace edm { + + /** + * Currently, ROOT doesn't use any latency-hiding optimizations for + * fast-cloning. This causes a significant slowdown when doing fast-cloning + * over a high-latency network (30ms latency makes this multiple factors slower). + * + * Accordingly, we allow sites to provide a separate hint on how to treat fast- + * cloning. The DuplicateTreeSentry allows us to implement it - given a tree + * we are about to clone, with the appropriate configs, this will re-open the + * file with lazy-download and re-open the tree. The new tree is appropriate + * for cloning. When the object is destroyed, the new file and tree are cleaned up. + * + */ + class DuplicateTreeSentry { + public: + DuplicateTreeSentry(TTree* tree) : tree_(tree) { dup(); } + DuplicateTreeSentry(DuplicateTreeSentry const&) = delete; // Disallow copying and moving + DuplicateTreeSentry& operator=(DuplicateTreeSentry const&) = delete; + + TTree* tree() const { return mytree_ ? mytree_.get() : tree_; } + + private: + struct CloseBeforeDelete { + void operator()(TFile* iFile) const { + if (iFile) { + iFile->Close(); + } + delete iFile; + } + }; + + void dup() { + edm::Service pSLC; + if (!pSLC.isAvailable()) { + return; + } + if (pSLC->sourceCacheHint() && *(pSLC->sourceCacheHint()) == "lazy-download") { + return; + } + if (!pSLC->sourceCloneCacheHint() || *(pSLC->sourceCloneCacheHint()) != "lazy-download") { + return; + } + edm::LogWarning("DuplicateTreeSentry") << "Re-opening file for fast-cloning"; + + TFile* file = tree_->GetCurrentFile(); + const TUrl* url = file->GetEndpointUrl(); + if (!url) { + return; + } + file_.reset(TFile::Open(url->GetUrl(), "READWRAP")); // May throw an exception. + if (!file_) { + return; + } + mytree_.reset(dynamic_cast(file_->Get(tree_->GetName()))); + if (!mytree_) { + return; + } + } + + /** + * Note this relies on the implicit delete ordering - mytree_ (if non-null) + * must be deleted before file_. Do not reorder the class members! + */ + std::unique_ptr file_; + TTree* tree_ = nullptr; + std::unique_ptr mytree_ = nullptr; + }; + + RootOutputRNTuple::RootOutputRNTuple(std::shared_ptr filePtr, + BranchType const& branchType, + int splitLevel, + int treeMaxVirtualSize, + std::string const& processName) + : filePtr_(filePtr), + tree_(processName.empty() + ? makeTTree(filePtr.get(), BranchTypeToProductTreeName(branchType), splitLevel) + : makeTTree(filePtr.get(), BranchTypeToProductTreeName(branchType, processName), splitLevel)), + producedBranches_(), + readBranches_(), + auxBranches_(), + unclonedReadBranches_(), + clonedReadBranchNames_(), + currentlyFastCloning_(), + fastCloneAuxBranches_(false) { + if (treeMaxVirtualSize >= 0) + tree_->SetMaxVirtualSize(treeMaxVirtualSize); + } + + TTree* RootOutputRNTuple::assignTTree(TFile* filePtr, TTree* tree) { + tree->SetDirectory(filePtr); + // Turn off autosaving because it is such a memory hog and we are not using + // this check-pointing feature anyway. + tree->SetAutoSave(std::numeric_limits::max()); + return tree; + } + + TTree* RootOutputRNTuple::makeTTree(TFile* filePtr, std::string const& name, int splitLevel) { + TTree* tree = new TTree(name.c_str(), "", splitLevel); + if (!tree) + throw edm::Exception(errors::FatalRootError) << "Failed to create the tree: " << name << "\n"; + if (tree->IsZombie()) + throw edm::Exception(errors::FatalRootError) << "Tree: " << name << " is a zombie." + << "\n"; + + return assignTTree(filePtr, tree); + } + + bool RootOutputRNTuple::checkSplitLevelsAndBasketSizes(TTree* inputTree) const { + assert(inputTree != nullptr); + + // Do the split level and basket size match in the input and output? + for (auto const& outputBranch : readBranches_) { + if (outputBranch != nullptr) { + TBranch* inputBranch = inputTree->GetBranch(outputBranch->GetName()); + + if (inputBranch != nullptr) { + if (inputBranch->GetSplitLevel() != outputBranch->GetSplitLevel() || + inputBranch->GetBasketSize() != outputBranch->GetBasketSize()) { + return false; + } + } + } + } + return true; + } + + namespace { + bool checkMatchingBranches(TBranchElement* inputBranch, TBranchElement* outputBranch) { + if (inputBranch->GetStreamerType() != outputBranch->GetStreamerType()) { + return false; + } + TObjArray* inputArray = inputBranch->GetListOfBranches(); + TObjArray* outputArray = outputBranch->GetListOfBranches(); + + if (outputArray->GetSize() < inputArray->GetSize()) { + return false; + } + TIter iter(outputArray); + TObject* obj = nullptr; + while ((obj = iter.Next()) != nullptr) { + TBranchElement* outBranch = dynamic_cast(obj); + if (outBranch) { + TBranchElement* inBranch = dynamic_cast(inputArray->FindObject(outBranch->GetName())); + if (!inBranch) { + return false; + } + if (!checkMatchingBranches(inBranch, outBranch)) { + return false; + } + } + } + return true; + } + } // namespace + + bool RootOutputRNTuple::checkIfFastClonable(TTree* inputTree) const { + if (inputTree == nullptr) + return false; + + // Do the sub-branches match in the input and output. Extra sub-branches in the input are OK for fast cloning, but not in the output. + for (auto const& outputBr : readBranches_) { + TBranchElement* outputBranch = dynamic_cast(outputBr); + if (outputBranch != nullptr) { + TBranchElement* inputBranch = dynamic_cast(inputTree->GetBranch(outputBranch->GetName())); + if (inputBranch != nullptr) { + // We have a matching top level branch. Do the recursive check on subbranches. + if (!checkMatchingBranches(inputBranch, outputBranch)) { + LogInfo("FastCloning") << "Fast Cloning disabled because a data member has been added to split branch: " + << inputBranch->GetName() << "\n."; + return false; + } + } + } + } + return true; + } + + namespace { + void setMatchingBranchSizes(TBranchElement* inputBranch, TBranchElement* outputBranch) { + if (inputBranch->GetStreamerType() != outputBranch->GetStreamerType()) { + return; + } + TObjArray* inputArray = inputBranch->GetListOfBranches(); + TObjArray* outputArray = outputBranch->GetListOfBranches(); + + if (outputArray->GetSize() < inputArray->GetSize()) { + return; + } + TIter iter(outputArray); + TObject* obj = nullptr; + while ((obj = iter.Next()) != nullptr) { + TBranchElement* outBranch = dynamic_cast(obj); + if (outBranch) { + TBranchElement* inBranch = dynamic_cast(inputArray->FindObject(outBranch->GetName())); + if (inBranch) { + outBranch->SetBasketSize(inBranch->GetBasketSize()); + setMatchingBranchSizes(inBranch, outBranch); + } + } + } + } + } // namespace + + void RootOutputRNTuple::setSubBranchBasketSizes(TTree* inputTree) const { + if (inputTree == nullptr) + return; + + for (auto const& outputBr : readBranches_) { + TBranchElement* outputBranch = dynamic_cast(outputBr); + if (outputBranch != nullptr) { + TBranchElement* inputBranch = dynamic_cast(inputTree->GetBranch(outputBranch->GetName())); + if (inputBranch != nullptr) { + // We have a matching top level branch. Do the recursion on the subbranches. + setMatchingBranchSizes(inputBranch, outputBranch); + } + } + } + } + + bool RootOutputRNTuple::checkEntriesInReadBranches(Long64_t expectedNumberOfEntries) const { + for (auto const& readBranch : readBranches_) { + if (readBranch->GetEntries() != expectedNumberOfEntries) { + return false; + } + } + return true; + } + + void RootOutputRNTuple::fastCloneTTree(TTree* in, std::string const& option) { + if (in->GetEntries() != 0) { + TObjArray* branches = tree_->GetListOfBranches(); + // If any products were produced (not just event products), the EventAuxiliary will be modified. + // In that case, don't fast copy auxiliary branches. Remove them, and add back after fast copying. + std::map auxIndexes; + bool mustRemoveSomeAuxs = false; + if (!fastCloneAuxBranches_) { + for (auto const& auxBranch : auxBranches_) { + int auxIndex = branches->IndexOf(auxBranch); + assert(auxIndex >= 0); + auxIndexes.insert(std::make_pair(auxIndex, auxBranch)); + branches->RemoveAt(auxIndex); + } + mustRemoveSomeAuxs = true; + } + + //Deal with any aux branches which can never be cloned + for (auto const& auxBranch : unclonedAuxBranches_) { + int auxIndex = branches->IndexOf(auxBranch); + assert(auxIndex >= 0); + auxIndexes.insert(std::make_pair(auxIndex, auxBranch)); + branches->RemoveAt(auxIndex); + mustRemoveSomeAuxs = true; + } + + if (mustRemoveSomeAuxs) { + branches->Compress(); + } + + DuplicateTreeSentry dupTree(in); + TTreeCloner cloner( + dupTree.tree(), tree_, option.c_str(), TTreeCloner::kNoWarnings | TTreeCloner::kIgnoreMissingTopLevel); + + if (!cloner.IsValid()) { + // Let's check why + static const char* okerror = "One of the export branch"; + if (strncmp(cloner.GetWarning(), okerror, strlen(okerror)) == 0) { + // That's fine we will handle it; + } else { + throw edm::Exception(errors::FatalRootError) << "invalid TTreeCloner (" << cloner.GetWarning() << ")\n"; + } + } + tree_->SetEntries(tree_->GetEntries() + in->GetEntries()); + Service rootHandler; + rootHandler->ignoreWarningsWhileDoing([&cloner] { cloner.Exec(); }); + + if (mustRemoveSomeAuxs) { + for (auto const& auxIndex : auxIndexes) { + // Add the auxiliary branches back after fast copying the rest of the tree. + Int_t last = branches->GetLast(); + if (last >= 0) { + branches->AddAtAndExpand(branches->At(last), last + 1); + for (Int_t ind = last - 1; ind >= auxIndex.first; --ind) { + branches->AddAt(branches->At(ind), ind + 1); + }; + branches->AddAt(auxIndex.second, auxIndex.first); + } else { + branches->Add(auxIndex.second); + } + } + } + } + } + + void RootOutputRNTuple::writeTTree(TTree* tree) { + if (tree->GetNbranches() != 0) { + // This is required when Fill is called on individual branches + // in the TTree instead of calling Fill once for the entire TTree. + tree->SetEntries(-1); + } + tree->AutoSave("FlushBaskets"); + } + + void RootOutputRNTuple::fillTTree(std::vector const& branches) { + for_all(branches, std::bind(&TBranch::Fill, std::placeholders::_1)); + } + + void RootOutputRNTuple::writeTree() { writeTTree(tree()); } + + void RootOutputRNTuple::maybeFastCloneTree(bool canFastClone, + bool canFastCloneAux, + TTree* tree, + std::string const& option) { + unclonedReadBranches_.clear(); + clonedReadBranchNames_.clear(); + currentlyFastCloning_ = canFastClone && !readBranches_.empty(); + if (currentlyFastCloning_) { + fastCloneAuxBranches_ = canFastCloneAux; + fastCloneTTree(tree, option); + for (auto const& branch : readBranches_) { + if (branch->GetEntries() == tree_->GetEntries()) { + clonedReadBranchNames_.insert(std::string(branch->GetName())); + } else { + unclonedReadBranches_.push_back(branch); + } + } + Service reportSvc; + reportSvc->reportFastClonedBranches(clonedReadBranchNames_, tree_->GetEntries()); + } + } + + void RootOutputRNTuple::fillTree() { + if (currentlyFastCloning_) { + if (!fastCloneAuxBranches_) + fillTTree(auxBranches_); + fillTTree(unclonedAuxBranches_); + fillTTree(producedBranches_); + fillTTree(unclonedReadBranches_); + } else { + // Isolate the fill operation so that IMT doesn't grab other large tasks + // that could lead to RNTupleTempOutputModule stalling + oneapi::tbb::this_task_arena::isolate([&] { tree_->Fill(); }); + } + } + + void RootOutputRNTuple::addBranch(std::string const& branchName, + std::string const& className, + void const*& pProd, + int splitLevel, + int basketSize, + bool produced) { + assert(splitLevel != ProductDescription::invalidSplitLevel); + assert(basketSize != ProductDescription::invalidBasketSize); + TBranch* branch = tree_->Branch(branchName.c_str(), className.c_str(), &pProd, basketSize, splitLevel); + assert(branch != nullptr); + /* + if(pProd != nullptr) { + // Delete the product that ROOT has allocated. + WrapperBase const* edp = static_cast(pProd); + delete edp; + pProd = nullptr; + } +*/ + if (produced) { + producedBranches_.push_back(branch); + } else { + readBranches_.push_back(branch); + } + } + + void RootOutputRNTuple::close() { + // The TFile was just closed. + // Just to play it safe, zero all pointers to quantities in the file. + auxBranches_.clear(); + unclonedAuxBranches_.clear(); + producedBranches_.clear(); + readBranches_.clear(); + unclonedReadBranches_.clear(); + tree_ = nullptr; // propagate_const has no reset() function + filePtr_ = nullptr; // propagate_const has no reset() function + } +} // namespace edm diff --git a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h new file mode 100644 index 0000000000000..ad8a77609012f --- /dev/null +++ b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h @@ -0,0 +1,124 @@ +#ifndef IOPool_Output_RootOutputRNTuple_h +#define IOPool_Output_RootOutputRNTuple_h + +/*---------------------------------------------------------------------- + +RootOutputRNTuple.h // used by ROOT output modules + +----------------------------------------------------------------------*/ + +#include +#include +#include +#include + +#include "FWCore/Utilities/interface/BranchType.h" +#include "FWCore/Utilities/interface/propagate_const.h" + +#include "TTree.h" + +class TFile; +class TBranch; + +namespace edm { + class RootOutputRNTuple { + public: + RootOutputRNTuple(std::shared_ptr filePtr, + BranchType const& branchType, + int splitLevel, + int treeMaxVirtualSize, + std::string const& processName = std::string()); + + ~RootOutputRNTuple() {} + + RootOutputRNTuple(RootOutputRNTuple const&) = delete; // Disallow copying and moving + RootOutputRNTuple& operator=(RootOutputRNTuple const&) = delete; // Disallow copying and moving + + template + void addAuxiliary(std::string const& branchName, T const*& pAux, int bufSize, bool allowCloning = true) { + if (allowCloning) { + auxBranches_.push_back(tree_->Branch(branchName.c_str(), &pAux, bufSize, 0)); + } else { + unclonedAuxBranches_.push_back(tree_->Branch(branchName.c_str(), &pAux, bufSize, 0)); + } + } + + template + void addAuxiliary(std::string const& branchName, T*& pAux, int bufSize, bool allowCloning = true) { + if (allowCloning) { + auxBranches_.push_back(tree_->Branch(branchName.c_str(), &pAux, bufSize, 0)); + } else { + unclonedAuxBranches_.push_back(tree_->Branch(branchName.c_str(), &pAux, bufSize, 0)); + } + } + + void fastCloneTTree(TTree* in, std::string const& option); + + static TTree* makeTTree(TFile* filePtr, std::string const& name, int splitLevel); + + static TTree* assignTTree(TFile* file, TTree* tree); + + static void writeTTree(TTree* tree); + + bool isValid() const; + + void addBranch(std::string const& branchName, + std::string const& className, + void const*& pProd, + int splitLevel, + int basketSize, + bool produced); + + bool checkSplitLevelsAndBasketSizes(TTree* inputTree) const; + + bool checkIfFastClonable(TTree* inputTree) const; + + void setSubBranchBasketSizes(TTree* inputTree) const; + + bool checkEntriesInReadBranches(Long64_t expectedNumberOfEntries) const; + + void maybeFastCloneTree(bool canFastClone, bool canFastCloneAux, TTree* tree, std::string const& option); + + void fillTree(); + + void writeTree(); + + TTree const* tree() const { return tree_.get(); } + + TTree* tree() { return tree_.get(); } + + void setEntries() { + if (tree_->GetNbranches() != 0) + tree_->SetEntries(-1); + } + + bool uncloned(std::string const& branchName) const { + return clonedReadBranchNames_.find(branchName) == clonedReadBranchNames_.end(); + } + + void close(); + + void optimizeBaskets(ULong64_t size) { tree_->OptimizeBaskets(size); } + + void setAutoFlush(Long64_t size) { tree_->SetAutoFlush(size); } + + private: + static void fillTTree(std::vector const& branches); + // We use bare pointers for pointers to some ROOT entities. + // Root owns them and uses bare pointers internally. + // Therefore, using smart pointers here will do no good. + edm::propagate_const> filePtr_; + edm::propagate_const tree_; + + std::vector producedBranches_; // does not include cloned branches + std::vector readBranches_; + std::vector auxBranches_; + std::vector unclonedAuxBranches_; + std::vector unclonedReadBranches_; + + std::set clonedReadBranchNames_; + bool currentlyFastCloning_; + bool fastCloneAuxBranches_; + }; +} // namespace edm +#endif diff --git a/FWIO/RNTupleTempOutput/src/TimeoutRNTupleTempOutputModule.cc b/FWIO/RNTupleTempOutput/src/TimeoutRNTupleTempOutputModule.cc new file mode 100644 index 0000000000000..5bf79a11ca3d9 --- /dev/null +++ b/FWIO/RNTupleTempOutput/src/TimeoutRNTupleTempOutputModule.cc @@ -0,0 +1,55 @@ +#include "FWIO/RNTupleTempOutput/interface/TimeoutRNTupleTempOutputModule.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" + +namespace edm::rntuple_temp { + + void TimeoutRNTupleTempOutputModule::write(EventForOutput const& e) { + eventsWrittenInCurrentFile++; + RNTupleTempOutputModule::write(e); + } + + TimeoutRNTupleTempOutputModule::TimeoutRNTupleTempOutputModule(ParameterSet const& ps) + : edm::one::OutputModuleBase::OutputModuleBase(ps), + RNTupleTempOutputModule(ps), + m_lastEvent(time(nullptr)), + eventsWrittenInCurrentFile(0), + m_timeout(-1) // we want the first event right away + {} + + void TimeoutRNTupleTempOutputModule::fillDescriptions(ConfigurationDescriptions& descriptions) { + ParameterSetDescription desc; + RNTupleTempOutputModule::fillDescription(desc); + descriptions.add("TimeoutRNTupleTempOutputModule", desc); + } + + bool TimeoutRNTupleTempOutputModule::shouldWeCloseFile() const { + time_t now(time(nullptr)); + if (RNTupleTempOutputModule::shouldWeCloseFile()) { + edm::LogVerbatim("TimeoutRNTupleTempOutputModule") + << " Closing file " << currentFileName() << " with " << eventsWrittenInCurrentFile << " events."; + eventsWrittenInCurrentFile = 0; + m_lastEvent = now; + return true; + } + // std::cout <<" Events "<< eventsWrittenInCurrentFile<<" time "<< now - m_lastEvent< + diff --git a/FWIO/RNTupleTempOutput/test/PoolDropRead_cfg.py b/FWIO/RNTupleTempOutput/test/PoolDropRead_cfg.py new file mode 100644 index 0000000000000..2555a64aca50e --- /dev/null +++ b/FWIO/RNTupleTempOutput/test/PoolDropRead_cfg.py @@ -0,0 +1,14 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTDROPREAD") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:PoolDropTest.root') +) + + + diff --git a/FWIO/RNTupleTempOutput/test/PoolDropTest_cfg.py b/FWIO/RNTupleTempOutput/test/PoolDropTest_cfg.py new file mode 100644 index 0000000000000..05d73dd308db3 --- /dev/null +++ b/FWIO/RNTupleTempOutput/test/PoolDropTest_cfg.py @@ -0,0 +1,24 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTDROP") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(20) +) +process.Thing = cms.EDProducer("ThingProducer") + +process.OtherThing = cms.EDProducer("OtherThingProducer") + +process.output = cms.OutputModule("RNTupleTempOutputModule", + outputCommands = cms.untracked.vstring('drop *', + 'keep *_dummy_*_*'), + fileName = cms.untracked.string('file:PoolDropTest.root') +) + +process.source = cms.Source("EmptySource") + +process.p = cms.Path(process.Thing*process.OtherThing) +process.ep = cms.EndPath(process.output) + + diff --git a/FWIO/RNTupleTempOutput/test/PoolMissingRead_cfg.py b/FWIO/RNTupleTempOutput/test/PoolMissingRead_cfg.py new file mode 100644 index 0000000000000..c01d8dadab749 --- /dev/null +++ b/FWIO/RNTupleTempOutput/test/PoolMissingRead_cfg.py @@ -0,0 +1,14 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTDROPREAD") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:PoolMissingTest.root') +) + + + diff --git a/FWIO/RNTupleTempOutput/test/PoolMissingTest_cfg.py b/FWIO/RNTupleTempOutput/test/PoolMissingTest_cfg.py new file mode 100644 index 0000000000000..a9830dd0f0b54 --- /dev/null +++ b/FWIO/RNTupleTempOutput/test/PoolMissingTest_cfg.py @@ -0,0 +1,22 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTMISSING") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(20) +) +process.Thing = cms.EDProducer("ThingProducer", + noPut = cms.untracked.bool(True) +) + +process.output = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('file:PoolMissingTest.root') +) + +process.source = cms.Source("EmptySource") + +process.p = cms.Path(process.Thing) +process.ep = cms.EndPath(process.output) + + diff --git a/FWIO/RNTupleTempOutput/test/PoolOutputAliasTestCheckResults.py b/FWIO/RNTupleTempOutput/test/PoolOutputAliasTestCheckResults.py new file mode 100644 index 0000000000000..8e9b50e093ba9 --- /dev/null +++ b/FWIO/RNTupleTempOutput/test/PoolOutputAliasTestCheckResults.py @@ -0,0 +1,8 @@ +import ROOT + +file = ROOT.TFile.Open("alias.root") +events = file.Get("Events") + +b = events.GetAlias("foo") +if not b: + raise RuntimeError("foo missing") diff --git a/FWIO/RNTupleTempOutput/test/PoolOutputAliasTest_cfg.py b/FWIO/RNTupleTempOutput/test/PoolOutputAliasTest_cfg.py new file mode 100644 index 0000000000000..edbc2923aee18 --- /dev/null +++ b/FWIO/RNTupleTempOutput/test/PoolOutputAliasTest_cfg.py @@ -0,0 +1,19 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") +process.maxEvents.input = 10 + +from FWCore.Modules.modules import EmptySource + +process.source = EmptySource() + +from FWCore.Integration.modules import ThingProducer + +process.thing = ThingProducer() + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule + +process.out = RNTupleTempOutputModule(fileName = "alias.root", + branchAliases = [cms.PSet(branch=cms.untracked.string("*_thing_*_*"), alias = cms.untracked.string("foo"))]) + +process.e = cms.EndPath(process.out, cms.Task(process.thing)) diff --git a/FWIO/RNTupleTempOutput/test/PoolOutputEmptyEventsTest_cfg.py b/FWIO/RNTupleTempOutput/test/PoolOutputEmptyEventsTest_cfg.py new file mode 100644 index 0000000000000..bf790aa5f64c3 --- /dev/null +++ b/FWIO/RNTupleTempOutput/test/PoolOutputEmptyEventsTest_cfg.py @@ -0,0 +1,26 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTOUTPUT") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(20) +) +process.Thing = cms.EDProducer("ThingProducer") + +process.OtherThing = cms.EDProducer("OtherThingProducer") + +process.output = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('file:PoolOutputEmptyEventsTest.root'), + SelectEvents = cms.untracked.PSet(SelectEvents=cms.vstring('p')) +) + +process.filter = cms.EDFilter("Prescaler", + prescaleFactor=cms.int32(100), + prescaleOffset=cms.int32(0) ) +process.source = cms.Source("EmptySource") + +process.p = cms.Path(process.filter+process.Thing*process.OtherThing) +process.ep = cms.EndPath(process.output) + + diff --git a/FWIO/RNTupleTempOutput/test/PoolOutputMergeWithEmptyFile_cfg.py b/FWIO/RNTupleTempOutput/test/PoolOutputMergeWithEmptyFile_cfg.py new file mode 100644 index 0000000000000..f6a96f6dc8be1 --- /dev/null +++ b/FWIO/RNTupleTempOutput/test/PoolOutputMergeWithEmptyFile_cfg.py @@ -0,0 +1,13 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.output = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('file:PoolOutputEmptyEventsMergeTest.root') +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring("file:PoolOutputEmptyEventsTest.root", + "file:PoolOutputTest.root")) +process.ep = cms.EndPath(process.output) diff --git a/FWIO/RNTupleTempOutput/test/PoolOutputRead_cfg.py b/FWIO/RNTupleTempOutput/test/PoolOutputRead_cfg.py new file mode 100644 index 0000000000000..f69908e637fff --- /dev/null +++ b/FWIO/RNTupleTempOutput/test/PoolOutputRead_cfg.py @@ -0,0 +1,14 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTOUTPUTREAD") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:PoolOutputTest.root') +) + + + diff --git a/FWIO/RNTupleTempOutput/test/PoolOutputTestOverrideGUID_cfg.py b/FWIO/RNTupleTempOutput/test/PoolOutputTestOverrideGUID_cfg.py new file mode 100644 index 0000000000000..5178955ca8ea9 --- /dev/null +++ b/FWIO/RNTupleTempOutput/test/PoolOutputTestOverrideGUID_cfg.py @@ -0,0 +1,38 @@ +import FWCore.ParameterSet.Config as cms + +import argparse +import sys +parser = argparse.ArgumentParser(prog=sys.argv[0], description="Test RNTupleTempOutputModule") +parser.add_argument("--guid", type=str, help="GUID that overrides") +parser.add_argument("--maxSize", type=int, default=None, help="Set maximum file size") +parser.add_argument("--input", type=str, default=[], nargs="*", help="Optional list of input files") + +args = parser.parse_args() + +process = cms.Process("TESTOUTPUTGUID") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents.input = 20 + +process.Thing = cms.EDProducer("ThingProducer") + +process.OtherThing = cms.EDProducer("OtherThingProducer") + +process.source = cms.Source("EmptySource") +if len(args.input) > 0: + process.maxEvents.input = -1 + process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring("file:"+x for x in args.input) + ) + +process.output = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('file:PoolOutputTestOverrideGUID.root'), + overrideGUID = cms.untracked.string(args.guid), +) +if args.maxSize is not None: + process.output.maxSize = cms.untracked.int32(args.maxSize) + +process.p = cms.Path(process.Thing*process.OtherThing) +process.ep = cms.EndPath(process.output) + + diff --git a/FWIO/RNTupleTempOutput/test/PoolOutputTestUnscheduledRead_cfg.py b/FWIO/RNTupleTempOutput/test/PoolOutputTestUnscheduledRead_cfg.py new file mode 100644 index 0000000000000..a9cfb9ac7cee6 --- /dev/null +++ b/FWIO/RNTupleTempOutput/test/PoolOutputTestUnscheduledRead_cfg.py @@ -0,0 +1,52 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTOUTPUTREAD") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) + +process.analyzeOther = cms.EDAnalyzer("OtherThingAnalyzer") + +# Test that the Run and Lumi products that we expect to be +# produced in the previous process are really there. +# It gets the products and checks that they contain +# the expected values. +# This tests that beginRun,beginLumi,endLumi, endRun +# get called for EDProducers with Unscheduled turned on. +process.test = cms.EDAnalyzer("TestMergeResults", + expectedBeginRunNew = cms.untracked.vint32( + 10001, 10002, 10003 # end run 1 + ), + + expectedEndRunNew = cms.untracked.vint32( + 100001, 100002, 100003 # end run 1 + ), + + expectedBeginLumiNew = cms.untracked.vint32( + 101, 102, 103 # end run 1 lumi 1 + ), + + expectedEndLumiNew = cms.untracked.vint32( + 1001, 1002, 1003 # end run 1 lumi 1 + ) +) + +process.getInt = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( + cms.InputTag("aliasForInt2"), + ), + expectedSum = cms.untracked.int32(220) +) + +process.tst = cms.Path(process.analyzeOther+process.test) + +process.path1 = cms.Path(process.getInt) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:PoolOutputTestUnscheduled.root') +) + + + diff --git a/FWIO/RNTupleTempOutput/test/PoolOutputTestUnscheduled_cfg.py b/FWIO/RNTupleTempOutput/test/PoolOutputTestUnscheduled_cfg.py new file mode 100644 index 0000000000000..1c01727596efc --- /dev/null +++ b/FWIO/RNTupleTempOutput/test/PoolOutputTestUnscheduled_cfg.py @@ -0,0 +1,60 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTOUTPUT") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(20) +) +process.Thing = cms.EDProducer("ThingProducer") + +process.OtherThing = cms.EDProducer("OtherThingProducer") + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") + +process.intProducer1 = cms.EDProducer("IntProducer", + ivalue = cms.int32(7) +) + +process.intProducer2 = cms.EDProducer("IntProducer", + ivalue = cms.int32(11) +) + +process.aliasForInt1 = cms.EDAlias( + intProducer1 = cms.VPSet( + cms.PSet(type = cms.string('edmtestIntProduct')) + ) +) + +process.aliasForInt2 = cms.EDAlias( + intProducer2 = cms.VPSet( + cms.PSet(type = cms.string('edmtestIntProduct')) + ) +) + +process.output = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('file:PoolOutputTestUnscheduled.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducer1_*_*', + 'drop *_aliasForInt1_*_*', + 'drop *_intProducer2_*_*' + ) +) + +process.getInt = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( + cms.InputTag("aliasForInt1"), + ), + expectedSum = cms.untracked.int32(140) +) + +process.source = cms.Source("EmptySource") + +process.t = cms.Task(process.Thing, process.OtherThing, process.thingWithMergeProducer, process.intProducer1, process.intProducer2) + +process.path1 = cms.Path(process.getInt, process.t) + +process.ep = cms.EndPath(process.output) + + diff --git a/FWIO/RNTupleTempOutput/test/PoolOutputTest_cfg.py b/FWIO/RNTupleTempOutput/test/PoolOutputTest_cfg.py new file mode 100644 index 0000000000000..9bde6836faea5 --- /dev/null +++ b/FWIO/RNTupleTempOutput/test/PoolOutputTest_cfg.py @@ -0,0 +1,31 @@ +import FWCore.ParameterSet.Config as cms + +import argparse +import sys +parser = argparse.ArgumentParser(prog=sys.argv[0], description="Test RNTupleTempOutputModule") +parser.add_argument("--firstLumi", type=int, default=None, help="Set first lumi to process ") + +args = parser.parse_args() + +process = cms.Process("TESTOUTPUT") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents.input = 20 + +process.Thing = cms.EDProducer("ThingProducer") + +process.OtherThing = cms.EDProducer("OtherThingProducer") + +process.output = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('file:PoolOutputTest.root') +) + +process.source = cms.Source("EmptySource") +if args.firstLumi is not None: + process.source.firstLuminosityBlock = cms.untracked.uint32(args.firstLumi) + process.output.fileName = "file:PoolOutputTestLumi{}.root".format(args.firstLumi) + +process.p = cms.Path(process.Thing*process.OtherThing) +process.ep = cms.EndPath(process.output) + + diff --git a/FWIO/RNTupleTempOutput/test/PoolTransientRead_cfg.py b/FWIO/RNTupleTempOutput/test/PoolTransientRead_cfg.py new file mode 100644 index 0000000000000..964a15d4b9a4f --- /dev/null +++ b/FWIO/RNTupleTempOutput/test/PoolTransientRead_cfg.py @@ -0,0 +1,14 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTTRANSIENTREAD") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:PoolTransientTest.root') +) + + + diff --git a/FWIO/RNTupleTempOutput/test/PoolTransientTest_cfg.py b/FWIO/RNTupleTempOutput/test/PoolTransientTest_cfg.py new file mode 100644 index 0000000000000..d29bf5cebe7d6 --- /dev/null +++ b/FWIO/RNTupleTempOutput/test/PoolTransientTest_cfg.py @@ -0,0 +1,25 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTTRANSIENT") +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(20) +) +process.TransientThing = cms.EDProducer("TransientIntProducer", + ivalue = cms.int32(1) +) + +process.ThingFromTransient = cms.EDProducer("IntProducerFromTransient", +) + +process.output = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('file:PoolTransientTest.root') +) + +process.source = cms.Source("EmptySource") + +process.p = cms.Path(process.TransientThing*process.ThingFromTransient) +process.ep = cms.EndPath(process.output) + + diff --git a/FWIO/RNTupleTempOutput/test/RNTupleTempOutputAliasTest.sh b/FWIO/RNTupleTempOutput/test/RNTupleTempOutputAliasTest.sh new file mode 100755 index 0000000000000..b39f8c0375fc0 --- /dev/null +++ b/FWIO/RNTupleTempOutput/test/RNTupleTempOutputAliasTest.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Pass in name and status +function die { echo $1: status $2 ; exit $2; } + +LOCAL_TEST_DIR=$SCRAM_TEST_PATH + +cmsRun ${LOCAL_TEST_DIR}/PoolOutputAliasTest_cfg.py || die 'Failure using PoolOutputAliasTest_cfg.py 1' $? +python3 ${LOCAL_TEST_DIR}/PoolOutputAliasTestCheckResults.py || die 'results check failed' $? \ No newline at end of file diff --git a/FWIO/RNTupleTempOutput/test/TestProvA_cfg.py b/FWIO/RNTupleTempOutput/test/TestProvA_cfg.py new file mode 100644 index 0000000000000..76beb5c1351e6 --- /dev/null +++ b/FWIO/RNTupleTempOutput/test/TestProvA_cfg.py @@ -0,0 +1,10 @@ +import FWCore.ParameterSet.Config as cms +process = cms.Process("A") +process.source = cms.Source("EmptySource") +process.thing = cms.EDProducer("ThingProducer") +process.other = cms.EDProducer("OtherThingProducer", thingTag=cms.InputTag("thing")) +process.p = cms.Path(process.thing*process.other) +process.out = cms.OutputModule("RNTupleTempOutputModule", fileName=cms.untracked.string("a.root")) +process.test = cms.OutputModule("ProvenanceCheckerOutputModule") +process.o = cms.EndPath(process.test+process.out) +process.maxEvents = cms.untracked.PSet(input=cms.untracked.int32(10)) diff --git a/FWIO/RNTupleTempOutput/test/TestProvB_cfg.py b/FWIO/RNTupleTempOutput/test/TestProvB_cfg.py new file mode 100644 index 0000000000000..bb9c80f9105b5 --- /dev/null +++ b/FWIO/RNTupleTempOutput/test/TestProvB_cfg.py @@ -0,0 +1,10 @@ +import FWCore.ParameterSet.Config as cms +process = cms.Process("B") +process.source = cms.Source("RNTupleTempSource", fileNames=cms.untracked.vstring("file:a.root")) +process.other = cms.EDProducer("OtherThingProducer", thingTag=cms.InputTag("thing")) +process.p = cms.Path(process.other) +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName=cms.untracked.string("b.root"), + outputCommands=cms.untracked.vstring("drop *","keep *_*_*_B")) +process.test = cms.OutputModule("ProvenanceCheckerOutputModule") +process.o = cms.EndPath(process.test+process.out) diff --git a/FWIO/RNTupleTempOutput/test/TestProvC_cfg.py b/FWIO/RNTupleTempOutput/test/TestProvC_cfg.py new file mode 100644 index 0000000000000..0f8bb57534e3c --- /dev/null +++ b/FWIO/RNTupleTempOutput/test/TestProvC_cfg.py @@ -0,0 +1,5 @@ +import FWCore.ParameterSet.Config as cms +process = cms.Process("C") +process.source = cms.Source("RNTupleTempSource", fileNames=cms.untracked.vstring("file:b.root")) +process.test = cms.OutputModule("ProvenanceCheckerOutputModule") +process.o = cms.EndPath(process.test) diff --git a/FWIO/RNTupleTempOutput/test/TestRNTupleTempOutput.sh b/FWIO/RNTupleTempOutput/test/TestRNTupleTempOutput.sh new file mode 100755 index 0000000000000..f41095960b04a --- /dev/null +++ b/FWIO/RNTupleTempOutput/test/TestRNTupleTempOutput.sh @@ -0,0 +1,76 @@ +#!/bin/sh +# Pass in name and status +function die { echo $1: status $2 ; exit $2; } + +LOCAL_TEST_DIR=$SCRAM_TEST_PATH + +cmsRun ${LOCAL_TEST_DIR}/PoolOutputTest_cfg.py || die 'Failure using PoolOutputTest_cfg.py 1' $? +GUID1=$(edmFileUtil -u PoolOutputTest.root | fgrep uuid | awk '{print $10}') +cmsRun ${LOCAL_TEST_DIR}/PoolOutputTest_cfg.py || die 'Failure using PoolOutputTest_cfg.py 2' $? +GUID2=$(edmFileUtil -u PoolOutputTest.root | fgrep uuid | awk '{print $10}') +if [ "x${GUID1}" == "x${GUID2}" ]; then + echo "GUID from two executions are the same: ${GUID1}" + exit 1 +fi + +cmsRun ${LOCAL_TEST_DIR}/PoolDropTest_cfg.py || die 'Failure using PoolDropTest_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/PoolMissingTest_cfg.py || die 'Failure using PoolMissingTest_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/PoolOutputRead_cfg.py || die 'Failure using PoolOutputRead_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/PoolDropRead_cfg.py || die 'Failure using PoolDropRead_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/PoolMissingRead_cfg.py || die 'Failure using PoolMissingRead_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/PoolTransientTest_cfg.py || die 'Failure using PoolTransientTest_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/PoolTransientRead_cfg.py || die 'Failure using PoolTransientRead_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/PoolOutputEmptyEventsTest_cfg.py || die 'Failure using PoolOutputEmptyEventsTest_cfg.py' $? +#reads file from above and from PoolOutputTest_cfg.py +cmsRun ${LOCAL_TEST_DIR}/PoolOutputMergeWithEmptyFile_cfg.py || die 'Failure using PoolOutputMergeWithEmptyFile_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/TestProvA_cfg.py || die 'Failure using TestProvA_cfg.py' $? +#reads file from above +cmsRun ${LOCAL_TEST_DIR}/TestProvB_cfg.py || die 'Failure using TestProvB_cfg.py' $? +#reads file from above +cmsRun ${LOCAL_TEST_DIR}/TestProvC_cfg.py || die 'Failure using TestProvC_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/PoolOutputTestUnscheduled_cfg.py || die 'Failure using PoolOutputTestUnscheduled_cfg.py' $? +cmsRun ${LOCAL_TEST_DIR}/PoolOutputTestUnscheduledRead_cfg.py || die 'Failure using PoolOutputTestUnscheduledRead_cfg.py' $? + +cmsRun ${LOCAL_TEST_DIR}/PoolOutputTestOverrideGUID_cfg.py --guid abcdef01-2345-6789-abcd-ef0123456789 || die 'Failure using PoolOutputTestOverrideGUID_cfg.py with valid GUID' $? +GUID=$(edmFileUtil -u PoolOutputTestOverrideGUID.root | fgrep uuid | awk '{print $10}') +if [ "x${GUID}" != "xabcdef01-2345-6789-abcd-ef0123456789" ]; then + echo "GUID in file '${GUID}' did not match 'abcdef01-2345-6789-abcd-ef0123456789'" + exit 1 +fi +cmsRun ${LOCAL_TEST_DIR}/PoolOutputTestOverrideGUID_cfg.py --guid ABCDEF01-2345-6789-abcd-ef0123456789 || die 'Failure using PoolOutputTestOverrideGUID_cfg.py with valid GUID (with some capital letteters)' $? +GUID=$(edmFileUtil -u PoolOutputTestOverrideGUID.root | fgrep uuid | awk '{print $10}') +if [ "x${GUID}" != "xABCDEF01-2345-6789-abcd-ef0123456789" ]; then + echo "GUID in file '${GUID}' did not match 'ABCDEF01-2345-6789-abcd-ef0123456789'" + exit 1 +fi + +cmsRun ${LOCAL_TEST_DIR}/PoolOutputTestOverrideGUID_cfg.py --guid abcdef01-2345-6789-abcd-ef01234567890 && die 'PoolOutputTestOverrideGUID_cfg.py with invalid GUID 1 did not fail' 1 +cmsRun ${LOCAL_TEST_DIR}/PoolOutputTestOverrideGUID_cfg.py --guid abcdef01-2345-6789-0abcd-ef0123456789 && die 'PoolOutputTestOverrideGUID_cfg.py with invalid GUID 2 did not fail' 1 +cmsRun ${LOCAL_TEST_DIR}/PoolOutputTestOverrideGUID_cfg.py --guid 0abcdef01-2345-6789-abcd-ef0123456789 && die 'PoolOutputTestOverrideGUID_cfg.py with invalid GUID 3 did not fail' 1 +cmsRun ${LOCAL_TEST_DIR}/PoolOutputTestOverrideGUID_cfg.py --guid abcdef012-345-6789-abcd-ef0123456789 && die 'PoolOutputTestOverrideGUID_cfg.py with invalid GUID 4 did not fail' 1 +cmsRun ${LOCAL_TEST_DIR}/PoolOutputTestOverrideGUID_cfg.py --guid abcdef01-2345-6789-abcd-ef012345678g && die 'PoolOutputTestOverrideGUID_cfg.py with invalid GUID 5 did not fail' 1 +cmsRun ${LOCAL_TEST_DIR}/PoolOutputTestOverrideGUID_cfg.py --guid abcdef01-2345-6789-abcd_ef0123456789 && die 'PoolOutputTestOverrideGUID_cfg.py with invalid GUID 6 did not fail' 1 + +cmsRun ${LOCAL_TEST_DIR}/PoolOutputTest_cfg.py --firstLumi 1 +cmsRun ${LOCAL_TEST_DIR}/PoolOutputTest_cfg.py --firstLumi 2 + +cmsRun ${LOCAL_TEST_DIR}/PoolOutputTestOverrideGUID_cfg.py --guid abcdef01-2345-6789-abcd-ef0123456789 --input PoolOutputTestLumi1.root PoolOutputTestLumi2.root --maxSize 1 || die 'Failure using PoolOutputTestOverrideGUID_cfg.py with valid GUID and two input files' $? +GUID1=$(edmFileUtil -u PoolOutputTestOverrideGUID.root | fgrep uuid | awk '{print $10}') +GUID2=$(edmFileUtil -u PoolOutputTestOverrideGUID001.root | fgrep uuid | awk '{print $10}') +if [ "x${GUID1}" != "xabcdef01-2345-6789-abcd-ef0123456789" ]; then + echo "GUID in first file '${GUID1}' did not match 'abcdef01-2345-6789-abcd-ef0123456789'" + exit 1 +fi +if [ "x${GUID1}" == "x${GUID2}" ]; then + echo "GUID from two output files are the same: ${GUID1}" + exit 1 +fi From 36aeb10542484f44e113480c5a07b93cd2c53eb4 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Fri, 19 Sep 2025 15:50:11 -0500 Subject: [PATCH 03/19] Replaced meta data TTrees with RNTuple - Also removed some backwards compatibility code --- FWIO/RNTupleTempInput/bin/BuildFile.xml | 27 + FWIO/RNTupleTempInput/bin/CollUtil.cc | 556 +++++++ FWIO/RNTupleTempInput/bin/CollUtil.h | 30 + .../bin/EdmRNTupleTempFileUtil.cpp | 324 ++++ .../bin/EdmRNTupleTempProvDump.cc | 1335 +++++++++++++++++ FWIO/RNTupleTempInput/src/InputFile.h | 6 +- FWIO/RNTupleTempInput/src/RootFile.cc | 219 +-- FWIO/RNTupleTempInput/src/RootFile.h | 6 - ...tReducedProcessHistoryHardwareResources.sh | 9 +- .../test/testReducedProcessHistoryVersion.sh | 8 +- .../interface/RNTupleTempOutputModule.h | 12 +- .../src/RNTupleTempOutputModule.cc | 29 +- FWIO/RNTupleTempOutput/src/RootOutputFile.cc | 259 ++-- FWIO/RNTupleTempOutput/src/RootOutputFile.h | 42 +- .../test/TestRNTupleTempOutput.sh | 12 +- 15 files changed, 2523 insertions(+), 351 deletions(-) create mode 100644 FWIO/RNTupleTempInput/bin/BuildFile.xml create mode 100644 FWIO/RNTupleTempInput/bin/CollUtil.cc create mode 100644 FWIO/RNTupleTempInput/bin/CollUtil.h create mode 100644 FWIO/RNTupleTempInput/bin/EdmRNTupleTempFileUtil.cpp create mode 100644 FWIO/RNTupleTempInput/bin/EdmRNTupleTempProvDump.cc diff --git a/FWIO/RNTupleTempInput/bin/BuildFile.xml b/FWIO/RNTupleTempInput/bin/BuildFile.xml new file mode 100644 index 0000000000000..18cdd94b10fb7 --- /dev/null +++ b/FWIO/RNTupleTempInput/bin/BuildFile.xml @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/FWIO/RNTupleTempInput/bin/CollUtil.cc b/FWIO/RNTupleTempInput/bin/CollUtil.cc new file mode 100644 index 0000000000000..be6e3d6fc1ae5 --- /dev/null +++ b/FWIO/RNTupleTempInput/bin/CollUtil.cc @@ -0,0 +1,556 @@ +#include "CollUtil.h" + +#include "DataFormats/Provenance/interface/BranchType.h" +#include "DataFormats/Provenance/interface/EventAuxiliary.h" +#include "DataFormats/Provenance/interface/FileFormatVersion.h" +#include "DataFormats/Provenance/interface/FileID.h" +#include "DataFormats/Provenance/interface/FileIndex.h" +#include "DataFormats/Provenance/interface/IndexIntoFile.h" + +#include "TBasket.h" +#include "TBranch.h" +#include "TFile.h" +#include "TIterator.h" +#include "TKey.h" +#include "TList.h" +#include "TObject.h" +#include "TTree.h" +#include "ROOT/RNTupleReader.hxx" + +#include +#include +#include + +namespace edm::rntuple_temp { + + // Get a file handler + TFile *openFileHdl(std::string const &fname) { + TFile *hdl = TFile::Open(fname.c_str(), "read"); + + if (nullptr == hdl) { + std::cout << "ERR Could not open file " << fname.c_str() << std::endl; + exit(1); + } + return hdl; + } + + // Print every tree in a file + void printTrees(TFile *hdl) { + hdl->ls(); + TList *keylist = hdl->GetListOfKeys(); + TIterator *iter = keylist->MakeIterator(); + TKey *key = nullptr; + while ((key = (TKey *)iter->Next())) { + TObject *obj = hdl->Get(key->GetName()); + if (obj->IsA() == TTree::Class()) { + obj->Print(); + } + } + return; + } + + // number of entries in a tree + Long64_t numEntries(TFile *hdl, std::string const &trname) { + TTree *tree = (TTree *)hdl->Get(trname.c_str()); + if (tree) { + return tree->GetEntries(); + } else { + std::cout << "ERR cannot find a TTree named \"" << trname << "\"" << std::endl; + return -1; + } + } + + namespace { + void addBranchSizes(TBranch *branch, Long64_t &size) { + size += branch->GetTotalSize(); // Includes size of branch metadata + // Now recurse through any subbranches. + Long64_t nB = branch->GetListOfBranches()->GetEntries(); + for (Long64_t i = 0; i < nB; ++i) { + TBranch *btemp = (TBranch *)branch->GetListOfBranches()->At(i); + addBranchSizes(btemp, size); + } + } + Long64_t branchCompressedSizes(TBranch *branch) { return branch->GetZipBytes("*"); } + + } // namespace + + void printBranchNames(TTree *tree) { + if (tree != nullptr) { + Long64_t nB = tree->GetListOfBranches()->GetEntries(); + for (Long64_t i = 0; i < nB; ++i) { + Long64_t size = 0LL; + TBranch *btemp = (TBranch *)tree->GetListOfBranches()->At(i); + addBranchSizes(btemp, size); + std::cout << "Branch " << i << " of " << tree->GetName() << " tree: " << btemp->GetName() + << " Total size = " << size << " Compressed size = " << branchCompressedSizes(btemp) << std::endl; + } + } else { + std::cout << "Missing Events tree?\n"; + } + } + + void longBranchPrint(TTree *tr) { + if (tr != nullptr) { + Long64_t nB = tr->GetListOfBranches()->GetEntries(); + for (Long64_t i = 0; i < nB; ++i) { + tr->GetListOfBranches()->At(i)->Print(); + } + } else { + std::cout << "Missing Events tree?\n"; + } + } + + namespace { + class BranchBasketBytes { + public: + BranchBasketBytes(TBranch const *branch) + : basketFirstEntry_(branch->GetBasketEntry()), + basketBytes_(branch->GetBasketBytes()), + branchName_(branch->GetName()), + maxBaskets_(branch->GetMaxBaskets()) {} + + bool isAlignedWithClusterBoundaries() const { return isAligned_; } + + std::string_view name() const { return branchName_; } + + // Processes "next cluster" for the branch, calculating the + // number of bytes and baskets in the cluster + // + // @param[in] clusterBegin Begin entry number for the cluster + // @param[in] clusterEnd End entry number (exclusive) for the cluster + // @param[out] nonAlignedBranches Branch name is added to the set if the basket boundary + // does not align with cluster boundary + // + // @return Tuple of the number of bytes and baskets in the cluster + std::tuple bytesInNextCluster(Long64_t clusterBegin, + Long64_t clusterEnd, + std::set &nonAlignedBranches) { + if (basketFirstEntry_[iBasket_] != clusterBegin) { + std::cout << "Branch " << branchName_ << " iBasket " << iBasket_ << " begin entry " + << basketFirstEntry_[iBasket_] << " does not align with cluster boundary, expected " << clusterBegin + << std::endl; + exit(1); + } + + Long64_t bytes = 0; + unsigned nbaskets = 0; + for (; iBasket_ < maxBaskets_ and basketFirstEntry_[iBasket_] < clusterEnd; ++iBasket_) { + bytes += basketBytes_[iBasket_]; + ++nbaskets; + } + if (basketFirstEntry_[iBasket_] != clusterEnd) { + nonAlignedBranches.insert(branchName_); + isAligned_ = false; + return std::tuple(0, 0); + } + return std::tuple(bytes, nbaskets); + } + + private: + Long64_t const *basketFirstEntry_; + Int_t const *basketBytes_; + std::string_view branchName_; + Int_t maxBaskets_; + Long64_t iBasket_ = 0; + bool isAligned_ = true; + }; + + std::vector makeBranchBasketBytes(TBranch *branch) { + std::vector ret; + + TObjArray *subBranches = branch->GetListOfBranches(); + if (subBranches and subBranches->GetEntries() > 0) { + // process sub-branches if there are any + auto const nbranches = subBranches->GetEntries(); + for (Long64_t iBranch = 0; iBranch < nbranches; ++iBranch) { + auto vec = makeBranchBasketBytes(dynamic_cast(subBranches->At(iBranch))); + ret.insert(ret.end(), std::make_move_iterator(vec.begin()), std::make_move_iterator(vec.end())); + } + } else { + ret.emplace_back(branch); + } + return ret; + } + + template + void processClusters(TTree *tr, T printer, const std::string &limitToBranch = "") { + TTree::TClusterIterator clusterIter = tr->GetClusterIterator(0); + Long64_t const nentries = tr->GetEntries(); + + // Keep the state of each branch basket index so that we don't + // have to iterate through everything on every cluster + std::vector processors; + { + TObjArray *branches = tr->GetListOfBranches(); + Long64_t const nbranches = branches->GetEntries(); + for (Long64_t iBranch = 0; iBranch < nbranches; ++iBranch) { + auto branch = dynamic_cast(branches->At(iBranch)); + if (limitToBranch.empty() or + std::string_view(branch->GetName()).find(limitToBranch) != std::string_view::npos) { + auto vec = makeBranchBasketBytes(branch); + processors.insert( + processors.end(), std::make_move_iterator(vec.begin()), std::make_move_iterator(vec.end())); + } + } + } + + printer.header(tr, processors); + // Record branches whose baskets do not align with cluster boundaries + std::set nonAlignedBranches; + { + Long64_t clusterBegin; + while ((clusterBegin = clusterIter()) < nentries) { + Long64_t clusterEnd = clusterIter.GetNextEntry(); + printer.beginCluster(clusterBegin, clusterEnd); + for (auto &p : processors) { + if (p.isAlignedWithClusterBoundaries()) { + auto const [bytes, baskets] = p.bytesInNextCluster(clusterBegin, clusterEnd, nonAlignedBranches); + printer.processBranch(bytes, baskets); + } + } + printer.endCluster(); + } + } + + if (not nonAlignedBranches.empty()) { + std::cout << "\nThe following branches had baskets whose entry boundaries did not align with the cluster " + "boundaries. Their baskets are excluded from the cluster size calculation above starting from the " + "first basket that did not align with a cluster boundary." + << std::endl; + for (auto &name : nonAlignedBranches) { + std::cout << " " << name << std::endl; + } + } + } + } // namespace + + void clusterPrint(TTree *tr) { + struct ClusterPrinter { + void header(TTree const *tr, std::vector const &branchProcessors) const { + std::cout << "Printing cluster boundaries in terms of tree entries of the tree " << tr->GetName() + << ". Note that the end boundary is exclusive." << std::endl; + std::cout << std::setw(15) << "Begin" << std::setw(15) << "End" << std::setw(15) << "Entries" << std::setw(15) + << "Max baskets" << std::setw(15) << "Bytes" << std::endl; + } + + void beginCluster(Long64_t clusterBegin, Long64_t clusterEnd) { + bytes_ = 0; + maxbaskets_ = 0; + std::cout << std::setw(15) << clusterBegin << std::setw(15) << clusterEnd << std::setw(15) + << (clusterEnd - clusterBegin); + } + + void processBranch(Long64_t bytes, unsigned int baskets) { + bytes_ += bytes; + maxbaskets_ = std::max(baskets, maxbaskets_); + } + + void endCluster() const { std::cout << std::setw(15) << maxbaskets_ << std::setw(15) << bytes_ << std::endl; } + + Long64_t bytes_ = 0; + unsigned int maxbaskets_ = 0; + }; + processClusters(tr, ClusterPrinter{}); + } + + void basketPrint(TTree *tr, const std::string &branchName) { + struct BasketPrinter { + void header(TTree const *tr, std::vector const &branchProcessors) const { + std::cout << "Printing cluster boundaries in terms of tree entries of the tree " << tr->GetName() + << ". Note that the end boundary is exclusive." << std::endl; + std::cout << "\nBranches for which number of baskets in each cluster are printed\n"; + for (int i = 0; auto const &p : branchProcessors) { + std::cout << "[" << i << "] " << p.name() << std::endl; + ++i; + } + std::cout << "\n" + << std::setw(15) << "Begin" << std::setw(15) << "End" << std::setw(15) << "Entries" << std::setw(15); + for (auto i : std::views::iota(0U, branchProcessors.size())) { + std::cout << std::setw(5) << (std::string("[") + std::to_string(i) + "]"); + } + std::cout << std::endl; + } + + void beginCluster(Long64_t clusterBegin, Long64_t clusterEnd) const { + std::cout << std::setw(15) << clusterBegin << std::setw(15) << clusterEnd << std::setw(15) + << (clusterEnd - clusterBegin); + } + + void processBranch(Long64_t bytes, unsigned int baskets) const { std::cout << std::setw(5) << baskets; } + + void endCluster() const { std::cout << std::endl; } + }; + processClusters(tr, BasketPrinter{}, branchName); + } + + std::string getUuid(ROOT::RNTupleReader *uuidTree) { + FileID fid; + auto entry = uuidTree->GetModel().CreateEntry(); + assert(entry); + entry->BindRawPtr(poolNames::fileIdentifierBranchName(), &fid); + uuidTree->LoadEntry(0, *entry); + return fid.fid(); + } + + void printUuids(ROOT::RNTupleReader *uuidTree) { std::cout << "UUID: " << getUuid(uuidTree) << std::endl; } + + static void preIndexIntoFilePrintEventLists(TFile *, + FileFormatVersion const &fileFormatVersion, + TTree *metaDataTree) { + FileIndex fileIndex; + FileIndex *findexPtr = &fileIndex; + if (metaDataTree->FindBranch(poolNames::fileIndexBranchName().c_str()) != nullptr) { + TBranch *fndx = metaDataTree->GetBranch(poolNames::fileIndexBranchName().c_str()); + fndx->SetAddress(&findexPtr); + fndx->GetEntry(0); + } else { + std::cout << "FileIndex not found. If this input file was created with release 1_8_0 or later\n" + "this indicates a problem with the file. This condition should be expected with\n" + "files created with earlier releases and printout of the event list will fail.\n"; + return; + } + + std::cout << "\n" << fileIndex; + + std::cout << "\nFileFormatVersion = " << fileFormatVersion << ". "; + if (fileFormatVersion.fastCopyPossible()) + std::cout << "This version supports fast copy\n"; + else + std::cout << "This version does not support fast copy\n"; + + if (fileIndex.allEventsInEntryOrder()) { + std::cout << "Events are sorted such that fast copy is possible in the \"noEventSort = False\" mode\n"; + } else { + std::cout << "Events are sorted such that fast copy is NOT possible in the \"noEventSort = False\" mode\n"; + } + + fileIndex.sortBy_Run_Lumi_EventEntry(); + if (fileIndex.allEventsInEntryOrder()) { + std::cout << "Events are sorted such that fast copy is possible in the \"noEventSort\" mode\n"; + } else { + std::cout << "Events are sorted such that fast copy is NOT possible in the \"noEventSort\" mode\n"; + } + std::cout << "(Note that other factors can prevent fast copy from occurring)\n\n"; + } + + static void postIndexIntoFilePrintEventLists(TFile *tfl, + FileFormatVersion const &fileFormatVersion, + TTree *metaDataTree) { + IndexIntoFile indexIntoFile; + IndexIntoFile *findexPtr = &indexIntoFile; + if (metaDataTree->FindBranch(poolNames::indexIntoFileBranchName().c_str()) != nullptr) { + TBranch *fndx = metaDataTree->GetBranch(poolNames::indexIntoFileBranchName().c_str()); + fndx->SetAddress(&findexPtr); + fndx->GetEntry(0); + } else { + std::cout << "IndexIntoFile not found. If this input file was created with release 1_8_0 or later\n" + "this indicates a problem with the file. This condition should be expected with\n" + "files created with earlier releases and printout of the event list will fail.\n"; + return; + } + //need to read event # from the EventAuxiliary branch + TTree *eventsTree = dynamic_cast(tfl->Get(poolNames::eventTreeName().c_str())); + TBranch *eventAuxBranch = nullptr; + assert(nullptr != eventsTree); + char const *const kEventAuxiliaryBranchName = "EventAuxiliary"; + if (eventsTree->FindBranch(kEventAuxiliaryBranchName) != nullptr) { + eventAuxBranch = eventsTree->GetBranch(kEventAuxiliaryBranchName); + } else { + std::cout << "Failed to find " << kEventAuxiliaryBranchName + << " branch in Events TTree. Something is wrong with this file." << std::endl; + return; + } + EventAuxiliary eventAuxiliary; + EventAuxiliary *eAPtr = &eventAuxiliary; + eventAuxBranch->SetAddress(&eAPtr); + std::cout << "\nPrinting IndexIntoFile contents. This includes a list of all Runs, LuminosityBlocks\n" + << "and Events stored in the root file.\n\n"; + std::cout << std::setw(15) << "Run" << std::setw(15) << "Lumi" << std::setw(15) << "Event" << std::setw(15) + << "TTree Entry" + << "\n"; + + for (IndexIntoFile::IndexIntoFileItr it = indexIntoFile.begin(IndexIntoFile::firstAppearanceOrder), + itEnd = indexIntoFile.end(IndexIntoFile::firstAppearanceOrder); + it != itEnd; + ++it) { + IndexIntoFile::EntryType t = it.getEntryType(); + std::cout << std::setw(15) << it.run() << std::setw(15) << it.lumi(); + EventNumber_t eventNum = 0; + std::string type; + switch (t) { + case IndexIntoFile::kRun: + type = "(Run)"; + break; + case IndexIntoFile::kLumi: + type = "(Lumi)"; + break; + case IndexIntoFile::kEvent: + eventAuxBranch->GetEntry(it.entry()); + eventNum = eventAuxiliary.id().event(); + break; + default: + break; + } + std::cout << std::setw(15) << eventNum << std::setw(15) << it.entry() << " " << type << std::endl; + } + + std::cout << "\nFileFormatVersion = " << fileFormatVersion << ". "; + if (fileFormatVersion.fastCopyPossible()) + std::cout << "This version supports fast copy\n"; + else + std::cout << "This version does not support fast copy\n"; + + if (indexIntoFile.iterationWillBeInEntryOrder(IndexIntoFile::firstAppearanceOrder)) { + std::cout << "Events are sorted such that fast copy is possible in the \"noEventSort = false\" mode\n"; + } else { + std::cout << "Events are sorted such that fast copy is NOT possible in the \"noEventSort = false\" mode\n"; + } + + // This will not work unless the other nonpersistent parts of the Index are filled first + // I did not have time to implement this yet. + // if (indexIntoFile.iterationWillBeInEntryOrder(IndexIntoFile::numericalOrder)) { + // std::cout << "Events are sorted such that fast copy is possible in the \"noEventSort\" mode\n"; + // } else { + // std::cout << "Events are sorted such that fast copy is NOT possible in the \"noEventSort\" mode\n"; + // } + std::cout << "(Note that other factors can prevent fast copy from occurring)\n\n"; + } + + void printEventLists(TFile *tfl) { + TTree *metaDataTree = dynamic_cast(tfl->Get(poolNames::metaDataTreeName().c_str())); + assert(nullptr != metaDataTree); + + FileFormatVersion fileFormatVersion; + FileFormatVersion *fftPtr = &fileFormatVersion; + if (metaDataTree->FindBranch(poolNames::fileFormatVersionBranchName().c_str()) != nullptr) { + TBranch *fft = metaDataTree->GetBranch(poolNames::fileFormatVersionBranchName().c_str()); + fft->SetAddress(&fftPtr); + fft->GetEntry(0); + } + if (fileFormatVersion.hasIndexIntoFile()) { + postIndexIntoFilePrintEventLists(tfl, fileFormatVersion, metaDataTree); + } else { + preIndexIntoFilePrintEventLists(tfl, fileFormatVersion, metaDataTree); + } + } + + static void preIndexIntoFilePrintEventsInLumis(TFile *, + FileFormatVersion const &fileFormatVersion, + TTree *metaDataTree) { + FileIndex fileIndex; + FileIndex *findexPtr = &fileIndex; + if (metaDataTree->FindBranch(poolNames::fileIndexBranchName().c_str()) != nullptr) { + TBranch *fndx = metaDataTree->GetBranch(poolNames::fileIndexBranchName().c_str()); + fndx->SetAddress(&findexPtr); + fndx->GetEntry(0); + } else { + std::cout << "FileIndex not found. If this input file was created with release 1_8_0 or later\n" + "this indicates a problem with the file. This condition should be expected with\n" + "files created with earlier releases and printout of the event list will fail.\n"; + return; + } + + std::cout << "\n" + << std::setw(15) << "Run" << std::setw(15) << "Lumi" << std::setw(15) << "# Events" + << "\n"; + unsigned long nEvents = 0; + unsigned long runID = 0; + unsigned long lumiID = 0; + for (std::vector::const_iterator it = fileIndex.begin(), itEnd = fileIndex.end(); it != itEnd; + ++it) { + if (it->getEntryType() == FileIndex::kEvent) { + ++nEvents; + } else if (it->getEntryType() == FileIndex::kLumi) { + if (runID != it->run_ || lumiID != it->lumi_) { + //print the previous one + if (lumiID != 0) { + std::cout << std::setw(15) << runID << std::setw(15) << lumiID << std::setw(15) << nEvents << "\n"; + } + nEvents = 0; + runID = it->run_; + lumiID = it->lumi_; + } + } + } + //print the last one + if (lumiID != 0) { + std::cout << std::setw(15) << runID << std::setw(15) << lumiID << std::setw(15) << nEvents << "\n"; + } + + std::cout << "\n"; + } + + static void postIndexIntoFilePrintEventsInLumis(TFile *tfl, + FileFormatVersion const &fileFormatVersion, + TTree *metaDataTree) { + IndexIntoFile indexIntoFile; + IndexIntoFile *findexPtr = &indexIntoFile; + if (metaDataTree->FindBranch(poolNames::indexIntoFileBranchName().c_str()) != nullptr) { + TBranch *fndx = metaDataTree->GetBranch(poolNames::indexIntoFileBranchName().c_str()); + fndx->SetAddress(&findexPtr); + fndx->GetEntry(0); + } else { + std::cout << "IndexIntoFile not found. If this input file was created with release 1_8_0 or later\n" + "this indicates a problem with the file. This condition should be expected with\n" + "files created with earlier releases and printout of the event list will fail.\n"; + return; + } + std::cout << "\n" + << std::setw(15) << "Run" << std::setw(15) << "Lumi" << std::setw(15) << "# Events" + << "\n"; + + unsigned long nEvents = 0; + unsigned long runID = 0; + unsigned long lumiID = 0; + + for (IndexIntoFile::IndexIntoFileItr it = indexIntoFile.begin(IndexIntoFile::firstAppearanceOrder), + itEnd = indexIntoFile.end(IndexIntoFile::firstAppearanceOrder); + it != itEnd; + ++it) { + IndexIntoFile::EntryType t = it.getEntryType(); + switch (t) { + case IndexIntoFile::kRun: + break; + case IndexIntoFile::kLumi: + if (runID != it.run() || lumiID != it.lumi()) { + //print the previous one + if (lumiID != 0) { + std::cout << std::setw(15) << runID << std::setw(15) << lumiID << std::setw(15) << nEvents << "\n"; + } + nEvents = 0; + runID = it.run(); + lumiID = it.lumi(); + } + break; + case IndexIntoFile::kEvent: + ++nEvents; + break; + default: + break; + } + } + //print the last one + if (lumiID != 0) { + std::cout << std::setw(15) << runID << std::setw(15) << lumiID << std::setw(15) << nEvents << "\n"; + } + std::cout << "\n"; + } + + void printEventsInLumis(TFile *tfl) { + TTree *metaDataTree = dynamic_cast(tfl->Get(poolNames::metaDataTreeName().c_str())); + assert(nullptr != metaDataTree); + + FileFormatVersion fileFormatVersion; + FileFormatVersion *fftPtr = &fileFormatVersion; + if (metaDataTree->FindBranch(poolNames::fileFormatVersionBranchName().c_str()) != nullptr) { + TBranch *fft = metaDataTree->GetBranch(poolNames::fileFormatVersionBranchName().c_str()); + fft->SetAddress(&fftPtr); + fft->GetEntry(0); + } + if (fileFormatVersion.hasIndexIntoFile()) { + postIndexIntoFilePrintEventsInLumis(tfl, fileFormatVersion, metaDataTree); + } else { + preIndexIntoFilePrintEventsInLumis(tfl, fileFormatVersion, metaDataTree); + } + } +} // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempInput/bin/CollUtil.h b/FWIO/RNTupleTempInput/bin/CollUtil.h new file mode 100644 index 0000000000000..34236ab39b871 --- /dev/null +++ b/FWIO/RNTupleTempInput/bin/CollUtil.h @@ -0,0 +1,30 @@ +#ifndef Modules_CollUtil_h +#define Modules_CollUtil_h + +#include "Rtypes.h" + +#include + +class TFile; +class TTree; + +namespace ROOT { + class RNTupleReader; +} + +namespace edm::rntuple_temp { + + TFile *openFileHdl(const std::string &fname); + void printTrees(TFile *hdl); + Long64_t numEntries(TFile *hdl, const std::string &trname); + void printBranchNames(TTree *tree); + void longBranchPrint(TTree *tr); + void clusterPrint(TTree *tr); + void basketPrint(TTree *tr, const std::string &branchName); + std::string getUuid(ROOT::RNTupleReader *uuidTree); + void printUuids(ROOT::RNTupleReader *uuidTree); + void printEventLists(TFile *tfl); + void printEventsInLumis(TFile *tfl); +} // namespace edm::rntuple_temp + +#endif diff --git a/FWIO/RNTupleTempInput/bin/EdmRNTupleTempFileUtil.cpp b/FWIO/RNTupleTempInput/bin/EdmRNTupleTempFileUtil.cpp new file mode 100644 index 0000000000000..3638218c39ab9 --- /dev/null +++ b/FWIO/RNTupleTempInput/bin/EdmRNTupleTempFileUtil.cpp @@ -0,0 +1,324 @@ +//---------------------------------------------------------------------- +// EdmFileUtil.cpp +// + +#include +#include +#include +#include +#include +#include +#include +#include +#include "CollUtil.h" +#include "DataFormats/Provenance/interface/BranchType.h" +#include "FWCore/Catalog/interface/InputFileCatalog.h" +#include "FWCore/Catalog/interface/SiteLocalConfig.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/PluginManager/interface/PluginManager.h" +#include "FWCore/PluginManager/interface/standard.h" +#include "FWCore/Services/interface/setupSiteLocalConfig.h" +#include "FWCore/Utilities/interface/Adler32Calculator.h" +#include "FWCore/Utilities/interface/Exception.h" +#include "FWCore/ServiceRegistry/interface/ServiceRegistry.h" + +#include "TFile.h" +#include "TError.h" +#include "ROOT/RNTuple.hxx" +#include "ROOT/RNTupleReader.hxx" + +int main(int argc, char* argv[]) { + gErrorIgnoreLevel = kError; + + // Add options here + + boost::program_options::options_description desc("Allowed options"); + desc.add_options()("help,h", "print help message")( + "file,f", boost::program_options::value >(), "data file (-f or -F required)")( + "Files,F", + boost::program_options::value(), + "text file containing names of data files, one per line")( + "catalog,c", boost::program_options::value(), "catalog")("decodeLFN,d", "Convert LFN to PFN")( + "uuid,u", "Print uuid")("adler32,a", "Print adler32 checksum.")("allowRecovery", + "Allow root to auto-recover corrupted files")( + "JSON,j", "JSON output format. Any arguments listed below are ignored")("ls,l", "list file content")( + "map,m", "Print TFile::Map(\"extended\"). The output can be HUGE.")("print,P", "Print all")( + "verbose,v", "Verbose printout")("printBranchDetails,b", "Call Print()sc for all branches")( + "printBaskets", + boost::program_options::value(), + "Print detailed information about baskets and clusters for the given branch")( + "printClusters", "Print detailed information about baskets and clusters for all branches")( + "tree,t", + boost::program_options::value(), + "Select tree used with -P, -b, --printClusters, and --printBaskets options")( + "events,e", + "Print list of all Events, Runs, and LuminosityBlocks in the file sorted by run number, luminosity block number, " + "and event number. Also prints the entry numbers and whether it is possible to use fast copy with the file.")( + "eventsInLumis", "Print how many Events are in each LuminosityBlock."); + + // What trees do we require for this to be a valid collection? + std::vector expectedTrees; + expectedTrees.push_back(edm::poolNames::metaDataTreeName()); + expectedTrees.push_back(edm::poolNames::eventTreeName()); + + boost::program_options::positional_options_description p; + p.add("file", -1); + + boost::program_options::variables_map vm; + + try { + boost::program_options::store( + boost::program_options::command_line_parser(argc, argv).options(desc).positional(p).run(), vm); + } catch (boost::program_options::error const& x) { + std::cerr << "Option parsing failure:\n" << x.what() << "\n\n"; + std::cerr << desc << "\n"; + return 1; + } + + boost::program_options::notify(vm); + + if (vm.count("help")) { + std::cout << desc << "\n"; + return 1; + } + + int rc = 0; + try { + auto operate = edm::setupSiteLocalConfig(); + + std::vector in = + (vm.count("file") ? vm["file"].as >() : std::vector()); + if (vm.count("Files")) { + std::ifstream ifile(vm["Files"].as().c_str()); + std::istream_iterator beginItr(ifile); + if (ifile.fail()) { + std::cout << "File '" << vm["Files"].as() << "' not found, not opened, or empty\n"; + return 1; + } + std::istream_iterator endItr; + copy(beginItr, endItr, back_inserter(in)); + } + if (in.empty()) { + std::cout << "Data file(s) not set.\n"; + std::cout << desc << "\n"; + return 1; + } + std::string catalogIn = (vm.count("catalog") ? vm["catalog"].as() : std::string()); + bool decodeLFN = vm.count("decodeLFN"); + bool uuid = vm.count("uuid"); + bool adler32 = vm.count("adler32"); + bool allowRecovery = vm.count("allowRecovery"); + bool json = vm.count("JSON"); + bool more = !json; + bool verbose = more && (vm.count("verbose") > 0 ? true : false); + bool events = more && (vm.count("events") > 0 ? true : false); + bool eventsInLumis = more && (vm.count("eventsInLumis") > 0 ? true : false); + bool ls = more && (vm.count("ls") > 0 ? true : false); + bool printMap = vm.count("map"); + bool tree = more && (vm.count("tree") > 0 ? true : false); + bool print = more && (vm.count("print") > 0 ? true : false); + bool printBranchDetails = more && (vm.count("printBranchDetails") > 0 ? true : false); + bool printClusters = more && (vm.count("printClusters") > 0 ? true : false); + std::string printBaskets = (vm.count("printBaskets") ? vm["printBaskets"].as() : std::string()); + bool onlyDecodeLFN = + decodeLFN && !(uuid || adler32 || allowRecovery || json || events || tree || ls || print || printBranchDetails); + std::string selectedTree = tree ? vm["tree"].as() : edm::poolNames::eventTreeName(); + + if (events || eventsInLumis) { + try { + edmplugin::PluginManager::configure(edmplugin::standard::config()); + } catch (std::exception& e) { + std::cout << "exception caught in EdmFileUtil while configuring the PluginManager\n" << e.what(); + return 1; + } + } + + edm::InputFileCatalog catalog(in, catalogIn, true); + std::vector const& filesIn = catalog.fileNames(0); + + if (json) { + std::cout << '[' << std::endl; + } + + // now run.. + // Allow user to input multiple files + for (unsigned int j = 0; j < in.size(); ++j) { + // We _only_ want the LFN->PFN conversion. No need to open the file, + // just check the catalog and move on + if (onlyDecodeLFN) { + std::cout << filesIn[j] << std::endl; + continue; + } + + // open a data file + if (!json) + std::cout << in[j] << "\n"; + std::string const& lfn = in[j]; + std::unique_ptr tfile{edm::rntuple_temp::openFileHdl(filesIn[j])}; + if (tfile == nullptr) + return 1; + + std::string const& pfn = filesIn[j]; + + if (verbose) + std::cout << "ECU:: Opened " << pfn << std::endl; + + std::string datafile = decodeLFN ? pfn : lfn; + + // First check that this file is not auto-recovered + // Stop the job unless specified to do otherwise + + bool isRecovered = tfile->TestBit(TFile::kRecovered); + if (isRecovered) { + if (allowRecovery) { + if (!json) { + std::cout << pfn << " appears not to have been closed correctly and has been autorecovered \n"; + std::cout << "Proceeding anyway\n"; + } + } else { + std::cout << pfn << " appears not to have been closed correctly and has been autorecovered \n"; + std::cout << "Stopping. Use --allowRecovery to try ignoring this\n"; + return 1; + } + } else { + if (verbose) + std::cout << "ECU:: Collection not autorecovered. Continuing\n"; + } + + // Ok. Do we have the expected trees? + for (unsigned int i = 0; i < expectedTrees.size(); ++i) { + TTree* t = (TTree*)tfile->Get(expectedTrees[i].c_str()); + if (t == nullptr) { + std::cout << "Tree " << expectedTrees[i] << " appears to be missing. Not a valid collection\n"; + std::cout << "Exiting\n"; + return 1; + } else { + if (verbose) + std::cout << "ECU:: Found Tree " << expectedTrees[i] << std::endl; + } + } + + if (verbose) + std::cout << "ECU:: Found all expected trees\n"; + + std::ostringstream auout; + if (adler32) { + unsigned int const EDMFILEUTILADLERBUFSIZE = 10 * 1024 * 1024; // 10MB buffer + static char buffer[EDMFILEUTILADLERBUFSIZE]; + size_t bufToRead = EDMFILEUTILADLERBUFSIZE; + uint32_t a = 1, b = 0; + size_t fileSize = tfile->GetSize(); + tfile->Seek(0, TFile::kBeg); + + for (size_t offset = 0; offset < fileSize; offset += EDMFILEUTILADLERBUFSIZE) { + // true on last loop + if (fileSize - offset < EDMFILEUTILADLERBUFSIZE) + bufToRead = fileSize - offset; + tfile->ReadBuffer((char*)buffer, bufToRead); + cms::Adler32(buffer, bufToRead, a, b); + } + uint32_t adler32sum = (b << 16) | a; + if (json) { + auout << ",\"adler32sum\":" << adler32sum; + } else { + auout << ", " << std::hex << adler32sum << " adler32sum"; + } + } + + if (uuid) { + auto* paramsTree = tfile->Get(edm::poolNames::metaDataTreeName().c_str()); + assert(paramsTree); + auto reader = ROOT::RNTupleReader::Open(*paramsTree); + assert(reader); + if (json) { + auout << ",\"uuid\":\"" << edm::rntuple_temp::getUuid(reader.get()) << '"'; + } else { + auout << ", " << edm::rntuple_temp::getUuid(reader.get()) << " uuid"; + } + } + + // Ok. How many events? + int nruns = edm::rntuple_temp::numEntries(tfile.get(), edm::poolNames::runTreeName()); + int nlumis = edm::rntuple_temp::numEntries(tfile.get(), edm::poolNames::luminosityBlockTreeName()); + int nevents = edm::rntuple_temp::numEntries(tfile.get(), edm::poolNames::eventTreeName()); + if (json) { + if (j > 0) + std::cout << ',' << std::endl; + std::cout << "{\"file\":\"" << datafile << '"' << ",\"runs\":" << nruns << ",\"lumis\":" << nlumis + << ",\"events\":" << nevents << ",\"bytes\":" << tfile->GetSize() << auout.str() << '}' << std::endl; + } else { + std::cout << datafile << " (" << nruns << " runs, " << nlumis << " lumis, " << nevents << " events, " + << tfile->GetSize() << " bytes" << auout.str() << ")" << std::endl; + } + + if (json) { + // Remainder of arguments not supported in JSON yet. + continue; + } + + // Look at the collection contents + if (ls) { + tfile->ls(); + } + + // Print Map() + if (printMap) { + tfile->Map("extended"); + } + + if (print or printBranchDetails or printClusters or not printBaskets.empty()) { + TTree* printTree = (TTree*)tfile->Get(selectedTree.c_str()); + if (printTree == nullptr) { + std::cout << "Tree " << selectedTree << " appears to be missing. Could not find it in the file.\n"; + std::cout << "Exiting\n"; + return 1; + } + // Print out each tree + if (print) { + edm::rntuple_temp::printBranchNames(printTree); + } + + if (printBranchDetails) { + edm::rntuple_temp::longBranchPrint(printTree); + } + + if (printClusters) { + edm::rntuple_temp::clusterPrint(printTree); + } + + if (not printBaskets.empty()) { + edm::rntuple_temp::basketPrint(printTree, printBaskets); + } + } + + // Print out event lists + if (events) { + edm::rntuple_temp::printEventLists(tfile.get()); + } + + if (eventsInLumis) { + edm::rntuple_temp::printEventsInLumis(tfile.get()); + } + + tfile->Close(); + } + if (json) { + std::cout << ']' << std::endl; + } + } catch (cms::Exception const& e) { + std::cout << "cms::Exception caught in " + << "EdmFileUtil" << '\n' + << e.explainSelf(); + rc = 1; + } catch (std::exception const& e) { + std::cout << "Standard library exception caught in " + << "EdmFileUtil" << '\n' + << e.what(); + rc = 1; + } catch (...) { + std::cout << "Unknown exception caught in " + << "EdmFileUtil"; + rc = 2; + } + return rc; +} diff --git a/FWIO/RNTupleTempInput/bin/EdmRNTupleTempProvDump.cc b/FWIO/RNTupleTempInput/bin/EdmRNTupleTempProvDump.cc new file mode 100644 index 0000000000000..0b5a187954431 --- /dev/null +++ b/FWIO/RNTupleTempInput/bin/EdmRNTupleTempProvDump.cc @@ -0,0 +1,1335 @@ +#include "DataFormats/Common/interface/setIsMergeable.h" +#include "DataFormats/Provenance/interface/BranchIDListHelper.h" +#include "DataFormats/Provenance/interface/BranchType.h" +#include "DataFormats/Provenance/interface/branchIDToProductID.h" +#include "DataFormats/Provenance/interface/EventSelectionID.h" +#include "DataFormats/Provenance/interface/History.h" +#include "DataFormats/Provenance/interface/ParameterSetBlob.h" +#include "DataFormats/Provenance/interface/ParameterSetID.h" +#include "DataFormats/Provenance/interface/ProcessHistoryRegistry.h" +#include "DataFormats/Provenance/interface/ProductRegistry.h" +#include "DataFormats/Provenance/interface/Parentage.h" +#include "DataFormats/Provenance/interface/ProductProvenance.h" +#include "DataFormats/Provenance/interface/StoredProductProvenance.h" +#include "DataFormats/Provenance/interface/ParentageRegistry.h" +#include "FWCore/Catalog/interface/InputFileCatalog.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/ParameterSet/interface/Registry.h" +#include "FWCore/Services/interface/setupSiteLocalConfig.h" + +#include "FWCore/Utilities/interface/Algorithms.h" +#include "FWCore/Utilities/interface/Exception.h" +#include "FWCore/Utilities/interface/propagate_const.h" + +#include "TError.h" +#include "TFile.h" +#include "TTree.h" + +#include "ROOT/RNTuple.hxx" +#include "ROOT/RNTupleReader.hxx" + +#include "boost/program_options.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static std::ostream& prettyPrint(std::ostream& oStream, + edm::ParameterSet const& iPSet, + std::string const& iIndent, + std::string const& iIndentDelta); + +static std::string const triggerResults = std::string("TriggerResults"); +static std::string const triggerPaths = std::string("@trigger_paths"); +static std::string const source = std::string("source"); +static std::string const input = std::string("@main_input"); + +namespace { + struct ModuleInformation { + std::set processSimpleIDs; + std::set products; + }; + // key is ParameterSetID string representation + using IdToBranches = std::map; + // key is data product's (process name, module label) + using ModuleToIdBranches = std::map, IdToBranches>; + + using ParameterSetMap = std::map; + + class HistoryNode { + public: + HistoryNode() = default; + + HistoryNode(edm::ProcessConfiguration const& iConfig, unsigned int iSimpleId) + : config_(iConfig), simpleId_(iSimpleId) {} + + edm::ParameterSetID const& parameterSetID() const { return config_.parameterSetID(); } + + std::string const& processName() const { return config_.processName(); } + + void print(std::ostream& os) const { os << config_.processName() << " [" << simpleId_ << "]"; } + + edm::ProcessConfigurationID configurationID() const { return config_.id(); } + + private: + edm::ProcessConfiguration config_; + unsigned int simpleId_ = 0; + }; + + class HistoryChain { + public: + explicit HistoryChain(edm::ProcessHistoryID id) : historyId_(id) {} + + void addChild(HistoryNode const& child) { children_.push_back(child); } + + void reserve(std::size_t s) { children_.reserve(s); } + + std::size_t size() const { return children_.size(); } + + typedef std::vector::const_iterator const_iterator; + typedef std::vector::iterator iterator; + + iterator begin() { return children_.begin(); } + iterator end() { return children_.end(); } + + const_iterator begin() const { return children_.begin(); } + const_iterator end() const { return children_.end(); } + + void printHistory(bool showHistoryID) const; + + private: + std::vector children_; + edm::ProcessHistoryID historyId_; + }; + + std::ostream& operator<<(std::ostream& os, HistoryNode const& node) { + node.print(os); + return os; + } + + std::string formatHeader(std::string_view text) { + constexpr std::string_view decoration = "---------"; + return fmt::format("{}{}{}", decoration, text, decoration); + } +} // namespace + +void HistoryChain::printHistory(bool showHistoryID) const { + constexpr std::string_view indent = " "; + std::cout << "History: {" << std::endl; + if (showHistoryID) { + std::cout << indent << "History id: " << historyId_ << std::endl; + } + for (auto const& item : children_) { + std::cout << indent << item << std::endl; + if (showHistoryID) { + std::cout << indent << indent << "Configuration id: " << item.configurationID() << std::endl; + } + } + std::cout << "}" << std::endl; +} + +std::string eventSetupComponent(char const* iType, + std::string const& iCompName, + edm::ParameterSet const& iProcessConfig, + std::string const& iProcessName) { + std::ostringstream result; + edm::ParameterSet const& pset = iProcessConfig.getParameterSet(iCompName); + std::string name(pset.getParameter("@module_label")); + if (name.empty()) { + name = pset.getParameter("@module_type"); + } + + result << iType << ": " << name << " " << iProcessName << "\n" + << " parameters: "; + prettyPrint(result, pset, " ", " "); + return result.str(); +} + +std::string nonProducerComponent(std::string const& iCompName, + edm::ParameterSet const& iProcessConfig, + std::string const& iProcessName) { + std::ostringstream result; + edm::ParameterSet const& pset = iProcessConfig.getParameterSet(iCompName); + std::string label(pset.getParameter("@module_label")); + + result << "Module: " << label << " " << iProcessName << "\n" + << " parameters: "; + prettyPrint(result, pset, " ", " "); + return result.str(); +} + +static void appendToSet(std::set& iSet, std::vector const& iFrom) { + for (auto const& n : iFrom) { + iSet.insert(n); + } +} + +static std::string topLevelPSet(std::string const& iName, + edm::ParameterSet const& iProcessConfig, + std::string const& iProcessName) { + std::ostringstream result; + edm::ParameterSet const& pset = iProcessConfig.getParameterSet(iName); + + result << "PSet: " << iName << " " << iProcessName << "\n" + << " parameters: "; + prettyPrint(result, pset, " ", " "); + return result.str(); +} + +namespace { + std::unique_ptr makeTFileWithLookup(std::string const& filename) { + // See if it is a logical file name. + auto operate = edm::setupSiteLocalConfig(); + std::string override; + std::vector fileNames; + fileNames.push_back(filename); + edm::InputFileCatalog catalog(fileNames, override, true); + if (catalog.fileNames(0)[0] == filename) { + throw cms::Exception("FileNotFound", "RootFile::RootFile()") + << "File " << filename << " was not found or could not be opened.\n"; + } + // filename is a valid LFN. + std::unique_ptr result(TFile::Open(catalog.fileNames(0)[0].c_str())); + if (!result.get()) { + throw cms::Exception("FileNotFound", "RootFile::RootFile()") + << "File " << fileNames[0] << " was not found or could not be opened.\n"; + } + return result; + } + + // Open the input file, returning the TFile object that represents it. + // The returned unique_ptr will not be null. The argument must not be null. + // We first try the file name as a PFN, so that the catalog and related + // services are not loaded unless needed. + std::unique_ptr makeTFile(std::string const& filename) { + gErrorIgnoreLevel = kFatal; + std::unique_ptr result(TFile::Open(filename.c_str())); + gErrorIgnoreLevel = kError; + if (!result.get()) { + // Try again with catalog. + return makeTFileWithLookup(filename); + } + return result; + } +} // namespace + +static std::ostream& prettyPrint(std::ostream& os, + edm::ParameterSetEntry const& psetEntry, + std::string const& iIndent, + std::string const& iIndentDelta) { + char const* trackiness = (psetEntry.isTracked() ? "tracked" : "untracked"); + os << "PSet " << trackiness << " = ("; + prettyPrint(os, psetEntry.pset(), iIndent + iIndentDelta, iIndentDelta); + os << ")"; + return os; +} + +static std::ostream& prettyPrint(std::ostream& os, + edm::VParameterSetEntry const& vpsetEntry, + std::string const& iIndent, + std::string const& iIndentDelta) { + std::vector const& vps = vpsetEntry.vpset(); + os << "VPSet " << (vpsetEntry.isTracked() ? "tracked" : "untracked") << " = ({" << std::endl; + std::string newIndent = iIndent + iIndentDelta; + std::string start; + std::string const between(",\n"); + for (auto const& item : vps) { + os << start << newIndent; + prettyPrint(os, item, newIndent, iIndentDelta); + start = between; + } + if (!vps.empty()) { + os << std::endl; + } + os << iIndent << "})"; + return os; +} + +static std::ostream& prettyPrint(std::ostream& oStream, + edm::ParameterSet const& iPSet, + std::string const& iIndent, + std::string const& iIndentDelta) { + std::string newIndent = iIndent + iIndentDelta; + + oStream << "{" << std::endl; + for (auto const& item : iPSet.tbl()) { + // indent a bit + oStream << newIndent << item.first << ": " << item.second << std::endl; + } + for (auto const& item : iPSet.psetTable()) { + // indent a bit + edm::ParameterSetEntry const& pe = item.second; + oStream << newIndent << item.first << ": "; + prettyPrint(oStream, pe, iIndent, iIndentDelta); + oStream << std::endl; + } + for (auto const& item : iPSet.vpsetTable()) { + // indent a bit + edm::VParameterSetEntry const& pe = item.second; + oStream << newIndent << item.first << ": "; + prettyPrint(oStream, pe, newIndent, iIndentDelta); + oStream << std::endl; + } + oStream << iIndent << "}"; + + return oStream; +} + +class ProvenanceDumper { +public: + // It is illegal to call this constructor with a null pointer; a + // legal C-style string is required. + ProvenanceDumper(std::string const& filename, + bool showDependencies, + bool extendedAncestors, + bool extendedDescendants, + bool excludeESModules, + bool showAllModules, + bool showTopLevelPSets, + bool showHistoryID, + std::vector const& findMatch, + bool dontPrintProducts, + std::string const& dumpPSetID, + int productIDEntry, + bool sort); + + ProvenanceDumper(ProvenanceDumper const&) = delete; // Disallow copying and moving + ProvenanceDumper& operator=(ProvenanceDumper const&) = delete; // Disallow copying and moving + + // Write the provenenace information to the given stream. + void dump(); + void printErrors(std::ostream& os); + int exitCode() const; + +private: + void addAncestors(edm::BranchID const& branchID, + std::set& ancestorBranchIDs, + std::ostringstream& sout, + std::map>& perProductParentage) const; + + void addDescendants(edm::BranchID const& branchID, + std::set& descendantBranchIDs, + std::ostringstream& sout, + std::map>& parentToChildren) const; + + std::string filename_; + edm::propagate_const> inputFile_; + int exitCode_; + std::stringstream errorLog_; + int errorCount_; + edm::ProductRegistry reg_; + edm::ProcessConfigurationVector phc_; + edm::ProcessHistoryVector phv_; + ParameterSetMap psm_; + std::vector histories_; + bool showDependencies_; + bool extendedAncestors_; + bool extendedDescendants_; + bool excludeESModules_; + bool showOtherModules_; + bool productRegistryPresent_; + bool showTopLevelPSets_; + bool showHistoryID_; + std::vector findMatch_; + bool dontPrintProducts_; + std::string dumpPSetID_; + int const productIDEntry_; + bool const sort_; + + // The "simple ID" unsigned int is a counter for processes that have + // the same name but with different ProcessConfigurationID. The + // "simple ID" is used in the process history and configuration + // printouts for human-friendlier identification of processes than + // the ProcessConfigurationID. + using ProcessSimpleIDsType = std::map; + + void work_(); + ProcessSimpleIDsType dumpProcessHistory_(); + void dumpOtherModulesHistory_(ModuleToIdBranches const&); + void dumpEventSetupHistory_(); + void dumpTopLevelPSetsHistory_(); + + bool matchToFindMatch_(std::string_view str) const { + if (findMatch_.empty()) + return true; + for (auto const& stringToFind : findMatch_) { + if (str.find(stringToFind) == std::string::npos) { + return false; + } + } + return true; + } + + void dumpEventFilteringParameterSets_(TFile* file, ProcessSimpleIDsType const& processSimpleIDs); + void dumpEventFilteringParameterSets(edm::EventSelectionIDVector const& ids, + ProcessSimpleIDsType const& processSimpleIDs); + void dumpParameterSetForID_(edm::ParameterSetID const& id); + std::optional>> makeBranchIDListHelper(); +}; + +ProvenanceDumper::ProvenanceDumper(std::string const& filename, + bool showDependencies, + bool extendedAncestors, + bool extendedDescendants, + bool excludeESModules, + bool showOtherModules, + bool showTopLevelPSets, + bool showHistoryID, + std::vector const& findMatch, + bool dontPrintProducts, + std::string const& dumpPSetID, + int productIDEntry, + bool sort) + : filename_(filename), + inputFile_(makeTFile(filename)), + exitCode_(0), + errorLog_(), + errorCount_(0), + showDependencies_(showDependencies), + extendedAncestors_(extendedAncestors), + extendedDescendants_(extendedDescendants), + excludeESModules_(excludeESModules), + showOtherModules_(showOtherModules), + productRegistryPresent_(true), + showTopLevelPSets_(showTopLevelPSets), + showHistoryID_(showHistoryID), + findMatch_(findMatch), + dontPrintProducts_(dontPrintProducts), + dumpPSetID_(dumpPSetID), + productIDEntry_(productIDEntry), + sort_(sort) {} + +void ProvenanceDumper::dump() { work_(); } + +void ProvenanceDumper::printErrors(std::ostream& os) { + if (errorCount_ > 0) + os << errorLog_.str() << std::endl; +} + +int ProvenanceDumper::exitCode() const { return exitCode_; } + +void ProvenanceDumper::dumpEventFilteringParameterSets(edm::EventSelectionIDVector const& ids, + ProcessSimpleIDsType const& processSimpleIDs) { + edm::EventSelectionIDVector::size_type num_ids = ids.size(); + if (num_ids == 0) { + std::cout << "No event filtering information is available.\n"; + } else { + std::cout << "Event filtering information for " << num_ids << " processing steps is available.\n" + << "The ParameterSets are printed out with the oldest process first.\n"; + std::string const indent = " "; + for (edm::EventSelectionIDVector::size_type i = 0; i != num_ids; ++i) { + auto found = psm_.find(ids[i]); + if (found == psm_.end()) { + std::cout << "PSet id " << ids[i] << " not found" << std::endl; + continue; + } + std::cout << "Event filtering:\n" << indent << "PSet id: " << ids[i] << "\n" << indent << "parameters: "; + edm::ParameterSet ps(found->second.pset()); + prettyPrint(std::cout, ps, indent, indent); + std::cout << std::endl; + } + } +} + +void ProvenanceDumper::dumpEventFilteringParameterSets_(TFile* file, ProcessSimpleIDsType const& processSimpleIDs) { + TTree* history = dynamic_cast(file->Get(edm::poolNames::eventHistoryTreeName().c_str())); + if (history != nullptr) { + edm::History h; + edm::History* ph = &h; + + history->SetBranchAddress(edm::poolNames::eventHistoryBranchName().c_str(), &ph); + if (history->GetEntry(0) <= 0) { + std::cout << "No event filtering information is available; the event history tree has no entries\n"; + } else { + dumpEventFilteringParameterSets(h.eventSelectionIDs(), processSimpleIDs); + } + } else { + TTree* events = dynamic_cast(file->Get(edm::poolNames::eventTreeName().c_str())); + assert(events != nullptr); + TBranch* eventSelectionsBranch = events->GetBranch(edm::poolNames::eventSelectionsBranchName().c_str()); + if (eventSelectionsBranch == nullptr) + return; + edm::EventSelectionIDVector ids; + edm::EventSelectionIDVector* pids = &ids; + eventSelectionsBranch->SetAddress(&pids); + if (eventSelectionsBranch->GetEntry(0) <= 0) { + std::cout << "No event filtering information is available; the event selections branch has no entries\n"; + } else { + dumpEventFilteringParameterSets(ids, processSimpleIDs); + } + } +} + +void ProvenanceDumper::dumpParameterSetForID_(edm::ParameterSetID const& id) { + std::cout << "ParameterSetID: " << id << '\n'; + if (id.isValid()) { + ParameterSetMap::const_iterator i = psm_.find(id); + if (i == psm_.end()) { + std::cout << "We are unable to find the corresponding ParameterSet\n"; + edm::ParameterSet empty; + empty.registerIt(); + if (id == empty.id()) { + std::cout << "But it would have been empty anyway\n"; + } + } else { + edm::ParameterSet ps(i->second.pset()); + prettyPrint(std::cout, ps, " ", " "); + std::cout << '\n'; + } + } else { + std::cout << "This ID is not valid\n"; + } + std::cout << " -------------------------\n"; +} + +ProvenanceDumper::ProcessSimpleIDsType ProvenanceDumper::dumpProcessHistory_() { + std::cout << formatHeader("Processing histories") << std::endl; + + ProcessSimpleIDsType simpleIDs; + // expect the outer vector to not grow large so that linear search is still acceptable + std::vector> processesByName; + histories_.reserve(phv_.size()); + for (auto const& ph : phv_) { + // loop over the history entries to find ProcessConfigurations that are the same + // use a simple count ID for each process that have the same name, but different ProcessConfigurationID + histories_.emplace_back(ph.id()); + HistoryChain& chain = histories_.back(); + chain.reserve(ph.size()); + + for (auto const& pc : ph) { + unsigned int& id = simpleIDs[pc.id()]; + if (id == 0) { + // first time seeing this ID + auto found = std::ranges::find_if(processesByName, [&name = pc.processName()](auto const& pcs) { + return pcs.front()->processName() == name; + }); + if (found == processesByName.end()) { + processesByName.emplace_back(std::initializer_list({&pc})); + id = 1; + } else { + found->emplace_back(&pc); + id = found->size(); + } + } + + chain.addChild(HistoryNode(pc, id)); + } + chain.printHistory(showHistoryID_); + } + + std::cout << formatHeader("Processes") << std::endl; + auto concatenate = [](std::ostream& os, std::vector const& vs) { + if (not vs.empty()) { + os << vs.front(); + for (auto it = vs.begin() + 1; it != vs.end(); ++it) { + os << ", " << *it; + } + } + }; + constexpr std::string_view indent = " "; + for (auto const& processes : processesByName) { + for (auto const* pc : processes) { + auto const& hwresources = pc->hardwareResourcesDescription(); + std::cout << "Process: " << pc->processName() << " [" << simpleIDs.at(pc->id()) << "]\n" + << indent << "PSet id: " << pc->parameterSetID() << "\n" + << indent << "version: '" << pc->releaseVersion() << "'\n" + << indent << "microarchitecture: " << hwresources.microarchitecture << "\n" + << indent << "accelerators: "; + concatenate(std::cout, hwresources.selectedAccelerators); + std::cout << "\n" << indent << "CPU models: "; + concatenate(std::cout, hwresources.cpuModels); + if (not hwresources.gpuModels.empty()) { + std::cout << "\n" << indent << "GPU models: "; + concatenate(std::cout, hwresources.gpuModels); + } + std::cout << "\n" << std::endl; + } + } + return simpleIDs; +} + +void ProvenanceDumper::dumpOtherModulesHistory_(ModuleToIdBranches const& iModules) { + for (auto const& chain : histories_) { + for (auto const& node : chain) { + //Get ParameterSet for process + ParameterSetMap::const_iterator itFind = psm_.find(node.parameterSetID()); + if (itFind == psm_.end()) { + errorLog_ << "No ParameterSetID for " << node.parameterSetID() << std::endl; + continue; + } + + edm::ParameterSet processConfig(itFind->second.pset()); + std::vector moduleStrings; + //get all modules + std::vector modules = processConfig.getParameter>("@all_modules"); + for (auto& moduleLabel : modules) { + //if we didn't already handle this from the branches + if (iModules.end() == iModules.find(std::make_pair(node.processName(), moduleLabel))) { + std::string retValue(nonProducerComponent(moduleLabel, processConfig, node.processName())); + if (matchToFindMatch_(retValue)) { + moduleStrings.push_back(std::move(retValue)); + } + } + } + if (sort_) { + std::sort(moduleStrings.begin(), moduleStrings.end()); + } + std::copy(moduleStrings.begin(), moduleStrings.end(), std::ostream_iterator(std::cout, "\n")); + } + } +} + +void ProvenanceDumper::dumpEventSetupHistory_() { + for (auto const& chain : histories_) { + for (auto const& node : chain) { + //Get ParameterSet for process + ParameterSetMap::const_iterator itFind = psm_.find(node.parameterSetID()); + if (itFind == psm_.end()) { + errorLog_ << "No ParameterSetID for " << node.parameterSetID() << std::endl; + continue; + } + + edm::ParameterSet processConfig(itFind->second.pset()); + std::vector sourceStrings, moduleStrings; + + //get the sources + auto const sources = processConfig.getParameter>("@all_essources"); + for (auto const& sourceLabel : sources) { + std::string retValue = eventSetupComponent("ESSource", sourceLabel, processConfig, node.processName()); + if (matchToFindMatch_(retValue)) { + sourceStrings.push_back(std::move(retValue)); + } + } + + //get the modules + auto const modules = processConfig.getParameter>("@all_esmodules"); + for (auto const& moduleLabel : modules) { + std::string retValue = eventSetupComponent("ESModule", moduleLabel, processConfig, node.processName()); + if (matchToFindMatch_(retValue)) { + moduleStrings.push_back(std::move(retValue)); + } + } + + if (sort_) { + std::sort(sourceStrings.begin(), sourceStrings.end()); + std::sort(moduleStrings.begin(), moduleStrings.end()); + } + + std::copy(sourceStrings.begin(), sourceStrings.end(), std::ostream_iterator(std::cout, "\n")); + std::copy(moduleStrings.begin(), moduleStrings.end(), std::ostream_iterator(std::cout, "\n")); + } + } +} + +void ProvenanceDumper::dumpTopLevelPSetsHistory_() { + for (auto const& chain : histories_) { + for (auto const& node : chain) { + //Get ParameterSet for process + ParameterSetMap::const_iterator itFind = psm_.find(node.parameterSetID()); + if (itFind == psm_.end()) { + errorLog_ << "No ParameterSetID for " << node.parameterSetID() << std::endl; + continue; + } + + //Get ParameterSet for process + edm::ParameterSet processConfig(itFind->second.pset()); + //Need to get the names of PSets which are used by the framework (e.g. names of modules) + std::set namesToExclude; + appendToSet(namesToExclude, processConfig.getParameter>("@all_modules")); + appendToSet(namesToExclude, processConfig.getParameter>("@all_sources")); + appendToSet(namesToExclude, processConfig.getParameter>("@all_loopers")); + appendToSet(namesToExclude, processConfig.getParameter>("@all_esmodules")); + appendToSet(namesToExclude, processConfig.getParameter>("@all_essources")); + appendToSet(namesToExclude, processConfig.getParameter>("@all_esprefers")); + if (processConfig.existsAs>("all_aliases")) { + appendToSet(namesToExclude, processConfig.getParameter>("@all_aliases")); + } + + std::vector allNames{}; + processConfig.getParameterSetNames(allNames); + + std::vector results; + for (auto const& name : allNames) { + if (name.empty() || '@' == name[0] || namesToExclude.find(name) != namesToExclude.end()) { + continue; + } + std::string retValue = topLevelPSet(name, processConfig, node.processName()); + if (matchToFindMatch_(retValue)) { + results.push_back(std::move(retValue)); + } + } + if (sort_) { + std::sort(results.begin(), results.end()); + } + std::copy(results.begin(), results.end(), std::ostream_iterator(std::cout, "\n")); + } + } +} + +std::optional>> +ProvenanceDumper::makeBranchIDListHelper() { + // BranchID-to-ProductID mapping disabled + if (productIDEntry_ < 0) { + return {}; + } + + TTree* metaTree = dynamic_cast(inputFile_->Get(edm::poolNames::metaDataTreeName().c_str())); + if (nullptr == metaTree) { + //std::cerr << "Did not find " << edm::poolNames::metaDataTreeName() << " tree" << std::endl; + return {}; + } + + TBranch* branchIDListsBranch = metaTree->GetBranch(edm::poolNames::branchIDListBranchName().c_str()); + if (nullptr == branchIDListsBranch) { + /* + std::cerr << "Did not find " << edm::poolNames::branchIDListBranchName() << " from " + << edm::poolNames::metaDataTreeName() << " tree" << std::endl; + */ + return {}; + } + + edm::BranchIDLists branchIDLists; + edm::BranchIDLists* branchIDListsPtr = &branchIDLists; + branchIDListsBranch->SetAddress(&branchIDListsPtr); + if (branchIDListsBranch->GetEntry(0) <= 0) { + //std::cerr << "Failed to read an entry from " << edm::poolNames::branchIDListBranchName() << std::endl; + return {}; + } + + edm::BranchIDListHelper branchIDListHelper; + branchIDListHelper.updateFromInput(branchIDLists); + + TTree* events = dynamic_cast(inputFile_->Get(edm::poolNames::eventTreeName().c_str())); + assert(events != nullptr); + TBranch* branchListIndexesBranch = events->GetBranch(edm::poolNames::branchListIndexesBranchName().c_str()); + if (nullptr == branchListIndexesBranch) { + /* + std::cerr << "Did not find " << edm::poolNames::branchListIndexesBranchName() << " from " + << edm::poolNames::eventTreeName() << " tree" << std::endl; + */ + return {}; + } + edm::BranchListIndexes branchListIndexes; + edm::BranchListIndexes* pbranchListIndexes = &branchListIndexes; + branchListIndexesBranch->SetAddress(&pbranchListIndexes); + if (branchListIndexesBranch->GetEntry(productIDEntry_) <= 0 or branchListIndexes.empty()) { + /* + std::cerr << "Failed to read entry from " << edm::poolNames::branchListIndexesBranchName() << ", or it is empty" + << std::endl; + */ + return {}; + } + + if (not branchIDListHelper.fixBranchListIndexes(branchListIndexes)) { + //std::cerr << "Call to branchIDListHelper.fixBranchListIndexes() failed" << std::endl; + return {}; + } + + // Fill in helper map for Branch to ProductID mapping + auto branchListIndexToProcessIndex = edm::makeBranchListIndexToProcessIndex(branchListIndexes); + + return std::tuple(std::move(branchIDListHelper), std::move(branchListIndexToProcessIndex)); +} + +void ProvenanceDumper::work_() { + auto metaRNTuple = inputFile_->Get(edm::poolNames::metaDataTreeName().c_str()); + //TTree* meta = dynamic_cast(inputFile_->Get(edm::poolNames::metaDataTreeName().c_str())); + assert(metaRNTuple); + + auto reader = ROOT::RNTupleReader::Open(*metaRNTuple); + + auto meta = reader->GetModel().CreateEntry(); + edm::ProductRegistry* pReg = ®_; + meta->BindRawPtr(edm::poolNames::productDescriptionBranchName(), pReg); + + { + auto rntuple = inputFile_->Get("ParameterSets"); + assert(rntuple); + auto psets = ROOT::RNTupleReader::Open(*rntuple); + assert(psets.get()); + auto entry = psets->GetModel().CreateBareEntry(); + + std::pair idToBlob; + entry->BindRawPtr("IdToParameterSetsBlobs", &idToBlob); + + for (ROOT::NTupleSize_t i = 0; i < psets->GetNEntries(); ++i) { + psets->LoadEntry(i, *entry); + psm_.insert(idToBlob); + } + } + + edm::ProcessHistoryVector* pPhv = &phv_; + meta->BindRawPtr(edm::poolNames::processHistoryBranchName(), pPhv); + + reader->LoadEntry(0, *meta); + //meta->GetEntry(0); + assert(nullptr != pReg); + + edm::pset::Registry& psetRegistry = *edm::pset::Registry::instance(); + for (auto const& item : psm_) { + edm::ParameterSet pset(item.second.pset()); + pset.setID(item.first); + psetRegistry.insertMapped(pset); + } + + if (!phv_.empty()) { + // (Re-)Sort according to reduced history ID in order to have a + // stable order with respect to hardware differences + std::ranges::stable_sort(phv_, {}, [](auto const& history) { + auto copy = history; + copy.reduce(); + return copy.id(); + }); + + for (auto const& history : phv_) { + for (auto const& process : history) { + phc_.push_back(process); + } + } + edm::sort_all(phc_); + phc_.erase(std::unique(phc_.begin(), phc_.end()), phc_.end()); + } + + if (!dumpPSetID_.empty()) { + edm::ParameterSetID psetID; + try { + psetID = edm::ParameterSetID(dumpPSetID_); + } catch (cms::Exception const& x) { + throw cms::Exception("Command Line Argument") + << "Illegal ParameterSetID string. It should contain 32 hexadecimal characters"; + } + dumpParameterSetForID_(psetID); + return; + } + + // Helper to map BranchID to ProductID (metadata tree needed also for parentage information) + auto branchIDListHelperAndToProcessIndex = makeBranchIDListHelper(); + + //Prepare the parentage information if requested + std::map> perProductParentage; + + if (showDependencies_ || extendedAncestors_ || extendedDescendants_) { + TTree* parentageTree = dynamic_cast(inputFile_->Get(edm::poolNames::parentageTreeName().c_str())); + if (nullptr == parentageTree) { + std::cerr << "ERROR, no Parentage tree available so cannot show dependencies, ancestors, or descendants.\n"; + std::cerr << "Possibly this is not a standard EDM format file. For example, dependency, ancestor, and\n"; + std::cerr << "descendant options to edmProvDump will not work with nanoAOD format files.\n\n"; + showDependencies_ = false; + extendedAncestors_ = false; + extendedDescendants_ = false; + } else { + edm::ParentageRegistry& registry = *edm::ParentageRegistry::instance(); + + std::vector orderedParentageIDs; + orderedParentageIDs.reserve(parentageTree->GetEntries()); + for (Long64_t i = 0, numEntries = parentageTree->GetEntries(); i < numEntries; ++i) { + edm::Parentage parentageBuffer; + edm::Parentage* pParentageBuffer = &parentageBuffer; + parentageTree->SetBranchAddress(edm::poolNames::parentageBranchName().c_str(), &pParentageBuffer); + parentageTree->GetEntry(i); + registry.insertMapped(parentageBuffer); + orderedParentageIDs.push_back(parentageBuffer.id()); + } + parentageTree->SetBranchAddress(edm::poolNames::parentageBranchName().c_str(), nullptr); + + TTree* eventMetaTree = + dynamic_cast(inputFile_->Get(edm::BranchTypeToMetaDataTreeName(edm::InEvent).c_str())); + if (nullptr == eventMetaTree) { + eventMetaTree = dynamic_cast(inputFile_->Get(edm::BranchTypeToProductTreeName(edm::InEvent).c_str())); + } + if (nullptr == eventMetaTree) { + std::cerr << "ERROR, no '" << edm::BranchTypeToProductTreeName(edm::InEvent) + << "' Tree in file so can not show dependencies\n"; + showDependencies_ = false; + extendedAncestors_ = false; + extendedDescendants_ = false; + } else { + TBranch* storedProvBranch = + eventMetaTree->GetBranch(edm::BranchTypeToProductProvenanceBranchName(edm::InEvent).c_str()); + + if (nullptr != storedProvBranch) { + std::vector info; + std::vector* pInfo = &info; + storedProvBranch->SetAddress(&pInfo); + for (Long64_t i = 0, numEntries = eventMetaTree->GetEntries(); i < numEntries; ++i) { + storedProvBranch->GetEntry(i); + for (auto const& item : info) { + edm::BranchID bid(item.branchID_); + perProductParentage[bid].insert(orderedParentageIDs.at(item.parentageIDIndex_)); + } + } + } else { + //backwards compatible check + TBranch* productProvBranch = + eventMetaTree->GetBranch(edm::BranchTypeToBranchEntryInfoBranchName(edm::InEvent).c_str()); + if (nullptr != productProvBranch) { + std::vector info; + std::vector* pInfo = &info; + productProvBranch->SetAddress(&pInfo); + for (Long64_t i = 0, numEntries = eventMetaTree->GetEntries(); i < numEntries; ++i) { + productProvBranch->GetEntry(i); + for (auto const& item : info) { + perProductParentage[item.branchID()].insert(item.parentageID()); + } + } + } else { + std::cerr << " ERROR, could not find provenance information so can not show dependencies\n"; + showDependencies_ = false; + extendedAncestors_ = false; + extendedDescendants_ = false; + } + } + } + } + } + + std::map> parentToChildren; + edm::ParentageRegistry& registry = *edm::ParentageRegistry::instance(); + + if (extendedDescendants_) { + for (auto const& itParentageSet : perProductParentage) { + edm::BranchID childBranchID = itParentageSet.first; + for (auto const& itParentageID : itParentageSet.second) { + edm::Parentage const* parentage = registry.getMapped(itParentageID); + if (nullptr != parentage) { + for (auto const& branch : parentage->parents()) { + parentToChildren[branch].insert(childBranchID); + } + } else { + std::cerr << " ERROR:parentage info not in registry ParentageID=" << itParentageID << std::endl; + } + } + } + } + + ProcessSimpleIDsType const& processSimpleIDs = dumpProcessHistory_(); + + std::cout << formatHeader("Event filtering") << std::endl; + dumpEventFilteringParameterSets_(inputFile_.get(), processSimpleIDs); + + if (productRegistryPresent_) { + std::cout << formatHeader("Producers with data in file") << std::endl; + } + + ModuleToIdBranches moduleToIdBranches; + std::map branchIDToBranchName; + + for (auto const& processConfig : phc_) { + edm::ParameterSet const* processParameterSet = + edm::pset::Registry::instance()->getMapped(processConfig.parameterSetID()); + if (nullptr == processParameterSet || processParameterSet->empty()) { + continue; + } + for (auto& item : reg_.productListUpdator()) { + auto& product = item.second; + if (product.processName() != processConfig.processName()) { + continue; + } + //force it to rebuild the branch name + product.init(); + setIsMergeable(product); + + if (showDependencies_ || extendedAncestors_ || extendedDescendants_) { + branchIDToBranchName[product.branchID()] = product.branchName(); + } + /* + std::cout << product.branchName() + << " id " << product.productID() << std::endl; + */ + std::string moduleLabel = product.moduleLabel(); + if (moduleLabel == source) { + moduleLabel = input; + } else if (moduleLabel == triggerResults) { + moduleLabel = triggerPaths; + } + + std::stringstream s; + + if (processParameterSet->existsAs(moduleLabel)) { + edm::ParameterSet const& moduleParameterSet = processParameterSet->getParameterSet(moduleLabel); + if (!moduleParameterSet.isRegistered()) { + edm::ParameterSet moduleParameterSetCopy = processParameterSet->getParameterSet(moduleLabel); + moduleParameterSetCopy.registerIt(); + s << moduleParameterSetCopy.id(); + } else { + s << moduleParameterSet.id(); + } + auto& moduleInformation = + moduleToIdBranches[std::make_pair(product.processName(), product.moduleLabel())][s.str()]; + moduleInformation.processSimpleIDs.insert(processSimpleIDs.at(processConfig.id())); + moduleInformation.products.insert(product); + } + } + } + + for (auto const& item : moduleToIdBranches) { + std::ostringstream sout; + auto const& processName = item.first.first; + auto const& moduleLabel = item.first.second; + sout << "Module: " << moduleLabel << " " << processName << std::endl; + std::set allBranchIDsForLabelAndProcess; + IdToBranches const& idToBranches = item.second; + for (auto const& idBranch : idToBranches) { + auto const& psetID = idBranch.first; + auto const& moduleInformation = idBranch.second; + + sout << " Process: " << processName; + for (auto const& simpleID : moduleInformation.processSimpleIDs) { + sout << " [" << simpleID << "]"; + } + sout << std::endl; + sout << " PSet id:" << psetID << std::endl; + if (!dontPrintProducts_) { + sout << " products: {" << std::endl; + } + std::set branchIDs; + for (auto const& branch : moduleInformation.products) { + if (!dontPrintProducts_) { + sout << " " << branch.branchName(); + edm::ProductID id; + if (branchIDListHelperAndToProcessIndex) { + sout << " ProductID " + << edm::branchIDToProductID(branch.branchID(), + std::get<0>(*branchIDListHelperAndToProcessIndex), + std::get<1>(*branchIDListHelperAndToProcessIndex)); + } else { + sout << " BranchID " << branch.branchID(); + } + sout << std::endl; + } + branchIDs.insert(branch.branchID()); + allBranchIDsForLabelAndProcess.insert(branch.branchID()); + } + sout << " }" << std::endl; + edm::ParameterSetID psid(idBranch.first); + ParameterSetMap::const_iterator itpsm = psm_.find(psid); + if (psm_.end() == itpsm) { + ++errorCount_; + errorLog_ << "No ParameterSetID for " << psid << std::endl; + exitCode_ = 1; + } else { + sout << " parameters: "; + prettyPrint(sout, edm::ParameterSet((*itpsm).second.pset()), " ", " "); + sout << std::endl; + } + if (showDependencies_) { + sout << " dependencies: {" << std::endl; + std::set parentageIDs; + for (auto const& branch : branchIDs) { + //Save these BranchIDs + std::set const& temp = perProductParentage[branch]; + parentageIDs.insert(temp.begin(), temp.end()); + } + for (auto const& parentID : parentageIDs) { + edm::Parentage const* parentage = registry.getMapped(parentID); + if (nullptr != parentage) { + for (auto const& branch : parentage->parents()) { + sout << " " << branchIDToBranchName[branch] << std::endl; + } + } else { + sout << " ERROR:parentage info not in registry ParentageID=" << parentID << std::endl; + } + } + if (parentageIDs.empty()) { + sout << " no dependencies recorded (event may not contain data from this module)" << std::endl; + } + sout << " }" << std::endl; + } + } // end loop over PSetIDs + if (extendedAncestors_) { + sout << " extendedAncestors: {" << std::endl; + std::set ancestorBranchIDs; + for (auto const& branchID : allBranchIDsForLabelAndProcess) { + addAncestors(branchID, ancestorBranchIDs, sout, perProductParentage); + } + for (auto const& ancestorBranchID : ancestorBranchIDs) { + sout << " " << branchIDToBranchName[ancestorBranchID] << "\n"; + } + sout << " }" << std::endl; + } + + if (extendedDescendants_) { + sout << " extendedDescendants: {" << std::endl; + std::set descendantBranchIDs; + for (auto const& branchID : allBranchIDsForLabelAndProcess) { + addDescendants(branchID, descendantBranchIDs, sout, parentToChildren); + } + for (auto const& descendantBranchID : descendantBranchIDs) { + sout << " " << branchIDToBranchName[descendantBranchID] << "\n"; + } + sout << " }" << std::endl; + } + if (matchToFindMatch_(sout.str())) { + std::cout << sout.str() << std::endl; + } + } // end loop over module label/process + + if (productRegistryPresent_ && showOtherModules_) { + std::cout << formatHeader("Other Modules") << std::endl; + dumpOtherModulesHistory_(moduleToIdBranches); + } else if (!productRegistryPresent_) { + std::cout << formatHeader("All Modules") << std::endl; + dumpOtherModulesHistory_(moduleToIdBranches); + } + + if (!excludeESModules_) { + std::cout << formatHeader("EventSetup") << std::endl; + dumpEventSetupHistory_(); + } + + if (showTopLevelPSets_) { + std::cout << formatHeader("Top Level PSets") << std::endl; + dumpTopLevelPSetsHistory_(); + } + if (errorCount_ != 0) { + exitCode_ = 1; + } +} + +void ProvenanceDumper::addAncestors(edm::BranchID const& branchID, + std::set& ancestorBranchIDs, + std::ostringstream& sout, + std::map>& perProductParentage) const { + edm::ParentageRegistry& registry = *edm::ParentageRegistry::instance(); + + std::set const& parentIDs = perProductParentage[branchID]; + for (auto const& parentageID : parentIDs) { + edm::Parentage const* parentage = registry.getMapped(parentageID); + if (nullptr != parentage) { + for (auto const& branch : parentage->parents()) { + if (ancestorBranchIDs.insert(branch).second) { + addAncestors(branch, ancestorBranchIDs, sout, perProductParentage); + } + } + } else { + sout << " ERROR:parentage info not in registry ParentageID=" << parentageID << std::endl; + } + } +} + +void ProvenanceDumper::addDescendants(edm::BranchID const& branchID, + std::set& descendantBranchIDs, + std::ostringstream& sout, + std::map>& parentToChildren) const { + for (auto const& childBranchID : parentToChildren[branchID]) { + if (descendantBranchIDs.insert(childBranchID).second) { + addDescendants(childBranchID, descendantBranchIDs, sout, parentToChildren); + } + } +} + +static char const* const kSortOpt = "sort"; +static char const* const kSortCommandOpt = "sort,s"; +static char const* const kDependenciesOpt = "dependencies"; +static char const* const kDependenciesCommandOpt = "dependencies,d"; +static char const* const kExtendedAncestorsOpt = "extendedAncestors"; +static char const* const kExtendedAncestorsCommandOpt = "extendedAncestors,x"; +static char const* const kExtendedDescendantsOpt = "extendedDescendants"; +static char const* const kExtendedDescendantsCommandOpt = "extendedDescendants,c"; +static char const* const kExcludeESModulesOpt = "excludeESModules"; +static char const* const kExcludeESModulesCommandOpt = "excludeESModules,e"; +static char const* const kShowAllModulesOpt = "showAllModules"; +static char const* const kShowAllModulesCommandOpt = "showAllModules,a"; +static char const* const kFindMatchOpt = "findMatch"; +static char const* const kFindMatchCommandOpt = "findMatch,f"; +static char const* const kDontPrintProductsOpt = "dontPrintProducts"; +static char const* const kDontPrintProductsCommandOpt = "dontPrintProducts,p"; +static char const* const kShowTopLevelPSetsOpt = "showTopLevelPSets"; +static char const* const kShowTopLevelPSetsCommandOpt = "showTopLevelPSets,t"; +static char const* const kShowHistoryIDOpt = "showHistoryID"; +static char const* const kHelpOpt = "help"; +static char const* const kHelpCommandOpt = "help,h"; +static char const* const kFileNameOpt = "input-file"; +static char const* const kDumpPSetIDOpt = "dumpPSetID"; +static char const* const kDumpPSetIDCommandOpt = "dumpPSetID,i"; +static char const* const kProductIDEntryOpt = "productIDEntry"; + +int main(int argc, char* argv[]) { + using namespace boost::program_options; + + std::string descString(argv[0]); + descString += " [options] "; + descString += "\nAllowed options"; + options_description desc(descString); + + // clang-format off + desc.add_options()(kHelpCommandOpt, "show help message")(kSortCommandOpt, "alphabetially sort EventSetup components")( + kDependenciesCommandOpt, "print what data each EDProducer is directly dependent upon")( + kExtendedAncestorsCommandOpt, "print what data each EDProducer is dependent upon including indirect dependences")( + kExtendedDescendantsCommandOpt, + "print what data depends on the data each EDProducer produces including indirect dependences")( + kExcludeESModulesCommandOpt, "do not print ES module information")( + kShowAllModulesCommandOpt, "show all modules (not just those that created data in the file)")( + kShowTopLevelPSetsCommandOpt, "show all top level PSets") + (kShowHistoryIDOpt, "show process history and configuration IDs")( + kFindMatchCommandOpt, + boost::program_options::value>(), + "show only modules whose information contains the matching string (or all the matching strings, this option can " + "be repeated with different strings)")(kDontPrintProductsCommandOpt, "do not print products produced by module")( + kDumpPSetIDCommandOpt, + value(), + "print the parameter set associated with the parameter set ID string (and print nothing else)")( + kProductIDEntryOpt, + value(), + "show ProductID instead of BranchID using the specified entry in the Events tree"); + // clang-format on + + //we don't want users to see these in the help messages since this + // name only exists since the parser needs it + options_description hidden; + hidden.add_options()(kFileNameOpt, value(), "file name"); + + //full list of options for the parser + options_description cmdline_options; + cmdline_options.add(desc).add(hidden); + + positional_options_description p; + p.add(kFileNameOpt, -1); + + variables_map vm; + try { + store(command_line_parser(argc, argv).options(cmdline_options).positional(p).run(), vm); + notify(vm); + } catch (error const& iException) { + std::cerr << iException.what() << std::endl; + return 1; + } + + if (vm.count(kHelpOpt)) { + std::cout << desc << std::endl; + return 0; + } + + bool sort = false; + if (vm.count(kSortOpt)) { + sort = true; + } + + bool showDependencies = false; + if (vm.count(kDependenciesOpt)) { + showDependencies = true; + } + + bool extendedAncestors = false; + if (vm.count(kExtendedAncestorsOpt)) { + extendedAncestors = true; + } + + bool extendedDescendants = false; + if (vm.count(kExtendedDescendantsOpt)) { + extendedDescendants = true; + } + + bool excludeESModules = false; + if (vm.count(kExcludeESModulesOpt)) { + excludeESModules = true; + } + + bool showAllModules = false; + if (vm.count(kShowAllModulesOpt)) { + showAllModules = true; + } + + bool showTopLevelPSets = false; + if (vm.count(kShowTopLevelPSetsOpt)) { + showTopLevelPSets = true; + } + + bool showHistoryID = false; + if (vm.count(kShowHistoryIDOpt)) { + showHistoryID = true; + } + + std::string fileName; + if (vm.count(kFileNameOpt)) { + try { + fileName = vm[kFileNameOpt].as(); + } catch (boost::bad_any_cast const& e) { + std::cout << e.what() << std::endl; + return 2; + } + } else { + std::cout << "Data file not specified." << std::endl; + std::cout << desc << std::endl; + return 2; + } + + std::string dumpPSetID; + if (vm.count(kDumpPSetIDOpt)) { + try { + dumpPSetID = vm[kDumpPSetIDOpt].as(); + } catch (boost::bad_any_cast const& e) { + std::cout << e.what() << std::endl; + return 2; + } + } + + std::vector findMatch; + if (vm.count(kFindMatchOpt)) { + try { + findMatch = vm[kFindMatchOpt].as>(); + } catch (boost::bad_any_cast const& e) { + std::cout << e.what() << std::endl; + return 2; + } + } + + bool dontPrintProducts = false; + if (vm.count(kDontPrintProductsOpt)) { + dontPrintProducts = true; + } + + int productIDEntry = -1; + if (vm.count(kProductIDEntryOpt)) { + try { + productIDEntry = vm[kProductIDEntryOpt].as(); + } catch (boost::bad_any_cast const& e) { + std::cout << e.what() << std::endl; + return 2; + } + } + + //silence ROOT warnings about missing dictionaries + gErrorIgnoreLevel = kError; + + ProvenanceDumper dumper(fileName, + showDependencies, + extendedAncestors, + extendedDescendants, + excludeESModules, + showAllModules, + showTopLevelPSets, + showHistoryID, + findMatch, + dontPrintProducts, + dumpPSetID, + productIDEntry, + sort); + int exitCode(0); + try { + dumper.dump(); + exitCode = dumper.exitCode(); + } catch (cms::Exception const& x) { + std::cerr << "cms::Exception caught\n"; + std::cerr << x.what() << '\n'; + exitCode = 2; + } catch (std::exception& x) { + std::cerr << "std::exception caught\n"; + std::cerr << x.what() << '\n'; + exitCode = 3; + } catch (...) { + std::cerr << "Unknown exception caught\n"; + exitCode = 4; + } + + dumper.printErrors(std::cerr); + return exitCode; +} diff --git a/FWIO/RNTupleTempInput/src/InputFile.h b/FWIO/RNTupleTempInput/src/InputFile.h index 758e4252cf813..acbde5f9de918 100644 --- a/FWIO/RNTupleTempInput/src/InputFile.h +++ b/FWIO/RNTupleTempInput/src/InputFile.h @@ -48,8 +48,10 @@ namespace edm::rntuple_temp { static void reportReadBranch(InputType inputType, std::string const& branchname); TObject* Get(char const* name) { return file_->Get(name); } - template - T* Get(char const* name) { return file_->Get(name); } + template + T* Get(char const* name) { + return file_->Get(name); + } std::unique_ptr createCacheWithSize(TTree& iTree, unsigned int cacheSize) { iTree.SetCacheSize(static_cast(cacheSize)); std::unique_ptr newCache(dynamic_cast(file_->GetCacheRead(&iTree))); diff --git a/FWIO/RNTupleTempInput/src/RootFile.cc b/FWIO/RNTupleTempInput/src/RootFile.cc index 00e27a0ef8906..27e8f0c567c57 100644 --- a/FWIO/RNTupleTempInput/src/RootFile.cc +++ b/FWIO/RNTupleTempInput/src/RootFile.cc @@ -174,8 +174,6 @@ namespace edm::rntuple_temp { indexIntoFileIter_(indexIntoFileBegin_), storedMergeableRunProductMetadata_((inputType == InputType::Primary) ? new StoredMergeableRunProductMetadata : nullptr), - eventProcessHistoryIDs_(), - eventProcessHistoryIter_(eventProcessHistoryIDs_.begin()), savedRunAuxiliary_(), skipAnyEvents_(processingOptions.skipAnyEvents), noRunLumiSort_(processingOptions.noRunLumiSort), @@ -210,15 +208,12 @@ namespace edm::rntuple_temp { processingMode_(processingOptions.processingMode), runHelper_(crossFileInfo.runHelper), newBranchToOldBranch_(), - eventHistoryTree_(nullptr), eventToProcessBlockIndexesBranch_( inputType == InputType::Primary ? eventTree_.tree()->GetBranch(poolNames::eventToProcessBlockIndexesBranchName().c_str()) : nullptr), - history_(), productDependencies_(new ProductDependencies), duplicateChecker_(crossFileInfo.duplicateChecker), - provenanceAdaptor_(), provenanceReaderMaker_(), eventProductProvenanceRetrievers_(), parentageIDLookup_(), @@ -232,70 +227,51 @@ namespace edm::rntuple_temp { treePointers_[InLumi] = &lumiTree_; treePointers_[InRun] = &runTree_; - // Read the metadata tree. + // Read the metadata RNTuple. // We use a smart pointer so the tree will be deleted after use, and not kept for the life of the file. - std::unique_ptr metaDataTree(dynamic_cast(filePtr_->Get(poolNames::metaDataTreeName().c_str()))); - if (nullptr == metaDataTree.get()) { + std::unique_ptr metaDataRNTuple{filePtr_->Get(poolNames::metaDataTreeName().c_str())}; + if (nullptr == metaDataRNTuple.get()) { throw Exception(errors::FileReadError) - << "Could not find tree " << poolNames::metaDataTreeName() << " in the input file.\n"; + << "Could not find rntuple " << poolNames::metaDataTreeName() << " in the input file.\n"; } - // To keep things simple, we just read in every possible branch that exists. - // We don't pay attention to which branches exist in which file format versions + auto metaDataReader = ROOT::RNTupleReader::Open(*metaDataRNTuple); - FileFormatVersion* fftPtr = &fileFormatVersion_; - if (metaDataTree->FindBranch(poolNames::fileFormatVersionBranchName().c_str()) != nullptr) { - TBranch* fft = metaDataTree->GetBranch(poolNames::fileFormatVersionBranchName().c_str()); - fft->SetAddress(&fftPtr); - rootrntuple::getEntry(fft, 0); - metaDataTree->SetBranchAddress(poolNames::fileFormatVersionBranchName().c_str(), &fftPtr); - } + //If call CreateBareEntry and we do not bind all top level fields we get a seg fault + auto metaDataEntry = metaDataReader->GetModel().CreateEntry(); + metaDataEntry->BindRawPtr(poolNames::fileFormatVersionBranchName(), &fileFormatVersion_); - FileID* fidPtr = &fid_; - if (metaDataTree->FindBranch(poolNames::fileIdentifierBranchName().c_str()) != nullptr) { - metaDataTree->SetBranchAddress(poolNames::fileIdentifierBranchName().c_str(), &fidPtr); - } + metaDataEntry->BindRawPtr(poolNames::fileIdentifierBranchName(), &fid_); - IndexIntoFile* iifPtr = &indexIntoFile_; - if (metaDataTree->FindBranch(poolNames::indexIntoFileBranchName().c_str()) != nullptr) { - metaDataTree->SetBranchAddress(poolNames::indexIntoFileBranchName().c_str(), &iifPtr); - } + metaDataEntry->BindRawPtr(poolNames::indexIntoFileBranchName(), &indexIntoFile_); storedProcessBlockHelper_ = std::make_unique(); - StoredProcessBlockHelper& storedProcessBlockHelper = *storedProcessBlockHelper_; - StoredProcessBlockHelper* pStoredProcessBlockHelper = storedProcessBlockHelper_.get(); if (inputType == InputType::Primary) { - if (metaDataTree->FindBranch(poolNames::processBlockHelperBranchName().c_str()) != nullptr) { - metaDataTree->SetBranchAddress(poolNames::processBlockHelperBranchName().c_str(), &pStoredProcessBlockHelper); - } + metaDataEntry->BindRawPtr(poolNames::processBlockHelperBranchName(), storedProcessBlockHelper_.get()); } StoredMergeableRunProductMetadata* smrc = nullptr; if (inputType == InputType::Primary) { smrc = &*storedMergeableRunProductMetadata_; - if (metaDataTree->FindBranch(poolNames::mergeableRunProductMetadataBranchName().c_str()) != nullptr) { - metaDataTree->SetBranchAddress(poolNames::mergeableRunProductMetadataBranchName().c_str(), &smrc); - } + metaDataEntry->BindRawPtr(poolNames::mergeableRunProductMetadataBranchName(), smrc); } // Need to read to a temporary registry so we can do a translation of the BranchKeys. // This preserves backward compatibility against friendly class name algorithm changes. ProductRegistry inputProdDescReg; - ProductRegistry* ppReg = &inputProdDescReg; - metaDataTree->SetBranchAddress(poolNames::productDescriptionBranchName().c_str(), (&ppReg)); + metaDataEntry->BindRawPtr(poolNames::productDescriptionBranchName(), &inputProdDescReg); using PsetMap = std::map; PsetMap psetMap; { auto rntuple = filePtr_->Get("ParameterSets"); - if(not rntuple) { - throw Exception(errors::FileReadError) - << "Could not find RNTuple 'ParameterSets' in the input file.\n"; + if (not rntuple) { + throw Exception(errors::FileReadError) << "Could not find RNTuple 'ParameterSets' in the input file.\n"; } auto psets = ROOT::RNTupleReader::Open(*rntuple); assert(psets.get()); auto entry = psets->GetModel().CreateBareEntry(); - + std::pair idToBlob; entry->BindRawPtr("IdToParameterSetsBlobs", &idToBlob); @@ -305,69 +281,22 @@ namespace edm::rntuple_temp { } } - // backward compatibility - ProcessHistoryRegistry::collection_type pHistMap; - ProcessHistoryRegistry::collection_type* pHistMapPtr = &pHistMap; - if (metaDataTree->FindBranch(poolNames::processHistoryMapBranchName().c_str()) != nullptr) { - metaDataTree->SetBranchAddress(poolNames::processHistoryMapBranchName().c_str(), &pHistMapPtr); - } - ProcessHistoryRegistry::vector_type pHistVector; - ProcessHistoryRegistry::vector_type* pHistVectorPtr = &pHistVector; - if (metaDataTree->FindBranch(poolNames::processHistoryBranchName().c_str()) != nullptr) { - metaDataTree->SetBranchAddress(poolNames::processHistoryBranchName().c_str(), &pHistVectorPtr); - } - - // backward compatibility - ProcessConfigurationVector processConfigurations; - ProcessConfigurationVector* procConfigVectorPtr = &processConfigurations; - if (metaDataTree->FindBranch(poolNames::processConfigurationBranchName().c_str()) != nullptr) { - metaDataTree->SetBranchAddress(poolNames::processConfigurationBranchName().c_str(), &procConfigVectorPtr); - } + metaDataEntry->BindRawPtr(poolNames::processHistoryBranchName(), &pHistVector); auto branchIDListsAPtr = std::make_unique(); - BranchIDLists* branchIDListsPtr = branchIDListsAPtr.get(); - if (metaDataTree->FindBranch(poolNames::branchIDListBranchName().c_str()) != nullptr) { - metaDataTree->SetBranchAddress(poolNames::branchIDListBranchName().c_str(), &branchIDListsPtr); - } + metaDataEntry->BindRawPtr(poolNames::branchIDListBranchName(), branchIDListsAPtr.get()); - ThinnedAssociationsHelper* thinnedAssociationsHelperPtr; // must remain in scope through getEntry() if (inputType != InputType::SecondarySource) { fileThinnedAssociationsHelper_ = std::make_unique(); // propagate_const has no reset() function - thinnedAssociationsHelperPtr = fileThinnedAssociationsHelper_.get(); - if (metaDataTree->FindBranch(poolNames::thinnedAssociationsHelperBranchName().c_str()) != nullptr) { - metaDataTree->SetBranchAddress(poolNames::thinnedAssociationsHelperBranchName().c_str(), - &thinnedAssociationsHelperPtr); - } + metaDataEntry->BindRawPtr(poolNames::thinnedAssociationsHelperBranchName(), fileThinnedAssociationsHelper_.get()); } - ProductDependencies* productDependenciesBuffer = productDependencies_.get(); - if (metaDataTree->FindBranch(poolNames::productDependenciesBranchName().c_str()) != nullptr) { - metaDataTree->SetBranchAddress(poolNames::productDependenciesBranchName().c_str(), &productDependenciesBuffer); - } - - // backward compatibility - std::vector* eventHistoryIDsPtr = &eventProcessHistoryIDs_; - if (metaDataTree->FindBranch(poolNames::eventHistoryBranchName().c_str()) != nullptr) { - metaDataTree->SetBranchAddress(poolNames::eventHistoryBranchName().c_str(), &eventHistoryIDsPtr); - } - - if (metaDataTree->FindBranch(poolNames::moduleDescriptionMapBranchName().c_str()) != nullptr) { - if (metaDataTree->GetBranch(poolNames::moduleDescriptionMapBranchName().c_str())->GetSplitLevel() != 0) { - metaDataTree->SetBranchStatus((poolNames::moduleDescriptionMapBranchName() + ".*").c_str(), false); - } else { - metaDataTree->SetBranchStatus(poolNames::moduleDescriptionMapBranchName().c_str(), false); - } - } + metaDataEntry->BindRawPtr(poolNames::productDependenciesBranchName(), productDependencies_.get()); // Here we read the metadata tree - rootrntuple::getEntry(metaDataTree.get(), 0); - - eventProcessHistoryIter_ = eventProcessHistoryIDs_.begin(); - - // Here we read the event history tree, if we have one. - readEventHistoryTree(); + metaDataReader->LoadEntry(0, *metaDataEntry); ParameterSetConverter::ParameterSetIdConverter psetIdConverter; if (!fileFormatVersion().triggerPathsTracked()) { @@ -397,31 +326,23 @@ namespace edm::rntuple_temp { } std::shared_ptr mutableBranchIDLists; if (!fileFormatVersion().splitProductIDs()) { - // Old provenance format input file. Create a provenance adaptor. - // propagate_const has no reset() function - provenanceAdaptor_ = std::make_unique( - inputProdDescReg, pHistMap, pHistVector, processConfigurations, psetIdConverter, true); - // Fill in the branchIDLists branch from the provenance adaptor - mutableBranchIDLists = provenanceAdaptor_->releaseBranchIDLists(); + assert(false); } else { if (!fileFormatVersion().triggerPathsTracked()) { - // New provenance format, but change in ParameterSet Format. Create a provenance adaptor. - // propagate_const has no reset() function - provenanceAdaptor_ = std::make_unique( - inputProdDescReg, pHistMap, pHistVector, processConfigurations, psetIdConverter, false); + assert(false); } // New provenance format input file. The branchIDLists branch was read directly from the input file. - if (metaDataTree->FindBranch(poolNames::branchIDListBranchName().c_str()) == nullptr) { - throw Exception(errors::EventCorruption) << "Failed to find branchIDLists branch in metaData tree.\n"; - } + //if (metaDataTree->FindBranch(poolNames::branchIDListBranchName().c_str()) == nullptr) { + // throw Exception(errors::EventCorruption) << "Failed to find branchIDLists branch in metaData tree.\n"; + // } mutableBranchIDLists.reset(branchIDListsAPtr.release()); } if (fileFormatVersion().hasThinnedAssociations()) { - if (metaDataTree->FindBranch(poolNames::thinnedAssociationsHelperBranchName().c_str()) == nullptr) { - throw Exception(errors::EventCorruption) - << "Failed to find thinnedAssociationsHelper branch in metaData tree.\n"; - } + //if (metaDataTree->FindBranch(poolNames::thinnedAssociationsHelperBranchName().c_str()) == nullptr) { + // throw Exception(errors::EventCorruption) + // << "Failed to find thinnedAssociationsHelper branch in metaData tree.\n"; + //} } if (!fileOptions.bypassVersionCheck) { @@ -455,6 +376,7 @@ namespace edm::rntuple_temp { // Insert the new branch description into the product registry. inputProdDescReg.copyProduct(newBD); // Fix up other per file metadata. + ProcessConfigurationVector processConfigurations; //was only needed for backwards compatibility daqProvenanceHelper_->fixMetaData(processConfigurations, pHistVector); daqProvenanceHelper_->fixMetaData(*mutableBranchIDLists); daqProvenanceHelper_->fixMetaData(*productDependencies_); @@ -496,11 +418,10 @@ namespace edm::rntuple_temp { ? IndexIntoFile::entryOrder : (processingOptions.noEventSort ? IndexIntoFile::firstAppearanceOrder : IndexIntoFile::numericalOrder)); runHelper_->setForcedRunOffset(indexIntoFileBegin_ == indexIntoFileEnd_ ? 1 : indexIntoFileBegin_.run()); - eventProcessHistoryIter_ = eventProcessHistoryIDs_.begin(); - makeProcessBlockRootRNTuples(fileOptions.filePtr, ttreeOptions, inputType, storedProcessBlockHelper); + makeProcessBlockRootRNTuples(fileOptions.filePtr, ttreeOptions, inputType, *storedProcessBlockHelper_); - setPresenceInProductRegistry(inputProdDescReg, storedProcessBlockHelper); + setPresenceInProductRegistry(inputProdDescReg, *storedProcessBlockHelper_); auto newReg = std::make_unique(); @@ -529,14 +450,14 @@ namespace edm::rntuple_temp { productChoices.productSelectorRules, productChoices.dropDescendantsOfDroppedProducts, inputType, - storedProcessBlockHelper, + *storedProcessBlockHelper_, crossFileInfo.processBlockHelper); if (inputType == InputType::SecondaryFile) { crossFileInfo.thinnedAssociationsHelper->updateFromSecondaryInput(*fileThinnedAssociationsHelper_, *productChoices.associationsFromSecondary); } else if (inputType == InputType::Primary) { - crossFileInfo.processBlockHelper->initializeFromPrimaryInput(storedProcessBlockHelper); + crossFileInfo.processBlockHelper->initializeFromPrimaryInput(*storedProcessBlockHelper_); crossFileInfo.thinnedAssociationsHelper->updateFromPrimaryInput(*fileThinnedAssociationsHelper_); } @@ -569,7 +490,7 @@ namespace edm::rntuple_temp { std::vector nBranches(treePointers_.size(), 0); for (auto const& product : prodList) { if (product.second.branchType() == InProcess) { - std::vector const& processes = storedProcessBlockHelper.processesWithProcessBlockProducts(); + std::vector const& processes = storedProcessBlockHelper_->processesWithProcessBlockProducts(); auto it = std::find(processes.begin(), processes.end(), product.second.processName()); if (it != processes.end()) { auto index = std::distance(processes.begin(), it); @@ -589,7 +510,7 @@ namespace edm::rntuple_temp { for (auto const& product : prodList) { ProductDescription const& prod = product.second; if (prod.branchType() == InProcess) { - std::vector const& processes = storedProcessBlockHelper.processesWithProcessBlockProducts(); + std::vector const& processes = storedProcessBlockHelper_->processesWithProcessBlockProducts(); auto it = std::find(processes.begin(), processes.end(), prod.processName()); if (it != processes.end()) { auto index = std::distance(processes.begin(), it); @@ -669,21 +590,23 @@ namespace edm::rntuple_temp { void RootFile::readParentageTree(InputType inputType) { // New format file // We use a smart pointer so the tree will be deleted after use, and not kept for the life of the file. - std::unique_ptr parentageTree(dynamic_cast(filePtr_->Get(poolNames::parentageTreeName().c_str()))); - if (nullptr == parentageTree.get()) { + std::unique_ptr parentageRNTuple{ + filePtr_->Get(poolNames::parentageTreeName().c_str())}; + if (nullptr == parentageRNTuple.get()) { throw Exception(errors::FileReadError) - << "Could not find tree " << poolNames::parentageTreeName() << " in the input file.\n"; + << "Could not find RNTuple " << poolNames::parentageTreeName() << " in the input file.\n"; } + auto parentageReader = ROOT::RNTupleReader::Open(*parentageRNTuple); + auto entry = parentageReader->GetModel().CreateBareEntry(); Parentage parents; - Parentage* pParentageBuffer = &parents; - parentageTree->SetBranchAddress(poolNames::parentageBranchName().c_str(), &pParentageBuffer); + entry->BindRawPtr(poolNames::parentageBranchName(), &parents); ParentageRegistry& registry = *ParentageRegistry::instance(); - parentageIDLookup_.reserve(parentageTree->GetEntries()); - for (Long64_t i = 0, numEntries = parentageTree->GetEntries(); i < numEntries; ++i) { - rootrntuple::getEntry(parentageTree.get(), i); + parentageIDLookup_.reserve(parentageReader->GetNEntries()); + for (Long64_t i = 0, numEntries = parentageReader->GetNEntries(); i < numEntries; ++i) { + parentageReader->LoadEntry(i, *entry); if (daqProvenanceHelper_) { ParentageID const oldID = parents.id(); daqProvenanceHelper_->fixMetaData(parents.parentsForUpdate()); @@ -698,7 +621,6 @@ namespace edm::rntuple_temp { } parentageIDLookup_.push_back(parents.id()); } - parentageTree->SetBranchAddress(poolNames::parentageBranchName().c_str(), nullptr); } void RootFile::setIfFastClonable(int remainingEvents, int remainingLumis) { @@ -1268,7 +1190,6 @@ namespace edm::rntuple_temp { void RootFile::close() { // Just to play it safe, zero all pointers to objects in the InputFile to be closed. - eventHistoryTree_ = nullptr; for (auto& treePointer : treePointers_) { treePointer->close(); treePointer = nullptr; @@ -1329,28 +1250,10 @@ namespace edm::rntuple_temp { // store this History object in a different tree than the event // data tree, this is too hard to do in this first version. if (fileFormatVersion().eventHistoryBranch()) { - // Lumi block number was not in EventID for the relevant releases. - EventID id(evtAux.id().run(), 0, evtAux.id().event()); - if (eventProcessHistoryIter_->eventID() != id) { - EventProcessHistoryID target(id, ProcessHistoryID()); - eventProcessHistoryIter_ = lower_bound_all(eventProcessHistoryIDs_, target); - assert(eventProcessHistoryIter_->eventID() == id); - } - evtAux.setProcessHistoryID(eventProcessHistoryIter_->processHistoryID()); - ++eventProcessHistoryIter_; + assert(false); } else if (fileFormatVersion().eventHistoryTree()) { + assert(false); // for backward compatibility. - History* pHistory = history_.get(); - TBranch* eventHistoryBranch = eventHistoryTree_->GetBranch(poolNames::eventHistoryBranchName().c_str()); - if (!eventHistoryBranch) { - throw Exception(errors::EventCorruption) << "Failed to find history branch in event history tree.\n"; - } - eventHistoryBranch->SetAddress(&pHistory); - rootrntuple::getEntry(eventHistoryTree_, eventTree_.entryNumber()); - eventHistoryBranch->SetAddress(nullptr); - evtAux.setProcessHistoryID(history_->processHistoryID()); - eventSelectionIDs.swap(history_->eventSelectionIDs()); - branchListIndexes.swap(history_->branchListIndexes()); } else if (fileFormatVersion().noMetaDataTrees()) { // Current format EventSelectionIDVector* pESV = &eventSelectionIDs; @@ -1362,18 +1265,11 @@ namespace edm::rntuple_temp { assert(branchListIndexesBranch != nullptr); eventTree_.fillBranchEntry(branchListIndexesBranch, pBLI); } - if (provenanceAdaptor_) { - evtAux.setProcessHistoryID(provenanceAdaptor_->convertID(evtAux.processHistoryID())); - for (auto& esID : eventSelectionIDs) { - esID = provenanceAdaptor_->convertID(esID); - } - } if (daqProvenanceHelper_) { evtAux.setProcessHistoryID(daqProvenanceHelper_->mapProcessHistoryID(evtAux.processHistoryID())); } if (!fileFormatVersion().splitProductIDs()) { // old format. branchListIndexes_ must be filled in from the ProvenanceAdaptor. - provenanceAdaptor_->branchListIndexes(branchListIndexes); } if (branchIDListHelper_) { return branchIDListHelper_->fixBranchListIndexes(branchListIndexes, assertOnFailure); @@ -1392,9 +1288,6 @@ namespace edm::rntuple_temp { lumiTree_.fillAux(pLumiAux); conversion(lumiAux, *lumiAuxiliary); } - if (provenanceAdaptor_) { - lumiAuxiliary->setProcessHistoryID(provenanceAdaptor_->convertID(lumiAuxiliary->processHistoryID())); - } if (daqProvenanceHelper_) { lumiAuxiliary->setProcessHistoryID(daqProvenanceHelper_->mapProcessHistoryID(lumiAuxiliary->processHistoryID())); } @@ -1415,9 +1308,6 @@ namespace edm::rntuple_temp { runTree_.fillAux(pRunAux); conversion(runAux, *runAuxiliary); } - if (provenanceAdaptor_) { - runAuxiliary->setProcessHistoryID(provenanceAdaptor_->convertID(runAuxiliary->processHistoryID())); - } if (daqProvenanceHelper_) { runAuxiliary->setProcessHistoryID(daqProvenanceHelper_->mapProcessHistoryID(runAuxiliary->processHistoryID())); } @@ -1859,17 +1749,6 @@ namespace edm::rntuple_temp { return true; } - void RootFile::readEventHistoryTree() { - // Read in the event history tree, if we have one... - if (fileFormatVersion().eventHistoryTree()) { - history_ = std::make_unique(); // propagate_const has no reset() function - eventHistoryTree_ = dynamic_cast(filePtr_->Get(poolNames::eventHistoryTreeName().c_str())); - if (!eventHistoryTree_) { - throw Exception(errors::EventCorruption) << "Failed to find the event history tree.\n"; - } - } - } - void RootFile::initializeDuplicateChecker( std::vector> const& indexesIntoFiles, std::vector>::size_type currentIndexIntoFile) { diff --git a/FWIO/RNTupleTempInput/src/RootFile.h b/FWIO/RNTupleTempInput/src/RootFile.h index e176cd08362e4..97a52a8b90607 100644 --- a/FWIO/RNTupleTempInput/src/RootFile.h +++ b/FWIO/RNTupleTempInput/src/RootFile.h @@ -245,7 +245,6 @@ namespace edm { void readParentageTree(InputType inputType); void readEntryDescriptionTree(EntryDescriptionMap& entryDescriptionMap, InputType inputType); // backward compatibility - void readEventHistoryTree(); bool isDuplicateEvent(); void initializeDuplicateChecker(std::vector> const& indexesIntoFiles, @@ -282,8 +281,6 @@ namespace edm { IndexIntoFile::IndexIntoFileItr indexIntoFileEnd_; IndexIntoFile::IndexIntoFileItr indexIntoFileIter_; edm::propagate_const> storedMergeableRunProductMetadata_; - std::vector eventProcessHistoryIDs_; // backward compatibility - std::vector::const_iterator eventProcessHistoryIter_; // backward compatibility edm::propagate_const> savedRunAuxiliary_; bool skipAnyEvents_; bool noRunLumiSort_; @@ -311,13 +308,10 @@ namespace edm { InputSource::ProcessingMode processingMode_; edm::propagate_const runHelper_; std::map newBranchToOldBranch_; - edm::propagate_const eventHistoryTree_; // backward compatibility EventToProcessBlockIndexes eventToProcessBlockIndexes_; edm::propagate_const eventToProcessBlockIndexesBranch_; - edm::propagate_const> history_; // backward compatibility edm::propagate_const> productDependencies_; edm::propagate_const> duplicateChecker_; - edm::propagate_const> provenanceAdaptor_; // backward comatibility edm::propagate_const> provenanceReaderMaker_; std::vector>> eventProductProvenanceRetrievers_; std::vector parentageIDLookup_; diff --git a/FWIO/RNTupleTempInput/test/testReducedProcessHistoryHardwareResources.sh b/FWIO/RNTupleTempInput/test/testReducedProcessHistoryHardwareResources.sh index 3a750962c2fda..03e9a43f7936b 100755 --- a/FWIO/RNTupleTempInput/test/testReducedProcessHistoryHardwareResources.sh +++ b/FWIO/RNTupleTempInput/test/testReducedProcessHistoryHardwareResources.sh @@ -12,16 +12,15 @@ function runSuccess { runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --accelerators test-one --firstEvent 1 --output test-one.root runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --accelerators test-two --firstEvent 101 --output test-two.root -#edmProvDump test-one.root | grep -q "accelerators: test-one" || die "Did not find test-one from test-one.root provenance" $? -#edmProvDump test-two.root | grep -q "accelerators: test-two" || die "Did not find test-two from test-two.root provenance" $? +edmRNTupleTempProvDump test-one.root | grep -q "accelerators: test-one" || die "Did not find test-one from test-one.root provenance" $? +edmRNTupleTempProvDump test-two.root | grep -q "accelerators: test-two" || die "Did not find test-two from test-two.root provenance" $? runSuccess ${SCRAM_TEST_PATH}/test_merge_two_files.py test-one.root test-two.root runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_files.root -#can't handle RNTUple -#edmProvDump merged_files.root | grep -q "accelerators: test-one" || die "Did not find test-one from merged_files.root provenance" $? -#edmProvDump merged_files.root | grep -q "accelerators: test-two" || die "Did not find test-two from merged_files.root provenance" $? +edmRNTupleTempProvDump merged_files.root | grep -q "accelerators: test-one" || die "Did not find test-one from merged_files.root provenance" $? +edmRNTupleTempProvDump merged_files.root | grep -q "accelerators: test-two" || die "Did not find test-two from merged_files.root provenance" $? exit 0 diff --git a/FWIO/RNTupleTempInput/test/testReducedProcessHistoryVersion.sh b/FWIO/RNTupleTempInput/test/testReducedProcessHistoryVersion.sh index 2b2ced3dcd98e..8efd7d07bd711 100755 --- a/FWIO/RNTupleTempInput/test/testReducedProcessHistoryVersion.sh +++ b/FWIO/RNTupleTempInput/test/testReducedProcessHistoryVersion.sh @@ -21,8 +21,8 @@ VERSION3="${VERSION_ARR[0]}_${VERSION_ARR[1]}_$((${VERSION_ARR[2]}+1))_0" runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --version ${VERSION1} --firstEvent 1 --output version1.root runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --version ${VERSION2} --firstEvent 101 --output version2.root -#edmProvDump version1.root | grep -q "version: '${VERSION1}'" || die "Did not find ${VERSION1} from version.root provenance" $? -#edmProvDump version2.root | grep -q "version: '${VERSION2}'" || die "Did not find ${VERSION2} from version.root provenance" $? +edmRNTupleTempProvDump version1.root | grep -q "version: '${VERSION1}'" || die "Did not find ${VERSION1} from version.root provenance" $? +edmRNTupleTempProvDump version2.root | grep -q "version: '${VERSION2}'" || die "Did not find ${VERSION2} from version.root provenance" $? runSuccess ${SCRAM_TEST_PATH}/test_merge_two_files.py version1.root version2.root @@ -32,7 +32,7 @@ runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_fi # Check that changing the minor version leads to new lumi runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --version ${VERSION3} --firstEvent 201 --output version3_lumi.root -#edmProvDump version3_lumi.root | grep -q "version: '${VERSION3}'" || die "Did not find ${VERSION3} from version3_lumi.root provenance" $? +edmRNTupleTempProvDump version3_lumi.root | grep -q "version: '${VERSION3}'" || die "Did not find ${VERSION3} from version3_lumi.root provenance" $? runSuccess ${SCRAM_TEST_PATH}/test_merge_two_files.py version1.root version3_lumi.root --output merged_files3_lumi.root --bypassVersionCheck @@ -44,7 +44,7 @@ runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_fi # Check that changing the minor version leads to new run runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --version ${VERSION3} --firstEvent 201 --lumi 2 --output version3_run.root -#edmProvDump version3_run.root | grep -q "version: '${VERSION3}'" || die "Did not find ${VERSION3} from version3_lumi.root provenance" $? +edmRNTupleTempProvDump version3_run.root | grep -q "version: '${VERSION3}'" || die "Did not find ${VERSION3} from version3_lumi.root provenance" $? runSuccess ${SCRAM_TEST_PATH}/test_merge_two_files.py version1.root version3_run.root --output merged_files3_run.root --bypassVersionCheck diff --git a/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h b/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h index 10d19608a766f..6a6d04c77dd64 100644 --- a/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h +++ b/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h @@ -182,19 +182,11 @@ namespace edm::rntuple_temp { void fillDependencyGraph(); void startEndFile(); - void writeFileFormatVersion(); - void writeFileIdentifier(); - void writeIndexIntoFile(); - void writeStoredMergeableRunProductMetadata(); - void writeProcessHistoryRegistry(); + void writeMetaData(); + void writeParameterSetRegistry(); - void writeProductDescriptionRegistry(); void writeParentageRegistry(); - void writeBranchIDListRegistry(); - void writeThinnedAssociationsHelper(); - void writeProductDependencies(); void writeEventAuxiliary(); - void writeProcessBlockHelper(); void finishEndFile(); void fillSelectedItemList(BranchType branchtype, diff --git a/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc b/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc index b23945983b8c5..fdd339809448c 100644 --- a/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc +++ b/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc @@ -355,18 +355,10 @@ namespace edm::rntuple_temp { fillDependencyGraph(); branchParents_.clear(); startEndFile(); - writeFileFormatVersion(); - writeFileIdentifier(); - writeIndexIntoFile(); - writeStoredMergeableRunProductMetadata(); - writeProcessHistoryRegistry(); + writeMetaData(); + writeParameterSetRegistry(); - writeProductDescriptionRegistry(); writeParentageRegistry(); - writeBranchIDListRegistry(); - writeThinnedAssociationsHelper(); - writeProductDependencies(); //productDependencies used here - writeProcessBlockHelper(); productDependencies_.clear(); finishEndFile(); @@ -376,24 +368,11 @@ namespace edm::rntuple_temp { // At some later date, we may move functionality from finishEndFile() to here. void RNTupleTempOutputModule::startEndFile() {} - void RNTupleTempOutputModule::writeFileFormatVersion() { rootOutputFile_->writeFileFormatVersion(); } - void RNTupleTempOutputModule::writeFileIdentifier() { rootOutputFile_->writeFileIdentifier(); } - void RNTupleTempOutputModule::writeIndexIntoFile() { rootOutputFile_->writeIndexIntoFile(); } - void RNTupleTempOutputModule::writeStoredMergeableRunProductMetadata() { - rootOutputFile_->writeStoredMergeableRunProductMetadata(); - } - void RNTupleTempOutputModule::writeProcessHistoryRegistry() { rootOutputFile_->writeProcessHistoryRegistry(); } + void RNTupleTempOutputModule::writeMetaData() { rootOutputFile_->writeMetaData(*reg_); } + void RNTupleTempOutputModule::writeParameterSetRegistry() { rootOutputFile_->writeParameterSetRegistry(); } - void RNTupleTempOutputModule::writeProductDescriptionRegistry() { - assert(reg_); - rootOutputFile_->writeProductDescriptionRegistry(*reg_); - } void RNTupleTempOutputModule::writeParentageRegistry() { rootOutputFile_->writeParentageRegistry(); } - void RNTupleTempOutputModule::writeBranchIDListRegistry() { rootOutputFile_->writeBranchIDListRegistry(); } - void RNTupleTempOutputModule::writeThinnedAssociationsHelper() { rootOutputFile_->writeThinnedAssociationsHelper(); } - void RNTupleTempOutputModule::writeProductDependencies() { rootOutputFile_->writeProductDependencies(); } void RNTupleTempOutputModule::writeEventAuxiliary() { rootOutputFile_->writeEventAuxiliary(); } - void RNTupleTempOutputModule::writeProcessBlockHelper() { rootOutputFile_->writeProcessBlockHelper(); } void RNTupleTempOutputModule::finishEndFile() { rootOutputFile_->finishEndFile(); rootOutputFile_ = nullptr; diff --git a/FWIO/RNTupleTempOutput/src/RootOutputFile.cc b/FWIO/RNTupleTempOutput/src/RootOutputFile.cc index c4d14b4105e25..4181c9f753dff 100644 --- a/FWIO/RNTupleTempOutput/src/RootOutputFile.cc +++ b/FWIO/RNTupleTempOutput/src/RootOutputFile.cc @@ -50,7 +50,6 @@ #include "ROOT/RNTuple.hxx" #include "ROOT/RNTupleWriter.hxx" - #include "Compression.h" #include @@ -101,8 +100,6 @@ namespace edm::rntuple_temp { indexIntoFile_(), storedMergeableRunProductMetadata_(processesWithSelectedMergeableRunProducts), nEventsInLumi_(0), - metaDataTree_(nullptr), - parentageTree_(nullptr), lumiAux_(), runAux_(), pEventAux_(nullptr), @@ -196,9 +193,6 @@ namespace edm::rntuple_temp { branchesWithStoredHistory_.insert(item.branchID()); } } - // Don't split metadata tree or event description tree - metaDataTree_ = RootOutputRNTuple::makeTTree(filePtr_.get(), poolNames::metaDataTreeName(), 0); - parentageTree_ = RootOutputRNTuple::makeTTree(filePtr_.get(), poolNames::parentageTreeName(), 0); if (overrideGUID.empty()) { fid_ = FileID(createGlobalIdentifier()); @@ -583,11 +577,49 @@ namespace edm::rntuple_temp { treePointers_[ttreeIndex]->optimizeBaskets(10ULL * 1024 * 1024); } + void RootOutputFile::writeMetaData(ProductRegistry const& iReg) { + auto model = ROOT::RNTupleModel::CreateBare(); + { + model->AddField(setupFileFormatVersion()); + model->AddField(setupFileIdentifier()); + model->AddField(setupIndexIntoFile()); + model->AddField(setupStoredMergeableRunProductMetadata()); + model->AddField(setupProcessHistoryRegistry()); + model->AddField(setupProductDescriptionRegistry()); + model->AddField(setupBranchIDListRegistry()); + model->AddField(setupThinnedAssociationsHelper()); + model->AddField(setupProductDependencies()); + model->AddField(setupProcessBlockHelper()); + } + + auto writeOptions = ROOT::RNTupleWriteOptions(); + //writeOptions.SetCompression(convert(iConfig.compressionAlgo), iConfig.compressionLevel); + auto metaData = + ROOT::RNTupleWriter::Append(std::move(model), poolNames::metaDataTreeName(), *filePtr_, writeOptions); + + auto rentry = metaData->CreateEntry(); + + writeFileFormatVersion(*rentry); + writeFileIdentifier(*rentry); + writeIndexIntoFile(*rentry); + writeStoredMergeableRunProductMetadata(*rentry); + writeProcessHistoryRegistry(*rentry); + writeProductDescriptionRegistry(*rentry, iReg); + writeBranchIDListRegistry(*rentry); + writeThinnedAssociationsHelper(*rentry); + writeProductDependencies(*rentry); + writeProcessBlockHelper(*rentry); + metaData->Fill(*rentry); + } + void RootOutputFile::writeParentageRegistry() { - Parentage const* desc(nullptr); + auto model = ROOT::RNTupleModel::CreateBare(); + model->AddField(ROOT::RFieldBase::Create(poolNames::parentageBranchName(), "edm::Parentage").Unwrap()); - if (!parentageTree_->Branch(poolNames::parentageBranchName().c_str(), &desc, om_->basketSize(), 0)) - throw Exception(errors::FatalRootError) << "Failed to create a branch for Parentages in the output file"; + auto writeOptions = ROOT::RNTupleWriteOptions(); + //writeOptions.SetCompression(convert(iConfig.compressionAlgo), iConfig.compressionLevel); + auto parentageWriter = + ROOT::RNTupleWriter::Append(std::move(model), poolNames::parentageTreeName(), *filePtr_, writeOptions); ParentageRegistry& ptReg = *ParentageRegistry::instance(); @@ -595,33 +627,70 @@ namespace edm::rntuple_temp { for (auto const& parentageID : parentageIDs_) { orderedIDs[parentageID.second] = parentageID.first; } + auto rentry = parentageWriter->CreateEntry(); + //now put them into the TTree in the correct order for (auto const& orderedID : orderedIDs) { - desc = ptReg.getMapped(orderedID); - //NOTE: some old format files have missing Parentage info - // so a null value of desc can't be fatal. - // Root will default construct an object in that case. - parentageTree_->Fill(); + rentry->BindRawPtr(poolNames::parentageBranchName(), const_cast(ptReg.getMapped(orderedID))); + parentageWriter->Fill(*rentry); } } + ///////// + + std::unique_ptr RootOutputFile::setupFileFormatVersion() { + return ROOT::RFieldBase::Create(poolNames::fileFormatVersionBranchName(), "edm::FileFormatVersion").Unwrap(); + } + + std::unique_ptr RootOutputFile::setupFileIdentifier() { + return ROOT::RFieldBase::Create(poolNames::fileIdentifierBranchName(), "edm::FileID").Unwrap(); + } + + std::unique_ptr RootOutputFile::setupIndexIntoFile() { + return ROOT::RFieldBase::Create(poolNames::indexIntoFileBranchName(), "edm::IndexIntoFile").Unwrap(); + } + + std::unique_ptr RootOutputFile::setupStoredMergeableRunProductMetadata() { + return ROOT::RFieldBase::Create(poolNames::mergeableRunProductMetadataBranchName(), + "edm::StoredMergeableRunProductMetadata") + .Unwrap(); + } + + std::unique_ptr RootOutputFile::setupProcessHistoryRegistry() { + return ROOT::RFieldBase::Create(poolNames::processHistoryBranchName(), "std::vector").Unwrap(); + } - void RootOutputFile::writeFileFormatVersion() { - FileFormatVersion fileFormatVersion(getFileFormatVersion()); - FileFormatVersion* pFileFmtVsn = &fileFormatVersion; - TBranch* b = - metaDataTree_->Branch(poolNames::fileFormatVersionBranchName().c_str(), &pFileFmtVsn, om_->basketSize(), 0); - assert(b); - b->Fill(); + std::unique_ptr RootOutputFile::setupBranchIDListRegistry() { + return ROOT::RFieldBase::Create(poolNames::branchIDListBranchName(), "std::vector>") + .Unwrap(); } - void RootOutputFile::writeFileIdentifier() { - FileID* fidPtr = &fid_; - TBranch* b = metaDataTree_->Branch(poolNames::fileIdentifierBranchName().c_str(), &fidPtr, om_->basketSize(), 0); - assert(b); - b->Fill(); + std::unique_ptr RootOutputFile::setupThinnedAssociationsHelper() { + return ROOT::RFieldBase::Create(poolNames::thinnedAssociationsHelperBranchName(), "edm::ThinnedAssociationsHelper") + .Unwrap(); } - void RootOutputFile::writeIndexIntoFile() { + std::unique_ptr RootOutputFile::setupProductDescriptionRegistry() { + return ROOT::RFieldBase::Create(poolNames::productDescriptionBranchName(), "edm::ProductRegistry").Unwrap(); + } + std::unique_ptr RootOutputFile::setupProductDependencies() { + return ROOT::RFieldBase::Create(poolNames::productDependenciesBranchName(), "edm::ProductDependencies").Unwrap(); + } + + std::unique_ptr RootOutputFile::setupProcessBlockHelper() { + return ROOT::RFieldBase::Create(poolNames::processBlockHelperBranchName(), "edm::StoredProcessBlockHelper").Unwrap(); + } + + //////// + void RootOutputFile::writeFileFormatVersion(ROOT::REntry& rentry) { + auto fileFormatVersion = std::make_shared(getFileFormatVersion()); + rentry.BindValue(poolNames::fileFormatVersionBranchName(), fileFormatVersion); + } + + void RootOutputFile::writeFileIdentifier(ROOT::REntry& rentry) { + rentry.BindRawPtr(poolNames::fileIdentifierBranchName(), &fid_); + } + + void RootOutputFile::writeIndexIntoFile(ROOT::REntry& rentry) { if (eventTree_.checkEntriesInReadBranches(eventEntryNumber_) == false) { Exception ex(errors::OtherCMS); ex << "The number of entries in at least one output TBranch whose entries\n" @@ -632,69 +701,36 @@ namespace edm::rntuple_temp { throw ex; } indexIntoFile_.sortVector_Run_Or_Lumi_Entries(); - IndexIntoFile* iifPtr = &indexIntoFile_; - TBranch* b = metaDataTree_->Branch(poolNames::indexIntoFileBranchName().c_str(), &iifPtr, om_->basketSize(), 0); - assert(b); - b->Fill(); + rentry.BindRawPtr(poolNames::indexIntoFileBranchName(), &indexIntoFile_); } - void RootOutputFile::writeStoredMergeableRunProductMetadata() { + void RootOutputFile::writeStoredMergeableRunProductMetadata(ROOT::REntry& rentry) { storedMergeableRunProductMetadata_.optimizeBeforeWrite(); - StoredMergeableRunProductMetadata* ptr = &storedMergeableRunProductMetadata_; - TBranch* b = - metaDataTree_->Branch(poolNames::mergeableRunProductMetadataBranchName().c_str(), &ptr, om_->basketSize(), 0); - assert(b); - b->Fill(); - } - - void RootOutputFile::writeProcessHistoryRegistry() { - fillProcessHistoryBranch(metaDataTree_.get(), om_->basketSize(), processHistoryRegistry_); + rentry.BindRawPtr(poolNames::mergeableRunProductMetadataBranchName(), &storedMergeableRunProductMetadata_); } - void RootOutputFile::writeBranchIDListRegistry() { - BranchIDLists const* p = om_->branchIDLists(); - TBranch* b = metaDataTree_->Branch(poolNames::branchIDListBranchName().c_str(), &p, om_->basketSize(), 0); - assert(b); - b->Fill(); + void RootOutputFile::writeProcessHistoryRegistry(ROOT::REntry& rentry) { + auto procHistoryVector = std::make_shared(); + for (auto const& ph : processHistoryRegistry_) { + procHistoryVector->push_back(ph.second); + } + rentry.BindValue(poolNames::processHistoryBranchName(), procHistoryVector); } - void RootOutputFile::writeThinnedAssociationsHelper() { - ThinnedAssociationsHelper const* p = om_->thinnedAssociationsHelper(); - TBranch* b = - metaDataTree_->Branch(poolNames::thinnedAssociationsHelperBranchName().c_str(), &p, om_->basketSize(), 0); - assert(b); - b->Fill(); + void RootOutputFile::writeBranchIDListRegistry(ROOT::REntry& rentry) { + rentry.BindRawPtr(poolNames::branchIDListBranchName(), const_cast(om_->branchIDLists())); } - void RootOutputFile::writeParameterSetRegistry() { - auto model = ROOT::RNTupleModel::CreateBare(); - { - auto field = - ROOT::RFieldBase::Create("IdToParameterSetsBlobs", "std::pair,edm::ParameterSetBlob>").Unwrap(); - model->AddField(std::move(field)); - } - auto writeOptions = ROOT::RNTupleWriteOptions(); - //writeOptions.SetCompression(convert(iConfig.compressionAlgo), iConfig.compressionLevel); - auto parameterSets = ROOT::RNTupleWriter::Append(std::move(model), poolNames::parameterSetsTreeName(), *filePtr_, writeOptions); - - std::pair idToBlob; - - auto rentry = parameterSets->CreateEntry(); - rentry->BindRawPtr("IdToParameterSetsBlobs", static_cast(&idToBlob)); - - for (auto const& pset : *pset::Registry::instance()) { - idToBlob.first = pset.first; - idToBlob.second.pset() = pset.second.toString(); - - parameterSets->Fill(*rentry); - } + void RootOutputFile::writeThinnedAssociationsHelper(ROOT::REntry& rentry) { + auto* p = const_cast(om_->thinnedAssociationsHelper()); + rentry.BindRawPtr(poolNames::thinnedAssociationsHelperBranchName(), p); } - void RootOutputFile::writeProductDescriptionRegistry(ProductRegistry const& iReg) { + void RootOutputFile::writeProductDescriptionRegistry(ROOT::REntry& rentry, ProductRegistry const& iReg) { // Make a local copy of the ProductRegistry, removing any transient or pruned products. using ProductList = ProductRegistry::ProductList; - ProductRegistry pReg(iReg.productList()); - ProductList& pList = const_cast(pReg.productList()); + auto pReg = std::make_shared(iReg.productList()); + ProductList& pList = const_cast(pReg->productList()); for (auto const& prod : pList) { if (prod.second.branchID() != prod.second.originalBranchID()) { if (branchesWithStoredHistory_.find(prod.second.branchID()) != branchesWithStoredHistory_.end()) { @@ -715,18 +751,45 @@ namespace edm::rntuple_temp { } } - ProductRegistry* ppReg = &pReg; - TBranch* b = metaDataTree_->Branch(poolNames::productDescriptionBranchName().c_str(), &ppReg, om_->basketSize(), 0); - assert(b); - b->Fill(); + rentry.BindValue(poolNames::productDescriptionBranchName(), pReg); } - void RootOutputFile::writeProductDependencies() { + void RootOutputFile::writeProductDependencies(ROOT::REntry& rentry) { ProductDependencies& pDeps = const_cast(om_->productDependencies()); - ProductDependencies* ppDeps = &pDeps; - TBranch* b = - metaDataTree_->Branch(poolNames::productDependenciesBranchName().c_str(), &ppDeps, om_->basketSize(), 0); - assert(b); - b->Fill(); + rentry.BindRawPtr(poolNames::productDependenciesBranchName(), &pDeps); + } + + void RootOutputFile::writeProcessBlockHelper(ROOT::REntry& rentry) { + if (!om_->outputProcessBlockHelper().processesWithProcessBlockProducts().empty()) { + auto storedProcessBlockHelper = std::make_shared( + om_->outputProcessBlockHelper().processesWithProcessBlockProducts()); + om_->outputProcessBlockHelper().fillCacheIndices(*storedProcessBlockHelper); + + rentry.BindValue(poolNames::processBlockHelperBranchName(), storedProcessBlockHelper); + } + } + void RootOutputFile::writeParameterSetRegistry() { + auto model = ROOT::RNTupleModel::CreateBare(); + { + auto field = + ROOT::RFieldBase::Create("IdToParameterSetsBlobs", "std::pair,edm::ParameterSetBlob>").Unwrap(); + model->AddField(std::move(field)); + } + auto writeOptions = ROOT::RNTupleWriteOptions(); + //writeOptions.SetCompression(convert(iConfig.compressionAlgo), iConfig.compressionLevel); + auto parameterSets = + ROOT::RNTupleWriter::Append(std::move(model), poolNames::parameterSetsTreeName(), *filePtr_, writeOptions); + + std::pair idToBlob; + + auto rentry = parameterSets->CreateEntry(); + rentry->BindRawPtr("IdToParameterSetsBlobs", &idToBlob); + + for (auto const& pset : *pset::Registry::instance()) { + idToBlob.first = pset.first; + idToBlob.second.pset() = pset.second.toString(); + + parameterSets->Fill(*rentry); + } } // For duplicate removal and to determine if fast cloning is possible, the input @@ -766,31 +829,10 @@ namespace edm::rntuple_temp { } } - void RootOutputFile::writeProcessBlockHelper() { - if (!om_->outputProcessBlockHelper().processesWithProcessBlockProducts().empty()) { - StoredProcessBlockHelper storedProcessBlockHelper( - om_->outputProcessBlockHelper().processesWithProcessBlockProducts()); - om_->outputProcessBlockHelper().fillCacheIndices(storedProcessBlockHelper); - - StoredProcessBlockHelper* pStoredProcessBlockHelper = &storedProcessBlockHelper; - TBranch* b = metaDataTree_->Branch( - poolNames::processBlockHelperBranchName().c_str(), &pStoredProcessBlockHelper, om_->basketSize(), 0); - assert(b); - b->Fill(); - } - } - void RootOutputFile::finishEndFile() { std::string_view status = "beginning"; std::string_view value = ""; try { - metaDataTree_->SetEntries(-1); - status = "writeTTree() for metadata"; - RootOutputRNTuple::writeTTree(metaDataTree_); - - status = "writeTTree() for parentage"; - RootOutputRNTuple::writeTTree(parentageTree_); - // Create branch aliases for all the branches in the // events/lumis/runs/processblock trees. The loop is over // all types of data products. @@ -812,7 +854,6 @@ namespace edm::rntuple_temp { // Just to play it safe, zero all pointers to objects in the TFile to be closed. status = "closing TTrees"; value = ""; - metaDataTree_ = parentageTree_ = nullptr; for (auto& treePointer : treePointers_) { treePointer->close(); treePointer = nullptr; diff --git a/FWIO/RNTupleTempOutput/src/RootOutputFile.h b/FWIO/RNTupleTempOutput/src/RootOutputFile.h index ec7e5650a451b..63af40fb5b4d5 100644 --- a/FWIO/RNTupleTempOutput/src/RootOutputFile.h +++ b/FWIO/RNTupleTempOutput/src/RootOutputFile.h @@ -1,5 +1,5 @@ -#ifndef IOPool_Output_RootOutputFile_h -#define IOPool_Output_RootOutputFile_h +#ifndef FWIO_RNTupleTempOutput_RootOutputFile_h +#define FWIO_RNTupleTempOutput_RootOutputFile_h ////////////////////////////////////////////////////////////////////// // @@ -41,6 +41,9 @@ class TTree; class TFile; class TClass; +#include "ROOT/RFieldBase.hxx" +#include "ROOT/REntry.hxx" + namespace edm { class OccurrenceForOutput; } @@ -62,19 +65,10 @@ namespace edm::rntuple_temp { void writeLuminosityBlock(LuminosityBlockForOutput const& lb); void writeRun(RunForOutput const& r); void writeProcessBlock(ProcessBlockForOutput const&); - void writeFileFormatVersion(); - void writeFileIdentifier(); - void writeIndexIntoFile(); - void writeStoredMergeableRunProductMetadata(); - void writeProcessHistoryRegistry(); void writeParameterSetRegistry(); - void writeProductDescriptionRegistry(ProductRegistry const&); void writeParentageRegistry(); - void writeBranchIDListRegistry(); - void writeThinnedAssociationsHelper(); - void writeProductDependencies(); + void writeMetaData(ProductRegistry const&); void writeEventAuxiliary(); - void writeProcessBlockHelper(); void finishEndFile(); void beginInputFile(FileBlock const& fb, int remainingEvents); @@ -84,6 +78,28 @@ namespace edm::rntuple_temp { std::string const& fileName() const { return file_; } private: + std::unique_ptr setupFileFormatVersion(); + std::unique_ptr setupFileIdentifier(); + std::unique_ptr setupIndexIntoFile(); + std::unique_ptr setupStoredMergeableRunProductMetadata(); + std::unique_ptr setupProcessHistoryRegistry(); + std::unique_ptr setupProductDescriptionRegistry(); + std::unique_ptr setupBranchIDListRegistry(); + std::unique_ptr setupThinnedAssociationsHelper(); + std::unique_ptr setupProductDependencies(); + std::unique_ptr setupProcessBlockHelper(); + + void writeFileFormatVersion(ROOT::REntry&); + void writeFileIdentifier(ROOT::REntry&); + void writeIndexIntoFile(ROOT::REntry&); + void writeStoredMergeableRunProductMetadata(ROOT::REntry&); + void writeProcessHistoryRegistry(ROOT::REntry&); + void writeProductDescriptionRegistry(ROOT::REntry&, ProductRegistry const&); + void writeBranchIDListRegistry(ROOT::REntry&); + void writeThinnedAssociationsHelper(ROOT::REntry&); + void writeProductDependencies(ROOT::REntry&); + void writeProcessBlockHelper(ROOT::REntry&); + void setBranchAliases(TTree* tree, SelectedProducts const& branches, std::string const& processName) const; void fillBranches(BranchType const& branchType, @@ -124,8 +140,6 @@ namespace edm::rntuple_temp { IndexIntoFile indexIntoFile_; StoredMergeableRunProductMetadata storedMergeableRunProductMetadata_; unsigned long nEventsInLumi_; - edm::propagate_const metaDataTree_; - edm::propagate_const parentageTree_; LuminosityBlockAuxiliary lumiAux_; RunAuxiliary runAux_; EventAuxiliary const* pEventAux_; diff --git a/FWIO/RNTupleTempOutput/test/TestRNTupleTempOutput.sh b/FWIO/RNTupleTempOutput/test/TestRNTupleTempOutput.sh index f41095960b04a..44c6347e466cb 100755 --- a/FWIO/RNTupleTempOutput/test/TestRNTupleTempOutput.sh +++ b/FWIO/RNTupleTempOutput/test/TestRNTupleTempOutput.sh @@ -5,9 +5,9 @@ function die { echo $1: status $2 ; exit $2; } LOCAL_TEST_DIR=$SCRAM_TEST_PATH cmsRun ${LOCAL_TEST_DIR}/PoolOutputTest_cfg.py || die 'Failure using PoolOutputTest_cfg.py 1' $? -GUID1=$(edmFileUtil -u PoolOutputTest.root | fgrep uuid | awk '{print $10}') +GUID1=$(edmRNTupleTempFileUtil -u PoolOutputTest.root | fgrep uuid | awk '{print $10}') cmsRun ${LOCAL_TEST_DIR}/PoolOutputTest_cfg.py || die 'Failure using PoolOutputTest_cfg.py 2' $? -GUID2=$(edmFileUtil -u PoolOutputTest.root | fgrep uuid | awk '{print $10}') +GUID2=$(edmRNTupleTempFileUtil -u PoolOutputTest.root | fgrep uuid | awk '{print $10}') if [ "x${GUID1}" == "x${GUID2}" ]; then echo "GUID from two executions are the same: ${GUID1}" exit 1 @@ -41,13 +41,13 @@ cmsRun ${LOCAL_TEST_DIR}/PoolOutputTestUnscheduled_cfg.py || die 'Failure using cmsRun ${LOCAL_TEST_DIR}/PoolOutputTestUnscheduledRead_cfg.py || die 'Failure using PoolOutputTestUnscheduledRead_cfg.py' $? cmsRun ${LOCAL_TEST_DIR}/PoolOutputTestOverrideGUID_cfg.py --guid abcdef01-2345-6789-abcd-ef0123456789 || die 'Failure using PoolOutputTestOverrideGUID_cfg.py with valid GUID' $? -GUID=$(edmFileUtil -u PoolOutputTestOverrideGUID.root | fgrep uuid | awk '{print $10}') +GUID=$(edmRNTupleTempFileUtil -u PoolOutputTestOverrideGUID.root | fgrep uuid | awk '{print $10}') if [ "x${GUID}" != "xabcdef01-2345-6789-abcd-ef0123456789" ]; then echo "GUID in file '${GUID}' did not match 'abcdef01-2345-6789-abcd-ef0123456789'" exit 1 fi cmsRun ${LOCAL_TEST_DIR}/PoolOutputTestOverrideGUID_cfg.py --guid ABCDEF01-2345-6789-abcd-ef0123456789 || die 'Failure using PoolOutputTestOverrideGUID_cfg.py with valid GUID (with some capital letteters)' $? -GUID=$(edmFileUtil -u PoolOutputTestOverrideGUID.root | fgrep uuid | awk '{print $10}') +GUID=$(edmRNTupleTempFileUtil -u PoolOutputTestOverrideGUID.root | fgrep uuid | awk '{print $10}') if [ "x${GUID}" != "xABCDEF01-2345-6789-abcd-ef0123456789" ]; then echo "GUID in file '${GUID}' did not match 'ABCDEF01-2345-6789-abcd-ef0123456789'" exit 1 @@ -64,8 +64,8 @@ cmsRun ${LOCAL_TEST_DIR}/PoolOutputTest_cfg.py --firstLumi 1 cmsRun ${LOCAL_TEST_DIR}/PoolOutputTest_cfg.py --firstLumi 2 cmsRun ${LOCAL_TEST_DIR}/PoolOutputTestOverrideGUID_cfg.py --guid abcdef01-2345-6789-abcd-ef0123456789 --input PoolOutputTestLumi1.root PoolOutputTestLumi2.root --maxSize 1 || die 'Failure using PoolOutputTestOverrideGUID_cfg.py with valid GUID and two input files' $? -GUID1=$(edmFileUtil -u PoolOutputTestOverrideGUID.root | fgrep uuid | awk '{print $10}') -GUID2=$(edmFileUtil -u PoolOutputTestOverrideGUID001.root | fgrep uuid | awk '{print $10}') +GUID1=$(edmRNTupleTempFileUtil -u PoolOutputTestOverrideGUID.root | fgrep uuid | awk '{print $10}') +GUID2=$(edmRNTupleTempFileUtil -u PoolOutputTestOverrideGUID001.root | fgrep uuid | awk '{print $10}') if [ "x${GUID1}" != "xabcdef01-2345-6789-abcd-ef0123456789" ]; then echo "GUID in first file '${GUID1}' did not match 'abcdef01-2345-6789-abcd-ef0123456789'" exit 1 From f6b725e340ec7ff3b979d405589e8b5b2b28a057 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Tue, 23 Sep 2025 16:09:14 -0500 Subject: [PATCH 04/19] Added FWIO/RNTupleTempTests items - based on IOPool/Tests - includes changes needed to make tests work --- FWIO/RNTupleTempInput/bin/CollUtil.cc | 52 +- FWIO/RNTupleTempInput/bin/CollUtil.h | 6 + .../bin/EdmRNTupleTempFileUtil.cpp | 57 +- FWIO/RNTupleTempInput/src/RootFile.cc | 15 +- FWIO/RNTupleTempInput/src/RootFile.h | 2 + FWIO/RNTupleTempOutput/src/RootOutputFile.cc | 4 +- .../test/AssociationMapReadTest_cfg.py | 31 + .../test/AssociationMapTest_cfg.py | 54 + FWIO/RNTupleTempTests/test/BuildFile.xml | 119 ++ .../test/DetSetVectorThinningTest1_cfg.py | 230 +++ .../test/DetSetVectorThinningTest2_cfg.py | 50 + .../test/EventHistory_1_cfg.py | 37 + .../test/EventHistory_2_cfg.py | 51 + .../test/EventHistory_3_cfg.py | 43 + .../test/EventHistory_4_cfg.py | 13 + .../test/EventHistory_5_cfg.py | 16 + .../test/EventHistory_6_cfg.py | 116 ++ .../test/Run_MissingDictionary.sh | 11 + .../test/SlimmingTest1_cfg.py | 411 +++++ .../test/SlimmingTest2A_cfg.py | 93 + .../test/SlimmingTest2B_cfg.py | 81 + .../test/SlimmingTest2C_cfg.py | 56 + .../test/SlimmingTest2D_cfg.py | 53 + .../test/SlimmingTest2E_cfg.py | 56 + .../test/SlimmingTest2F_cfg.py | 51 + .../test/SlimmingTest2G_cfg.py | 42 + .../test/SlimmingTest2H_cfg.py | 25 + .../test/SlimmingTest2I_cfg.py | 50 + .../test/SlimmingTest3B_cfg.py | 57 + .../test/SlimmingTest3C_cfg.py | 28 + .../test/SlimmingTest3D_cfg.py | 28 + .../test/SlimmingTest3E_cfg.py | 28 + .../test/SlimmingTest3F_cfg.py | 38 + .../test/SlimmingTest3I_cfg.py | 26 + .../test/SlimmingTest4B_cfg.py | 28 + .../test/SlimmingTest4F_cfg.py | 28 + .../test/ThinningTest1_cfg.py | 497 ++++++ .../test/ThinningTest2_cfg.py | 142 ++ .../test/ThinningTest3_cfg.py | 273 +++ .../test/ThinningTest4Slimming_cfg.py | 36 + .../test/ThinningTest4_cfg.py | 252 +++ .../test/ThinningTest_dropOnInput_cfg.py | 219 +++ .../test/concurrentIOVsAndRuns.sh | 13 + .../RNTupleTempTests/test/eventHistoryTest.sh | 42 + .../test/inconsistent_products_prod_cfg.py | 59 + .../test/inconsistent_products_test_cfg.py | 44 + .../test/makeEmptyRootFile.py | 13 + .../test/provenance_check_cfg.py | 13 + .../test/provenance_prod_cfg.py | 48 + FWIO/RNTupleTempTests/test/provenance_test.sh | 21 + .../test/ref_alias_compare_drop_alias_cfg.py | 33 + .../ref_alias_compare_drop_original_cfg.py | 33 + .../test/ref_alias_compare_read_alias_cfg.py | 26 + .../ref_alias_compare_read_original_cfg.py | 25 + FWIO/RNTupleTempTests/test/ref_merge_cfg.py | 27 + .../test/ref_merge_prod_cfg.py | 63 + .../test/ref_merge_test_cfg.py | 21 + FWIO/RNTupleTempTests/test/run_RefAlias.sh | 20 + FWIO/RNTupleTempTests/test/run_RefMerge.sh | 33 + FWIO/RNTupleTempTests/test/run_RunMerge.sh | 119 ++ FWIO/RNTupleTempTests/test/run_RunMerge2.sh | 50 + .../test/run_SeriesOfProcesses.sh | 28 + .../test/run_TestDropOnInput.sh | 31 + FWIO/RNTupleTempTests/test/run_TestEDAlias.sh | 17 + .../test/run_TestExistingDictionary.sh | 16 + .../RNTupleTempTests/test/run_TestFallback.sh | 21 + FWIO/RNTupleTempTests/test/run_TestGetBy.sh | 41 + .../test/run_TestGetByLabel.sh | 24 + FWIO/RNTupleTempTests/test/run_TestOutput.sh | 69 + .../test/run_TestProcessBlock.sh | 292 ++++ .../test/run_ThinningTests.sh | 44 + .../test/run_inconsistent_products.sh | 82 + .../test/run_testMaxEventsOutput.sh | 21 + .../test/run_unscheduledFailOnOutput.sh | 20 + .../test/testConcurrentIOVsAndRunsRead_cfg.py | 48 + .../test/testConcurrentIOVsAndRuns_cfg.py | 53 + .../test/testDropOnInput1_1_cfg.py | 100 ++ .../test/testDropOnInput1_2_cfg.py | 100 ++ .../test/testDropOnInput2_cfg.py | 59 + .../test/testDropOnInput3_cfg.py | 143 ++ .../test/testDropOnInputRead2001_cfg.py | 63 + .../test/testDropOnInputRead2_cfg.py | 46 + .../test/testDropOnInputRead3_cfg.py | 60 + .../test/testDuplicateProcess_cfg.py | 15 + .../test/testEDAliasAnalyze_cfg.py | 15 + .../test/testEDAliasTask_cfg.py | 35 + .../testExistingDictionaryCheckingRead_cfg.py | 16 + .../testExistingDictionaryChecking_cfg.py | 22 + .../test/testGetBy1Mod_cfg.py | 30 + FWIO/RNTupleTempTests/test/testGetBy1_cfg.py | 111 ++ FWIO/RNTupleTempTests/test/testGetBy2_cfg.py | 51 + FWIO/RNTupleTempTests/test/testGetBy3_cfg.py | 213 +++ .../test/testGetByLabelStep1_cfg.py | 15 + .../test/testGetByLabelStep2_cfg.py | 52 + .../test/testGetByMerge_cfg.py | 33 + .../test/testGetByRunsLumisMode_cfg.py | 41 + .../test/testGetByRunsMode_cfg.py | 45 + .../test/testGetByWithEmptyRun_cfg.py | 43 + .../test/testLooperEventNavigation.txt | 36 + .../test/testLooperEventNavigation1_cfg.py | 76 + .../test/testLooperEventNavigation2.txt | 24 + .../test/testLooperEventNavigation2_cfg.py | 41 + .../test/testLooperEventNavigation3.txt | 6 + .../test/testLooperEventNavigation3_cfg.py | 42 + .../test/testLooperEventNavigation_cfg.py | 67 + .../test/testMaxEventsOutput_cfg.py | 41 + ...stMaybeUninitializedIntProductPart1_cfg.py | 29 + ...stMaybeUninitializedIntProductPart2_cfg.py | 23 + .../test/testMissingDictionaryChecking_cfg.py | 34 + .../test/testNoProcessFallback1_cfg.py | 26 + .../test/testNoProcessFallback2_cfg.py | 25 + .../test/testNoProcessFallback3_cfg.py | 131 ++ .../testNoProcessFallbackNoCurrent_cfg.py | 90 + FWIO/RNTupleTempTests/test/testOutput1_cfg.py | 43 + FWIO/RNTupleTempTests/test/testOutput2_cfg.py | 45 + .../test/testProcessBlock1_cfg.py | 98 ++ .../test/testProcessBlock2Dropped_cfg.py | 74 + .../test/testProcessBlock2_cfg.py | 45 + .../test/testProcessBlock3_cfg.py | 36 + .../test/testProcessBlock4_cfg.py | 36 + .../test/testProcessBlock5_cfg.py | 37 + .../test/testProcessBlockDropOnInput_cfg.py | 82 + .../test/testProcessBlockDropOnOutput2_cfg.py | 92 + .../test/testProcessBlockDropOnOutput_cfg.py | 53 + .../test/testProcessBlockDummy_cfg.py | 27 + .../test/testProcessBlockFailMerge_cfg.py | 32 + .../test/testProcessBlockLooper_cfg.py | 75 + .../test/testProcessBlockMerge2_cfg.py | 59 + .../test/testProcessBlockMerge3_cfg.py | 43 + ...testProcessBlockMergeOfMergedFiles2_cfg.py | 37 + .../testProcessBlockMergeOfMergedFiles_cfg.py | 95 + .../test/testProcessBlockMerge_cfg.py | 87 + ...estProcessBlockNOMergeOfMergedFiles_cfg.py | 93 + .../test/testProcessBlockNonStrict2_cfg.py | 83 + .../test/testProcessBlockNonStrict3_cfg.py | 86 + .../test/testProcessBlockNonStrict_cfg.py | 79 + .../test/testProcessBlockRead2_cfg.py | 51 + .../testProcessBlockReadDropOnOutput2_cfg.py | 42 + .../testProcessBlockReadDropOnOutput_cfg.py | 42 + .../testProcessBlockReadThreeFileInput_cfg.py | 55 + .../test/testProcessBlockRead_cfg.py | 277 +++ .../test/testProcessBlockTEST_cfg.py | 81 + .../testProcessBlockThreeFileInput_cfg.py | 70 + .../test/testRunMergeCOPY100_cfg.py | 56 + .../test/testRunMergeCOPY101_cfg.py | 56 + .../test/testRunMergeCOPY1_cfg.py | 210 +++ .../test/testRunMergeCOPY_cfg.py | 146 ++ .../test/testRunMergeFastCloning_cfg.py | 45 + .../test/testRunMergeMERGE100_cfg.py | 54 + .../test/testRunMergeMERGE101_cfg.py | 57 + .../test/testRunMergeMERGE102_cfg.py | 54 + .../test/testRunMergeMERGE1_cfg.py | 116 ++ .../test/testRunMergeMERGE2_cfg.py | 141 ++ .../test/testRunMergeMERGE3_cfg.py | 103 ++ .../test/testRunMergeMERGE3x_cfg.py | 101 ++ .../test/testRunMergeMERGE4_cfg.py | 121 ++ .../test/testRunMergeMERGE5_cfg.py | 23 + .../test/testRunMergeMERGE6_cfg.py | 106 ++ .../test/testRunMergeMERGE_cfg.py | 123 ++ .../test/testRunMergeNoRunLumiSort_cfg.py | 117 ++ .../test/testRunMergePROD0_cfg.py | 178 ++ .../test/testRunMergePROD100_cfg.py | 31 + .../test/testRunMergePROD101_cfg.py | 31 + .../test/testRunMergePROD102_cfg.py | 31 + .../test/testRunMergePROD11_cfg.py | 225 +++ .../test/testRunMergePROD1_cfg.py | 179 ++ .../test/testRunMergePROD2EXTRA_cfg.py | 31 + .../test/testRunMergePROD2_cfg.py | 184 ++ .../test/testRunMergePROD3EXTRA_cfg.py | 27 + .../test/testRunMergePROD3_cfg.py | 179 ++ .../test/testRunMergePROD4_cfg.py | 145 ++ .../test/testRunMergePROD5_cfg.py | 144 ++ .../test/testRunMergePROD6_cfg.py | 146 ++ .../test/testRunMergePROD7_cfg.py | 150 ++ .../test/testRunMergePickEvents_cfg.py | 51 + .../test/testRunMergePickEventsx_cfg.py | 43 + .../test/testRunMergeSPLIT100_cfg.py | 27 + .../test/testRunMergeSPLIT101_cfg.py | 27 + .../test/testRunMergeSPLIT102_cfg.py | 27 + .../test/testRunMergeSPLIT103_cfg.py | 27 + .../test/testRunMergeTEST100_cfg.py | 137 ++ .../test/testRunMergeTEST101_cfg.py | 109 ++ .../test/testRunMergeTEST11_cfg.py | 122 ++ .../test/testRunMergeTEST1_cfg.py | 248 +++ .../test/testRunMergeTEST2_cfg.py | 87 + .../test/testRunMergeTEST3_cfg.py | 293 ++++ .../test/testRunMergeTEST4_cfg.py | 139 ++ .../test/testRunMergeTEST5_cfg.py | 24 + .../test/testRunMergeTEST6_cfg.py | 157 ++ .../test/testRunMergeTESTFAIL_cfg.py | 30 + .../test/testRunMergeTEST_cfg.py | 293 ++++ .../test/testSeriesOfProcessesHLT_cfg.py | 115 ++ .../testSeriesOfProcessesPROD2TEST_cfg.py | 64 + ...iesOfProcessesPROD2TEST_unscheduled_cfg.py | 66 + .../testSeriesOfProcessesPROD3TEST_cfg.py | 49 + .../test/testSeriesOfProcessesPROD_cfg.py | 96 ++ .../test/testSeriesOfProcessesTEST1_cfg.py | 50 + .../test/testSeriesOfProcessesTEST2_cfg.py | 29 + .../test/testSeriesOfProcessesTEST3_cfg.py | 26 + .../test/testSeriesOfProcessesTEST_cfg.py | 127 ++ .../test/unit_test_outputs/testGetBy1.log | 1523 +++++++++++++++++ .../test/unit_test_outputs/testGetBy2.log | 872 ++++++++++ .../testLooperEventNavigationOutput.txt | 123 ++ ...led_fail_on_output_IgnoreCompletely_cfg.py | 4 + .../unscheduled_fail_on_output_Rethrow_cfg.py | 4 + ...eduled_fail_on_output_TryToContinue_cfg.py | 4 + .../test/unscheduled_fail_on_output_cfg.py | 18 + ..._output_no_dependency_TryToContinue_cfg.py | 8 + ...eduled_fail_on_output_read_found_events.py | 14 + ...scheduled_fail_on_output_read_no_events.py | 14 + .../RNTupleTempTests/test/useEmptyRootFile.py | 11 + 211 files changed, 17416 insertions(+), 21 deletions(-) create mode 100644 FWIO/RNTupleTempTests/test/AssociationMapReadTest_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/AssociationMapTest_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/BuildFile.xml create mode 100644 FWIO/RNTupleTempTests/test/DetSetVectorThinningTest1_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/DetSetVectorThinningTest2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/EventHistory_1_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/EventHistory_2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/EventHistory_3_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/EventHistory_4_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/EventHistory_5_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/EventHistory_6_cfg.py create mode 100755 FWIO/RNTupleTempTests/test/Run_MissingDictionary.sh create mode 100644 FWIO/RNTupleTempTests/test/SlimmingTest1_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/SlimmingTest2A_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/SlimmingTest2B_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/SlimmingTest2C_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/SlimmingTest2D_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/SlimmingTest2E_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/SlimmingTest2F_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/SlimmingTest2G_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/SlimmingTest2H_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/SlimmingTest2I_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/SlimmingTest3B_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/SlimmingTest3C_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/SlimmingTest3D_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/SlimmingTest3E_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/SlimmingTest3F_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/SlimmingTest3I_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/SlimmingTest4B_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/SlimmingTest4F_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/ThinningTest1_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/ThinningTest2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/ThinningTest3_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/ThinningTest4Slimming_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/ThinningTest4_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/ThinningTest_dropOnInput_cfg.py create mode 100755 FWIO/RNTupleTempTests/test/concurrentIOVsAndRuns.sh create mode 100755 FWIO/RNTupleTempTests/test/eventHistoryTest.sh create mode 100644 FWIO/RNTupleTempTests/test/inconsistent_products_prod_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/inconsistent_products_test_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/makeEmptyRootFile.py create mode 100644 FWIO/RNTupleTempTests/test/provenance_check_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/provenance_prod_cfg.py create mode 100755 FWIO/RNTupleTempTests/test/provenance_test.sh create mode 100644 FWIO/RNTupleTempTests/test/ref_alias_compare_drop_alias_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/ref_alias_compare_drop_original_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/ref_alias_compare_read_alias_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/ref_alias_compare_read_original_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/ref_merge_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/ref_merge_prod_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/ref_merge_test_cfg.py create mode 100755 FWIO/RNTupleTempTests/test/run_RefAlias.sh create mode 100755 FWIO/RNTupleTempTests/test/run_RefMerge.sh create mode 100755 FWIO/RNTupleTempTests/test/run_RunMerge.sh create mode 100755 FWIO/RNTupleTempTests/test/run_RunMerge2.sh create mode 100755 FWIO/RNTupleTempTests/test/run_SeriesOfProcesses.sh create mode 100755 FWIO/RNTupleTempTests/test/run_TestDropOnInput.sh create mode 100755 FWIO/RNTupleTempTests/test/run_TestEDAlias.sh create mode 100755 FWIO/RNTupleTempTests/test/run_TestExistingDictionary.sh create mode 100755 FWIO/RNTupleTempTests/test/run_TestFallback.sh create mode 100755 FWIO/RNTupleTempTests/test/run_TestGetBy.sh create mode 100755 FWIO/RNTupleTempTests/test/run_TestGetByLabel.sh create mode 100755 FWIO/RNTupleTempTests/test/run_TestOutput.sh create mode 100755 FWIO/RNTupleTempTests/test/run_TestProcessBlock.sh create mode 100755 FWIO/RNTupleTempTests/test/run_ThinningTests.sh create mode 100755 FWIO/RNTupleTempTests/test/run_inconsistent_products.sh create mode 100755 FWIO/RNTupleTempTests/test/run_testMaxEventsOutput.sh create mode 100755 FWIO/RNTupleTempTests/test/run_unscheduledFailOnOutput.sh create mode 100644 FWIO/RNTupleTempTests/test/testConcurrentIOVsAndRunsRead_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testConcurrentIOVsAndRuns_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testDropOnInput1_1_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testDropOnInput1_2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testDropOnInput2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testDropOnInput3_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testDropOnInputRead2001_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testDropOnInputRead2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testDropOnInputRead3_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testDuplicateProcess_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testEDAliasAnalyze_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testEDAliasTask_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testExistingDictionaryCheckingRead_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testExistingDictionaryChecking_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testGetBy1Mod_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testGetBy1_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testGetBy2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testGetBy3_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testGetByLabelStep1_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testGetByLabelStep2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testGetByMerge_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testGetByRunsLumisMode_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testGetByRunsMode_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testGetByWithEmptyRun_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testLooperEventNavigation.txt create mode 100644 FWIO/RNTupleTempTests/test/testLooperEventNavigation1_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testLooperEventNavigation2.txt create mode 100644 FWIO/RNTupleTempTests/test/testLooperEventNavigation2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testLooperEventNavigation3.txt create mode 100644 FWIO/RNTupleTempTests/test/testLooperEventNavigation3_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testLooperEventNavigation_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testMaxEventsOutput_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testMaybeUninitializedIntProductPart1_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testMaybeUninitializedIntProductPart2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testMissingDictionaryChecking_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testNoProcessFallback1_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testNoProcessFallback2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testNoProcessFallback3_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testNoProcessFallbackNoCurrent_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testOutput1_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testOutput2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlock1_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlock2Dropped_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlock2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlock3_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlock4_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlock5_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockDropOnInput_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockDropOnOutput2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockDropOnOutput_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockDummy_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockFailMerge_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockLooper_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockMerge2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockMerge3_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockMergeOfMergedFiles2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockMergeOfMergedFiles_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockMerge_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockNOMergeOfMergedFiles_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockNonStrict2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockNonStrict3_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockNonStrict_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockRead2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockReadDropOnOutput2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockReadDropOnOutput_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockReadThreeFileInput_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockRead_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockTEST_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testProcessBlockThreeFileInput_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeCOPY100_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeCOPY101_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeCOPY1_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeCOPY_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeFastCloning_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeMERGE100_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeMERGE101_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeMERGE102_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeMERGE1_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeMERGE2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeMERGE3_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeMERGE3x_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeMERGE4_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeMERGE5_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeMERGE6_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeMERGE_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeNoRunLumiSort_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergePROD0_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergePROD100_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergePROD101_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergePROD102_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergePROD11_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergePROD1_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergePROD2EXTRA_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergePROD2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergePROD3EXTRA_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergePROD3_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergePROD4_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergePROD5_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergePROD6_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergePROD7_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergePickEvents_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergePickEventsx_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeSPLIT100_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeSPLIT101_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeSPLIT102_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeSPLIT103_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeTEST100_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeTEST101_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeTEST11_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeTEST1_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeTEST2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeTEST3_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeTEST4_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeTEST5_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeTEST6_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeTESTFAIL_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testRunMergeTEST_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testSeriesOfProcessesHLT_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testSeriesOfProcessesPROD2TEST_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testSeriesOfProcessesPROD2TEST_unscheduled_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testSeriesOfProcessesPROD3TEST_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testSeriesOfProcessesPROD_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testSeriesOfProcessesTEST1_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testSeriesOfProcessesTEST2_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testSeriesOfProcessesTEST3_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/testSeriesOfProcessesTEST_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/unit_test_outputs/testGetBy1.log create mode 100644 FWIO/RNTupleTempTests/test/unit_test_outputs/testGetBy2.log create mode 100644 FWIO/RNTupleTempTests/test/unit_test_outputs/testLooperEventNavigationOutput.txt create mode 100644 FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_IgnoreCompletely_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_Rethrow_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_TryToContinue_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_no_dependency_TryToContinue_cfg.py create mode 100644 FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_read_found_events.py create mode 100644 FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_read_no_events.py create mode 100644 FWIO/RNTupleTempTests/test/useEmptyRootFile.py diff --git a/FWIO/RNTupleTempInput/bin/CollUtil.cc b/FWIO/RNTupleTempInput/bin/CollUtil.cc index be6e3d6fc1ae5..fde8b4078ff7e 100644 --- a/FWIO/RNTupleTempInput/bin/CollUtil.cc +++ b/FWIO/RNTupleTempInput/bin/CollUtil.cc @@ -16,6 +16,7 @@ #include "TObject.h" #include "TTree.h" #include "ROOT/RNTupleReader.hxx" +#include "ROOT/RNTupleDescriptor.hxx" #include #include @@ -71,7 +72,6 @@ namespace edm::rntuple_temp { } } Long64_t branchCompressedSizes(TBranch *branch) { return branch->GetZipBytes("*"); } - } // namespace void printBranchNames(TTree *tree) { @@ -100,6 +100,53 @@ namespace edm::rntuple_temp { } } + namespace { + Long64_t storageForField(ROOT::RFieldDescriptor const &iField, ROOT::RNTupleDescriptor const &iDesc) { + Long64_t storage = 0; + for (auto &col : iDesc.GetColumnIterable(iField)) { + if (col.IsAliasColumn()) { + continue; + } + auto id = col.GetPhysicalId(); + + for (auto &cluster : iDesc.GetClusterIterable()) { + auto columnRange = cluster.GetColumnRange(id); + if (columnRange.IsSuppressed()) { + continue; + } + const auto &pageRange = cluster.GetPageRange(id); + for (const auto &page : pageRange.GetPageInfos()) { + storage += page.GetLocator().GetNBytesOnStorage(); + } + } + } + return storage; + } + + Long64_t storageForFieldAndSubFields(ROOT::RFieldDescriptor const &iField, ROOT::RNTupleDescriptor const &iDesc) { + Long64_t storage = storageForField(iField, iDesc); + for (auto const &sub : iDesc.GetFieldIterable(iField)) { + storage += storageForFieldAndSubFields(sub, iDesc); + } + return storage; + } + } // namespace + + void printFieldNames(ROOT::RNTupleReader &reader) { + auto &desc = reader.GetDescriptor(); + auto topLevel = desc.GetTopLevelFields(); + unsigned int index = 0; + for (auto const &topField : topLevel) { + Long64_t storage = storageForFieldAndSubFields(topField, desc); + std::cout << "Field " << index << " of " << desc.GetName() << " RNTuple: " << topField.GetFieldName() + << " Bytes On Storage = " << storage << std::endl; + + ++index; + } + } + + void longFieldPrint(ROOT::RNTupleReader &reader) { reader.PrintInfo(ROOT::ENTupleInfo::kStorageDetails); } + namespace { class BranchBasketBytes { public: @@ -283,6 +330,9 @@ namespace edm::rntuple_temp { processClusters(tr, BasketPrinter{}, branchName); } + void clusterPrint(ROOT::RNTupleReader &) {} + void pagePrint(ROOT::RNTupleReader &, std::string const &) {} + std::string getUuid(ROOT::RNTupleReader *uuidTree) { FileID fid; auto entry = uuidTree->GetModel().CreateEntry(); diff --git a/FWIO/RNTupleTempInput/bin/CollUtil.h b/FWIO/RNTupleTempInput/bin/CollUtil.h index 34236ab39b871..5b23ba5d800ea 100644 --- a/FWIO/RNTupleTempInput/bin/CollUtil.h +++ b/FWIO/RNTupleTempInput/bin/CollUtil.h @@ -25,6 +25,12 @@ namespace edm::rntuple_temp { void printUuids(ROOT::RNTupleReader *uuidTree); void printEventLists(TFile *tfl); void printEventsInLumis(TFile *tfl); + + void printFieldNames(ROOT::RNTupleReader &tree); + void longFieldPrint(ROOT::RNTupleReader &tr); + void clusterPrint(ROOT::RNTupleReader &tr); + void pagePrint(ROOT::RNTupleReader &tr, const std::string &branchName); + } // namespace edm::rntuple_temp #endif diff --git a/FWIO/RNTupleTempInput/bin/EdmRNTupleTempFileUtil.cpp b/FWIO/RNTupleTempInput/bin/EdmRNTupleTempFileUtil.cpp index 3638218c39ab9..f5c7820306553 100644 --- a/FWIO/RNTupleTempInput/bin/EdmRNTupleTempFileUtil.cpp +++ b/FWIO/RNTupleTempInput/bin/EdmRNTupleTempFileUtil.cpp @@ -24,6 +24,7 @@ #include "TFile.h" #include "TError.h" +#include "TTree.h" #include "ROOT/RNTuple.hxx" #include "ROOT/RNTupleReader.hxx" @@ -267,27 +268,53 @@ int main(int argc, char* argv[]) { } if (print or printBranchDetails or printClusters or not printBaskets.empty()) { - TTree* printTree = (TTree*)tfile->Get(selectedTree.c_str()); - if (printTree == nullptr) { - std::cout << "Tree " << selectedTree << " appears to be missing. Could not find it in the file.\n"; + auto container = tfile->GetObjectUnchecked(selectedTree.c_str()); + if (container == nullptr) { + std::cout << "RNTuple " << selectedTree << " appears to be missing. Could not find it in the file.\n"; std::cout << "Exiting\n"; return 1; } - // Print out each tree - if (print) { - edm::rntuple_temp::printBranchNames(printTree); - } + auto* printRNTuple = tfile->Get(selectedTree.c_str()); + if (printRNTuple) { + auto reader = ROOT::RNTupleReader::Open(*printRNTuple); + if (print) { + edm::rntuple_temp::printFieldNames(*reader); + } - if (printBranchDetails) { - edm::rntuple_temp::longBranchPrint(printTree); - } + if (printBranchDetails) { + edm::rntuple_temp::longFieldPrint(*reader); + } - if (printClusters) { - edm::rntuple_temp::clusterPrint(printTree); - } + if (printClusters) { + edm::rntuple_temp::clusterPrint(*reader); + } + + if (not printBaskets.empty()) { + edm::rntuple_temp::pagePrint(*reader, printBaskets); + } + } else { + TTree* printTree = (TTree*)tfile->Get(selectedTree.c_str()); + if (printTree == nullptr) { + std::cout << "Tree " << selectedTree << " appears to be missing. Could not find it in the file.\n"; + std::cout << "Exiting\n"; + return 1; + } + // Print out each tree + if (print) { + edm::rntuple_temp::printBranchNames(printTree); + } + + if (printBranchDetails) { + edm::rntuple_temp::longBranchPrint(printTree); + } + + if (printClusters) { + edm::rntuple_temp::clusterPrint(printTree); + } - if (not printBaskets.empty()) { - edm::rntuple_temp::basketPrint(printTree, printBaskets); + if (not printBaskets.empty()) { + edm::rntuple_temp::basketPrint(printTree, printBaskets); + } } } diff --git a/FWIO/RNTupleTempInput/src/RootFile.cc b/FWIO/RNTupleTempInput/src/RootFile.cc index 27e8f0c567c57..2bcb33aa1276b 100644 --- a/FWIO/RNTupleTempInput/src/RootFile.cc +++ b/FWIO/RNTupleTempInput/src/RootFile.cc @@ -246,7 +246,9 @@ namespace edm::rntuple_temp { metaDataEntry->BindRawPtr(poolNames::indexIntoFileBranchName(), &indexIntoFile_); storedProcessBlockHelper_ = std::make_unique(); - if (inputType == InputType::Primary) { + if (inputType == InputType::Primary and + metaDataReader->GetModel().GetFieldNames().end() != + metaDataReader->GetModel().GetFieldNames().find(poolNames::processBlockHelperBranchName())) { metaDataEntry->BindRawPtr(poolNames::processBlockHelperBranchName(), storedProcessBlockHelper_.get()); } @@ -478,9 +480,13 @@ namespace edm::rntuple_temp { processingOrderMerge(*processHistoryRegistry_, processingOrder); newReg->setProcessOrder(processingOrder); - // freeze the product registry - newReg->setFrozen(inputType != InputType::Primary); - productRegistry_.reset(newReg.release()); + if (not processingOrder.empty()) { + // freeze the product registry + newReg->setFrozen(inputType != InputType::Primary); + productRegistry_.reset(newReg.release()); + } else { + productRegistry_ = std::make_shared(); + } } // Set up information from the product registry. @@ -541,6 +547,7 @@ namespace edm::rntuple_temp { RootFile::~RootFile() {} + bool RootFile::empty() const { return runTree_.entries() == 0; } void RootFile::readEntryDescriptionTree(EntryDescriptionMap& entryDescriptionMap, InputType inputType) { // Called only for old format files. // We use a smart pointer so the tree will be deleted after use, and not kept for the life of the file. diff --git a/FWIO/RNTupleTempInput/src/RootFile.h b/FWIO/RNTupleTempInput/src/RootFile.h index 97a52a8b90607..9ae610a1b61a0 100644 --- a/FWIO/RNTupleTempInput/src/RootFile.h +++ b/FWIO/RNTupleTempInput/src/RootFile.h @@ -157,6 +157,8 @@ namespace edm { std::array const& hasNewlyDroppedBranch() const { return hasNewlyDroppedBranch_; } bool branchListIndexesUnchanged() const { return branchListIndexesUnchanged_; } bool modifiedIDs() const { return daqProvenanceHelper_.get() != nullptr; } + //Are there no data stored in the file? + bool empty() const; std::shared_ptr createFileBlock(); void updateFileBlock(FileBlock&); diff --git a/FWIO/RNTupleTempOutput/src/RootOutputFile.cc b/FWIO/RNTupleTempOutput/src/RootOutputFile.cc index 4181c9f753dff..72624c1300a47 100644 --- a/FWIO/RNTupleTempOutput/src/RootOutputFile.cc +++ b/FWIO/RNTupleTempOutput/src/RootOutputFile.cc @@ -589,7 +589,9 @@ namespace edm::rntuple_temp { model->AddField(setupBranchIDListRegistry()); model->AddField(setupThinnedAssociationsHelper()); model->AddField(setupProductDependencies()); - model->AddField(setupProcessBlockHelper()); + if (!om_->outputProcessBlockHelper().processesWithProcessBlockProducts().empty()) { + model->AddField(setupProcessBlockHelper()); + } } auto writeOptions = ROOT::RNTupleWriteOptions(); diff --git a/FWIO/RNTupleTempTests/test/AssociationMapReadTest_cfg.py b/FWIO/RNTupleTempTests/test/AssociationMapReadTest_cfg.py new file mode 100644 index 0000000000000..a9bddf705fc6c --- /dev/null +++ b/FWIO/RNTupleTempTests/test/AssociationMapReadTest_cfg.py @@ -0,0 +1,31 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READ") + +process.load("FWCore.MessageLogger.MessageLogger_cfi") + +process.MessageLogger.cerr.enableStatistics = False + + +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:AssociationMapTest.root' + ) +) + +process.test = cms.EDAnalyzer("AssociationMapAnalyzer", + inputTag1 = cms.InputTag("intvec1"), + inputTag2 = cms.InputTag("intvec2"), + associationMapTag1 = cms.InputTag("associationMapProducer"), + associationMapTag2 = cms.InputTag("associationMapProducer", "twoArg"), + associationMapTag3 = cms.InputTag("associationMapProducer"), + associationMapTag4 = cms.InputTag("associationMapProducer", "handleArg"), + associationMapTag5 = cms.InputTag("associationMapProducer"), + associationMapTag6 = cms.InputTag("associationMapProducer"), + associationMapTag7 = cms.InputTag("associationMapProducer"), + associationMapTag8 = cms.InputTag("associationMapProducer", "twoArg") +) + +process.p = cms.Path(process.test) diff --git a/FWIO/RNTupleTempTests/test/AssociationMapTest_cfg.py b/FWIO/RNTupleTempTests/test/AssociationMapTest_cfg.py new file mode 100644 index 0000000000000..b5a251bee2ff8 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/AssociationMapTest_cfg.py @@ -0,0 +1,54 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD") + +process.load("FWCore.MessageLogger.MessageLogger_cfi") + +process.MessageLogger.cerr.enableStatistics = False + + +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.source = cms.Source("EmptySource") + +process.intvec1 = cms.EDProducer("IntVectorProducer", + count = cms.int32(9), + ivalue = cms.int32(11), + delta = cms.int32(1) +) + +process.intvec2 = cms.EDProducer("IntVectorProducer", + count = cms.int32(9), + ivalue = cms.int32(21), + delta = cms.int32(1) +) + +process.associationMapProducer = cms.EDProducer("AssociationMapProducer", + inputTag1 = cms.InputTag("intvec1"), + inputTag2 = cms.InputTag("intvec2") +) + +process.test = cms.EDAnalyzer("AssociationMapAnalyzer", + inputTag1 = cms.InputTag("intvec1"), + inputTag2 = cms.InputTag("intvec2"), + associationMapTag1 = cms.InputTag("associationMapProducer"), + associationMapTag2 = cms.InputTag("associationMapProducer", "twoArg"), + associationMapTag3 = cms.InputTag("associationMapProducer"), + associationMapTag4 = cms.InputTag("associationMapProducer", "handleArg"), + associationMapTag5 = cms.InputTag("associationMapProducer"), + associationMapTag6 = cms.InputTag("associationMapProducer"), + associationMapTag7 = cms.InputTag("associationMapProducer"), + associationMapTag8 = cms.InputTag("associationMapProducer", "twoArg") +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('AssociationMapTest.root') +) + +process.p = cms.Path(process.intvec1 * process.intvec2 * process.associationMapProducer * process.test) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/BuildFile.xml b/FWIO/RNTupleTempTests/test/BuildFile.xml new file mode 100644 index 0000000000000..2d099ec70f267 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/BuildFile.xml @@ -0,0 +1,119 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/FWIO/RNTupleTempTests/test/DetSetVectorThinningTest1_cfg.py b/FWIO/RNTupleTempTests/test/DetSetVectorThinningTest1_cfg.py new file mode 100644 index 0000000000000..84a83a8a1efb1 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/DetSetVectorThinningTest1_cfg.py @@ -0,0 +1,230 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1) +) + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.source = cms.Source("EmptySource") + +process.WhatsItESProducer = cms.ESProducer("WhatsItESProducer") + +process.DoodadESSource = cms.ESSource("DoodadESSource") + +process.thingProducer = cms.EDProducer( + "DetSetVectorThingProducer", + offsetDelta = cms.int32(100), + nThings = cms.int32(50), + detSets = cms.vint32(1,2,3) +) + +process.anotherThingProducer = cms.EDProducer("DetSetVectorThingProducer", + offsetDelta = cms.int32(100), + nThings = cms.int32(50), + detSets = cms.vint32(1,2,3) +) + +process.thirdThingProducer = cms.EDProducer("DetSetVectorThingProducer", + offsetDelta = cms.int32(100), + nThings = cms.int32(50), + detSets = cms.vint32(1,2,3) +) + +process.trackOfThingsProducerA = cms.EDProducer("TrackOfDSVThingsProducer", + inputTag = cms.InputTag('thingProducer'), + keysToReference = cms.vuint32( + list(range(0,9)) + + list(range(50,59)) + + list(range(100,109)) + ), + nTracks = cms.uint32(8*3) +) + +process.trackOfAnotherThingsProducerA = cms.EDProducer("TrackOfDSVThingsProducer", + inputTag = cms.InputTag('anotherThingProducer'), + keysToReference = cms.vuint32( + list(range(0,5)) + + list(range(50,55)) + + list(range(100,105)) + ), + nTracks = cms.uint32(4*3) +) + +process.trackOfThirdThingsProducerA = cms.EDProducer("TrackOfDSVThingsProducer", + inputTag = cms.InputTag('thirdThingProducer'), + keysToReference = cms.vuint32( + list(range(0,3)) + + list(range(50,53)) + + list(range(100,103)) + ), + nTracks = cms.uint32(2*3) +) + +process.thinningThingProducerA = cms.EDProducer("ThinningDSVThingProducer", + inputTag = cms.InputTag('thingProducer'), + trackTag = cms.InputTag('trackOfThingsProducerA'), + expectedDetSets = cms.uint32(3), + expectedDetSetSize = cms.uint32(50), +) + +process.slimmingThingProducerA = cms.EDProducer("SlimmingDSVThingProducer", + inputTag = cms.InputTag('thingProducer'), + trackTag = cms.InputTag('trackOfThingsProducerA'), + offsetToThinnedKey = cms.uint32(0), + expectedDetSets = cms.uint32(3), + expectedDetSetSize = cms.uint32(50), +) + +process.thinningAnotherThingProducerA = cms.EDProducer("ThinningDSVThingProducer", + inputTag = cms.InputTag('anotherThingProducer'), + trackTag = cms.InputTag('trackOfThingsProducerA'), + expectedDetSets = cms.uint32(3), + expectedDetSetSize = cms.uint32(50), + thinnedRefSetIgnoreInvalidParentRef = cms.bool(True), +) + +process.thinningAnotherThingProducerA2 = cms.EDProducer("ThinningDSVThingProducer", + inputTag = cms.InputTag('thingProducer'), + trackTag = cms.InputTag('trackOfAnotherThingsProducerA'), + expectedDetSets = cms.uint32(3), + expectedDetSetSize = cms.uint32(50), + thinnedRefSetIgnoreInvalidParentRef = cms.bool(True), +) + +process.thinningAnotherThingProducerA3 = cms.EDProducer("ThinningDSVThingProducer", + inputTag = cms.InputTag('anotherThingProducer'), + trackTag = cms.InputTag('trackOfThirdThingsProducerA'), + expectedDetSets = cms.uint32(3), + expectedDetSetSize = cms.uint32(50), + thinnedRefSetIgnoreInvalidParentRef = cms.bool(True), +) + +process.testA = cms.EDAnalyzer("ThinningDSVTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerA'), + associationTag = cms.InputTag('thinningThingProducerA'), + trackTag = cms.InputTag('trackOfThingsProducerA'), + expectedParentContent = cms.VPSet( + cms.PSet(id = cms.uint32(1), values = cms.vint32(range(0,50))), + cms.PSet(id = cms.uint32(2), values = cms.vint32(range(50,100))), + cms.PSet(id = cms.uint32(3), values = cms.vint32(range(100,150))), + ), + expectedThinnedContent = cms.VPSet( + cms.PSet(id = cms.uint32(1), values = cms.vint32(range(0,9))), + cms.PSet(id = cms.uint32(2), values = cms.vint32(range(50,59))), + cms.PSet(id = cms.uint32(3), values = cms.vint32(range(100,109))), + ), + expectedIndexesIntoParent = cms.vuint32( + list(range(0,9)) + + list(range(50,59)) + + list(range(100,109)) + ), + expectedNumberOfTracks = cms.uint32(8*3), + expectedValues = cms.vint32( + list(range(0,9)) + + list(range(50,59)) + + list(range(100,109)) + ) +) + +process.slimmingTestA = cms.EDAnalyzer("ThinningDSVTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('slimmingThingProducerA'), + associationTag = cms.InputTag('slimmingThingProducerA'), + trackTag = cms.InputTag('trackOfThingsProducerA'), + thinnedSlimmedCount = cms.int32(1), + expectedParentContent = cms.VPSet( + cms.PSet(id = cms.uint32(1), values = cms.vint32(range(0,50))), + cms.PSet(id = cms.uint32(2), values = cms.vint32(range(50,100))), + cms.PSet(id = cms.uint32(3), values = cms.vint32(range(100,150))), + ), + expectedThinnedContent = cms.VPSet( + cms.PSet(id = cms.uint32(1), values = cms.vint32(range(0,9))), + cms.PSet(id = cms.uint32(2), values = cms.vint32(range(50,59))), + cms.PSet(id = cms.uint32(3), values = cms.vint32(range(100,109))), + ), + expectedIndexesIntoParent = cms.vuint32( + list(range(0,9)) + + list(range(50,59)) + + list(range(100,109)) + ), + expectedNumberOfTracks = cms.uint32(8*3), + expectedValues = cms.vint32( + list(range(0,9)) + + list(range(50,59)) + + list(range(100,109)) + ) +) + +process.anotherTestA = cms.EDAnalyzer("ThinningDSVTestAnalyzer", + parentTag = cms.InputTag('anotherThingProducer'), + thinnedTag = cms.InputTag('thinningAnotherThingProducerA'), + associationTag = cms.InputTag('thinningAnotherThingProducerA'), + trackTag = cms.InputTag('trackOfThingsProducerA'), + expectedParentContent = cms.VPSet(), + expectedThinnedContent = cms.VPSet(), + expectedIndexesIntoParent = cms.vuint32(), + expectedNumberOfTracks = cms.uint32(8*3), + expectedValues = cms.vint32() +) + +process.anotherTestA2 = cms.EDAnalyzer("ThinningDSVTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningAnotherThingProducerA2'), + associationTag = cms.InputTag('thinningAnotherThingProducerA2'), + trackTag = cms.InputTag('trackOfAnotherThingsProducerA'), + expectedParentContent = cms.VPSet(), + expectedThinnedContent = cms.VPSet(), + expectedIndexesIntoParent = cms.vuint32(), + expectedNumberOfTracks = cms.uint32(4*3), + expectedValues = cms.vint32() +) + +process.anotherTestA3 = cms.EDAnalyzer("ThinningDSVTestAnalyzer", + parentTag = cms.InputTag('anotherThingProducer'), + thinnedTag = cms.InputTag('thinningAnotherThingProducerA3'), + associationTag = cms.InputTag('thinningAnotherThingProducerA3'), + trackTag = cms.InputTag('trackOfThirdThingsProducerA'), + expectedParentContent = cms.VPSet(), + expectedThinnedContent = cms.VPSet(), + expectedIndexesIntoParent = cms.vuint32(), + expectedNumberOfTracks = cms.uint32(2*3), + expectedValues = cms.vint32() +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testDetSetVectorThinningTest1.root'), + outputCommands = cms.untracked.vstring( + 'drop *', + 'keep *_trackOfThingsProducerA_*_*', + 'keep *_slimmingThingProducerA_*_*', + ) +) + +process.p = cms.Path( + process.thingProducer + * process.anotherThingProducer + * process.thirdThingProducer + * process.trackOfThingsProducerA + * process.trackOfAnotherThingsProducerA + * process.trackOfThirdThingsProducerA + * process.thinningThingProducerA + * process.slimmingThingProducerA + * process.thinningAnotherThingProducerA + * process.thinningAnotherThingProducerA2 + * process.thinningAnotherThingProducerA3 + * process.testA + * process.slimmingTestA + * process.anotherTestA + * process.anotherTestA2 + * process.anotherTestA3 +) + +process.ep = cms.EndPath( + process.out +) diff --git a/FWIO/RNTupleTempTests/test/DetSetVectorThinningTest2_cfg.py b/FWIO/RNTupleTempTests/test/DetSetVectorThinningTest2_cfg.py new file mode 100644 index 0000000000000..445bc04879e42 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/DetSetVectorThinningTest2_cfg.py @@ -0,0 +1,50 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1) +) + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testDetSetVectorThinningTest1.root') +) + +process.slimmingTestA = cms.EDAnalyzer("ThinningDSVTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('slimmingThingProducerA'), + associationTag = cms.InputTag('slimmingThingProducerA'), + trackTag = cms.InputTag('trackOfThingsProducerA'), + parentWasDropped = cms.bool(True), + thinnedSlimmedCount = cms.int32(1), + refSlimmedCount = cms.int32(1), + expectedParentContent = cms.VPSet( + cms.PSet(id = cms.uint32(1), values = cms.vint32(range(0,50))), + cms.PSet(id = cms.uint32(2), values = cms.vint32(range(50,100))), + cms.PSet(id = cms.uint32(3), values = cms.vint32(range(100,150))), + ), + expectedThinnedContent = cms.VPSet( + cms.PSet(id = cms.uint32(1), values = cms.vint32(range(0,9))), + cms.PSet(id = cms.uint32(2), values = cms.vint32(range(50,59))), + cms.PSet(id = cms.uint32(3), values = cms.vint32(range(100,109))), + ), + expectedIndexesIntoParent = cms.vuint32( + list(range(0,9)) + + list(range(50,59)) + + list(range(100,109)) + ), + expectedNumberOfTracks = cms.uint32(8*3), + expectedValues = cms.vint32( + list(range(0,9)) + + list(range(50,59)) + + list(range(100,109)) + ) +) + +process.p = cms.Path( + process.slimmingTestA +) diff --git a/FWIO/RNTupleTempTests/test/EventHistory_1_cfg.py b/FWIO/RNTupleTempTests/test/EventHistory_1_cfg.py new file mode 100644 index 0000000000000..53bbc747e3d3e --- /dev/null +++ b/FWIO/RNTupleTempTests/test/EventHistory_1_cfg.py @@ -0,0 +1,37 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("FIRST") + +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(100) +) +process.source = cms.Source("EmptySource") + +process.intdeque = cms.EDProducer("IntDequeProducer", + count = cms.int32(12), + ivalue = cms.int32(21) +) + +process.intlist = cms.EDProducer("IntListProducer", + count = cms.int32(4), + ivalue = cms.int32(3) +) + +process.intset = cms.EDProducer("IntSetProducer", + start = cms.int32(100), + stop = cms.int32(110) +) + +process.intvec = cms.EDProducer("IntVectorProducer", + count = cms.int32(9), + ivalue = cms.int32(11) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testEventHistory_1.root') +) + +process.p1 = cms.Path(process.intdeque+process.intlist+process.intset+process.intvec) +process.ep1 = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/EventHistory_2_cfg.py b/FWIO/RNTupleTempTests/test/EventHistory_2_cfg.py new file mode 100644 index 0000000000000..597755454657a --- /dev/null +++ b/FWIO/RNTupleTempTests/test/EventHistory_2_cfg.py @@ -0,0 +1,51 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("SECOND") + +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testEventHistory_1.root') +) + +process.intdeque = cms.EDProducer("IntDequeProducer", + count = cms.int32(12), + ivalue = cms.int32(21) +) + +process.intlist = cms.EDProducer("IntListProducer", + count = cms.int32(4), + ivalue = cms.int32(3) +) + +process.intset = cms.EDProducer("IntSetProducer", + start = cms.int32(100), + stop = cms.int32(110) +) + +process.intvec = cms.EDProducer("IntVectorProducer", + count = cms.int32(9), + ivalue = cms.int32(11) +) + +process.filt55 = cms.EDFilter("TestFilterModule", + acceptValue = cms.untracked.int32(55) +) + +process.filt75 = cms.EDFilter("TestFilterModule", + acceptValue = cms.untracked.int32(75) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + SelectEvents = cms.untracked.PSet( + SelectEvents = cms.vstring('f55') + ), + fileName = cms.untracked.string('testEventHistory_2.root') +) + +process.s = cms.Sequence(process.intdeque+process.intlist+process.intset+process.intvec) +process.f55 = cms.Path(process.s*process.filt55) +process.f75 = cms.Path(process.s*process.filt75) +process.ep2 = cms.EndPath(process.out) + +process.sched = cms.Schedule(process.f55, process.f75, process.ep2) diff --git a/FWIO/RNTupleTempTests/test/EventHistory_3_cfg.py b/FWIO/RNTupleTempTests/test/EventHistory_3_cfg.py new file mode 100644 index 0000000000000..848efdddc3ce2 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/EventHistory_3_cfg.py @@ -0,0 +1,43 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("THIRD") + +process.load("FWCore.Framework.test.cmsExceptionsFatal_cff") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testEventHistory_2.root') +) + +process.intdeque = cms.EDProducer("IntDequeProducer", + count = cms.int32(12), + ivalue = cms.int32(21) +) + +process.intlist = cms.EDProducer("IntListProducer", + count = cms.int32(4), + ivalue = cms.int32(3) +) + +process.intset = cms.EDProducer("IntSetProducer", + start = cms.int32(100), + stop = cms.int32(110) +) + +process.intvec = cms.EDProducer("IntVectorProducer", + count = cms.int32(9), + ivalue = cms.int32(11) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testEventHistory_3.root'), + outputCommands = cms.untracked.vstring('keep *', 'drop *_intvec_*_THIRD') +) + +process.outother = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testEventHistory_other.root') +) + +process.p3 = cms.Path(process.intdeque+process.intlist+process.intset) +process.ep31 = cms.EndPath(process.out) +process.ep32 = cms.EndPath(process.intvec*process.intset*process.outother*process.out*process.outother) +process.epother = cms.EndPath(process.outother) diff --git a/FWIO/RNTupleTempTests/test/EventHistory_4_cfg.py b/FWIO/RNTupleTempTests/test/EventHistory_4_cfg.py new file mode 100644 index 0000000000000..f2dcecf2cd6b1 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/EventHistory_4_cfg.py @@ -0,0 +1,13 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("FOURTH") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testEventHistory_3.root') +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testEventHistory_4.root') +) + +process.ep4 = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/EventHistory_5_cfg.py b/FWIO/RNTupleTempTests/test/EventHistory_5_cfg.py new file mode 100644 index 0000000000000..e6ada76f026a4 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/EventHistory_5_cfg.py @@ -0,0 +1,16 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("FIFTH") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testEventHistory_4.root') +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + SelectEvents = cms.untracked.PSet( + SelectEvents = cms.vstring('f55:SECOND','f75:SECOND') + ), + fileName = cms.untracked.string('testEventHistory_5.root') +) + +process.ep4 = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/EventHistory_6_cfg.py b/FWIO/RNTupleTempTests/test/EventHistory_6_cfg.py new file mode 100644 index 0000000000000..f9e9b7bdb0447 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/EventHistory_6_cfg.py @@ -0,0 +1,116 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("SIXTH") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testEventHistory_5.root') +) + +process.historytest = cms.EDAnalyzer("HistoryAnalyzer", + # Why does the filter module (from step 3) pass 56 events, when I + # give it a rate of 55%? (after 100 events are created the filter + # starts counting at 1 and passes events 1 to 55 then 100 also ...) + expectedCount = cms.int32(56), + # this does not count the current process + expectedSize = cms.int32(4), + # check SelectEventInfo from previous processes + expectedSelectEventsInfo = cms.VPSet( + cms.PSet( + EndPathPositions = cms.vint32(0), + EndPaths = cms.vstring('ep1'), + InProcessHistory = cms.bool(True), + SelectEvents = cms.vstring() + ), + cms.PSet( + EndPathPositions = cms.vint32(0), + EndPaths = cms.vstring('ep2'), + InProcessHistory = cms.bool(True), + SelectEvents = cms.vstring('f55') + ), + cms.PSet( + EndPathPositions = cms.vint32(0, 2), + EndPaths = cms.vstring('ep31', 'ep32'), + InProcessHistory = cms.bool(True), + SelectEvents = cms.vstring() + ), + cms.PSet( + EndPathPositions = cms.vint32(0), + EndPaths = cms.vstring('ep4'), + InProcessHistory = cms.bool(False), + SelectEvents = cms.vstring('f55:SECOND','f75:SECOND') + ) + ), + # check the deletion modules from the top level + # ParameterSet of the current process. The following + # should be deleted + # OutputModules + # EDAnalyzers not on endPaths + # if unscheduled, EDFilters and EDProducers not on paths or end paths + # The module ParameterSet should be removed + # The label should be removed from @all_modules and all end paths + # empty end paths should be removed from @end_paths and @paths + expectedPaths = cms.vstring('p1', 'p2', 'ep62'), + expectedEndPaths = cms.vstring('ep62'), + expectedModules = cms.vstring('analyzerOnPath', 'filterOnEndPath', 'filterOnPath', 'historytest', 'producerOnEndPath', 'producerOnPath'), + expectedDroppedEndPaths = cms.vstring('ep61', 'ep63'), + expectedDroppedModules = cms.vstring('dummyanalyzer', 'dummyfilter', 'dummyproducer', 'dummyout'), + expectedDropFromProcPSet = cms.vstring('out', 'out2', 'analyzerOnEndPath'), + expectedModulesOnEndPaths = cms.PSet( + ep62 = cms.vstring('producerOnEndPath', 'filterOnEndPath', 'historytest') + ) +) + +process.producerOnPath = cms.EDProducer("IntProducer", + ivalue = cms.int32(2) +) + +process.producerOnEndPath = cms.EDProducer("IntProducer", + ivalue = cms.int32(2) +) + +process.filterOnPath = cms.EDFilter("TestFilterModule", + acceptValue = cms.untracked.int32(55) +) + +process.filterOnEndPath = cms.EDFilter("TestFilterModule", + acceptValue = cms.untracked.int32(55) +) + +process.analyzerOnPath = cms.EDAnalyzer("NonAnalyzer") + +process.analyzerOnEndPath = cms.EDAnalyzer("NonAnalyzer") + +process.dummyanalyzer = cms.EDAnalyzer("NonAnalyzer") + +process.dummyproducer = cms.EDProducer("IntProducer", + ivalue = cms.int32(2) +) + +process.dummyfilter = cms.EDFilter("TestFilterModule", + acceptValue = cms.untracked.int32(55) +) + +process.dummyout = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('dummy_6.root') +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + SelectEvents = cms.untracked.PSet( + SelectEvents = cms.vstring('f55:SECOND','f75:SECOND') + ), + fileName = cms.untracked.string('testEventHistory_6.root') +) + +process.out2 = cms.OutputModule("RNTupleTempOutputModule", + SelectEvents = cms.untracked.PSet( + SelectEvents = cms.vstring('f55:SECOND','f75:SECOND') + ), + fileName = cms.untracked.string('testEventHistory_62.root') +) + +process.p1 = cms.Path(process.historytest*process.filterOnPath*process.producerOnPath) +process.p2 = cms.Path(process.historytest*process.analyzerOnPath) + +process.ep61 = cms.EndPath(process.out) +process.ep62 = cms.EndPath(process.producerOnEndPath*process.filterOnEndPath*process.out*process.historytest) +process.ep63 = cms.EndPath(process.analyzerOnEndPath*process.out2*process.out) diff --git a/FWIO/RNTupleTempTests/test/Run_MissingDictionary.sh b/FWIO/RNTupleTempTests/test/Run_MissingDictionary.sh new file mode 100755 index 0000000000000..16f749f9dcb6b --- /dev/null +++ b/FWIO/RNTupleTempTests/test/Run_MissingDictionary.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -x +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} +# Pass in name and status +function die { echo $1: status $2 ; exit $2; } + +# It is intentional that this test throws an exception. The test fails if it does not. +cmsRun ${LOCAL_TEST_DIR}/testMissingDictionaryChecking_cfg.py &> testMissingDictionaryChecking.log && die 'Failed to get exception running testMissingDictionaryChecking_cfg.py' 1 +grep -q MissingDictionaryTestF testMissingDictionaryChecking.log || die 'Failed to print out exception message with missing dictionary listed' $? + +#grep -w ESProducer CatcheStdException.log diff --git a/FWIO/RNTupleTempTests/test/SlimmingTest1_cfg.py b/FWIO/RNTupleTempTests/test/SlimmingTest1_cfg.py new file mode 100644 index 0000000000000..ff65528d6e7f9 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/SlimmingTest1_cfg.py @@ -0,0 +1,411 @@ +# This process is the first step of a test that involves multiple +# processing steps. It tests the thinning and slimming collections and +# redirecting Refs, Ptrs, and RefToBases. +# +# From thingProducer collection (50 elements) +# - A: elements 0-19 +# - B: elements 0-10 +# - C: elements 11-19 +# - D: elements 0-5 +# - E: elements 6-10 +# - F: elements 6-8 +# - G: elements 9-10 +# - H: elements 0-19 +# - I: elements 11-15 +# +# thingProducer (0-49) +# |\- thinningThingProducerA (0-19) +# | \- thinningThingProducerAB (0-10) +# | |\- thinningThingProducerABE (6-10) +# | | \- thinningThingProducerABEF (6-8) slimmed (in job reading file F) +# | | +# | |\- thinningThingProducerABF (6-8) +# | | +# | \-- thinningThingProducerABG (9-10) +# | +# |\- thinningThingProducerB (0-10) slimmed +# | |\- thinningThingProducerBD (0-5) +# | | +# | |\- thinningThingProducerBE (6-10) slimmed +# | | |\- thinningThingProducerBEF (6-8) +# | | | +# | | \-- thinningThingProducerBEG (9-10) +# | | +# | \- thinningThingProducerBF (6-8) +# | +# |\- thinningThingProducerC (11-19) +# | \- thinnindThingProducerCI (11-15) +# | +# \-- thinningThingProducerH (0-19), no product because of event filtering +# +# cases to be tested +# A: everything can be put in one file, when reading thinned are read +# B: file including ABF, ABG, BF, BEG +# - ABF is preferred over BF for elements 6-8 +# - ABG is preferred over BEG for elements 9-10 +# - then drop ABF, ABG, and can get BF but not BEG +# - then drop BF, can get BEG +# C: file including BF, BEF +# - BF is preferred of BEF +# - then drop BF, and get BEF +# D: file including ABE, BD +# - ABE is preferred over BD for elements 0-5 (leading to product not found) +# - drop ABE, then BD works +# E: file including BD, BEF +# - BD is preferred over BEF for elements 6-8 (leading to product not found) +# - drop BD, then BEF works +# F: file including ABE +# - slim ABE further to ABEF +# - then try to merge with B file with BEG, should fail +# G: file including A +# - try to thin from thingProducer, should fail +# H: file including H, B +# - non-existing H is preferred, all Refs are Null +# I: file including B, CI +# - CI is preferred over B for elements 0-10 (leading to product not fond) +# - drop CU, then B works + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD") + +process.maxEvents.input = 3 + +process.source = cms.Source("EmptySource") + +process.WhatsItESProducer = cms.ESProducer("WhatsItESProducer") + +process.DoodadESSource = cms.ESSource("DoodadESSource") + +process.thingProducer = cms.EDProducer("ThingProducer", + offsetDelta = cms.int32(100), + nThings = cms.int32(50) +) + +process.trackOfThingsProducerA = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer'), + keysToReference = cms.vuint32(range(0, 20)) +) + +process.trackOfThingsProducerB = process.trackOfThingsProducerA.clone(keysToReference = range(0,11)) +process.trackOfThingsProducerC = process.trackOfThingsProducerA.clone(keysToReference = range(11,20)) +process.trackOfThingsProducerD = process.trackOfThingsProducerA.clone(keysToReference = range(0,6)) +process.trackOfThingsProducerE = process.trackOfThingsProducerA.clone(keysToReference = range(6,11)) +process.trackOfThingsProducerF = process.trackOfThingsProducerA.clone(keysToReference = range(6,9)) +process.trackOfThingsProducerG = process.trackOfThingsProducerA.clone(keysToReference = range(9,11)) +process.trackOfThingsProducerH = process.trackOfThingsProducerA.clone() +process.trackOfThingsProducerI = process.trackOfThingsProducerA.clone(keysToReference = range(11,16)) + +process.thinningThingProducerA = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thingProducer'), + trackTag = cms.InputTag('trackOfThingsProducerA'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(50) +) +process.thinningThingProducerAB = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerA'), + trackTag = cms.InputTag('trackOfThingsProducerB'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(20) +) +process.thinningThingProducerABE = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerAB'), + trackTag = cms.InputTag('trackOfThingsProducerE'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(11) +) +process.thinningThingProducerABF = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerAB'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(11) +) +process.thinningThingProducerABG = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerAB'), + trackTag = cms.InputTag('trackOfThingsProducerG'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(11) +) + +process.thinningThingProducerB = cms.EDProducer("SlimmingThingProducer", + inputTag = cms.InputTag('thingProducer'), + trackTag = cms.InputTag('trackOfThingsProducerB'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(50) +) +process.thinningThingProducerBD = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerB'), + trackTag = cms.InputTag('trackOfThingsProducerD'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(11), + slimmedValueFactor = cms.int32(10) +) +process.thinningThingProducerBE = cms.EDProducer("SlimmingThingProducer", + inputTag = cms.InputTag('thinningThingProducerB'), + trackTag = cms.InputTag('trackOfThingsProducerE'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(11), + slimmedValueFactor = cms.int32(10) +) +process.thinningThingProducerBEF = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerBE'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + offsetToThinnedKey = cms.uint32(6), + expectedCollectionSize = cms.uint32(5), + slimmedValueFactor = cms.int32(100) +) +process.thinningThingProducerBEG = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerBE'), + trackTag = cms.InputTag('trackOfThingsProducerG'), + offsetToThinnedKey = cms.uint32(6), + expectedCollectionSize = cms.uint32(5), + slimmedValueFactor = cms.int32(100) +) +process.thinningThingProducerBF = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerB'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(11), + slimmedValueFactor = cms.int32(10) +) + +process.thinningThingProducerC = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thingProducer'), + trackTag = cms.InputTag('trackOfThingsProducerC'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(50) +) +process.thinningThingProducerCI = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerC'), + trackTag = cms.InputTag('trackOfThingsProducerI'), + offsetToThinnedKey = cms.uint32(11), + offsetToValue = cms.uint32(11), + expectedCollectionSize = cms.uint32(9) +) + +process.thinningThingProducerH = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thingProducer'), + trackTag = cms.InputTag('trackOfThingsProducerH'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(50) +) +process.rejectingFilter = cms.EDFilter("TestFilterModule", + acceptValue = cms.untracked.int32(-1) +) + +process.testA = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerA'), + associationTag = cms.InputTag('thinningThingProducerA'), + trackTag = cms.InputTag('trackOfThingsProducerA'), + expectedParentContent = cms.vint32(range(0,50)), + expectedThinnedContent = cms.vint32(range(0,20)), + expectedIndexesIntoParent = cms.vuint32(range(0,20)), + expectedValues = cms.vint32(range(0,20)), +) + +process.testABF = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerAB'), + thinnedTag = cms.InputTag('thinningThingProducerABF'), + associationTag = cms.InputTag('thinningThingProducerABF'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + expectedParentContent = cms.vint32(range(0,11)), + expectedThinnedContent = cms.vint32(range(6,9)), + expectedIndexesIntoParent = cms.vuint32(range(6,9)), + expectedValues = cms.vint32(range(6,9)), +) + +process.testB = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerB'), + associationTag = cms.InputTag('thinningThingProducerB'), + trackTag = cms.InputTag('trackOfThingsProducerB'), + thinnedSlimmedCount = cms.int32(1), + expectedParentContent = cms.vint32(range(0,50)), + expectedThinnedContent = cms.vint32(range(0,11)), + expectedIndexesIntoParent = cms.vuint32(range(0,11)), + expectedValues = cms.vint32(range(0,11)), +) + +process.testBD = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerB'), + thinnedTag = cms.InputTag('thinningThingProducerBD'), + associationTag = cms.InputTag('thinningThingProducerBD'), + trackTag = cms.InputTag('trackOfThingsProducerD'), + parentSlimmedCount = cms.int32(1), + thinnedSlimmedCount = cms.int32(1), + expectedParentContent = cms.vint32(range(0,11)), + expectedThinnedContent = cms.vint32(range(0,6)), + expectedIndexesIntoParent = cms.vuint32(range(0,6)), + expectedValues = cms.vint32(range(0,6)), +) + +process.testBE = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerB'), + thinnedTag = cms.InputTag('thinningThingProducerBE'), + associationTag = cms.InputTag('thinningThingProducerBE'), + trackTag = cms.InputTag('trackOfThingsProducerE'), + parentSlimmedCount = cms.int32(1), + thinnedSlimmedCount = cms.int32(2), + expectedParentContent = cms.vint32(range(0,11)), + expectedThinnedContent = cms.vint32(range(6,11)), + expectedIndexesIntoParent = cms.vuint32(range(6,11)), + expectedValues = cms.vint32(range(6,11)), +) + +process.testBEG = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerBE'), + thinnedTag = cms.InputTag('thinningThingProducerBEG'), + associationTag = cms.InputTag('thinningThingProducerBEG'), + trackTag = cms.InputTag('trackOfThingsProducerG'), + parentSlimmedCount = cms.int32(2), + thinnedSlimmedCount = cms.int32(2), + expectedParentContent = cms.vint32(range(6,11)), + expectedThinnedContent = cms.vint32(range(9,11)), + expectedIndexesIntoParent = cms.vuint32(range(3,5)), + expectedValues = cms.vint32(range(9,11)), +) + + + +process.outA = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSlimmingTest1A.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_thingProducer_*_*', + ) +) + +process.outB = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSlimmingTest1B.root'), + outputCommands = cms.untracked.vstring( + 'drop *', + 'keep *_trackOfThingsProducerF_*_*', + 'keep *_trackOfThingsProducerG_*_*', + 'keep *_thinningThingProducerABF_*_*', + 'keep *_thinningThingProducerABG_*_*', + 'keep *_thinningThingProducerBF_*_*', + 'keep *_thinningThingProducerBEG_*_*', + ) +) + +process.outC = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSlimmingTest1C.root'), + outputCommands = cms.untracked.vstring( + 'drop *', + 'keep *_trackOfThingsProducerF_*_*', + 'keep *_thinningThingProducerBF_*_*', + 'keep *_thinningThingProducerBEF_*_*', + ) +) + +process.outD = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSlimmingTest1D.root'), + outputCommands = cms.untracked.vstring( + 'drop *', + 'keep *_trackOfThingsProducerD_*_*', + 'keep *_trackOfThingsProducerE_*_*', + 'keep *_thinningThingProducerABE_*_*', + 'keep *_thinningThingProducerBD_*_*', + ) +) + +process.outE = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSlimmingTest1E.root'), + outputCommands = cms.untracked.vstring( + 'drop *', + 'keep *_trackOfThingsProducerD_*_*', + 'keep *_trackOfThingsProducerF_*_*', + 'keep *_thinningThingProducerBD_*_*', + 'keep *_thinningThingProducerBEF_*_*', + ) +) + +process.outF = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSlimmingTest1F.root'), + outputCommands = cms.untracked.vstring( + 'drop *', + 'keep *_trackOfThingsProducerF_*_*', + 'keep *_thinningThingProducerABE_*_*', + ) +) + +process.outG = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSlimmingTest1G.root'), + outputCommands = cms.untracked.vstring( + 'drop *', + 'keep *_trackOfThingsProducerB_*_*', + 'keep *_thinningThingProducerA_*_*', + ) +) + +process.outH = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSlimmingTest1H.root'), + outputCommands = cms.untracked.vstring( + 'drop *', + 'keep *_thinningThingProducerB_*_*', + 'keep *_trackOfThingsProducerH_*_*', + 'keep *_thinningThingProducerH_*_*', + ) +) + +process.outI = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSlimmingTest1I.root'), + outputCommands = cms.untracked.vstring( + 'drop *', + 'keep *_trackOfThingsProducerB_*_*', + 'keep *_trackOfThingsProducerI_*_*', + 'keep *_thinningThingProducerB_*_*', + 'keep *_thinningThingProducerCI_*_*', + ) +) + + +process.p = cms.Path( + process.thingProducer + * process.trackOfThingsProducerA + * process.trackOfThingsProducerB + * process.trackOfThingsProducerC + * process.trackOfThingsProducerD + * process.trackOfThingsProducerE + * process.trackOfThingsProducerF + * process.trackOfThingsProducerG + * process.trackOfThingsProducerI + * process.thinningThingProducerA + * process.thinningThingProducerAB + * process.thinningThingProducerABE + * process.thinningThingProducerABF + * process.thinningThingProducerABG + * process.thinningThingProducerB + * process.thinningThingProducerBD + * process.thinningThingProducerBE + * process.thinningThingProducerBEF + * process.thinningThingProducerBEG + * process.thinningThingProducerBF + * process.thinningThingProducerC + * process.thinningThingProducerCI + * process.testA + * process.testABF + * process.testB + * process.testBD + * process.testBE + * process.testBEG +) +process.p2 = cms.Path( + process.thingProducer + * process.trackOfThingsProducerH + * process.rejectingFilter + * process.thinningThingProducerH +) + +process.ep = cms.EndPath( + process.outA + * process.outB + * process.outC + * process.outD + * process.outE + * process.outF + * process.outG + * process.outH + * process.outI +) diff --git a/FWIO/RNTupleTempTests/test/SlimmingTest2A_cfg.py b/FWIO/RNTupleTempTests/test/SlimmingTest2A_cfg.py new file mode 100644 index 0000000000000..5e67875b24093 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/SlimmingTest2A_cfg.py @@ -0,0 +1,93 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST2A") + +process.maxEvents.input = 3 + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSlimmingTest1A.root') +) + +process.testA = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerA'), + associationTag = cms.InputTag('thinningThingProducerA'), + trackTag = cms.InputTag('trackOfThingsProducerA'), + parentWasDropped = cms.bool(True), + expectedParentContent = cms.vint32(range(0,50)), + expectedThinnedContent = cms.vint32(range(0,20)), + expectedIndexesIntoParent = cms.vuint32(range(0,20)), + expectedValues = cms.vint32(range(0,20)), +) + +process.testABF = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerAB'), + thinnedTag = cms.InputTag('thinningThingProducerABF'), + associationTag = cms.InputTag('thinningThingProducerABF'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + expectedParentContent = cms.vint32(range(0,11)), + expectedThinnedContent = cms.vint32(range(6,9)), + expectedIndexesIntoParent = cms.vuint32(range(6,9)), + expectedValues = cms.vint32(range(6,9)), +) + +process.testB = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerB'), + associationTag = cms.InputTag('thinningThingProducerB'), + trackTag = cms.InputTag('trackOfThingsProducerB'), + parentWasDropped = cms.bool(True), + thinnedSlimmedCount = cms.int32(1), + expectedParentContent = cms.vint32(range(0,50)), + expectedThinnedContent = cms.vint32(range(0,11)), + expectedIndexesIntoParent = cms.vuint32(range(0,11)), + expectedValues = cms.vint32(range(0,11)), +) + +process.testBD = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerB'), + thinnedTag = cms.InputTag('thinningThingProducerBD'), + associationTag = cms.InputTag('thinningThingProducerBD'), + trackTag = cms.InputTag('trackOfThingsProducerD'), + parentSlimmedCount = cms.int32(1), + thinnedSlimmedCount = cms.int32(1), + expectedParentContent = cms.vint32(range(0,11)), + expectedThinnedContent = cms.vint32(range(0,6)), + expectedIndexesIntoParent = cms.vuint32(range(0,6)), + expectedValues = cms.vint32(range(0,6)), +) + +process.testBE = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerB'), + thinnedTag = cms.InputTag('thinningThingProducerBE'), + associationTag = cms.InputTag('thinningThingProducerBE'), + trackTag = cms.InputTag('trackOfThingsProducerE'), + parentSlimmedCount = cms.int32(1), + thinnedSlimmedCount = cms.int32(2), + expectedParentContent = cms.vint32(range(0,11)), + expectedThinnedContent = cms.vint32(range(6,11)), + expectedIndexesIntoParent = cms.vuint32(range(6,11)), + expectedValues = cms.vint32(range(6,11)), +) + +process.testBEG = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerBE'), + thinnedTag = cms.InputTag('thinningThingProducerBEG'), + associationTag = cms.InputTag('thinningThingProducerBEG'), + trackTag = cms.InputTag('trackOfThingsProducerG'), + parentSlimmedCount = cms.int32(2), + thinnedSlimmedCount = cms.int32(2), + expectedParentContent = cms.vint32(range(6,11)), + expectedThinnedContent = cms.vint32(range(9,11)), + expectedIndexesIntoParent = cms.vuint32(range(3,5)), + expectedValues = cms.vint32(range(9,11)), +) + +process.p = cms.Path( + process.testA + * process.testABF + * process.testB + * process.testBD + * process.testBE + * process.testBEG +) diff --git a/FWIO/RNTupleTempTests/test/SlimmingTest2B_cfg.py b/FWIO/RNTupleTempTests/test/SlimmingTest2B_cfg.py new file mode 100644 index 0000000000000..876c5dd3e2bee --- /dev/null +++ b/FWIO/RNTupleTempTests/test/SlimmingTest2B_cfg.py @@ -0,0 +1,81 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST2B") + +process.maxEvents.input = 3 + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSlimmingTest1B.root') +) + +process.testABF = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerAB'), + thinnedTag = cms.InputTag('thinningThingProducerABF'), + associationTag = cms.InputTag('thinningThingProducerABF'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + parentWasDropped = cms.bool(True), + expectedParentContent = cms.vint32(range(0,11)), + expectedThinnedContent = cms.vint32(range(6,9)), + expectedIndexesIntoParent = cms.vuint32(range(6,9)), + expectedValues = cms.vint32(range(6,9)), +) + +process.testABG = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerAB'), + thinnedTag = cms.InputTag('thinningThingProducerABG'), + associationTag = cms.InputTag('thinningThingProducerABG'), + trackTag = cms.InputTag('trackOfThingsProducerG'), + parentWasDropped = cms.bool(True), + expectedParentContent = cms.vint32(range(0,11)), + expectedThinnedContent = cms.vint32(range(9,11)), + expectedIndexesIntoParent = cms.vuint32(range(9,11)), + expectedValues = cms.vint32(range(9,11)), +) + +process.testBF = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerB'), + thinnedTag = cms.InputTag('thinningThingProducerBF'), + associationTag = cms.InputTag('thinningThingProducerBF'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + parentWasDropped = cms.bool(True), + parentSlimmedCount = cms.int32(1), + thinnedSlimmedCount = cms.int32(1), + expectedParentContent = cms.vint32(range(0,11)), + expectedThinnedContent = cms.vint32(range(6,9)), + expectedIndexesIntoParent = cms.vuint32(range(6,9)), + expectedValues = cms.vint32(range(6,9)), +) + +process.testBEG = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerBE'), + thinnedTag = cms.InputTag('thinningThingProducerBEG'), + associationTag = cms.InputTag('thinningThingProducerBEG'), + trackTag = cms.InputTag('trackOfThingsProducerG'), + parentWasDropped = cms.bool(True), + parentSlimmedCount = cms.int32(2), + thinnedSlimmedCount = cms.int32(2), + expectedParentContent = cms.vint32(range(6,11)), + expectedThinnedContent = cms.vint32(range(9,11)), + expectedIndexesIntoParent = cms.vuint32(range(3,5)), + expectedValues = cms.vint32(range(9,11)), +) + +process.outB = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSlimmingTest2B.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_thinningThingProducerABF_*_*', + 'drop *_thinningThingProducerABG_*_*', + ) +) + +process.p = cms.Path( + process.testABF + * process.testABG + * process.testBF + * process.testBEG +) + +process.ep = cms.EndPath( + process.outB +) diff --git a/FWIO/RNTupleTempTests/test/SlimmingTest2C_cfg.py b/FWIO/RNTupleTempTests/test/SlimmingTest2C_cfg.py new file mode 100644 index 0000000000000..647f213f5ff8a --- /dev/null +++ b/FWIO/RNTupleTempTests/test/SlimmingTest2C_cfg.py @@ -0,0 +1,56 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST2C") + +process.maxEvents.input = 3 + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSlimmingTest1C.root') +) + +process.testBF = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerB'), + thinnedTag = cms.InputTag('thinningThingProducerBF'), + associationTag = cms.InputTag('thinningThingProducerBF'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + parentWasDropped = cms.bool(True), + parentSlimmedCount = cms.int32(1), + thinnedSlimmedCount = cms.int32(1), + refSlimmedCount = cms.int32(1), + expectedParentContent = cms.vint32(range(0,11)), + expectedThinnedContent = cms.vint32(range(6,9)), + expectedIndexesIntoParent = cms.vuint32(range(6,9)), + expectedValues = cms.vint32(range(6,9)), +) + +process.testBEF = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerBE'), + thinnedTag = cms.InputTag('thinningThingProducerBEF'), + associationTag = cms.InputTag('thinningThingProducerBEF'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + parentWasDropped = cms.bool(True), + parentSlimmedCount = cms.int32(2), + thinnedSlimmedCount = cms.int32(2), + refSlimmedCount = cms.int32(1), + expectedParentContent = cms.vint32(range(6,11)), + expectedThinnedContent = cms.vint32(range(6,9)), + expectedIndexesIntoParent = cms.vuint32(range(0,3)), + expectedValues = cms.vint32(range(6,9)), +) + +process.outC = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSlimmingTest2C.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_thinningThingProducerBF_*_*', + ) +) + +process.p = cms.Path( + process.testBF + * process.testBEF +) + +process.ep = cms.EndPath( + process.outC +) diff --git a/FWIO/RNTupleTempTests/test/SlimmingTest2D_cfg.py b/FWIO/RNTupleTempTests/test/SlimmingTest2D_cfg.py new file mode 100644 index 0000000000000..421dfd6a0b849 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/SlimmingTest2D_cfg.py @@ -0,0 +1,53 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST2D") + +process.maxEvents.input = 3 + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSlimmingTest1D.root') +) + +process.testABE = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerAB'), + thinnedTag = cms.InputTag('thinningThingProducerABE'), + associationTag = cms.InputTag('thinningThingProducerABE'), + trackTag = cms.InputTag('trackOfThingsProducerE'), + parentWasDropped = cms.bool(True), + expectedParentContent = cms.vint32(range(0,11)), + expectedThinnedContent = cms.vint32(range(6,11)), + expectedIndexesIntoParent = cms.vuint32(range(6,11)), + expectedValues = cms.vint32(range(6,11)), +) + +process.testBD = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerB'), + thinnedTag = cms.InputTag('thinningThingProducerBD'), + associationTag = cms.InputTag('thinningThingProducerBD'), + trackTag = cms.InputTag('trackOfThingsProducerD'), + parentWasDropped = cms.bool(True), + parentSlimmedCount = cms.int32(1), + thinnedSlimmedCount = cms.int32(1), + refToParentIsAvailable = cms.bool(False), + expectedParentContent = cms.vint32(range(0,11)), + expectedThinnedContent = cms.vint32(range(0,6)), + expectedIndexesIntoParent = cms.vuint32(range(0,6)), + expectedValues = cms.vint32(range(0,6)), +) + +process.outD = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSlimmingTest2D.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_thinningThingProducerABE_*_*', + ) +) + +process.p = cms.Path( + process.testABE + * process.testBD +) + +process.ep = cms.EndPath( + process.outD +) diff --git a/FWIO/RNTupleTempTests/test/SlimmingTest2E_cfg.py b/FWIO/RNTupleTempTests/test/SlimmingTest2E_cfg.py new file mode 100644 index 0000000000000..a42eaa2c58719 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/SlimmingTest2E_cfg.py @@ -0,0 +1,56 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST2E") + +process.maxEvents.input = 3 + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSlimmingTest1E.root') +) + +process.testBD = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerB'), + thinnedTag = cms.InputTag('thinningThingProducerBD'), + associationTag = cms.InputTag('thinningThingProducerBD'), + trackTag = cms.InputTag('trackOfThingsProducerD'), + parentWasDropped = cms.bool(True), + parentSlimmedCount = cms.int32(1), + thinnedSlimmedCount = cms.int32(1), + refSlimmedCount = cms.int32(1), + expectedParentContent = cms.vint32(range(0,11)), + expectedThinnedContent = cms.vint32(range(0,6)), + expectedIndexesIntoParent = cms.vuint32(range(0,6)), + expectedValues = cms.vint32(range(0,6)), +) + +process.testBEF = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerBE'), + thinnedTag = cms.InputTag('thinningThingProducerBEF'), + associationTag = cms.InputTag('thinningThingProducerBEF'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + parentWasDropped = cms.bool(True), + parentSlimmedCount = cms.int32(2), + thinnedSlimmedCount = cms.int32(2), + refToParentIsAvailable = cms.bool(False), + expectedParentContent = cms.vint32(range(0,11)), + expectedThinnedContent = cms.vint32(range(6,9)), + expectedIndexesIntoParent = cms.vuint32(range(0,3)), + expectedValues = cms.vint32(range(6,9)), +) + +process.outE = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSlimmingTest2E.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_thinningThingProducerBD_*_*', + ) +) + +process.p = cms.Path( + process.testBD + * process.testBEF +) + +process.ep = cms.EndPath( + process.outE +) diff --git a/FWIO/RNTupleTempTests/test/SlimmingTest2F_cfg.py b/FWIO/RNTupleTempTests/test/SlimmingTest2F_cfg.py new file mode 100644 index 0000000000000..8f332a3ac5fef --- /dev/null +++ b/FWIO/RNTupleTempTests/test/SlimmingTest2F_cfg.py @@ -0,0 +1,51 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST2F") + +process.maxEvents.input = 3 + +process.WhatsItESProducer = cms.ESProducer("WhatsItESProducer") + +process.DoodadESSource = cms.ESSource("DoodadESSource") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSlimmingTest1F.root') +) + +process.thinningThingProducerABEF = cms.EDProducer("SlimmingThingProducer", + inputTag = cms.InputTag('thinningThingProducerABE'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + offsetToThinnedKey = cms.uint32(6), + offsetToValue = cms.uint32(6), + expectedCollectionSize = cms.uint32(5) +) + +process.testABEF = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerABE'), + thinnedTag = cms.InputTag('thinningThingProducerABEF'), + associationTag = cms.InputTag('thinningThingProducerABEF'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + thinnedSlimmedCount = cms.int32(1), + expectedParentContent = cms.vint32(range(6,11)), + expectedThinnedContent = cms.vint32(range(6,9)), + expectedIndexesIntoParent = cms.vuint32(range(0,3)), + expectedValues = cms.vint32(range(6,9)), +) + +process.outF = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSlimmingTest2F.root'), + outputCommands = cms.untracked.vstring( + 'drop *', + 'keep *_thinningThingProducerABEF_*_*', + 'keep *_trackOfThingsProducerF_*_*', + ) +) + +process.p = cms.Path( + process.thinningThingProducerABEF + * process.testABEF +) + +process.ep = cms.EndPath( + process.outF +) diff --git a/FWIO/RNTupleTempTests/test/SlimmingTest2G_cfg.py b/FWIO/RNTupleTempTests/test/SlimmingTest2G_cfg.py new file mode 100644 index 0000000000000..09f3cc7ec7791 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/SlimmingTest2G_cfg.py @@ -0,0 +1,42 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST2G") + +process.maxEvents.input = 3 + +process.WhatsItESProducer = cms.ESProducer("WhatsItESProducer") + +process.DoodadESSource = cms.ESSource("DoodadESSource") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSlimmingTest1G.root') +) + +# Here thinningThingProducerB leads to an association with +# default-constructed BranchID for the parent to be inserted. That +# should not lead to other failures as long as the thinning producer +# is not run. +process.rejectingFilter = cms.EDFilter("TestFilterModule", + acceptValue = cms.untracked.int32(-1) +) +process.thinningThingProducerB = cms.EDProducer("SlimmingThingProducer", + inputTag = cms.InputTag('doesNotExist', '', 'PROD'), + trackTag = cms.InputTag('trackOfThingsProducerB'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(50) +) + +process.thinningThingProducerAB = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerA'), + trackTag = cms.InputTag('trackOfThingsProducerB'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(20) +) + +process.p = cms.Path( + process.rejectingFilter * + process.thinningThingProducerB +) +process.p2 = cms.Path( + process.thinningThingProducerAB +) diff --git a/FWIO/RNTupleTempTests/test/SlimmingTest2H_cfg.py b/FWIO/RNTupleTempTests/test/SlimmingTest2H_cfg.py new file mode 100644 index 0000000000000..d1dbb11d4ef63 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/SlimmingTest2H_cfg.py @@ -0,0 +1,25 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST2H") + +process.maxEvents.input = 3 + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSlimmingTest1H.root') +) + +process.testH = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerH'), + associationTag = cms.InputTag('thinningThingProducerH'), + trackTag = cms.InputTag('trackOfThingsProducerH'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + associationShouldBeDropped = cms.bool(True), + refToParentIsAvailable = cms.bool(False), + expectedValues = cms.vint32(range(0,19)), +) + +process.p = cms.Path( + process.testH +) diff --git a/FWIO/RNTupleTempTests/test/SlimmingTest2I_cfg.py b/FWIO/RNTupleTempTests/test/SlimmingTest2I_cfg.py new file mode 100644 index 0000000000000..0b92c939dc450 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/SlimmingTest2I_cfg.py @@ -0,0 +1,50 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST2I") + +process.maxEvents.input = 3 + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSlimmingTest1I.root') +) + +process.testB = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerB'), + associationTag = cms.InputTag('thinningThingProducerB'), + trackTag = cms.InputTag('trackOfThingsProducerB'), + parentWasDropped = cms.bool(True), + refToParentIsAvailable = cms.bool(False), + thinnedSlimmedCount = cms.int32(1), + expectedThinnedContent = cms.vint32(range(0,11)), + expectedIndexesIntoParent = cms.vuint32(range(0,11)), + expectedValues = cms.vint32(range(0,11)), +) + +process.testCI = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerC'), + thinnedTag = cms.InputTag('thinningThingProducerCI'), + associationTag = cms.InputTag('thinningThingProducerCI'), + trackTag = cms.InputTag('trackOfThingsProducerI'), + parentWasDropped = cms.bool(True), + expectedThinnedContent = cms.vint32(range(11,16)), + expectedIndexesIntoParent = cms.vuint32(range(0,5)), + expectedValues = cms.vint32(range(11,16)), +) + +process.outI = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSlimmingTest2I.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_thinningThingProducerCI_*_*', + ) +) + +process.p = cms.Path( + process.testB + * process.testCI +) + +process.ep = cms.EndPath( + process.outI +) diff --git a/FWIO/RNTupleTempTests/test/SlimmingTest3B_cfg.py b/FWIO/RNTupleTempTests/test/SlimmingTest3B_cfg.py new file mode 100644 index 0000000000000..584f75354851b --- /dev/null +++ b/FWIO/RNTupleTempTests/test/SlimmingTest3B_cfg.py @@ -0,0 +1,57 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST3B") + +process.maxEvents.input = 3 + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSlimmingTest2B.root') +) + +process.testBF = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerB'), + thinnedTag = cms.InputTag('thinningThingProducerBF'), + associationTag = cms.InputTag('thinningThingProducerBF'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + parentWasDropped = cms.bool(True), + parentSlimmedCount = cms.int32(1), + thinnedSlimmedCount = cms.int32(1), + refSlimmedCount = cms.int32(1), + expectedParentContent = cms.vint32(range(0,10)), + expectedThinnedContent = cms.vint32(range(6,9)), + expectedIndexesIntoParent = cms.vuint32(range(6,9)), + expectedValues = cms.vint32(range(6,9)), +) + +process.testBEG = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerBE'), + thinnedTag = cms.InputTag('thinningThingProducerBEG'), + associationTag = cms.InputTag('thinningThingProducerBEG'), + trackTag = cms.InputTag('trackOfThingsProducerG'), + parentWasDropped = cms.bool(True), + refToParentIsAvailable = cms.bool(False), + parentSlimmedCount = cms.int32(2), + thinnedSlimmedCount = cms.int32(2), + refSlimmedCount = cms.int32(1), + expectedParentContent = cms.vint32(range(6,11)), + expectedThinnedContent = cms.vint32(range(9,11)), + expectedIndexesIntoParent = cms.vuint32(range(3,5)), + expectedValues = cms.vint32(range(9,11)), +) + +process.outB = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSlimmingTest3B.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_thinningThingProducerBF_*_*', + ) +) + +process.p = cms.Path( + process.testBF + * process.testBEG +) + +process.ep = cms.EndPath( + process.outB +) diff --git a/FWIO/RNTupleTempTests/test/SlimmingTest3C_cfg.py b/FWIO/RNTupleTempTests/test/SlimmingTest3C_cfg.py new file mode 100644 index 0000000000000..f70c084046cd6 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/SlimmingTest3C_cfg.py @@ -0,0 +1,28 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST3C") + +process.maxEvents.input = 3 + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSlimmingTest2C.root') +) + +process.testBEF = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerBE'), + thinnedTag = cms.InputTag('thinningThingProducerBEF'), + associationTag = cms.InputTag('thinningThingProducerBEF'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + parentWasDropped = cms.bool(True), + parentSlimmedCount = cms.int32(2), + thinnedSlimmedCount = cms.int32(2), + refSlimmedCount = cms.int32(2), + expectedParentContent = cms.vint32(range(6,11)), + expectedThinnedContent = cms.vint32(range(6,9)), + expectedIndexesIntoParent = cms.vuint32(range(0,3)), + expectedValues = cms.vint32(range(6,9)), +) + +process.p = cms.Path( + process.testBEF +) diff --git a/FWIO/RNTupleTempTests/test/SlimmingTest3D_cfg.py b/FWIO/RNTupleTempTests/test/SlimmingTest3D_cfg.py new file mode 100644 index 0000000000000..eb91a57576779 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/SlimmingTest3D_cfg.py @@ -0,0 +1,28 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST3D") + +process.maxEvents.input = 3 + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSlimmingTest2D.root') +) + +process.testBD = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerB'), + thinnedTag = cms.InputTag('thinningThingProducerBD'), + associationTag = cms.InputTag('thinningThingProducerBD'), + trackTag = cms.InputTag('trackOfThingsProducerD'), + parentWasDropped = cms.bool(True), + parentSlimmedCount = cms.int32(1), + thinnedSlimmedCount = cms.int32(1), + refSlimmedCount = cms.int32(1), + expectedParentContent = cms.vint32(range(0,11)), + expectedThinnedContent = cms.vint32(range(0,6)), + expectedIndexesIntoParent = cms.vuint32(range(0,6)), + expectedValues = cms.vint32(range(0,6)), +) + +process.p = cms.Path( + process.testBD +) diff --git a/FWIO/RNTupleTempTests/test/SlimmingTest3E_cfg.py b/FWIO/RNTupleTempTests/test/SlimmingTest3E_cfg.py new file mode 100644 index 0000000000000..d2d98b1f18b1a --- /dev/null +++ b/FWIO/RNTupleTempTests/test/SlimmingTest3E_cfg.py @@ -0,0 +1,28 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST3E") + +process.maxEvents.input = 3 + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSlimmingTest2E.root') +) + +process.testBEF = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerBE'), + thinnedTag = cms.InputTag('thinningThingProducerBEF'), + associationTag = cms.InputTag('thinningThingProducerBEF'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + parentWasDropped = cms.bool(True), + parentSlimmedCount = cms.int32(2), + thinnedSlimmedCount = cms.int32(2), + refSlimmedCount = cms.int32(2), + expectedParentContent = cms.vint32(range(0,11)), + expectedThinnedContent = cms.vint32(range(6,9)), + expectedIndexesIntoParent = cms.vuint32(range(0,3)), + expectedValues = cms.vint32(range(6,9)), +) + +process.p = cms.Path( + process.testBEF +) diff --git a/FWIO/RNTupleTempTests/test/SlimmingTest3F_cfg.py b/FWIO/RNTupleTempTests/test/SlimmingTest3F_cfg.py new file mode 100644 index 0000000000000..4a854ff27a9ba --- /dev/null +++ b/FWIO/RNTupleTempTests/test/SlimmingTest3F_cfg.py @@ -0,0 +1,38 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST3F") + +process.maxEvents.input = 3 + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSlimmingTest2F.root') +) + +process.testABEF = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerABE'), + thinnedTag = cms.InputTag('thinningThingProducerABEF'), + associationTag = cms.InputTag('thinningThingProducerABEF'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + parentWasDropped = cms.bool(True), + thinnedSlimmedCount = cms.int32(1), + refSlimmedCount = cms.int32(1), + expectedParentContent = cms.vint32(range(6,11)), + expectedThinnedContent = cms.vint32(range(6,9)), + expectedIndexesIntoParent = cms.vuint32(range(0,3)), + expectedValues = cms.vint32(range(6,9)), +) + +process.outF = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSlimmingTest3F.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + ) +) + +process.p = cms.Path( + process.testABEF +) + +process.ep = cms.EndPath( + process.outF +) diff --git a/FWIO/RNTupleTempTests/test/SlimmingTest3I_cfg.py b/FWIO/RNTupleTempTests/test/SlimmingTest3I_cfg.py new file mode 100644 index 0000000000000..2cee45fbdf255 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/SlimmingTest3I_cfg.py @@ -0,0 +1,26 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST3I") + +process.maxEvents.input = 3 + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSlimmingTest2I.root') +) + +process.testB = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerB'), + associationTag = cms.InputTag('thinningThingProducerB'), + trackTag = cms.InputTag('trackOfThingsProducerB'), + parentWasDropped = cms.bool(True), + thinnedSlimmedCount = cms.int32(1), + refSlimmedCount = cms.int32(1), + expectedThinnedContent = cms.vint32(range(0,11)), + expectedIndexesIntoParent = cms.vuint32(range(0,11)), + expectedValues = cms.vint32(range(0,11)), +) + +process.p = cms.Path( + process.testB +) diff --git a/FWIO/RNTupleTempTests/test/SlimmingTest4B_cfg.py b/FWIO/RNTupleTempTests/test/SlimmingTest4B_cfg.py new file mode 100644 index 0000000000000..bb15e7d21407e --- /dev/null +++ b/FWIO/RNTupleTempTests/test/SlimmingTest4B_cfg.py @@ -0,0 +1,28 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST4B") + +process.maxEvents.input = 3 + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSlimmingTest3B.root') +) + +process.testBEG = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerBE'), + thinnedTag = cms.InputTag('thinningThingProducerBEG'), + associationTag = cms.InputTag('thinningThingProducerBEG'), + trackTag = cms.InputTag('trackOfThingsProducerG'), + parentWasDropped = cms.bool(True), + parentSlimmedCount = cms.int32(2), + thinnedSlimmedCount = cms.int32(2), + refSlimmedCount = cms.int32(2), + expectedParentContent = cms.vint32(range(6,11)), + expectedThinnedContent = cms.vint32(range(9,11)), + expectedIndexesIntoParent = cms.vuint32(range(3,5)), + expectedValues = cms.vint32(range(9,11)), +) + +process.p = cms.Path( + process.testBEG +) diff --git a/FWIO/RNTupleTempTests/test/SlimmingTest4F_cfg.py b/FWIO/RNTupleTempTests/test/SlimmingTest4F_cfg.py new file mode 100644 index 0000000000000..4313064d62e6b --- /dev/null +++ b/FWIO/RNTupleTempTests/test/SlimmingTest4F_cfg.py @@ -0,0 +1,28 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST4F") + +process.maxEvents.input = 3 + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSlimmingTest3F.root'), + secondaryFileNames = cms.untracked.vstring('file:testSlimmingTest3B.root') +) + +process.testABEF = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerABE'), + thinnedTag = cms.InputTag('thinningThingProducerABEF'), + associationTag = cms.InputTag('thinningThingProducerABEF'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + parentWasDropped = cms.bool(True), + thinnedSlimmedCount = cms.int32(1), + refSlimmedCount = cms.int32(1), + expectedParentContent = cms.vint32(range(6,11)), + expectedThinnedContent = cms.vint32(range(6,9)), + expectedIndexesIntoParent = cms.vuint32(range(0,3)), + expectedValues = cms.vint32(range(6,9)), +) + +process.p = cms.Path( + process.testABEF +) diff --git a/FWIO/RNTupleTempTests/test/ThinningTest1_cfg.py b/FWIO/RNTupleTempTests/test/ThinningTest1_cfg.py new file mode 100644 index 0000000000000..1fdf249f20eb6 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/ThinningTest1_cfg.py @@ -0,0 +1,497 @@ +# This process is the first step of a test that involves multiple +# processing steps. It tests the thinning collections and +# redirecting Refs, Ptrs, and RefToBases. +# +# Produce 15 thinned collections +# +# Collection A contains Things 0-8 +# Collection B contains Things 0-3 and made from collection A +# Collection C contains Things 4-7 and made from collection A +# +# x Collection D contains Things 10-18 +# Collection E contains Things 10-14 and made from collection D +# Collection F contains Things 14-17 and made from collection D +# +# Collection G contains Things 20-28 +# x Collection H contains Things 20-23 and made from collection G +# x Collection I contains Things 24-27 and made from collection G +# +# x Collection J contains Things 30-38 +# x Collection K contains Things 30-33 and made from collection J +# x Collection L contains Things 34-37 and made from collection J +# +# x Collection M contains Things 40-48 +# x Collection N contains Things 40-43 and made from collection M +# Collection O contains Things 44-47 and made from collection M +# +# The collections marked with an x will get deleted in the next +# processing step. +# +# The Things kept are set by creating TracksOfThings which +# reference them and using those in the selection of a +# Thinning Producer. +# +# The ThinningTestAnalyzer checks that things are working as +# they are supposed to work. +# +# The ThinnedRefFromTestAnalyzer includes additional checks +# for thinnedRefFrom() + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1) +) + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.source = cms.Source("EmptySource") + +process.WhatsItESProducer = cms.ESProducer("WhatsItESProducer") + +process.DoodadESSource = cms.ESSource("DoodadESSource") + +process.thingProducer = cms.EDProducer("ThingProducer", + offsetDelta = cms.int32(100), + nThings = cms.int32(50) +) +process.thingProducer2 = cms.EDProducer("ThingProducer", + offsetDelta = cms.int32(100), + nThings = cms.int32(50) +) +process.thingProducerNotThinned = cms.EDProducer("ThingProducer", + offsetDelta = cms.int32(100), + nThings = cms.int32(50) +) + +process.thingProducer2alias = cms.EDAlias( + thingProducer2 = cms.VPSet( + cms.PSet(type = cms.string('edmtestThings')) + ) +) + +process.trackOfThingsProducerA = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer'), + keysToReference = cms.vuint32(0, 1, 2, 3, 4, 5, 6, 7, 8) +) + +process.trackOfThingsProducerB = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer'), + keysToReference = cms.vuint32(0, 1, 2, 3) +) + +process.trackOfThingsProducerC = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer'), + keysToReference = cms.vuint32(4, 5, 6, 7) +) + +process.trackOfThingsProducerD = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer'), + keysToReference = cms.vuint32(10, 11, 12, 13, 14, 15, 16, 17, 18) +) + +process.trackOfThingsProducerDPlus = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer'), + keysToReference = cms.vuint32(10, 11, 12, 13, 14, 15, 16, 17, 18, 21) +) + +process.trackOfThingsProducerE = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer'), + keysToReference = cms.vuint32(10, 11, 12, 13, 14) +) + +process.trackOfThingsProducerF = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer'), + keysToReference = cms.vuint32(14, 15, 16, 17) +) + +process.trackOfThingsProducerG = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer'), + keysToReference = cms.vuint32(20, 21, 22, 23, 24, 25, 26, 27, 28) +) + +process.trackOfThingsProducerH = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer'), + keysToReference = cms.vuint32(20, 21, 22, 23) +) + +process.trackOfThingsProducerI = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer'), + keysToReference = cms.vuint32(24, 25, 26, 27) +) + +process.trackOfThingsProducerJ = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer'), + keysToReference = cms.vuint32(30, 31, 32, 33, 34, 35, 36, 37, 38) +) + +process.trackOfThingsProducerK = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer'), + keysToReference = cms.vuint32(30, 31, 32, 33) +) + +process.trackOfThingsProducerL = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer'), + keysToReference = cms.vuint32(34, 35, 36, 37) +) + +process.trackOfThingsProducerM = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer'), + keysToReference = cms.vuint32(40, 41, 42, 43, 44, 45, 46, 47, 48) +) + +process.trackOfThingsProducerN = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer'), + keysToReference = cms.vuint32(40, 41, 42, 43) +) + +process.trackOfThingsProducerO = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer'), + keysToReference = cms.vuint32(44, 45, 46, 47) +) + +process.trackOfThingsProducerA2 = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer2'), + keysToReference = cms.vuint32(0, 1, 2, 3, 4, 5, 6, 7, 8) +) + +process.trackOfThingsProducerD2 = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer2'), + keysToReference = cms.vuint32(10, 11, 12, 13, 14, 15, 16, 17, 18) +) + +process.trackOfThingsProducerE2 = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer2'), + keysToReference = cms.vuint32(10, 11, 12, 13, 14) +) + +process.trackOfThingsProducerF2 = cms.EDProducer("TrackOfThingsProducer", + inputTag = cms.InputTag('thingProducer2'), + keysToReference = cms.vuint32(14, 15, 16, 17) +) + +process.thinningThingProducerA = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thingProducer'), + trackTag = cms.InputTag('trackOfThingsProducerA'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(50) +) + +process.thinningThingProducerB = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerA'), + trackTag = cms.InputTag('trackOfThingsProducerB'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(9) +) + +process.thinningThingProducerC = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerA'), + trackTag = cms.InputTag('trackOfThingsProducerC'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(9) +) + +process.thinningThingProducerD = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thingProducer'), + trackTag = cms.InputTag('trackOfThingsProducerD'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(50) +) + +process.thinningThingProducerE = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerD'), + trackTag = cms.InputTag('trackOfThingsProducerE'), + offsetToThinnedKey = cms.uint32(10), + expectedCollectionSize = cms.uint32(9) +) + +process.thinningThingProducerF = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerD'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + offsetToThinnedKey = cms.uint32(10), + expectedCollectionSize = cms.uint32(9) +) + +process.thinningThingProducerG = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thingProducer'), + trackTag = cms.InputTag('trackOfThingsProducerG'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(50) +) + +process.thinningThingProducerH = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerG'), + trackTag = cms.InputTag('trackOfThingsProducerH'), + offsetToThinnedKey = cms.uint32(20), + expectedCollectionSize = cms.uint32(9) +) + +process.thinningThingProducerI = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerG'), + trackTag = cms.InputTag('trackOfThingsProducerI'), + offsetToThinnedKey = cms.uint32(20), + expectedCollectionSize = cms.uint32(9) +) + +process.thinningThingProducerJ = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thingProducer'), + trackTag = cms.InputTag('trackOfThingsProducerJ'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(50) +) + +process.thinningThingProducerK = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerJ'), + trackTag = cms.InputTag('trackOfThingsProducerK'), + offsetToThinnedKey = cms.uint32(30), + expectedCollectionSize = cms.uint32(9) +) + +process.thinningThingProducerL = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerJ'), + trackTag = cms.InputTag('trackOfThingsProducerL'), + offsetToThinnedKey = cms.uint32(30), + expectedCollectionSize = cms.uint32(9) +) + +process.thinningThingProducerM = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thingProducer'), + trackTag = cms.InputTag('trackOfThingsProducerM'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(50) +) + +process.aliasM = cms.EDAlias( + thinningThingProducerM = cms.VPSet( + cms.PSet(type = cms.string('edmtestThings')), + # the next one should get ignored + cms.PSet(type = cms.string('edmThinnedAssociation')) + ) +) + +process.thinningThingProducerN = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerM'), + trackTag = cms.InputTag('trackOfThingsProducerN'), + offsetToThinnedKey = cms.uint32(40), + expectedCollectionSize = cms.uint32(9) +) + +process.aliasN = cms.EDAlias( + thinningThingProducerN = cms.VPSet( + cms.PSet(type = cms.string('edmtestThings')), + # the next one should get ignored + cms.PSet(type = cms.string('edmThinnedAssociation')) + ) +) + +process.thinningThingProducerO = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('aliasM'), + trackTag = cms.InputTag('trackOfThingsProducerO'), + offsetToThinnedKey = cms.uint32(40), + expectedCollectionSize = cms.uint32(9) +) + +process.aliasO = cms.EDAlias( + thinningThingProducerO = cms.VPSet( + cms.PSet(type = cms.string('edmtestThings')), + # the next one should get ignored + cms.PSet(type = cms.string('edmThinnedAssociation')) + ) +) + +process.thinningThingProducerA2 = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thingProducer2'), + trackTag = cms.InputTag('trackOfThingsProducerA2'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(50) +) + +process.slimmingThingProducerA = cms.EDProducer("SlimmingThingProducer", + inputTag = cms.InputTag('thingProducer'), + trackTag = cms.InputTag('trackOfThingsProducerA'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(50) +) + +process.slimmingThingProducerA2 = cms.EDProducer("SlimmingThingProducer", + inputTag = cms.InputTag('thingProducer2'), + trackTag = cms.InputTag('trackOfThingsProducerA2'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(50) +) + +process.testA = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerA'), + associationTag = cms.InputTag('thinningThingProducerA'), + trackTag = cms.InputTag('trackOfThingsProducerA'), + expectedParentContent = cms.vint32( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, 48, 49 + ), + expectedThinnedContent = cms.vint32(0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedIndexesIntoParent = cms.vuint32(0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedValues = cms.vint32(0, 1, 2, 3, 4, 5, 6, 7, 8) +) + +process.testB = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerA'), + thinnedTag = cms.InputTag('thinningThingProducerB'), + associationTag = cms.InputTag('thinningThingProducerB'), + trackTag = cms.InputTag('trackOfThingsProducerB'), + expectedParentContent = cms.vint32( 0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedThinnedContent = cms.vint32(0, 1, 2, 3), + expectedIndexesIntoParent = cms.vuint32(0, 1, 2, 3), + expectedValues = cms.vint32(0, 1, 2, 3) +) + +process.testC = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerA'), + thinnedTag = cms.InputTag('thinningThingProducerC'), + associationTag = cms.InputTag('thinningThingProducerC'), + trackTag = cms.InputTag('trackOfThingsProducerC'), + expectedParentContent = cms.vint32( 0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedThinnedContent = cms.vint32(4, 5, 6, 7), + expectedIndexesIntoParent = cms.vuint32(4, 5, 6, 7), + expectedValues = cms.vint32(4, 5, 6, 7) +) + +process.thinnedRefTestA = cms.EDAnalyzer("ThinnedRefFromTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerA'), + unrelatedTag = cms.InputTag('thingProducerNotThinned'), + trackTag = cms.InputTag('trackOfThingsProducerA') +) + +process.thinnedRefTestA2 = cms.EDAnalyzer("ThinnedRefFromTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerA'), + unrelatedTag = cms.InputTag('thingProducer2'), + trackTag = cms.InputTag('trackOfThingsProducerA') +) + +process.slimmingTestA = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('slimmingThingProducerA'), + associationTag = cms.InputTag('slimmingThingProducerA'), + trackTag = cms.InputTag('trackOfThingsProducerA'), + thinnedSlimmedCount = cms.int32(1), + expectedParentContent = cms.vint32( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, 48, 49 + ), + expectedThinnedContent = cms.vint32(0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedIndexesIntoParent = cms.vuint32(0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedValues = cms.vint32(0, 1, 2, 3, 4, 5, 6, 7, 8) +) + +process.slimmingTestA2 = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer2'), + thinnedTag = cms.InputTag('slimmingThingProducerA2'), + associationTag = cms.InputTag('slimmingThingProducerA2'), + trackTag = cms.InputTag('trackOfThingsProducerA2'), + thinnedSlimmedCount = cms.int32(1), + expectedParentContent = cms.vint32( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, 48, 49 + ), + expectedThinnedContent = cms.vint32(0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedIndexesIntoParent = cms.vuint32(0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedValues = cms.vint32(0, 1, 2, 3, 4, 5, 6, 7, 8) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testThinningTest1.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_thingProducer2_*_*', + 'drop *_thinningThingProducerM_*_*', + 'drop *_thinningThingProducerN_*_*', + 'drop *_thinningThingProducerO_*_*' + ) +) + +process.out2 = cms.OutputModule("EventStreamFileWriter", + fileName = cms.untracked.string('testThinningStreamerout.dat'), + compression_level = cms.untracked.int32(1), + use_compression = cms.untracked.bool(True), + max_event_size = cms.untracked.int32(7000000), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_thingProducer_*_*', + 'drop *_thingProducer2_*_*', + 'drop *_thingProducerNotThinned_*_*', + 'drop *_trackOfThingsProducerA2_*_*', + 'drop *_thinningThingProducerD_*_*', + 'drop *_thinningThingProducerH_*_*', + 'drop *_thinningThingProducerI_*_*', + 'drop *_thinningThingProducerJ_*_*', + 'drop *_thinningThingProducerK_*_*', + 'drop *_thinningThingProducerL_*_*', + 'drop *_thinningThingProducerM_*_*', + 'drop *_thinningThingProducerN_*_*', + 'drop *_thinningThingProducerO_*_*', + 'drop *_aliasM_*_*', + 'drop *_aliasN_*_*' + ) +) + +process.p = cms.Path(process.thingProducer * process.thingProducer2 * process.thingProducerNotThinned + * process.trackOfThingsProducerA + * process.trackOfThingsProducerB + * process.trackOfThingsProducerC + * process.trackOfThingsProducerD + * process.trackOfThingsProducerDPlus + * process.trackOfThingsProducerE + * process.trackOfThingsProducerF + * process.trackOfThingsProducerG + * process.trackOfThingsProducerH + * process.trackOfThingsProducerI + * process.trackOfThingsProducerJ + * process.trackOfThingsProducerK + * process.trackOfThingsProducerL + * process.trackOfThingsProducerM + * process.trackOfThingsProducerN + * process.trackOfThingsProducerO + * process.trackOfThingsProducerA2 + * process.trackOfThingsProducerD2 + * process.trackOfThingsProducerE2 + * process.trackOfThingsProducerF2 + * process.thinningThingProducerA + * process.thinningThingProducerB + * process.thinningThingProducerC + * process.thinningThingProducerD + * process.thinningThingProducerE + * process.thinningThingProducerF + * process.thinningThingProducerG + * process.thinningThingProducerH + * process.thinningThingProducerI + * process.thinningThingProducerJ + * process.thinningThingProducerK + * process.thinningThingProducerL + * process.thinningThingProducerM + * process.thinningThingProducerN + * process.thinningThingProducerO + * process.thinningThingProducerA2 + * process.slimmingThingProducerA + * process.slimmingThingProducerA2 + * process.testA + * process.testB + * process.testC + * process.thinnedRefTestA + * process.thinnedRefTestA2 + * process.slimmingTestA + * process.slimmingTestA2 + ) + +process.endPath = cms.EndPath(process.out * process.out2) diff --git a/FWIO/RNTupleTempTests/test/ThinningTest2_cfg.py b/FWIO/RNTupleTempTests/test/ThinningTest2_cfg.py new file mode 100644 index 0000000000000..9762f22c3e252 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/ThinningTest2_cfg.py @@ -0,0 +1,142 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1) +) + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testThinningTest1.root') +) + +process.WhatsItESProducer = cms.ESProducer("WhatsItESProducer") + +process.DoodadESSource = cms.ESSource("DoodadESSource") + +# Produce some products that have the same branch name except process +# These test disambiguation when there are multiple entries +# in the ThinnedAssociationsHelper with the same parent BranchID. +process.thingProducer = cms.EDProducer("ThingProducer", + offsetDelta = cms.int32(2000), + nThings = cms.int32(50) +) + +process.thinningThingProducerD = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thingProducer'), + trackTag = cms.InputTag('trackOfThingsProducerD'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(50) +) + +process.thinningThingProducerETEST = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerD'), + trackTag = cms.InputTag('trackOfThingsProducerE'), + offsetToThinnedKey = cms.uint32(10), + expectedCollectionSize = cms.uint32(9) +) + +process.thinningThingProducerFTEST = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerD'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + offsetToThinnedKey = cms.uint32(10), + expectedCollectionSize = cms.uint32(9) +) + +process.thinningThingProducerD2 = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thingProducer2alias'), + trackTag = cms.InputTag('trackOfThingsProducerD2'), + offsetToThinnedKey = cms.uint32(0), + expectedCollectionSize = cms.uint32(50) +) + +process.thinningThingProducerD2alias = cms.EDAlias( + thinningThingProducerD2 = cms.VPSet( + cms.PSet(type = cms.string('edmtestThings')) + ) +) + +process.testA = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer', '', 'PROD'), + thinnedTag = cms.InputTag('thinningThingProducerA'), + associationTag = cms.InputTag('thinningThingProducerA'), + trackTag = cms.InputTag('trackOfThingsProducerA'), + expectedParentContent = cms.vint32( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, 48, 49 + ), + expectedThinnedContent = cms.vint32(0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedIndexesIntoParent = cms.vuint32(0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedValues = cms.vint32(0, 1, 2, 3, 4, 5, 6, 7, 8) +) + +process.testB = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerA'), + thinnedTag = cms.InputTag('thinningThingProducerB'), + associationTag = cms.InputTag('thinningThingProducerB'), + trackTag = cms.InputTag('trackOfThingsProducerB'), + expectedParentContent = cms.vint32( 0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedThinnedContent = cms.vint32(0, 1, 2, 3), + expectedIndexesIntoParent = cms.vuint32(0, 1, 2, 3), + expectedValues = cms.vint32(0, 1, 2, 3) +) + +process.testC = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerA'), + thinnedTag = cms.InputTag('thinningThingProducerC'), + associationTag = cms.InputTag('thinningThingProducerC'), + trackTag = cms.InputTag('trackOfThingsProducerC'), + expectedParentContent = cms.vint32( 0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedThinnedContent = cms.vint32(4, 5, 6, 7), + expectedIndexesIntoParent = cms.vuint32(4, 5, 6, 7), + expectedValues = cms.vint32(4, 5, 6, 7) +) + +process.slimmingTestA = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer', '', 'PROD'), + thinnedTag = cms.InputTag('slimmingThingProducerA'), + associationTag = cms.InputTag('slimmingThingProducerA'), + trackTag = cms.InputTag('trackOfThingsProducerA'), + thinnedSlimmedCount = cms.int32(1), + expectedParentContent = cms.vint32( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, 48, 49 + ), + expectedThinnedContent = cms.vint32(0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedIndexesIntoParent = cms.vuint32(0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedValues = cms.vint32(0, 1, 2, 3, 4, 5, 6, 7, 8) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testThinningTest2.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_thingProducer_*_*', + 'drop *_thinningThingProducerD2_*_*', + 'drop *_thinningThingProducerD_*_*', + 'drop *_thinningThingProducerH_*_*', + 'drop *_thinningThingProducerI_*_*', + 'drop *_thinningThingProducerJ_*_*', + 'drop *_thinningThingProducerK_*_*', + 'drop *_thinningThingProducerL_*_*', + 'drop *_aliasM_*_*', + 'drop *_aliasN_*_*', + ) +) + +process.p = cms.Path(process.thinningThingProducerD2 * process.testA * process.testB * process.testC + * process.slimmingTestA + * process.thingProducer + * process.thinningThingProducerD + * process.thinningThingProducerETEST + * process.thinningThingProducerFTEST) + +process.endPath = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/ThinningTest3_cfg.py b/FWIO/RNTupleTempTests/test/ThinningTest3_cfg.py new file mode 100644 index 0000000000000..5473e5ae0f29f --- /dev/null +++ b/FWIO/RNTupleTempTests/test/ThinningTest3_cfg.py @@ -0,0 +1,273 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST3") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1) +) + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testThinningTest2.root') + #secondaryFileNames = cms.untracked.vstring('file:testSeriesOfProcessesPROD1.root') +) + +process.WhatsItESProducer = cms.ESProducer("WhatsItESProducer") + +process.DoodadESSource = cms.ESSource("DoodadESSource") + +process.thinningThingProducerE2 = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerD2alias'), + trackTag = cms.InputTag('trackOfThingsProducerE2'), + offsetToThinnedKey = cms.uint32(10), + expectedCollectionSize = cms.uint32(9) +) + +process.thinningThingProducerE2alias = cms.EDAlias( + thinningThingProducerE2 = cms.VPSet( + cms.PSet(type = cms.string('edmtestThings')) + ) +) + +process.thinningThingProducerF2 = cms.EDProducer("ThinningThingProducer", + inputTag = cms.InputTag('thinningThingProducerD2alias'), + trackTag = cms.InputTag('trackOfThingsProducerF2'), + offsetToThinnedKey = cms.uint32(10), + expectedCollectionSize = cms.uint32(9) +) + +process.thinningThingProducerF2alias = cms.EDAlias( + thinningThingProducerF2 = cms.VPSet( + cms.PSet(type = cms.string('edmtestThings')) + ) +) + +process.testA = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer', '', 'PROD'), + thinnedTag = cms.InputTag('thinningThingProducerA'), + associationTag = cms.InputTag('thinningThingProducerA'), + trackTag = cms.InputTag('trackOfThingsProducerA'), + parentWasDropped = cms.bool(True), + expectedThinnedContent = cms.vint32(0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedIndexesIntoParent = cms.vuint32(0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedValues = cms.vint32(0, 1, 2, 3, 4, 5, 6, 7, 8) +) + +process.testB = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerA'), + thinnedTag = cms.InputTag('thinningThingProducerB'), + associationTag = cms.InputTag('thinningThingProducerB'), + trackTag = cms.InputTag('trackOfThingsProducerB'), + expectedParentContent = cms.vint32( 0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedThinnedContent = cms.vint32(0, 1, 2, 3), + expectedIndexesIntoParent = cms.vuint32(0, 1, 2, 3), + expectedValues = cms.vint32(0, 1, 2, 3) +) + +process.testC = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerA'), + thinnedTag = cms.InputTag('thinningThingProducerC'), + associationTag = cms.InputTag('thinningThingProducerC'), + trackTag = cms.InputTag('trackOfThingsProducerC'), + expectedParentContent = cms.vint32( 0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedThinnedContent = cms.vint32(4, 5, 6, 7), + expectedIndexesIntoParent = cms.vuint32(4, 5, 6, 7), + expectedValues = cms.vint32(4, 5, 6, 7) +) + +process.testD = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer', '', 'PROD'), + thinnedTag = cms.InputTag('thinningThingProducerD', '', 'PROD'), + associationTag = cms.InputTag('thinningThingProducerD', '', 'PROD'), + trackTag = cms.InputTag('trackOfThingsProducerD'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + expectedIndexesIntoParent = cms.vuint32(10, 11, 12, 13, 14, 15, 16, 17, 18), + expectedValues = cms.vint32(10, 11, 12, 13, 14, 15, 16, 17, -1) +) + +process.testDPlus = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerD', '', 'PROD'), + associationTag = cms.InputTag('thinningThingProducerD', '', 'PROD'), + trackTag = cms.InputTag('trackOfThingsProducerDPlus'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + expectedIndexesIntoParent = cms.vuint32(10, 11, 12, 13, 14, 15, 16, 17, 18), + expectedValues = cms.vint32(10, 11, 12, 13, 14, 15, 16, 17, -1, 21) +) + +process.testE = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerD', '', 'PROD'), + thinnedTag = cms.InputTag('thinningThingProducerE'), + associationTag = cms.InputTag('thinningThingProducerE'), + trackTag = cms.InputTag('trackOfThingsProducerE'), + parentWasDropped = cms.bool(True), + expectedThinnedContent = cms.vint32(10, 11, 12, 13, 14), + expectedIndexesIntoParent = cms.vuint32(0, 1, 2, 3, 4), + expectedValues = cms.vint32(10, 11, 12, 13, 14) +) + +process.testF = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerD', '', 'PROD'), + thinnedTag = cms.InputTag('thinningThingProducerF'), + associationTag = cms.InputTag('thinningThingProducerF'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + parentWasDropped = cms.bool(True), + expectedThinnedContent = cms.vint32(14, 15, 16, 17), + expectedIndexesIntoParent = cms.vuint32(4, 5, 6, 7), + expectedValues = cms.vint32(14, 15, 16, 17) +) + +process.testG = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer', '', 'PROD'), + thinnedTag = cms.InputTag('thinningThingProducerG'), + associationTag = cms.InputTag('thinningThingProducerG'), + trackTag = cms.InputTag('trackOfThingsProducerG'), + parentWasDropped = cms.bool(True), + expectedThinnedContent = cms.vint32(20, 21, 22, 23, 24, 25, 26, 27, 28), + expectedIndexesIntoParent = cms.vuint32(20, 21, 22, 23, 24, 25, 26, 27, 28), + expectedValues = cms.vint32(20, 21, 22, 23, 24, 25, 26, 27, 28) +) + +process.testH = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerG'), + thinnedTag = cms.InputTag('thinningThingProducerH'), + associationTag = cms.InputTag('thinningThingProducerH'), + trackTag = cms.InputTag('trackOfThingsProducerH'), + thinnedWasDropped = cms.bool(True), + expectedParentContent = cms.vint32( 20, 21, 22, 23, 24, 25, 26, 27, 28), + associationShouldBeDropped = cms.bool(True), + expectedValues = cms.vint32(20, 21, 22, 23) +) + +process.testI = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerG'), + thinnedTag = cms.InputTag('thinningThingProducerI'), + associationTag = cms.InputTag('thinningThingProducerI'), + trackTag = cms.InputTag('trackOfThingsProducerI'), + thinnedWasDropped = cms.bool(True), + associationShouldBeDropped = cms.bool(True), + expectedParentContent = cms.vint32( 20, 21, 22, 23, 24, 25, 26, 27, 28), + expectedValues = cms.vint32(24, 25, 26, 27) +) + +process.testJ = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer', '', 'PROD'), + thinnedTag = cms.InputTag('thinningThingProducerJ'), + associationTag = cms.InputTag('thinningThingProducerJ'), + trackTag = cms.InputTag('trackOfThingsProducerJ'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + associationShouldBeDropped = cms.bool(True), + expectedValues = cms.vint32(-1, -1, -1, -1, -1, -1, -1, -1, -1) +) + +process.testK = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerJ'), + thinnedTag = cms.InputTag('thinningThingProducerK'), + associationTag = cms.InputTag('thinningThingProducerK'), + trackTag = cms.InputTag('trackOfThingsProducerK'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + associationShouldBeDropped = cms.bool(True), + expectedValues = cms.vint32(-1, -1, -1, -1) +) + +process.testL = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerJ'), + thinnedTag = cms.InputTag('thinningThingProducerL'), + associationTag = cms.InputTag('thinningThingProducerL'), + trackTag = cms.InputTag('trackOfThingsProducerL'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + associationShouldBeDropped = cms.bool(True), + expectedValues = cms.vint32(-1, -1, -1, -1) +) + +process.testM = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer', '', 'PROD'), + thinnedTag = cms.InputTag('thinningThingProducerM'), + associationTag = cms.InputTag('thinningThingProducerM'), + trackTag = cms.InputTag('trackOfThingsProducerM'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + expectedIndexesIntoParent = cms.vuint32(40, 41, 42, 43, 44, 45, 46, 47, 48), + expectedValues = cms.vint32(-1, -1, -1, -1, 44, 45, 46, 47, -1) +) + +process.testN = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerM'), + thinnedTag = cms.InputTag('thinningThingProducerN'), + associationTag = cms.InputTag('thinningThingProducerN'), + trackTag = cms.InputTag('trackOfThingsProducerN'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + associationShouldBeDropped = cms.bool(True), + expectedValues = cms.vint32(-1, -1, -1, -1) +) + +process.testO = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerM'), + thinnedTag = cms.InputTag('aliasO'), + associationTag = cms.InputTag('thinningThingProducerO'), + trackTag = cms.InputTag('trackOfThingsProducerO'), + parentWasDropped = cms.bool(True), + thinnedIsAlias = cms.bool(True), + expectedThinnedContent = cms.vint32(44, 45, 46, 47), + expectedIndexesIntoParent = cms.vuint32(4, 5, 6, 7), + expectedValues = cms.vint32(44, 45, 46, 47) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testThinningTest3.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_thingProducer2alias_*_*', + 'drop *_thinningThingProducerD2alias_*_*', + 'drop *_thinningThingProducerE2_*_*', + 'drop *_thinningThingProducerF2_*_*', + 'drop *_thinningThingProducerA_*_*', + 'drop *_thinningThingProducerB_*_*', + 'drop *_thinningThingProducerC_*_*', + 'drop *_thinningThingProducerE_*_*', + 'drop *_thinningThingProducerF_*_*', + 'drop *_thinningThingProducerG_*_*', + 'drop *_thinningThingProducerO_*_*' + ) +) + +process.outSlim = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testThinningTest3Slimming.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_thingProducer2alias_*_*', + 'drop *_aliasO_*_*', + 'drop *_thinningThingProducer*_*_*', + ) +) + +process.p = cms.Path(process.thinningThingProducerE2 * + process.thinningThingProducerF2 * + process.testA * + process.testB * + process.testC * + process.testD * + process.testDPlus * + process.testE * + process.testF * + process.testG * + process.testH * + process.testI * + process.testJ * + process.testK * + process.testL * + process.testM * + process.testN * + process.testO +) + +process.endPath = cms.EndPath(process.out*process.outSlim) diff --git a/FWIO/RNTupleTempTests/test/ThinningTest4Slimming_cfg.py b/FWIO/RNTupleTempTests/test/ThinningTest4Slimming_cfg.py new file mode 100644 index 0000000000000..9a2335d37afdb --- /dev/null +++ b/FWIO/RNTupleTempTests/test/ThinningTest4Slimming_cfg.py @@ -0,0 +1,36 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST4") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1) +) + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testThinningTest3Slimming.root') +) + +process.slimmingTestA = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer', '', 'PROD'), + thinnedTag = cms.InputTag('slimmingThingProducerA'), + associationTag = cms.InputTag('slimmingThingProducerA'), + trackTag = cms.InputTag('trackOfThingsProducerA'), + parentWasDropped = cms.bool(True), + thinnedSlimmedCount = cms.int32(1), + refSlimmedCount = cms.int32(1), + expectedParentContent = cms.vint32( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, + 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, 48, 49 + ), + expectedThinnedContent = cms.vint32(0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedIndexesIntoParent = cms.vuint32(0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedValues = cms.vint32(0, 1, 2, 3, 4, 5, 6, 7, 8) +) + +process.p = cms.Path(process.slimmingTestA) diff --git a/FWIO/RNTupleTempTests/test/ThinningTest4_cfg.py b/FWIO/RNTupleTempTests/test/ThinningTest4_cfg.py new file mode 100644 index 0000000000000..b7e860ddbf942 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/ThinningTest4_cfg.py @@ -0,0 +1,252 @@ +# Test 2 file input + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST4") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1) +) + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testThinningTest3.root'), + secondaryFileNames = cms.untracked.vstring('file:testThinningTest2.root'), + inputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_thingProducer2alias_*_*', + 'drop *_thinningThingProducerD2alias_*_*' + ), + dropDescendantsOfDroppedBranches = cms.untracked.bool(False) +) + +process.testA = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerA'), + associationTag = cms.InputTag('thinningThingProducerA'), + trackTag = cms.InputTag('trackOfThingsProducerA'), + parentWasDropped = cms.bool(True), + expectedThinnedContent = cms.vint32(0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedIndexesIntoParent = cms.vuint32(0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedValues = cms.vint32(0, 1, 2, 3, 4, 5, 6, 7, 8) +) + +process.testB = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerA'), + thinnedTag = cms.InputTag('thinningThingProducerB'), + associationTag = cms.InputTag('thinningThingProducerB'), + trackTag = cms.InputTag('trackOfThingsProducerB'), + expectedParentContent = cms.vint32( 0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedThinnedContent = cms.vint32(0, 1, 2, 3), + expectedIndexesIntoParent = cms.vuint32(0, 1, 2, 3), + expectedValues = cms.vint32(0, 1, 2, 3) +) + +process.testC = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerA'), + thinnedTag = cms.InputTag('thinningThingProducerC'), + associationTag = cms.InputTag('thinningThingProducerC'), + trackTag = cms.InputTag('trackOfThingsProducerC'), + expectedParentContent = cms.vint32( 0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedThinnedContent = cms.vint32(4, 5, 6, 7), + expectedIndexesIntoParent = cms.vuint32(4, 5, 6, 7), + expectedValues = cms.vint32(4, 5, 6, 7) +) + +process.testD = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerD'), + associationTag = cms.InputTag('thinningThingProducerD'), + trackTag = cms.InputTag('trackOfThingsProducerD'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + expectedIndexesIntoParent = cms.vuint32(10, 11, 12, 13, 14, 15, 16, 17, 18), + expectedValues = cms.vint32(10, 11, 12, 13, 14, 15, 16, 17, -1) +) + +process.testE = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerD'), + thinnedTag = cms.InputTag('thinningThingProducerE'), + associationTag = cms.InputTag('thinningThingProducerE'), + trackTag = cms.InputTag('trackOfThingsProducerE'), + parentWasDropped = cms.bool(True), + expectedThinnedContent = cms.vint32(10, 11, 12, 13, 14), + expectedIndexesIntoParent = cms.vuint32(0, 1, 2, 3, 4), + expectedValues = cms.vint32(10, 11, 12, 13, 14) +) + +process.testF = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerD'), + thinnedTag = cms.InputTag('thinningThingProducerF'), + associationTag = cms.InputTag('thinningThingProducerF'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + parentWasDropped = cms.bool(True), + expectedThinnedContent = cms.vint32(14, 15, 16, 17), + expectedIndexesIntoParent = cms.vuint32(4, 5, 6, 7), + expectedValues = cms.vint32(14, 15, 16, 17) +) + +process.testG = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerG'), + associationTag = cms.InputTag('thinningThingProducerG'), + trackTag = cms.InputTag('trackOfThingsProducerG'), + parentWasDropped = cms.bool(True), + expectedThinnedContent = cms.vint32(20, 21, 22, 23, 24, 25, 26, 27, 28), + expectedIndexesIntoParent = cms.vuint32(20, 21, 22, 23, 24, 25, 26, 27, 28), + expectedValues = cms.vint32(20, 21, 22, 23, 24, 25, 26, 27, 28) +) + +process.testH = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerG'), + thinnedTag = cms.InputTag('thinningThingProducerH'), + associationTag = cms.InputTag('thinningThingProducerH'), + trackTag = cms.InputTag('trackOfThingsProducerH'), + thinnedWasDropped = cms.bool(True), + expectedParentContent = cms.vint32( 20, 21, 22, 23, 24, 25, 26, 27, 28), + associationShouldBeDropped = cms.bool(True), + expectedValues = cms.vint32(20, 21, 22, 23) +) + +process.testI = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerG'), + thinnedTag = cms.InputTag('thinningThingProducerI'), + associationTag = cms.InputTag('thinningThingProducerI'), + trackTag = cms.InputTag('trackOfThingsProducerI'), + thinnedWasDropped = cms.bool(True), + associationShouldBeDropped = cms.bool(True), + expectedParentContent = cms.vint32( 20, 21, 22, 23, 24, 25, 26, 27, 28), + expectedValues = cms.vint32(24, 25, 26, 27) +) + +process.testJ = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerJ'), + associationTag = cms.InputTag('thinningThingProducerJ'), + trackTag = cms.InputTag('trackOfThingsProducerJ'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + associationShouldBeDropped = cms.bool(True), + expectedValues = cms.vint32(-1, -1, -1, -1, -1, -1, -1, -1, -1) +) + +process.testK = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerJ'), + thinnedTag = cms.InputTag('thinningThingProducerK'), + associationTag = cms.InputTag('thinningThingProducerK'), + trackTag = cms.InputTag('trackOfThingsProducerK'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + associationShouldBeDropped = cms.bool(True), + expectedValues = cms.vint32(-1, -1, -1, -1) +) + +process.testL = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerJ'), + thinnedTag = cms.InputTag('thinningThingProducerL'), + associationTag = cms.InputTag('thinningThingProducerL'), + trackTag = cms.InputTag('trackOfThingsProducerL'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + associationShouldBeDropped = cms.bool(True), + expectedValues = cms.vint32(-1, -1, -1, -1) +) + +process.testM = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerM'), + associationTag = cms.InputTag('thinningThingProducerM'), + trackTag = cms.InputTag('trackOfThingsProducerM'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + expectedIndexesIntoParent = cms.vuint32(40, 41, 42, 43, 44, 45, 46, 47, 48), + expectedValues = cms.vint32(-1, -1, -1, -1, 44, 45, 46, 47, -1) +) + +process.testN = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerM'), + thinnedTag = cms.InputTag('thinningThingProducerN'), + associationTag = cms.InputTag('thinningThingProducerN'), + trackTag = cms.InputTag('trackOfThingsProducerN'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + associationShouldBeDropped = cms.bool(True), + expectedValues = cms.vint32(-1, -1, -1, -1) +) + +process.testO = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerM'), + thinnedTag = cms.InputTag('aliasO'), + thinnedIsAlias = cms.bool(True), + associationTag = cms.InputTag('thinningThingProducerO'), + trackTag = cms.InputTag('trackOfThingsProducerO'), + parentWasDropped = cms.bool(True), + expectedThinnedContent = cms.vint32(44, 45, 46, 47), + expectedIndexesIntoParent = cms.vuint32(4, 5, 6, 7), + expectedValues = cms.vint32(44, 45, 46, 47) +) + +process.testD2 = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer2alias'), + thinnedTag = cms.InputTag('thinningThingProducerD2alias'), + associationTag = cms.InputTag('thinningThingProducerD2'), + trackTag = cms.InputTag('trackOfThingsProducerD2'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + expectedIndexesIntoParent = cms.vuint32(10, 11, 12, 13, 14, 15, 16, 17, 18), + expectedValues = cms.vint32(10, 11, 12, 13, 14, 15, 16, 17, -1) +) + +process.testE2 = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerD2alias'), + thinnedTag = cms.InputTag('thinningThingProducerE2alias'), + associationTag = cms.InputTag('thinningThingProducerE2'), + trackTag = cms.InputTag('trackOfThingsProducerE2'), + parentWasDropped = cms.bool(True), + expectedThinnedContent = cms.vint32(10, 11, 12, 13, 14), + expectedIndexesIntoParent = cms.vuint32(0, 1, 2, 3, 4), + expectedValues = cms.vint32(10, 11, 12, 13, 14) +) + +process.testF2 = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerD2alias'), + thinnedTag = cms.InputTag('thinningThingProducerF2alias'), + associationTag = cms.InputTag('thinningThingProducerF2'), + trackTag = cms.InputTag('trackOfThingsProducerF2'), + parentWasDropped = cms.bool(True), + expectedThinnedContent = cms.vint32(14, 15, 16, 17), + expectedIndexesIntoParent = cms.vuint32(4, 5, 6, 7), + expectedValues = cms.vint32(14, 15, 16, 17) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testThinningTest4.root'), + outputCommands = cms.untracked.vstring( + 'keep *' + ) +) + +process.p = cms.Path(process.testA * + process.testB * + process.testC * + process.testD * + process.testE * + process.testF * + process.testG * + process.testH * + process.testI * + process.testJ * + process.testK * + process.testL * + process.testM * + process.testN * + process.testO * + process.testD2 * + process.testE2 * + process.testF2 +) + +process.endPath = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/ThinningTest_dropOnInput_cfg.py b/FWIO/RNTupleTempTests/test/ThinningTest_dropOnInput_cfg.py new file mode 100644 index 0000000000000..a5516d90ec9f8 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/ThinningTest_dropOnInput_cfg.py @@ -0,0 +1,219 @@ +# Test drop on input + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTDROPONINPUT") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1) +) + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testThinningTest1.root'), + inputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_thingProducer_*_*', + 'drop *_thinningThingProducerD_*_*', + 'drop *_thinningThingProducerH_*_*', + 'drop *_thinningThingProducerI_*_*', + 'drop *_thinningThingProducerJ_*_*', + 'drop *_thinningThingProducerK_*_*', + 'drop *_thinningThingProducerL_*_*', + 'drop *_aliasM_*_*', + 'drop *_aliasN_*_*' + ), + dropDescendantsOfDroppedBranches = cms.untracked.bool(False) +) + +process.testA = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerA'), + associationTag = cms.InputTag('thinningThingProducerA'), + trackTag = cms.InputTag('trackOfThingsProducerA'), + parentWasDropped = cms.bool(True), + expectedThinnedContent = cms.vint32(0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedIndexesIntoParent = cms.vuint32(0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedValues = cms.vint32(0, 1, 2, 3, 4, 5, 6, 7, 8) +) + +process.testB = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerA'), + thinnedTag = cms.InputTag('thinningThingProducerB'), + associationTag = cms.InputTag('thinningThingProducerB'), + trackTag = cms.InputTag('trackOfThingsProducerB'), + expectedParentContent = cms.vint32( 0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedThinnedContent = cms.vint32(0, 1, 2, 3), + expectedIndexesIntoParent = cms.vuint32(0, 1, 2, 3), + expectedValues = cms.vint32(0, 1, 2, 3) +) + +process.testC = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerA'), + thinnedTag = cms.InputTag('thinningThingProducerC'), + associationTag = cms.InputTag('thinningThingProducerC'), + trackTag = cms.InputTag('trackOfThingsProducerC'), + expectedParentContent = cms.vint32( 0, 1, 2, 3, 4, 5, 6, 7, 8), + expectedThinnedContent = cms.vint32(4, 5, 6, 7), + expectedIndexesIntoParent = cms.vuint32(4, 5, 6, 7), + expectedValues = cms.vint32(4, 5, 6, 7) +) + +process.testD = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerD'), + associationTag = cms.InputTag('thinningThingProducerD'), + trackTag = cms.InputTag('trackOfThingsProducerD'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + expectedIndexesIntoParent = cms.vuint32(10, 11, 12, 13, 14, 15, 16, 17, 18), + expectedValues = cms.vint32(10, 11, 12, 13, 14, 15, 16, 17, -1) +) + +process.testE = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerD'), + thinnedTag = cms.InputTag('thinningThingProducerE'), + associationTag = cms.InputTag('thinningThingProducerE'), + trackTag = cms.InputTag('trackOfThingsProducerE'), + parentWasDropped = cms.bool(True), + expectedThinnedContent = cms.vint32(10, 11, 12, 13, 14), + expectedIndexesIntoParent = cms.vuint32(0, 1, 2, 3, 4), + expectedValues = cms.vint32(10, 11, 12, 13, 14) +) + +process.testF = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerD'), + thinnedTag = cms.InputTag('thinningThingProducerF'), + associationTag = cms.InputTag('thinningThingProducerF'), + trackTag = cms.InputTag('trackOfThingsProducerF'), + parentWasDropped = cms.bool(True), + expectedThinnedContent = cms.vint32(14, 15, 16, 17), + expectedIndexesIntoParent = cms.vuint32(4, 5, 6, 7), + expectedValues = cms.vint32(14, 15, 16, 17) +) + +process.testG = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerG'), + associationTag = cms.InputTag('thinningThingProducerG'), + trackTag = cms.InputTag('trackOfThingsProducerG'), + parentWasDropped = cms.bool(True), + expectedThinnedContent = cms.vint32(20, 21, 22, 23, 24, 25, 26, 27, 28), + expectedIndexesIntoParent = cms.vuint32(20, 21, 22, 23, 24, 25, 26, 27, 28), + expectedValues = cms.vint32(20, 21, 22, 23, 24, 25, 26, 27, 28) +) + +process.testH = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerG'), + thinnedTag = cms.InputTag('thinningThingProducerH'), + associationTag = cms.InputTag('thinningThingProducerH'), + trackTag = cms.InputTag('trackOfThingsProducerH'), + thinnedWasDropped = cms.bool(True), + expectedParentContent = cms.vint32( 20, 21, 22, 23, 24, 25, 26, 27, 28), + associationShouldBeDropped = cms.bool(True), + expectedValues = cms.vint32(20, 21, 22, 23) +) + +process.testI = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerG'), + thinnedTag = cms.InputTag('thinningThingProducerI'), + associationTag = cms.InputTag('thinningThingProducerI'), + trackTag = cms.InputTag('trackOfThingsProducerI'), + thinnedWasDropped = cms.bool(True), + associationShouldBeDropped = cms.bool(True), + expectedParentContent = cms.vint32( 20, 21, 22, 23, 24, 25, 26, 27, 28), + expectedValues = cms.vint32(24, 25, 26, 27) +) + +process.testJ = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerJ'), + associationTag = cms.InputTag('thinningThingProducerJ'), + trackTag = cms.InputTag('trackOfThingsProducerJ'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + associationShouldBeDropped = cms.bool(True), + expectedValues = cms.vint32(-1, -1, -1, -1, -1, -1, -1, -1, -1) +) + +process.testK = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerJ'), + thinnedTag = cms.InputTag('thinningThingProducerK'), + associationTag = cms.InputTag('thinningThingProducerK'), + trackTag = cms.InputTag('trackOfThingsProducerK'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + associationShouldBeDropped = cms.bool(True), + expectedValues = cms.vint32(-1, -1, -1, -1) +) + +process.testL = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerJ'), + thinnedTag = cms.InputTag('thinningThingProducerL'), + associationTag = cms.InputTag('thinningThingProducerL'), + trackTag = cms.InputTag('trackOfThingsProducerL'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + associationShouldBeDropped = cms.bool(True), + expectedValues = cms.vint32(-1, -1, -1, -1) +) + +process.testM = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thingProducer'), + thinnedTag = cms.InputTag('thinningThingProducerM'), + associationTag = cms.InputTag('thinningThingProducerM'), + trackTag = cms.InputTag('trackOfThingsProducerM'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + expectedIndexesIntoParent = cms.vuint32(40, 41, 42, 43, 44, 45, 46, 47, 48), + expectedValues = cms.vint32(-1, -1, -1, -1, 44, 45, 46, 47, -1) +) + +process.testN = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerM'), + thinnedTag = cms.InputTag('thinningThingProducerN'), + associationTag = cms.InputTag('thinningThingProducerN'), + trackTag = cms.InputTag('trackOfThingsProducerN'), + parentWasDropped = cms.bool(True), + thinnedWasDropped = cms.bool(True), + associationShouldBeDropped = cms.bool(True), + expectedValues = cms.vint32(-1, -1, -1, -1) +) + +process.testO = cms.EDAnalyzer("ThinningTestAnalyzer", + parentTag = cms.InputTag('thinningThingProducerM'), + thinnedTag = cms.InputTag('aliasO'), + associationTag = cms.InputTag('thinningThingProducerO'), + trackTag = cms.InputTag('trackOfThingsProducerO'), + thinnedIsAlias = cms.bool(True), + parentWasDropped = cms.bool(True), + expectedThinnedContent = cms.vint32(44, 45, 46, 47), + expectedIndexesIntoParent = cms.vuint32(4, 5, 6, 7), + expectedValues = cms.vint32(44, 45, 46, 47) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testThinningTestDropOnInput.root') +) + +process.p = cms.Path(process.testA * + process.testB * + process.testC * + process.testD * + process.testE * + process.testF * + process.testG * + process.testH * + process.testI * + process.testJ * + process.testK * + process.testL * + process.testM * + process.testN * + process.testO +) + +process.endPath = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/concurrentIOVsAndRuns.sh b/FWIO/RNTupleTempTests/test/concurrentIOVsAndRuns.sh new file mode 100755 index 0000000000000..fed3716625b99 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/concurrentIOVsAndRuns.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +function die { echo $1: status $2 ; exit $2; } +function diecat { echo "$1: status $2, log" ; cat $3; exit $2; } + +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} + +echo testConcurrentIOVsAndRuns_cfg.py +cmsRun ${LOCAL_TEST_DIR}/testConcurrentIOVsAndRuns_cfg.py || die 'Failed in testConcurrentIOVsAndRuns_cfg.py' $? + +echo testConcurrentIOVsAndRunsRead_cfg.py +cmsRun ${LOCAL_TEST_DIR}/testConcurrentIOVsAndRunsRead_cfg.py || die 'Failed in testConcurrentIOVsAndRunsRead_cfg.py' $? + diff --git a/FWIO/RNTupleTempTests/test/eventHistoryTest.sh b/FWIO/RNTupleTempTests/test/eventHistoryTest.sh new file mode 100755 index 0000000000000..11a685bb40eaf --- /dev/null +++ b/FWIO/RNTupleTempTests/test/eventHistoryTest.sh @@ -0,0 +1,42 @@ +#!/bin/sh + +# Pass in name and status +function die { echo $1: status $2 ; exit $2; } + +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} + +# Write a file for the FIRST process +cmsRun ${LOCAL_TEST_DIR}/EventHistory_1_cfg.py || die 'Failed in EventHistory_1' $? +echo "*************************************************" +echo "**************** Finished pass 1 ****************" +echo "*************************************************" + +# Read the first file, write the second. +cmsRun ${LOCAL_TEST_DIR}/EventHistory_2_cfg.py || die 'Failed in EventHistory_2' $? +echo "*************************************************" +echo "**************** Finished pass 2 ****************" +echo "*************************************************" + +# Read the second file, write the third. +cmsRun ${LOCAL_TEST_DIR}/EventHistory_3_cfg.py || die 'Failed in EventHistory_3' $? +echo "*************************************************" +echo "**************** Finished pass 3 ****************" +echo "*************************************************" + +# Read the third file, make sure the event data have the right history +cmsRun ${LOCAL_TEST_DIR}/EventHistory_4_cfg.py || die 'Failed in EventHistory_4' $? +echo "*************************************************" +echo "**************** Finished pass 4 ****************" +echo "*************************************************" + +# Read the fourth file, make sure the event data have the right history +cmsRun ${LOCAL_TEST_DIR}/EventHistory_5_cfg.py || die 'Failed in EventHistory_5' $? +echo "*************************************************" +echo "**************** Finished pass 5 ****************" +echo "*************************************************" + +# Read the fifth file, make sure the event data have the right history +cmsRun ${LOCAL_TEST_DIR}/EventHistory_6_cfg.py || die 'Failed in EventHistory_6' $? +echo "*************************************************" +echo "**************** Finished pass 6 ****************" +echo "*************************************************" diff --git a/FWIO/RNTupleTempTests/test/inconsistent_products_prod_cfg.py b/FWIO/RNTupleTempTests/test/inconsistent_products_prod_cfg.py new file mode 100644 index 0000000000000..316bab661ef17 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/inconsistent_products_prod_cfg.py @@ -0,0 +1,59 @@ +import FWCore.ParameterSet.Config as cms + +import argparse +import sys + +parser = argparse.ArgumentParser(prog=sys.argv[0], description='Test Processes with inconsistent data products.') + +parser.add_argument("--dropThing2", help="drop Thing2 from output", action="store_true") +parser.add_argument("--fileName", help="name of output file") +parser.add_argument("--noThing2Prod", help="do not include Thing2's producer", action="store_true") +parser.add_argument("--startEvent", help="starting event number", type=int, default=1) +parser.add_argument("--addAndStoreOther2", help='add the OtherThingProducer consuming thing2 and store it', action='store_true') +parser.add_argument("--thing2DependsOnThing1", help="have thing2 depend on thing1", action='store_true') + +args = parser.parse_args() + + +process = cms.Process("PROD") + +nEvents = 10 +from FWCore.Modules.modules import EmptySource +process.source = EmptySource(firstEvent = args.startEvent) + +process.maxEvents.input = nEvents + +from FWCore.Framework.modules import IntProducer, AddIntsProducer +process.thing1 = IntProducer(ivalue=1) + +process.t = cms.Task(process.thing1) +if not args.noThing2Prod: + if args.thing2DependsOnThing1: + process.thing2 = AddIntsProducer(labels=['thing1']) + else: + process.thing2 = IntProducer(ivalue=2) + process.t.add(process.thing2) + if args.addAndStoreOther2: + process.other2 = AddIntsProducer(labels=['thing2']) + process.t.add(process.other2) +elif args.addAndStoreOther2: + process.other2 = AddIntsProducer(labels=['thing1']) + process.t.add(process.other2) + +outputs = [] +if args.dropThing2: + outputs = ["keep *", + "drop *_thing2_*_*"] + + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.o = RNTupleTempOutputModule(outputCommands = outputs, + fileName = args.fileName + ) + +process.out = cms.EndPath(process.o, process.t) + + + + + diff --git a/FWIO/RNTupleTempTests/test/inconsistent_products_test_cfg.py b/FWIO/RNTupleTempTests/test/inconsistent_products_test_cfg.py new file mode 100644 index 0000000000000..e5ff838e28bd1 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/inconsistent_products_test_cfg.py @@ -0,0 +1,44 @@ +import FWCore.ParameterSet.Config as cms +import argparse +import sys + +parser = argparse.ArgumentParser(prog=sys.argv[0], description='Test Refs after merge.') + +parser.add_argument("--fileNames", nargs='+', help="files to read") +parser.add_argument("--nEventsToFail", help='number of events to fail before actually reading file', type=int) +parser.add_argument("--thing2Dropped", help="tell `other` to expect thing2 to be missing", action='store_true') +parser.add_argument("--thing2NotRun", help="the thing2 module was never run", action="store_true") +parser.add_argument("--other2Run", help="the other2 module was run", action="store_true") +parser.add_argument("--thing2DependsOnThing1", help="in previous job thing2 depends on thing1", action='store_true') +args = parser.parse_args() + +print(args) +process = cms.Process("TEST") + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource(fileNames = [f"file:{n}" for n in args.fileNames]) + +from FWCore.Integration.modules import TestFindProduct, TestParentage + +process.filt = cms.EDFilter("TestFilterModule", acceptValue = cms.untracked.int32(args.nEventsToFail)) +getTags=[] +missingTags=[] +if args.thing2Dropped or args.thing2NotRun: + missingTags=['thing2'] +else: + getTags=['thing2'] +process.other = TestFindProduct(getByTokenFirst=True, inputTags=getTags, inputTagsNotFound=missingTags) +expectedAncestors = [] +if args.thing2NotRun: + expectedAncestors = ['thing1'] +else: + if args.thing2DependsOnThing1: + expectedAncestors = ['thing2', 'thing1'] + else: + expectedAncestors = ['thing2'] + +process.e = cms.Path(~process.filt+process.other) +if args.other2Run: + process.parentage = TestParentage(inputTag='other2', expectedAncestors=expectedAncestors) + process.e +=process.parentage + diff --git a/FWIO/RNTupleTempTests/test/makeEmptyRootFile.py b/FWIO/RNTupleTempTests/test/makeEmptyRootFile.py new file mode 100644 index 0000000000000..d8eddb3343e6e --- /dev/null +++ b/FWIO/RNTupleTempTests/test/makeEmptyRootFile.py @@ -0,0 +1,13 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("WRITE") + +process.source = cms.Source("EmptySource") + +process.maxEvents.input = 10 + +process.out = cms.OutputModule("RNTupleTempOutputModule", + outputCommands = cms.untracked.vstring("drop *"), + fileName = cms.untracked.string("empty.root")) + +process.o = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/provenance_check_cfg.py b/FWIO/RNTupleTempTests/test/provenance_check_cfg.py new file mode 100644 index 0000000000000..7ea5fa2a6a115 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/provenance_check_cfg.py @@ -0,0 +1,13 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READ") + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource + +process.source = RNTupleTempSource(fileNames = ["file:prov.root", "file:prov_extra.root"]) + +from FWCore.Modules.modules import ProvenanceCheckerOutputModule, AsciiOutputModule +process.out = ProvenanceCheckerOutputModule() +process.prnt = AsciiOutputModule(verbosity = 2, allProvenance=True) + +process.e = cms.EndPath(process.out+process.prnt) diff --git a/FWIO/RNTupleTempTests/test/provenance_prod_cfg.py b/FWIO/RNTupleTempTests/test/provenance_prod_cfg.py new file mode 100644 index 0000000000000..aecd40aa71188 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/provenance_prod_cfg.py @@ -0,0 +1,48 @@ +import FWCore.ParameterSet.Config as cms +from argparse import ArgumentParser + +parser = ArgumentParser(description='Write streamer output file for provenance read test') +parser.add_argument("--consumeProd2", help="add an extra producer to the job and drop on output", action="store_true") +parser.add_argument("--diffRun", help="use a different run number", action="store_true") +parser.add_argument("--diffLumi", help="use a different LuminosityBlock number", action="store_true") +args = parser.parse_args() + + +process = cms.Process("OUTPUT") + +from FWCore.Modules.modules import EmptySource + +runNumber = 1 +lumiNumber = 1 +if args.diffRun: + runNumber = 2 +if args.diffLumi: + lumiNumber=2 +eventNumber = 1 +if args.consumeProd2: + eventNumber = 2 + +process.source = EmptySource(firstRun = runNumber, firstEvent = eventNumber, firstLuminosityBlock = lumiNumber ) + +from FWCore.Framework.modules import AddIntsProducer, IntProducer + +process.one = IntProducer(ivalue=1) +process.two = IntProducer(ivalue=2) +process.sum = AddIntsProducer(labels=['one']) +process.t = cms.Task(process.one, process.two, process.sum) + +baseOutFileName = "prov" +if args.consumeProd2 : + process.sum.untrackedLabels = ['two'] + baseOutFileName += "_extra" + + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule + +process.out = RNTupleTempOutputModule(fileName = baseOutFileName+".root", + outputCommands = ["drop *", "keep *_sum_*_*"]) + +from FWCore.Modules.modules import AsciiOutputModule +process.prnt = AsciiOutputModule(verbosity = 2, allProvenance = True) +process.e = cms.EndPath(process.out+process.prnt, process.t) +process.maxEvents.input = 1 diff --git a/FWIO/RNTupleTempTests/test/provenance_test.sh b/FWIO/RNTupleTempTests/test/provenance_test.sh new file mode 100755 index 0000000000000..88ac353f8e071 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/provenance_test.sh @@ -0,0 +1,21 @@ +#!/bin/sh + + +function die { echo $1: status $2 ; exit $2; } + +#The two jobs will have different ProductRegistries in their output files but have the same ProcessHistory. +# The ProductRegistry just differ because the internal dependencies between the data products is different +# and PoolOutputModule only stores provenance of 'dropped' data products IFF they are parents of a kept product. +# The check makes sure the provenance in the ProductRegistry is properly updated when the new file is read +cmsRun ${SCRAM_TEST_PATH}/provenance_prod_cfg.py || die 'Failed in provenance_prod_cfg.py' $? +cmsRun ${SCRAM_TEST_PATH}/provenance_prod_cfg.py --consumeProd2 || die 'Failed in provenance_prod_cfg.py --consumeProd2' $? +cmsRun ${SCRAM_TEST_PATH}/provenance_check_cfg.py || die 'Failed test of provenance' $? + +cmsRun ${SCRAM_TEST_PATH}/provenance_prod_cfg.py --consumeProd2 --diffRun || die 'Failed in provenance_prod_cfg.py --consumeProd2 --diffRun' $? +cmsRun ${SCRAM_TEST_PATH}/provenance_check_cfg.py || die 'Failed test of provenance with diffRun' $? + +cmsRun ${SCRAM_TEST_PATH}/provenance_prod_cfg.py --consumeProd2 --diffLumi || die 'Failed in provenance_prod_cfg.py --consumeProd2 --diffLumi' $? +cmsRun ${SCRAM_TEST_PATH}/provenance_check_cfg.py || die 'Failed test of provenance with diffLumi' $? + +cmsRun ${SCRAM_TEST_PATH}/provenance_prod_cfg.py --consumeProd2 --diffLumi --diffRun|| die 'Failed in provenance_prod_cfg.py --consumeProd2 --diffLumi --diffRun' $? +cmsRun ${SCRAM_TEST_PATH}/provenance_check_cfg.py || die 'Failed test of provenance with diffLumi & diffRun' $? \ No newline at end of file diff --git a/FWIO/RNTupleTempTests/test/ref_alias_compare_drop_alias_cfg.py b/FWIO/RNTupleTempTests/test/ref_alias_compare_drop_alias_cfg.py new file mode 100644 index 0000000000000..b33ba939ac82c --- /dev/null +++ b/FWIO/RNTupleTempTests/test/ref_alias_compare_drop_alias_cfg.py @@ -0,0 +1,33 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("Test") +process.maxEvents = cms.untracked.PSet(input = cms.untracked.int32(10)) + +process.source = cms.Source("EmptySource") + +process.thing = cms.EDProducer("ThingProducer") + +process.thingAlias = cms.EDAlias( thing = cms.VPSet( + cms.PSet(type = cms.string('edmtestThings'), + fromProductInstance = cms.string('*'), + toProductInstance = cms.string('*')))) + +process.otherThing1 = cms.EDProducer("OtherThingProducer", + thingTag=cms.InputTag("thing")) + +process.otherThing2 = cms.EDProducer("OtherThingProducer", + thingTag=cms.InputTag("thingAlias")) + +process.comparer = cms.EDAnalyzer("OtherThingRefComparer", + first = cms.untracked.InputTag("otherThing1:testUserTag"), + second = cms.untracked.InputTag("otherThing2:testUserTag") + ) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string("ref_alias_drop_alias.root"), + outputCommands = cms.untracked.vstring("keep *", + "drop *_thingAlias_*_*") +) + +process.p = cms.Path(process.thing+process.otherThing1+process.otherThing2+process.comparer) +process.o = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/ref_alias_compare_drop_original_cfg.py b/FWIO/RNTupleTempTests/test/ref_alias_compare_drop_original_cfg.py new file mode 100644 index 0000000000000..93a80dd9eb582 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/ref_alias_compare_drop_original_cfg.py @@ -0,0 +1,33 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("Test") +process.maxEvents = cms.untracked.PSet(input = cms.untracked.int32(10)) + +process.source = cms.Source("EmptySource") + +process.thing = cms.EDProducer("ThingProducer") + +process.thingAlias = cms.EDAlias( thing = cms.VPSet( + cms.PSet(type = cms.string('edmtestThings'), + fromProductInstance = cms.string('*'), + toProductInstance = cms.string('*')))) + +process.otherThing1 = cms.EDProducer("OtherThingProducer", + thingTag=cms.InputTag("thing")) + +process.otherThing2 = cms.EDProducer("OtherThingProducer", + thingTag=cms.InputTag("thingAlias")) + +process.comparer = cms.EDAnalyzer("OtherThingRefComparer", + first = cms.untracked.InputTag("otherThing1:testUserTag"), + second = cms.untracked.InputTag("otherThing2:testUserTag") + ) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string("ref_alias_drop_original.root"), + outputCommands = cms.untracked.vstring("keep *", + "drop *_thing_*_*") +) + +process.p = cms.Path(process.thing+process.otherThing1+process.otherThing2+process.comparer) +process.o = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/ref_alias_compare_read_alias_cfg.py b/FWIO/RNTupleTempTests/test/ref_alias_compare_read_alias_cfg.py new file mode 100644 index 0000000000000..49a61742ac851 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/ref_alias_compare_read_alias_cfg.py @@ -0,0 +1,26 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("Analyze") + +process.source = cms.Source("RNTupleTempSource", fileNames = cms.untracked.vstring("file:ref_alias_drop_original.root")) + +process.otherThing3 = cms.EDProducer("OtherThingProducer", + thingTag=cms.InputTag("thingAlias")) + +process.comparerA = cms.EDAnalyzer("OtherThingRefComparer", + first = cms.untracked.InputTag("otherThing1:testUserTag"), + second = cms.untracked.InputTag("otherThing2:testUserTag") + ) + +process.comparerB = cms.EDAnalyzer("OtherThingRefComparer", + first = cms.untracked.InputTag("otherThing1:testUserTag"), + second = cms.untracked.InputTag("otherThing3:testUserTag") + ) + +process.comparerC = cms.EDAnalyzer("OtherThingRefComparer", + first = cms.untracked.InputTag("otherThing2:testUserTag"), + second = cms.untracked.InputTag("otherThing3:testUserTag") + ) + +process.p = cms.Path(process.otherThing3+process.comparerA+process.comparerB+process.comparerC) + diff --git a/FWIO/RNTupleTempTests/test/ref_alias_compare_read_original_cfg.py b/FWIO/RNTupleTempTests/test/ref_alias_compare_read_original_cfg.py new file mode 100644 index 0000000000000..896943c3c7a60 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/ref_alias_compare_read_original_cfg.py @@ -0,0 +1,25 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("Analyze") + +process.source = cms.Source("RNTupleTempSource", fileNames = cms.untracked.vstring("file:ref_alias_drop_alias.root")) + +process.otherThing3 = cms.EDProducer("OtherThingProducer", + thingTag=cms.InputTag("thing")) + +process.comparerA = cms.EDAnalyzer("OtherThingRefComparer", + first = cms.untracked.InputTag("otherThing1:testUserTag"), + second = cms.untracked.InputTag("otherThing2:testUserTag") + ) + +process.comparerB = cms.EDAnalyzer("OtherThingRefComparer", + first = cms.untracked.InputTag("otherThing1:testUserTag"), + second = cms.untracked.InputTag("otherThing3:testUserTag") + ) + +process.comparerC = cms.EDAnalyzer("OtherThingRefComparer", + first = cms.untracked.InputTag("otherThing2:testUserTag"), + second = cms.untracked.InputTag("otherThing3:testUserTag") + ) + +process.p = cms.Path(process.otherThing3+process.comparerA+process.comparerB+process.comparerC) diff --git a/FWIO/RNTupleTempTests/test/ref_merge_cfg.py b/FWIO/RNTupleTempTests/test/ref_merge_cfg.py new file mode 100644 index 0000000000000..51c5e32285912 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/ref_merge_cfg.py @@ -0,0 +1,27 @@ +import FWCore.ParameterSet.Config as cms +import argparse +import sys + +parser = argparse.ArgumentParser(prog=sys.argv[0], description='Test ConditionalTasks.') + +parser.add_argument("--inFile1", help="first file to read") +parser.add_argument("--inFile2", help="second file to read") +parser.add_argument("--outFile", help="name of output file", default="ref_merge.root") + +args = parser.parse_args() + +process = cms.Process("MERGE") + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource(fileNames = [f"file:{args.inFile1}", + f"file:{args.inFile2}"] + ) + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.out = RNTupleTempOutputModule(fileName = args.outFile) + +from FWCore.Integration.modules import OtherThingAnalyzer +process.tester = OtherThingAnalyzer(other = ("d","testUserTag")) + +process.o = cms.EndPath(process.out+process.tester) + diff --git a/FWIO/RNTupleTempTests/test/ref_merge_prod_cfg.py b/FWIO/RNTupleTempTests/test/ref_merge_prod_cfg.py new file mode 100644 index 0000000000000..658346eeb2433 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/ref_merge_prod_cfg.py @@ -0,0 +1,63 @@ +import FWCore.ParameterSet.Config as cms + +import argparse +import sys + +parser = argparse.ArgumentParser(prog=sys.argv[0], description='Test ConditionalTasks.') + +parser.add_argument("--extraProducers", help="Add extra producers to configuration", action="store_true") +parser.add_argument("--fileName", help="name of output file") +parser.add_argument("--firstLumi", help="LuminosityBlock number for first lumi", type = int, default=1) +parser.add_argument("--keepAllProducts", help="Keep all products made in the job", action="store_true") +parser.add_argument("--dropThings", help="drop the Things collections so the refs will not function", action="store_true") + +args = parser.parse_args() + + +process = cms.Process("PROD") + +nEvents = 10 +from FWCore.Modules.modules import EmptySource +process.source = EmptySource( + firstLuminosityBlock = args.firstLumi, + firstEvent = nEvents*(args.firstLumi-1)+1 +) + +process.maxEvents.input = nEvents + +if args.extraProducers: + from FWCore.Framework.modules import IntProducer + process.a = IntProducer(ivalue = 1) + + process.b = IntProducer(ivalue = 2) + +from FWCore.Integration.modules import ThingProducer, OtherThingProducer, OtherThingAnalyzer +process.c = ThingProducer() + +process.d = OtherThingProducer(thingTag="c") + +outputs = [] +if not args.keepAllProducts: + outputs = ["drop *", + "keep edmtestOtherThings_*_*_*"] + if not args.dropThings: + outputs.append("keep edmtestThings_*_*_*") + + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.o = RNTupleTempOutputModule(outputCommands = outputs, + fileName = args.fileName + ) +if args.extraProducers: + process.p = cms.Path(process.a+process.b+process.c*process.d) +else: + process.p = cms.Path(process.c*process.d) + +process.tester = OtherThingAnalyzer(other = ("d","testUserTag")) + +process.out = cms.EndPath(process.o+process.tester) + + + + + diff --git a/FWIO/RNTupleTempTests/test/ref_merge_test_cfg.py b/FWIO/RNTupleTempTests/test/ref_merge_test_cfg.py new file mode 100644 index 0000000000000..27f359f85784c --- /dev/null +++ b/FWIO/RNTupleTempTests/test/ref_merge_test_cfg.py @@ -0,0 +1,21 @@ +import FWCore.ParameterSet.Config as cms +import argparse +import sys + +parser = argparse.ArgumentParser(prog=sys.argv[0], description='Test Refs after merge.') + +parser.add_argument("--fileName", help="file to read") +parser.add_argument("--promptRead", action="store_true", default=False, help="prompt read the event products") + +args = parser.parse_args() + +process = cms.Process("TEST") + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource(fileNames = [f"file:{args.fileName}"], delayReadingEventProducts = not args.promptRead) + +from FWCore.Integration.modules import OtherThingAnalyzer +process.tester = OtherThingAnalyzer(other = ("d","testUserTag")) + +process.e = cms.EndPath(process.tester) + diff --git a/FWIO/RNTupleTempTests/test/run_RefAlias.sh b/FWIO/RNTupleTempTests/test/run_RefAlias.sh new file mode 100755 index 0000000000000..7dc516879e7cb --- /dev/null +++ b/FWIO/RNTupleTempTests/test/run_RefAlias.sh @@ -0,0 +1,20 @@ +#!/bin/bash +test=ref_alias_compare_ + +function die { echo Failure $1: status $2 ; exit $2 ; } + +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} + + echo ${test}drop_alias_cfg.py ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}drop_alias_cfg.py || die "cmsRun ${test}drop_alias_cfg.py" $? + + echo ${test}drop_original_cfg.py ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}drop_original_cfg.py || die "cmsRun ${test}drop_original_cfg.py" $? + + echo ${test}read_alias_cfg.py------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}read_alias_cfg.py || die "cmsRun ${test}read_alias_cfg.py" $? + + echo ${test}read_original_cfg.py------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}read_original_cfg.py || die "cmsRun ${test}read_original_cfg.py" $? + +exit 0 diff --git a/FWIO/RNTupleTempTests/test/run_RefMerge.sh b/FWIO/RNTupleTempTests/test/run_RefMerge.sh new file mode 100755 index 0000000000000..eba8ff1e96902 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/run_RefMerge.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +test=ref_merge_ + +function die { echo Failure $1: status $2 ; exit $2 ; } + +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} + echo ${test}prod1 ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}prod_cfg.py --extraProducers --fileName 'ref_merge_prod1.root' || die "cmsRun ${test}prod_cfg.py --extraProducers" $? + + echo ${test}prod2 ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}prod_cfg.py --firstLumi 10 --fileName 'ref_merge_prod2.root'|| die "cmsRun ${test}prod_cfg.py" $? + + echo ${test}MERGE------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}cfg.py --inFile1 'ref_merge_prod1.root' --inFile2 'ref_merge_prod2.root' --outFile 'ref_merge.root' || die "cmsRun ${test}cfg.py" $? + + + echo ${test}MERGE promptRead------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}test_cfg.py --fileName 'ref_merge.root' --promptRead || die "cmsRun ${test}test_cfg.py" $? + + echo ${test}keepAllProd ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}prod_cfg.py --extraProducers --keepAllProducts --fileName 'ref_merge_prod_all.root' || die "cmsRun ${test}prod_cfg.py --keepAllProducts" $? + + echo ${test}MERGE_keepAll1st ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}cfg.py --inFile1 'ref_merge_prod_all.root' --inFile2 'ref_merge_prod2.root' --outFile 'ref_merge_all1st.root' || die "cmsRun ${test}cfg.py" $? + + echo ${test}test_all1st------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}test_cfg.py --fileName 'ref_merge_all1st.root' || die "cmsRun ${test}test_cfg.py all1st" $? + + #note having all be the second file does not work as PoolSource enforces that subsequent files must have a strict subset + # of the branches in the first file read + +exit 0 diff --git a/FWIO/RNTupleTempTests/test/run_RunMerge.sh b/FWIO/RNTupleTempTests/test/run_RunMerge.sh new file mode 100755 index 0000000000000..abe70fff3c628 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/run_RunMerge.sh @@ -0,0 +1,119 @@ +#!/bin/bash + +test=testRunMerge + +function die { echo Failure $1: status $2 ; exit $2 ; } + +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} + echo ${test}PROD0 ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}PROD0_cfg.py || die "cmsRun ${test}PROD0_cfg.py" $? + + echo ${test}PROD1 ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}PROD1_cfg.py || die "cmsRun ${test}PROD1_cfg.py" $? + + echo ${test}PROD2------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}PROD2_cfg.py || die "cmsRun ${test}PROD2_cfg.py" $? + + echo ${test}PROD2EXTRA------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}PROD2EXTRA_cfg.py || die "cmsRun ${test}PROD2EXTRA_cfg.py" $? + + echo ${test}PROD3------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}PROD3_cfg.py || die "cmsRun ${test}PROD3_cfg.py" $? + + echo ${test}PROD3EXTRA------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}PROD3EXTRA_cfg.py || die "cmsRun ${test}PROD3EXTRA_cfg.py" $? + + echo ${test}PROD4------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}PROD4_cfg.py || die "cmsRun ${test}PROD4_cfg.py" $? + + echo ${test}PROD5------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}PROD5_cfg.py || die "cmsRun ${test}PROD5_cfg.py" $? + + echo ${test}PROD6------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}PROD6_cfg.py || die "cmsRun ${test}PROD6_cfg.py" $? + + echo ${test}PROD7------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}PROD7_cfg.py || die "cmsRun ${test}PROD7_cfg.py" $? + + echo ${test}PROD11------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}PROD11_cfg.py || die "cmsRun ${test}PROD11_cfg.py" $? + + echo ${test}MERGE------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}MERGE_cfg.py || die "cmsRun ${test}MERGE_cfg.py" $? + + echo ${test}MERGE1------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}MERGE1_cfg.py || die "cmsRun ${test}MERGE1_cfg.py" $? + + echo ${test}MERGE2------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}MERGE2_cfg.py || die "cmsRun ${test}MERGE2_cfg.py" $? + + echo ${test}MERGE3------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}MERGE3_cfg.py || die "cmsRun ${test}MERGE3_cfg.py" $? + + echo ${test}MERGE3x------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}MERGE3x_cfg.py || die "cmsRun ${test}MERGE3x_cfg.py" $? + + echo ${test}MERGE4------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}MERGE4_cfg.py || die "cmsRun ${test}MERGE4_cfg.py" $? + + echo ${test}MERGE5------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}MERGE5_cfg.py || die "cmsRun ${test}MERGE5_cfg.py" $? + + echo ${test}TEST------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}TEST_cfg.py || die "cmsRun ${test}TEST_cfg.py" $? + + echo ${test}TESTFAIL------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}TESTFAIL_cfg.py 2>/dev/null && die "cmsRun ${test}TESTFAIL_cfg.py" 1 + + echo ${test}TEST1------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}TEST1_cfg.py || die "cmsRun ${test}TEST1_cfg.py" $? + + echo ${test}TEST2------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}TEST2_cfg.py || die "cmsRun ${test}TEST2_cfg.py" $? + + echo ${test}TEST3------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}TEST3_cfg.py || die "cmsRun ${test}TEST3_cfg.py" $? + + echo ${test}TEST4------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}TEST4_cfg.py || die "cmsRun ${test}TEST4_cfg.py" $? + + echo ${test}TEST5------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}TEST5_cfg.py || die "cmsRun ${test}TEST5_cfg.py" $? + + echo ${test}TEST11------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}TEST11_cfg.py || die "cmsRun ${test}TEST11_cfg.py" $? + + echo ${test}COPY------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}COPY_cfg.py || die "cmsRun ${test}COPY_cfg.py" $? + + echo ${test}COPY1------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}COPY1_cfg.py || die "cmsRun ${test}COPY1_cfg.py" $? + + echo ${test}MERGE6------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}MERGE6_cfg.py || die "cmsRun ${test}MERGE6_cfg.py" $? + + echo ${test}NoRunLumiSort------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}NoRunLumiSort_cfg.py || die "cmsRun ${test}NoRunLumiSort_cfg.py" $? + + echo ${test}TEST6------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}TEST6_cfg.py || die "cmsRun ${test}TEST6_cfg.py" $? + + echo ${test}PickEvents------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}PickEvents_cfg.py || die "cmsRun ${test}PickEvents_cfg.py" $? + + echo ${test}PickEventsx------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}PickEventsx_cfg.py || die "cmsRun ${test}PickEventsx_cfg.py" $? + + echo ${test}FastCloning------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}FastCloning_cfg.py 2> testFastCloning.txt + grep "Another exception was caught" testFastCloning.txt || die "cmsRun testRunMergeFastCloning_cfg.py" $? + + echo testLooperEventNavigation----------------------------------------------------- + cmsRun ${LOCAL_TEST_DIR}/testLooperEventNavigation_cfg.py < ${LOCAL_TEST_DIR}/testLooperEventNavigation.txt > testLooperEventNavigationOutput.txt || die "cmsRun testLooperEventNavigation_cfg.py " $? + diff ${LOCAL_TEST_DIR}/unit_test_outputs/testLooperEventNavigationOutput.txt testLooperEventNavigationOutput.txt || die "comparing testLooperEventNavigationOutput.txt" $? + + echo testLooperEventNavigation1----------------------------------------------------- + cmsRun ${LOCAL_TEST_DIR}/testLooperEventNavigation1_cfg.py < ${LOCAL_TEST_DIR}/testLooperEventNavigation.txt > testLooperEventNavigationOutput1.txt || die "cmsRun testLooperEventNavigation1_cfg.py " $? + diff ${LOCAL_TEST_DIR}/unit_test_outputs/testLooperEventNavigationOutput.txt testLooperEventNavigationOutput1.txt || die "comparing testLooperEventNavigationOutput1.txt" $? + +exit 0 diff --git a/FWIO/RNTupleTempTests/test/run_RunMerge2.sh b/FWIO/RNTupleTempTests/test/run_RunMerge2.sh new file mode 100755 index 0000000000000..edc9b8124f0f2 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/run_RunMerge2.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +test=testRunMerge + +function die { echo Failure $1: status $2 ; exit $2 ; } + +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} + echo ${test}PROD100 ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}PROD100_cfg.py || die "cmsRun ${test}PROD100_cfg.py" $? + + echo ${test}PROD101 ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}PROD101_cfg.py || die "cmsRun ${test}PROD101_cfg.py" $? + + echo ${test}PROD102 ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}PROD102_cfg.py || die "cmsRun ${test}PROD102_cfg.py" $? + + echo ${test}SPLIT100 ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}SPLIT100_cfg.py || die "cmsRun ${test}SPLIT100_cfg.py" $? + + echo ${test}SPLIT101 ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}SPLIT101_cfg.py || die "cmsRun ${test}SPLIT101_cfg.py" $? + + echo ${test}SPLIT102 ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}SPLIT102_cfg.py || die "cmsRun ${test}SPLIT102_cfg.py" $? + + echo ${test}SPLIT103 ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}SPLIT103_cfg.py || die "cmsRun ${test}SPLIT103_cfg.py" $? + + echo ${test}MERGE100 ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}MERGE100_cfg.py || die "cmsRun ${test}MERGE100_cfg.py" $? + + echo ${test}MERGE101 ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}MERGE101_cfg.py || die "cmsRun ${test}MERGE101_cfg.py" $? + + echo ${test}MERGE102 ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}MERGE102_cfg.py || die "cmsRun ${test}MERGE102_cfg.py" $? + + echo ${test}TEST100 ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}TEST100_cfg.py || die "cmsRun ${test}TEST100_cfg.py" $? + + echo ${test}TEST101 ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}TEST101_cfg.py || die "cmsRun ${test}TEST101_cfg.py" $? + + echo ${test}COPY100 ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}COPY100_cfg.py || die "cmsRun ${test}COPY100_cfg.py" $? + + echo ${test}COPY101 ------------------------------------------------------------ + cmsRun ${LOCAL_TEST_DIR}/${test}COPY101_cfg.py || die "cmsRun ${test}COPY101_cfg.py" $? + +exit 0 diff --git a/FWIO/RNTupleTempTests/test/run_SeriesOfProcesses.sh b/FWIO/RNTupleTempTests/test/run_SeriesOfProcesses.sh new file mode 100755 index 0000000000000..44e8fd9284293 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/run_SeriesOfProcesses.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +test=testSeriesOfProcesses + +function die { echo Failure $1: status $2 ; exit $2 ; } + +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} + + cmsRun ${LOCAL_TEST_DIR}/${test}HLT_cfg.py || die "cmsRun ${test}HLT_cfg.py" $? + + cmsRun ${LOCAL_TEST_DIR}/${test}PROD_cfg.py || die "cmsRun ${test}PROD_cfg.py" $? + + cmsRun ${LOCAL_TEST_DIR}/${test}TEST_cfg.py || die "cmsRun ${test}TEST_cfg.py" $? + + cmsRun ${LOCAL_TEST_DIR}/${test}TEST1_cfg.py || die "cmsRun ${test}TEST1_cfg.py" $? + + cmsRun ${LOCAL_TEST_DIR}/${test}TEST2_cfg.py 2> ${test}TEST2.txt + grep "Duplicate Process" ${test}TEST2.txt || die "cmsRun ${test}TEST2_cfg.py" $? + + cmsRun ${LOCAL_TEST_DIR}/${test}TEST3_cfg.py || die "Failure in history testing in ${test}" $? + + cmsRun ${LOCAL_TEST_DIR}/${test}PROD2TEST_cfg.py || die "cmsRun ${test}PROD2TEST_cfg.py" $? + + cmsRun ${LOCAL_TEST_DIR}/${test}PROD3TEST_cfg.py || die "cmsRun ${test}PROD3TEST_cfg.py" $? + + cmsRun ${LOCAL_TEST_DIR}/${test}PROD2TEST_unscheduled_cfg.py || die "cmsRun ${test}PROD2TEST_cfg.py" $? + +exit 0 diff --git a/FWIO/RNTupleTempTests/test/run_TestDropOnInput.sh b/FWIO/RNTupleTempTests/test/run_TestDropOnInput.sh new file mode 100755 index 0000000000000..44d73378e68a5 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/run_TestDropOnInput.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +test=testDropOnInput + +function die { echo Failure $1: status $2 ; exit $2 ; } + +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} + + echo "testDropOnInput1_1" + cmsRun ${LOCAL_TEST_DIR}/${test}1_1_cfg.py || die "cmsRun ${test}1_1_cfg.py" $? + + echo "testDropOnInput1_2" + cmsRun ${LOCAL_TEST_DIR}/${test}1_2_cfg.py || die "cmsRun ${test}1_2_cfg.py" $? + + echo "testDropOnInput2" + cmsRun ${LOCAL_TEST_DIR}/${test}2_cfg.py || die "cmsRun ${test}2_cfg.py" $? + + echo "testDropOnInput3" + cmsRun ${LOCAL_TEST_DIR}/${test}3_cfg.py || die "cmsRun ${test}3_cfg.py" $? + + echo "testDropOnInputRead2" + cmsRun ${LOCAL_TEST_DIR}/${test}Read2_cfg.py || die "cmsRun ${test}Read2_cfg.py" $? + + echo "testDropOnInputRead2001" + cmsRun ${LOCAL_TEST_DIR}/${test}Read2001_cfg.py || die "cmsRun ${test}Read2001_cfg.py" $? + + echo "testDropOnInputRead3" + cmsRun ${LOCAL_TEST_DIR}/${test}Read3_cfg.py || die "cmsRun ${test}Read3_cfg.py" $? + +exit 0 + diff --git a/FWIO/RNTupleTempTests/test/run_TestEDAlias.sh b/FWIO/RNTupleTempTests/test/run_TestEDAlias.sh new file mode 100755 index 0000000000000..032d1e127d4e6 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/run_TestEDAlias.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +test=testEDAlias + +function die { echo Failure $1: status $2 ; exit $2 ; } + +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} + + echo "*************************************************" + echo "EDAlias consumer in a Task" + cmsRun ${LOCAL_TEST_DIR}/${test}Task_cfg.py || die "cmsRun ${test}Task_cfg.py 1" $? + + echo "*************************************************" + echo "Test output" + cmsRun ${LOCAL_TEST_DIR}/${test}Analyze_cfg.py testEDAliasTask.root || die "cmsRun ${test}Analyze_cfg.py testEDAliasTask.root" $? + +exit 0 diff --git a/FWIO/RNTupleTempTests/test/run_TestExistingDictionary.sh b/FWIO/RNTupleTempTests/test/run_TestExistingDictionary.sh new file mode 100755 index 0000000000000..e392270a691b0 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/run_TestExistingDictionary.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +test=testExistingDictionaryChecking + +function die { echo Failure $1: status $2 ; exit $2 ; } + +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} + + echo "*************************************************" + echo "Producer" + cmsRun ${LOCAL_TEST_DIR}/${test}_cfg.py || die "cmsRun ${test}_cfg.py 1" $? + + echo "*************************************************" + echo "Consumer" + cmsRun ${LOCAL_TEST_DIR}/${test}Read_cfg.py || die "cmsRun ${test}Read_cfg.py 1" $? + diff --git a/FWIO/RNTupleTempTests/test/run_TestFallback.sh b/FWIO/RNTupleTempTests/test/run_TestFallback.sh new file mode 100755 index 0000000000000..05cf6b0f70bfa --- /dev/null +++ b/FWIO/RNTupleTempTests/test/run_TestFallback.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +test=testNoProcessFallback + +function die { echo Failure $1: status $2 ; exit $2 ; } + +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} + + echo "testFallback1" + cmsRun ${LOCAL_TEST_DIR}/${test}1_cfg.py > ${test}1.log 2>/dev/null || die "cmsRun ${test}1_cfg.py" $? + + echo "testFallback2" + cmsRun ${LOCAL_TEST_DIR}/${test}2_cfg.py > ${test}2.log 2>/dev/null || die "cmsRun ${test}2_cfg.py" $? + + echo "testFallback3" + cmsRun ${LOCAL_TEST_DIR}/${test}3_cfg.py || die "cmsRun ${test}3_cfg.py" $? + + echo "testFallbackNoCurrent" + cmsRun ${LOCAL_TEST_DIR}/${test}NoCurrent_cfg.py || die "cmsRun ${test}NoCurrent_cfg.py" $? + +exit 0 diff --git a/FWIO/RNTupleTempTests/test/run_TestGetBy.sh b/FWIO/RNTupleTempTests/test/run_TestGetBy.sh new file mode 100755 index 0000000000000..c3db4e7f4091e --- /dev/null +++ b/FWIO/RNTupleTempTests/test/run_TestGetBy.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +test=testGetBy + +function die { echo Failure $1: status $2 ; exit $2 ; } + +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} + + echo "testGetBy1" + cmsRun ${LOCAL_TEST_DIR}/${test}1_cfg.py > testGetBy1.log 2>/dev/null || die "cmsRun ${test}1_cfg.py" $? + diff ${LOCAL_TEST_DIR}/unit_test_outputs/testGetBy1.log testGetBy1.log || die "comparing testGetBy1.log" $? + + echo "testGetBy2" + cmsRun ${LOCAL_TEST_DIR}/${test}2_cfg.py > testGetBy2.log 2>/dev/null || die "cmsRun ${test}2_cfg.py" $? + grep -v 'Initiating request to open file\|Successfully opened file\|Closed file' testGetBy2.log > testGetBy2_1.log + diff ${LOCAL_TEST_DIR}/unit_test_outputs/testGetBy2.log testGetBy2_1.log || die "comparing testGetBy2.log" $? + + echo "testGetBy3" + cmsRun ${LOCAL_TEST_DIR}/${test}3_cfg.py || die "cmsRun ${test}3_cfg.py" $? + + #It is intentional that this cmsRun process throws an exception + echo "testDuplicateProcess" + cmsRun ${LOCAL_TEST_DIR}/testDuplicateProcess_cfg.py &> testDuplicateProcess.log && die 'Failed to get exception running testDuplicateProcess_cfg.py' 1 + grep -q "Duplicate Process" testDuplicateProcess.log || die 'Failed to print out exception message for duplicate process name' $? + + echo "testGetBy1Mod" + cmsRun ${LOCAL_TEST_DIR}/${test}1Mod_cfg.py > testGetBy1Mod.log 2>/dev/null || die "cmsRun ${test}1Mod_cfg.py" $? + + echo "testGetByMerge" + cmsRun ${LOCAL_TEST_DIR}/${test}Merge_cfg.py > testGetByMerge.log 2>/dev/null || die "cmsRun ${test}Merge_cfg.py" $? + + echo "testGetByRunsMode_cfg.py" + cmsRun ${LOCAL_TEST_DIR}/testGetByRunsMode_cfg.py || die "cmsRun testGetByRunsMode_cfg.py" $? + + echo "testGetByRunsLumisMode_cfg.py" + cmsRun ${LOCAL_TEST_DIR}/testGetByRunsLumisMode_cfg.py || die "cmsRun testGetByRunsLumisMode_cfg.py" $? + + echo "testGetByWithEmptyRun_cfg.py" + cmsRun ${LOCAL_TEST_DIR}/testGetByWithEmptyRun_cfg.py || die "cmsRun testGetByWithEmptyRun_cfg.py" $? + +exit 0 diff --git a/FWIO/RNTupleTempTests/test/run_TestGetByLabel.sh b/FWIO/RNTupleTempTests/test/run_TestGetByLabel.sh new file mode 100755 index 0000000000000..e253d84769458 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/run_TestGetByLabel.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +function die { echo Failure $1: status $2 ; exit $2 ; } + +TEST_NAME=TestGetByLabel +if [ -d ${TEST_NAME} ]; then + rm -fR ${TEST_NAME} +fi +mkdir ${TEST_NAME} + +pushd ${TEST_NAME} + +TEST_PATH=${LOCALTOP}/src/IOPool/Tests/test + +cmsRun ${TEST_PATH}/testGetByLabelStep1_cfg.py || die "Failed cmsRun testGetByLabel_step1_cfg.py" $1 + +cmsRun ${TEST_PATH}/testGetByLabelStep2_cfg.py || die "Failed cmsRun testGetByLabel_step2_cfg.py" $1 +cmsRun ${TEST_PATH}/testGetByLabelStep2_cfg.py --noConsumes || die "Failed cmsRun testGetByLabel_step2_cfg.py --noConsumes" $1 +cmsRun ${TEST_PATH}/testGetByLabelStep2_cfg.py --thing || die "Failed cmsRun testGetByLabel_step2_cfg.py --thing" $1 +cmsRun ${TEST_PATH}/testGetByLabelStep2_cfg.py --thing --noConsumes || die "Failed cmsRun testGetByLabel_step2_cfg.py --thing --noConsumes" $1 +cmsRun ${TEST_PATH}/testGetByLabelStep2_cfg.py --otherInt || die "Failed cmsRun testGetByLabel_step2_cfg.py --otherInt" $1 +cmsRun ${TEST_PATH}/testGetByLabelStep2_cfg.py --otherInt --noConsumes || die "Failed cmsRun testGetByLabel_step2_cfg.py --otherInt --noConsumes" $1 + +popd diff --git a/FWIO/RNTupleTempTests/test/run_TestOutput.sh b/FWIO/RNTupleTempTests/test/run_TestOutput.sh new file mode 100755 index 0000000000000..ce1f47af9c212 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/run_TestOutput.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +test=testOutput + +function die { echo Failure $1: status $2 ; exit $2 ; } + +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} + + echo "testOutput1" + cmsRun ${LOCAL_TEST_DIR}/${test}1_cfg.py 2> testOutput1.log || die "cmsRun ${test}1_cfg.py" $? + + # Check that all the transitions that were supposed to occur + # in a global output module actually did occur + grep "global write event" testOutput1.log > /dev/null || die "grep failed to find 'global write event'" $? + grep "global writeLuminosityBlock" testOutput1.log > /dev/null || die "grep failed to find 'global writeLuminosityBlock'" $? + grep "global writeRun" testOutput1.log > /dev/null || die "grep failed to find 'global writeRun'" $? + grep "global writeProcessBlock" testOutput1.log > /dev/null || die "grep failed to find 'global writeProcessBlock'" $? + grep "global respondToOpenInputFile" testOutput1.log > /dev/null || die "grep failed to find 'global respondToOpenInputFile'" $? + grep "global respondToCloseInputFile" testOutput1.log > /dev/null || die "grep failed to find 'global respondToCloseInputFile'" $? + grep "global globalBeginRun" testOutput1.log > /dev/null || die "grep failed to find 'global globalBeginRun'" $? + grep "global globalEndRun" testOutput1.log > /dev/null || die "grep failed to find 'global globalEndRun'" $? + grep "global globalBeginLuminosityBlock" testOutput1.log > /dev/null || die "grep failed to find 'global globalBeginLuminosityBlock'" $? + grep "global globalEndLuminosityBlock" testOutput1.log > /dev/null || die "grep failed to find 'global globalEndLuminosityBlock'" $? + + # Check the branch ID for the EDAliases was placed correctly in the BranchIDLists + grep "global branchID 3673681161" testOutput1.log > /dev/null || die "grep failed to find 'global branchID 3673681161'" $? + + # Repeat checks for the limited module + grep "limited write event" testOutput1.log > /dev/null || die "grep failed to find 'limited write event'" $? + grep "limited writeLuminosityBlock" testOutput1.log > /dev/null || die "grep failed to find 'limited writeLuminosityBlock'" $? + grep "limited writeRun" testOutput1.log > /dev/null || die "grep failed to find 'limited writeRun'" $? + grep "limited writeProcessBlock" testOutput1.log > /dev/null || die "grep failed to find 'limited writeProcessBlock'" $? + grep "limited respondToOpenInputFile" testOutput1.log > /dev/null || die "grep failed to find 'limited respondToOpenInputFile'" $? + grep "limited respondToCloseInputFile" testOutput1.log > /dev/null || die "grep failed to find 'limited respondToCloseInputFile'" $? + grep "limited globalBeginRun" testOutput1.log > /dev/null || die "grep failed to find 'limited globalBeginRun'" $? + grep "limited globalEndRun" testOutput1.log > /dev/null || die "grep failed to find 'limited globalEndRun'" $? + grep "limited globalBeginLuminosityBlock" testOutput1.log > /dev/null || die "grep failed to find 'limited globalBeginLuminosityBlock'" $? + grep "limited globalEndLuminosityBlock" testOutput1.log > /dev/null || die "grep failed to find 'limited globalEndLuminosityBlock'" $? + grep "limited branchID 3673681161" testOutput1.log > /dev/null || die "grep failed to find 'limited branchID 3673681161'" $? + + # Above we tested using EmptySource. Repeat reading a file using PoolSource + echo "testOutput2" + cmsRun ${LOCAL_TEST_DIR}/${test}2_cfg.py 2> testOutput2.log || die "cmsRun ${test}2_cfg.py" $? + + grep "global write event" testOutput2.log > /dev/null || die "grep failed to find 'global write event'" $? + grep "global writeLuminosityBlock" testOutput2.log > /dev/null || die "grep failed to find 'global writeLuminosityBlock'" $? + grep "global writeRun" testOutput2.log > /dev/null || die "grep failed to find 'global writeRun'" $? + grep "global writeProcessBlock" testOutput2.log > /dev/null || die "grep failed to find 'global writeProcessBlock'" $? + grep "global respondToOpenInputFile" testOutput2.log > /dev/null || die "grep failed to find 'global respondToOpenInputFile'" $? + grep "global respondToCloseInputFile" testOutput2.log > /dev/null || die "grep failed to find 'global respondToCloseInputFile'" $? + grep "global globalBeginRun" testOutput2.log > /dev/null || die "grep failed to find 'global globalBeginRun'" $? + grep "global globalEndRun" testOutput2.log > /dev/null || die "grep failed to find 'global globalEndRun'" $? + grep "global globalBeginLuminosityBlock" testOutput2.log > /dev/null || die "grep failed to find 'global globalBeginLuminosityBlock'" $? + grep "global globalEndLuminosityBlock" testOutput2.log > /dev/null || die "grep failed to find 'global globalEndLuminosityBlock'" $? + grep "global branchID 4057644746" testOutput2.log > /dev/null || die "grep failed to find 'global branchID 4057644746'" $? + + grep "limited write event" testOutput2.log > /dev/null || die "grep failed to find 'limited write event'" $? + grep "limited writeLuminosityBlock" testOutput2.log > /dev/null || die "grep failed to find 'limited writeLuminosityBlock'" $? + grep "limited writeRun" testOutput2.log > /dev/null || die "grep failed to find 'limited writeRun'" $? + grep "limited writeProcessBlock" testOutput2.log > /dev/null || die "grep failed to find 'limited writeProcessBlock'" $? + grep "limited respondToOpenInputFile" testOutput2.log > /dev/null || die "grep failed to find 'limited respondToOpenInputFile'" $? + grep "limited respondToCloseInputFile" testOutput2.log > /dev/null || die "grep failed to find 'limited respondToCloseInputFile'" $? + grep "limited globalBeginRun" testOutput2.log > /dev/null || die "grep failed to find 'limited globalBeginRun'" $? + grep "limited globalEndRun" testOutput2.log > /dev/null || die "grep failed to find 'limited globalEndRun'" $? + grep "limited globalBeginLuminosityBlock" testOutput2.log > /dev/null || die "grep failed to find 'limited globalBeginLuminosityBlock'" $? + grep "limited globalEndLuminosityBlock" testOutput2.log > /dev/null || die "grep failed to find 'limited globalEndLuminosityBlock'" $? + grep "limited branchID 4057644746" testOutput2.log > /dev/null || die "grep failed to find 'limited branchID 4057644746'" $? + +exit 0 diff --git a/FWIO/RNTupleTempTests/test/run_TestProcessBlock.sh b/FWIO/RNTupleTempTests/test/run_TestProcessBlock.sh new file mode 100755 index 0000000000000..b7ef24cefa445 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/run_TestProcessBlock.sh @@ -0,0 +1,292 @@ +#!/bin/bash +set -x +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} + +function die { echo Failure $1: status $2 ; exit $2 ; } + +# The tests executed by this bash script are all related and +# it seemed clearest to include them all in the same file. +# These tests are divided into distinct groups. Each time +# the script runs, it will execute only one group of tests. +# The script requires that its first command line argument +# specifies the group to be run. The "if" conditional statements +# below implement this. The BuildFile directs scram to run +# this script once for each group when unit tests are run. +# The BuildFile also specifies the dependencies between the +# groups. In some cases, one group cannot run until another +# group of tests has finished. The purpose of this is to +# allow maximum concurrency while running the tests so the +# tests can run faster. + +if [ $1 -eq 1 ] +then + echo "testProcessBlock1" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlock1_cfg.py &> testProcessBlock1.log || die "cmsRun testProcessBlock1_cfg.py" $? + + # The MetaData ProcessBlock branch and the TTree should exist to hold the ProcessBlock + # data. The Events branch should not exist because there were not any ProcessBlock branches + # saved from an input file. Test that here: + edmRNTupleTempFileUtil -l -t MetaData -P file:testProcessBlock1.root > testProcessBlock1ContentsM.txt + grep "Field.* ProcessBlockHelper " testProcessBlock1ContentsM.txt || die "Check for existence of ProcessBlockHelper branch" $? + grep "TTree.*ProcessBlocksPROD1" testProcessBlock1ContentsM.txt || die "Check for existence of ProcessBlocksPROD1 TTree" $? + edmRNTupleTempFileUtil -t Events -P file:testProcessBlock1.root > testProcessBlock1ContentsE.txt + grep "Branch.* EventToProcessBlockIndexes " testProcessBlock1ContentsE.txt && die "Check for non-existence of eventToProcessBlockIndexes branch" 1 +fi + +if [ $1 -eq 2 ] +then + echo "testProcessBlock2" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlock2_cfg.py &> testProcessBlock2.log || die "cmsRun testProcessBlock2_cfg.py" $? +fi + +if [ $1 -eq 3 ] +then + echo "testProcessBlock3" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlock3_cfg.py &> testProcessBlock3.log || die "cmsRun testProcessBlock3_cfg.py" $? +fi + +if [ $1 -eq 4 ] +then + echo "testProcessBlock4" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlock4_cfg.py &> testProcessBlock4.log || die "cmsRun testProcessBlock4_cfg.py" $? +fi + +if [ $1 -eq 5 ] +then + echo "testProcessBlockMerge" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlockMerge_cfg.py &> testProcessBlockMerge.log || die "cmsRun testProcessBlockMerge_cfg.py" $? + + # The ProcessBlock Branches and TTrees should exist in this case. Test that here: + edmRNTupleTempFileUtil -l -t MetaData -P file:testProcessBlockMerge.root > testProcessBlockMContentsM.txt + grep "Field.* ProcessBlockHelper " testProcessBlockMContentsM.txt || die "Check for existence of ProcessBlockHelper branch" $? + grep "TTree.*ProcessBlocksPROD1" testProcessBlockMContentsM.txt || die "Check for existence of ProcessBlocksPROD1 TTree" $? + grep "TTree.*ProcessBlocksMERGE" testProcessBlockMContentsM.txt || die "Check for existence of ProcessBlocksMERGE TTree" $? + edmRNTupleTempFileUtil -t Events -P file:testProcessBlockMerge.root > testProcessBlockMContentsE.txt + grep "Branch.* EventToProcessBlockIndexes " testProcessBlockMContentsE.txt || die "Check for existence of eventToProcessBlockIndexes branch" $? +fi + +if [ $1 -eq 6 ] +then + echo "testProcessBlockTEST" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlockTEST_cfg.py &> testProcessBlockTEST.log || die "cmsRun testProcessBlockTEST_cfg.py" $? + + echo "testProcessBlockRead" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlockRead_cfg.py &> testProcessBlockRead.log || die "cmsRun testProcessBlockRead_cfg.py" $? + grep "InputProcessBlockIntAnalyzer::accessInputProcessBlock" testProcessBlockRead.log || die "Check that InputProcessBlockIntAnalyzer::accessInputProcessBlock was called" $? + grep "InputProcessBlockIntFilter::accessInputProcessBlock" testProcessBlockRead.log || die "Check that InputProcessBlockIntFilter::accessInputProcessBlock was called" $? + grep "InputProcessBlockIntProducer::accessInputProcessBlock" testProcessBlockRead.log || die "Check that InputProcessBlockIntProducer::accessInputProcessBlock was called" $? +fi + +if [ $1 -eq 7 ] +then + echo "testProcessBlock2Dropped" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlock2Dropped_cfg.py &> testProcessBlock2Dropped.log || die "cmsRun testProcessBlock2Dropped_cfg.py" $? + + # The ProcessBlock Branches and TTrees should not exist in this case because + # all the ProcessBlock products are dropped. Test that here: + edmRNTupleTempFileUtil -l -t MetaData -P file:testProcessBlock2Dropped.root > testProcessBlock2DroppedContentsM.txt + grep "Field.* ProcessBlockHelper " testProcessBlock2DroppedContentsM.txt && die "Check for non-existence of ProcessBlockHelper branch" 1 + grep "TTree.*ProcessBlocksPROD1" testProcessBlock2DroppedContentsM.txt && die "Check for non-existence of ProcessBlocksPROD1 TTree" 1 + edmRNTupleTempFileUtil -t Events -P file:testProcessBlock2Dropped.root > testProcessBlock2DroppedContentsE.txt + grep "Branch.* EventToProcessBlockIndexes " testProcessBlock2DroppedContentsE.txt && die "Check for non-existence of eventToProcessBlockIndexes branch" 1 +fi + +if [ $1 -eq 8 ] +then + # This one intentionally fails because the product content of the + # files does not match (strict merging requirements for ProcessBlocks) + echo "testProcessBlockFailMerge" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlockFailMerge_cfg.py &> testProcessBlockFailMerge.log && die "cmsRun testProcessBlockFailMerge_cfg.py" 1 +fi + +if [ $1 -eq 9 ] +then + echo "testProcessBlockMerge2" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlockMerge2_cfg.py &> testProcessBlockMerge2.log || die "cmsRun testProcessBlockMerge2_cfg.py" $? +fi + +if [ $1 -eq 10 ] +then + echo "testProcessBlockMergeOfMergedFiles" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlockMergeOfMergedFiles_cfg.py &> testProcessBlockMergeOfMergedFiles.log || die "cmsRun testProcessBlockMergeOfMergedFiles_cfg.py" $? +fi + +if [ $1 -eq 11 ] +then + echo "testProcessBlockNOMergeOfMergedFiles" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlockNOMergeOfMergedFiles_cfg.py &> testProcessBlockNOMergeOfMergedFiles.log || die "cmsRun testProcessBlockNOMergeOfMergedFiles_cfg.py" $? +fi + +if [ $1 -eq 12 ] +then + echo "testProcessBlockRead2" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlockRead2_cfg.py &> testProcessBlockRead2.log || die "cmsRun testProcessBlockRead2_cfg.py" $? +fi + +if [ $1 -eq 17 ] +then + echo "testProcessBlockLooper" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlockLooper_cfg.py &> testProcessBlockLooper.log || die "cmsRun testProcessBlockLooper_cfg.py" $? +fi + +if [ $1 -eq 18 ] +then + echo "testProcessBlock5" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlock5_cfg.py &> testProcessBlock5.log || die "cmsRun testProcessBlock5_cfg.py" $? + + echo "testProcessBlockDummy" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlockDummy_cfg.py &> testProcessBlockDummy.log || die "cmsRun testProcessBlockDummy_cfg.py" $? + + echo "testProcessBlockMerge3" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlockMerge3_cfg.py &> testProcessBlockMerge3.log || die "cmsRun testProcessBlockMerge3_cfg.py" $? + + echo "testProcessBlockMergeOfMergedFiles2" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlockMergeOfMergedFiles2_cfg.py &> testProcessBlockMergeOfMergedFiles2.log || die "cmsRun testProcessBlockMergeOfMergedFiles2_cfg.py" $? +fi + +if [ $1 -eq 19 ] +then + echo "testProcessBlockDropOnInput" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlockDropOnInput_cfg.py &> testProcessBlockDropOnInput.log || die "cmsRun testProcessBlockDropOnInput_cfg.py" $? +fi + +if [ $1 -eq 20 ] +then + echo "testProcessBlockThreeFileInput" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlockThreeFileInput_cfg.py &> testProcessBlockThreeFileInput.log || die "cmsRun testProcessBlockThreeFileInput_cfg.py" $? + + echo "testProcessBlockReadThreeFileInput" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlockReadThreeFileInput_cfg.py &> testProcessBlockReadThreeFileInput.log || die "cmsRun testProcessBlockReadThreeFileInput_cfg.py" $? +fi + +if [ $1 -eq 21 ] +then + echo "testLooperEventNavigation2" + cmsRun ${LOCAL_TEST_DIR}/testLooperEventNavigation2_cfg.py < ${LOCAL_TEST_DIR}/testLooperEventNavigation2.txt &> testLooperEventNavigation2.log || die "cmsRun testLooperEventNavigation2_cfg.py" $? +fi + +if [ $1 -eq 22 ] +then + echo "testLooperEventNavigation3" + cmsRun ${LOCAL_TEST_DIR}/testLooperEventNavigation3_cfg.py < ${LOCAL_TEST_DIR}/testLooperEventNavigation3.txt &> testLooperEventNavigation3.log || die "cmsRun testLooperEventNavigation3_cfg.py" $? +fi + +if [ $1 -eq 23 ] +then + echo "testProcessBlockDropOnOutput" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlockDropOnOutput_cfg.py &> testProcessBlockDropOnOutput.log || die "cmsRun testProcessBlockDropOnOutput_cfg.py" $? + + echo "testProcessBlockReadDropOnOutput" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlockReadDropOnOutput_cfg.py &> testProcessBlockReadDropOnOutput.log || die "cmsRun testProcessBlockReadDropOnOutput_cfg.py" $? +fi + +if [ $1 -eq 24 ] +then + echo "testProcessBlockDropOnOutput2" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlockDropOnOutput2_cfg.py &> testProcessBlockDropOnOutput2.log || die "cmsRun testProcessBlockDropOnOutput2_cfg.py" $? + + echo "testProcessBlockReadDropOnOutput2" + cmsRun ${LOCAL_TEST_DIR}/testProcessBlockReadDropOnOutput2_cfg.py &> testProcessBlockReadDropOnOutput2.log || die "cmsRun testProcessBlockReadDropOnOutput2_cfg.py" $? +fi + +# The next three tests would be relevant if we disabled the strict merging requirement +# in ProductRegistry.cc for ProcessBlock products (a one line code change). As long +# as we always enforce the strict merging requirement these tests will fail, but they +# would be useful if we decide to allow that requirement to be disabled in the future. +# I ran them manually with the ProductRegistry.cc modified to disable the requirement +# and in May 2021 these tests passed. In addition to uncommenting the tests here, they +# would also need to be added in the BuildFile with the proper dependency (both 25 +# and 26 depend on 19 at the moment) + +#if [ $1 -eq 25 ] +#then +# echo "testProcessBlockNonStrict" +# cmsRun ${LOCAL_TEST_DIR}/testProcessBlockNonStrict_cfg.py &> testProcessBlockNonStrict.log || die "cmsRun testProcessBlockNonStrict_cfg.py" $? +# +# echo "testProcessBlockNonStrict2" +# cmsRun ${LOCAL_TEST_DIR}/testProcessBlockNonStrict2_cfg.py &> testProcessBlockNonStrict2.log || die "cmsRun testProcessBlockNonStrict2_cfg.py" $? +#fi + +#if [ $1 -eq 26 ] +#then +# echo "testProcessBlockNonStrict3" +# cmsRun ${LOCAL_TEST_DIR}/testProcessBlockNonStrict3_cfg.py &> testProcessBlockNonStrict3.log || die "cmsRun testProcessBlockNonStrict3_cfg.py" $? +#fi + +if [ $1 -eq 100 ] +then + rm testProcessBlock1ContentsM.txt + rm testProcessBlock1ContentsE.txt + rm testProcessBlockMContentsM.txt + rm testProcessBlockMContentsE.txt + rm testProcessBlock2DroppedContentsM.txt + rm testProcessBlock2DroppedContentsE.txt + + rm testProcessBlock1.log + rm testProcessBlock2.log + rm testProcessBlock3.log + rm testProcessBlock4.log + rm testProcessBlockMerge.log + rm testProcessBlockTEST.log + rm testProcessBlockRead.log + rm testProcessBlock2Dropped.log + rm testProcessBlockFailMerge.log + rm testProcessBlockMerge2.log + rm testProcessBlockMergeOfMergedFiles.log + rm testProcessBlockNOMergeOfMergedFiles.log + rm testProcessBlockRead2.log + rm testProcessBlockLooper.log + rm testProcessBlock5.log + rm testProcessBlockDummy.log + rm testProcessBlockMerge3.log + rm testProcessBlockMergeOfMergedFiles2.log + rm testProcessBlockDropOnInput.log + rm testProcessBlockThreeFileInput.log + rm testProcessBlockReadThreeFileInput.log + rm testLooperEventNavigation2.log + rm testLooperEventNavigation3.log + rm testProcessBlockDropOnOutput.log + rm testProcessBlockReadDropOnOutput.log + rm testProcessBlockDropOnOutput2.log + rm testProcessBlockReadDropOnOutput2.log + + rm testProcessBlock1.root + rm testProcessBlock2.root + rm testProcessBlock3.root + rm testProcessBlock4.root + rm testProcessBlockMerge.root + rm testProcessBlockTest.root + rm testProcessBlockRead.root + rm testProcessBlock2Dropped.root + rm testProcessBlockFailMerge.root + rm testProcessBlockMerge2.root + rm testProcessBlockMergeOfMergedFiles.root + rm testProcessBlockNOMergeOfMergedFiles.root + rm testProcessBlockRead2.root + rm testProcessBlockLooperTest.root + rm testProcessBlock5.root + rm testProcessBlockDummy.root + rm testProcessBlockMerge3.root + rm testProcessBlockMergeOfMergedFiles2.root + rm testProcessBlockDropOnInput.root + rm testProcessBlockThreeFileInput.root + rm testProcessBlockReadThreeFileInput.root + rm testProcessBlockDropOnOutput.root + rm testProcessBlockDropOnOutput2.root + rm testProcessBlockDropOnOutput2_2.root + rm testProcessBlockReadDropOnOutput.root + rm testProcessBlockReadDropOnOutput2.root + rm testProcessBlockNOMergeOfMergedFiles001.root + rm testProcessBlockLooperTest001.root + rm testProcessBlockLooperTest002.root + + #rm testProcessBlockNonStrict.log + #rm testProcessBlockNonStrict2.log + #rm testProcessBlockNonStrict3.log + #rm testProcessBlockNonStrict.root + #rm testProcessBlockNonStrict2.root + #rm testProcessBlockNonStrict3.root + +fi + +exit 0 diff --git a/FWIO/RNTupleTempTests/test/run_ThinningTests.sh b/FWIO/RNTupleTempTests/test/run_ThinningTests.sh new file mode 100755 index 0000000000000..5ef4f503c8f10 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/run_ThinningTests.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +function die { echo Failure $1: status $2 ; exit $2 ; } + +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} + + cmsRun ${LOCAL_TEST_DIR}/ThinningTest1_cfg.py || die "cmsRun ThinningTest1_cfg.py" $? + + cmsRun ${LOCAL_TEST_DIR}/ThinningTest2_cfg.py || die "cmsRun ThinningTest2_cfg.py" $? + + cmsRun ${LOCAL_TEST_DIR}/ThinningTest3_cfg.py || die "cmsRun ThinningTest3_cfg.py" $? + + cmsRun ${LOCAL_TEST_DIR}/ThinningTest4_cfg.py || die "cmsRun ThinningTest4_cfg.py" $? + + cmsRun ${LOCAL_TEST_DIR}/ThinningTest_dropOnInput_cfg.py || die "cmsRun ThinningTest_dropOnInput_cfg.py" $? + + cmsRun ${LOCAL_TEST_DIR}/ThinningTest4Slimming_cfg.py || die "cmsRun ThinningTest4Slimming_cfg.py" $? + + cmsRun ${LOCAL_TEST_DIR}/DetSetVectorThinningTest1_cfg.py || die "cmsRun DetSetVectorThinningTest1_cfg.py" $? + cmsRun ${LOCAL_TEST_DIR}/DetSetVectorThinningTest2_cfg.py || die "cmsRun DetSetVectorThinningTest2_cfg.py" $? + + cmsRun ${LOCAL_TEST_DIR}/SlimmingTest1_cfg.py || die "cmsRun SlimmingTest1_cfg.py" $? + + cmsRun ${LOCAL_TEST_DIR}/SlimmingTest2A_cfg.py || die "cmsRun SlimmingTest2A_cfg.py" $? + cmsRun ${LOCAL_TEST_DIR}/SlimmingTest2B_cfg.py || die "cmsRun SlimmingTest2B_cfg.py" $? + cmsRun ${LOCAL_TEST_DIR}/SlimmingTest2C_cfg.py || die "cmsRun SlimmingTest2C_cfg.py" $? + cmsRun ${LOCAL_TEST_DIR}/SlimmingTest2D_cfg.py || die "cmsRun SlimmingTest2D_cfg.py" $? + cmsRun ${LOCAL_TEST_DIR}/SlimmingTest2E_cfg.py || die "cmsRun SlimmingTest2E_cfg.py" $? + cmsRun ${LOCAL_TEST_DIR}/SlimmingTest2F_cfg.py || die "cmsRun SlimmingTest2E_cfg.py" $? + cmsRun ${LOCAL_TEST_DIR}/SlimmingTest2G_cfg.py || die "cmsRun SlimmingTest2G_cfg.py" $? + cmsRun ${LOCAL_TEST_DIR}/SlimmingTest2H_cfg.py || die "cmsRun SlimmingTest2H_cfg.py" $? + cmsRun ${LOCAL_TEST_DIR}/SlimmingTest2I_cfg.py || die "cmsRun SlimmingTest2I_cfg.py" $? + + cmsRun ${LOCAL_TEST_DIR}/SlimmingTest3B_cfg.py || die "cmsRun SlimmingTest3B_cfg.py" $? + cmsRun ${LOCAL_TEST_DIR}/SlimmingTest3C_cfg.py || die "cmsRun SlimmingTest3C_cfg.py" $? + cmsRun ${LOCAL_TEST_DIR}/SlimmingTest3D_cfg.py || die "cmsRun SlimmingTest3D_cfg.py" $? + cmsRun ${LOCAL_TEST_DIR}/SlimmingTest3E_cfg.py || die "cmsRun SlimmingTest3E_cfg.py" $? + cmsRun ${LOCAL_TEST_DIR}/SlimmingTest3F_cfg.py || die "cmsRun SlimmingTest3F_cfg.py" $? + cmsRun ${LOCAL_TEST_DIR}/SlimmingTest3I_cfg.py || die "cmsRun SlimmingTest3I_cfg.py" $? + + cmsRun ${LOCAL_TEST_DIR}/SlimmingTest4B_cfg.py || die "cmsRun SlimmingTest4B_cfg.py" $? + cmsRun ${LOCAL_TEST_DIR}/SlimmingTest4F_cfg.py && die "cmsRun SlimmingTest4F_cfg.py" 1 + +exit 0 diff --git a/FWIO/RNTupleTempTests/test/run_inconsistent_products.sh b/FWIO/RNTupleTempTests/test/run_inconsistent_products.sh new file mode 100755 index 0000000000000..97afd55a7a019 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/run_inconsistent_products.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +test=inconsistent_products_ + +function die { echo Failure $1: status $2 ; exit $2 ; } + +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} +echo ${test}prod_allThings ------------------------------------------------------------ +cmsRun ${LOCAL_TEST_DIR}/${test}prod_cfg.py --startEvent=11 --fileName='allThings.root' || die "cmsRun ${test}prod_cfg.py" $? + +echo ${test}prod_dropThing2 ------------------------------------------------------------ +cmsRun ${LOCAL_TEST_DIR}/${test}prod_cfg.py --dropThing2 --fileName 'dropThing2.root'|| die "cmsRun ${test}prod_cfg.py --dropThing2" $? + +echo ${test}test_dropThing2------------------------------------------------------------ +#If file which contains the branch is read second, things fail +cmsRun ${LOCAL_TEST_DIR}/${test}test_cfg.py --nEventsToFail=10 --fileNames dropThing2.root allThings.root && die "cmsRun ${test}cfg.py dropThing2 --fileNames dropThing2.root allThings.root" 1 + +echo ${test}test_dropThing2_2nd------------------------------------------------------------ +#If file which contains the branch is read first, things succeed +cmsRun ${LOCAL_TEST_DIR}/${test}test_cfg.py --nEventsToFail=10 --thing2Dropped --fileNames allThings.root dropThing2.root || die "cmsRun ${test}cfg.py --thing2Dropped --fileNames allThings.root dropThing2.root" $? + +echo ${test}prod_dropThing2_other2 ------------------------------------------------------------ +cmsRun ${LOCAL_TEST_DIR}/${test}prod_cfg.py --dropThing2 --addAndStoreOther2 --fileName 'dropThing2_other2.root'|| die "cmsRun ${test}prod_cfg.py --dropThing2 --addAndStoreOther2" $? + +echo ${test}prod_allThings ------------------------------------------------------------ +cmsRun ${LOCAL_TEST_DIR}/${test}prod_cfg.py --startEvent=11 --addAndStoreOther2 --fileName='allThings_other2.root' || die "cmsRun ${test}prod_cfg.py --addAndStoreOther2" $? + +echo ${test}test_dropThing2_other2_allThings------------------------------------------------------------ +#If file which contains the branch is read second, things fail +cmsRun ${LOCAL_TEST_DIR}/${test}test_cfg.py --nEventsToFail=10 --fileNames dropThing2_other2.root allThings.root && die "cmsRun ${test}cfg.py --fileNames dropThing2_other2.root allThings.root" 1 + +echo ${test}test_dropThing2_other2------------------------------------------------------------ +#If file which contains the branch is read second, things fail +cmsRun ${LOCAL_TEST_DIR}/${test}test_cfg.py --nEventsToFail=10 --fileNames dropThing2_other2.root allThings_other2.root && die "cmsRun ${test}cfg.py --fileNames dropThing2_other2.root allThings_other2.root" 1 + +echo ${test}test_dropThing2_other2_second_allThingsFirst------------------------------------------------------------ +#If file which contains the branch is read second, things fail, here the branch is other2 +cmsRun ${LOCAL_TEST_DIR}/${test}test_cfg.py --nEventsToFail=10 --thing2Dropped --fileNames allThings.root dropThing2_other2.root && die "cmsRun ${test}cfg.py --thing2Dropped --fileNames allThings.root dropThing2_other2.root" 1 + +echo ${test}test_dropThing2_other2_second------------------------------------------------------------ +#If file which contains the branch is read first, things succeed +cmsRun ${LOCAL_TEST_DIR}/${test}test_cfg.py --nEventsToFail=10 --thing2Dropped --other2Run --fileNames allThings_other2.root dropThing2_other2.root || die "cmsRun ${test}cfg.py --thing2Dropped --fileNames allThings_other2.root dropThing2_other2.root" $? + +echo ${test}prod_noThing2_other2 ------------------------------------------------------------ +cmsRun ${LOCAL_TEST_DIR}/${test}prod_cfg.py --noThing2Prod --addAndStoreOther2 --fileName 'noThing2_other2.root'|| die "cmsRun ${test}prod_cfg.py --dropThing2 --addAndStoreOther2" $? + +echo ${test}test_noThing2_other2_second------------------------------------------------------------ +#If file which contains the branch is read first, things succeed +cmsRun ${LOCAL_TEST_DIR}/${test}test_cfg.py --nEventsToFail=10 --thing2NotRun --other2Run --fileNames allThings_other2.root noThing2_other2.root || die "cmsRun ${test}cfg.py --thing2Dropped --fileNames allThings_other2.root noThing2_other2.root" $? + +echo ${test}test_noThing2_other2_first------------------------------------------------------------ +#If file which contains the branch is read second, things fails +cmsRun ${LOCAL_TEST_DIR}/${test}test_cfg.py --nEventsToFail=10 --other2Run --fileNames noThing2_other2.root allThings_other2.root && die "cmsRun ${test}cfg.py --thing2Dropped --fileNames noThing2_other2 allThings_other2.root" 1 + +echo ${test}prod_dropThing2_other2_start11 ------------------------------------------------------------ +#if thing2 is dropped and we still run other2, then other2 will read from thing1 instead thereby changing the provenance +cmsRun ${LOCAL_TEST_DIR}/${test}prod_cfg.py --startEvent=11 --dropThing2 --addAndStoreOther2 --fileName='dropThing2_other2_start11.root' || die "cmsRun ${test}prod_cfg.py --startEvent=11 --dropThing2 --addAndStoreOther2" $? + +#branch structures are the same but in one other2 depends on thing1 while in the other file other2 depends on thing2 +echo ${test}test_dropThing2_other2_first_noThing2_other2_second------------------------------------------------------------ +cmsRun ${LOCAL_TEST_DIR}/${test}test_cfg.py --nEventsToFail=10 --thing2NotRun --other2Run --fileNames dropThing2_other2_start11.root noThing2_other2.root || die "cmsRun ${test}cfg.py --thing2NotRun --other2Run --fileNames dropThing2_other2_start11.root noThing2_other2.root" $? + +echo ${test}test_noThing2_other2_first_dropThing2_other2_second------------------------------------------------------------ +cmsRun ${LOCAL_TEST_DIR}/${test}test_cfg.py --nEventsToFail=10 --other2Run --thing2Dropped --fileNames noThing2_other2.root dropThing2_other2_start11.root || die "cmsRun ${test}cfg.py --other2Run --thing2Dropped --fileNames noThing2_other2 dropThing2_other2_start11.root" $? + +echo ${test}prod_dropThing2_other2_depThing1_start11 ------------------------------------------------------------ + +#if thing2 is dropped and we still run other2, then other2 will read from thing1 instead thereby changing the provenance +echo ${test}prod_dropThing2_other2_depThing1_start11------------------------------------------------------------ +cmsRun ${LOCAL_TEST_DIR}/${test}prod_cfg.py --startEvent=11 --dropThing2 --thing2DependsOnThing1 --addAndStoreOther2 --fileName='dropThing2_other2_depThing1_start11.root' || die "cmsRun ${test}prod_cfg.py --startEvent=11 --dropThing2 --thing2DependsOnThing1 --addAndStoreOther2" $? + +#branch structures are the same but in one other2 depends on thing1 while in the other file other2 depends on thing2 +echo ${test}test_dropThing2_other2_first_noThing2_other2_second------------------------------------------------------------ +cmsRun ${LOCAL_TEST_DIR}/${test}test_cfg.py --nEventsToFail=10 --thing2NotRun --other2Run --fileNames dropThing2_other2_depThing1_start11.root noThing2_other2.root || die "cmsRun ${test}cfg.py --thing2NotRun --other2Run --fileNames dropThing2_other2_depThing1_start11.root noThing2_other2.root" $? + +echo ${test}test_noThing2_other2_first_dropThing2_other2_second------------------------------------------------------------ +cmsRun ${LOCAL_TEST_DIR}/${test}test_cfg.py --nEventsToFail=10 --other2Run --thing2Dropped --thing2DependsOnThing1 --fileNames noThing2_other2.root dropThing2_other2_depThing1_start11.root || die "cmsRun ${test}cfg.py --other2Run --thing2Dropped --fileNames noThing2_other2 dropThing2_other2_depThing1_start11.root" $? + + + + +exit 0 diff --git a/FWIO/RNTupleTempTests/test/run_testMaxEventsOutput.sh b/FWIO/RNTupleTempTests/test/run_testMaxEventsOutput.sh new file mode 100755 index 0000000000000..f9ddfdff08d7e --- /dev/null +++ b/FWIO/RNTupleTempTests/test/run_testMaxEventsOutput.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# Pass in name and status +function die { echo $1: status $2 ; exit $2; } + +LOCAL_TEST_DIR="${SCRAM_TEST_PATH}" + +# Test maxEvents output parameter +F3=${LOCAL_TEST_DIR}/testMaxEventsOutput_cfg.py +echo $F3 +(cmsRun $F3 ) || die "Failure running cmsRun $F3" $? +# 6th word on the line containing the string "events" +# output by edmRNTupleTempFileUtil +nEvents=`edmRNTupleTempFileUtil file:testMaxEventsOutput.root | grep events | awk ' {print $6; exit} '` +if [ "$nEvents" -lt 6 ] || [ "$nEvents" -gt 9 ]; then +echo "maxEvents output test failed, nEvents = " $nEvents +exit 1 +fi +echo "number of events written = " $nEvents + +exit 0 diff --git a/FWIO/RNTupleTempTests/test/run_unscheduledFailOnOutput.sh b/FWIO/RNTupleTempTests/test/run_unscheduledFailOnOutput.sh new file mode 100755 index 0000000000000..490c71584e6a4 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/run_unscheduledFailOnOutput.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +test=unscheduled_fail_on_output_ +LOCAL_TEST_DIR=${SCRAM_TEST_PATH} +CFG_DIR=${LOCAL_TEST_DIR} + +function die { echo Failure $1: status $2 ; exit $2 ; } + + cmsRun ${CFG_DIR}/${test}Rethrow_cfg.py && die "cmsRun ${test}Rethrow_cfg.py did not fail" 1 + + cmsRun ${CFG_DIR}/${test}IgnoreCompletely_cfg.py || die "cmsRun ${test}IgnoreCompletely_cfg.py" $? + cmsRun ${CFG_DIR}/${test}read_found_events.py || die "cmsRun ${test}read_found_events.py for IgnoreCompletely" $? + + cmsRun ${CFG_DIR}/${test}TryToContinue_cfg.py || die "cmsRun ${test}TryToComplete_cfg.py" $? + cmsRun ${CFG_DIR}/${test}read_no_events.py || die "cmsRun ${test}read_no_events.py" $? + + cmsRun ${CFG_DIR}/${test}no_dependency_TryToContinue_cfg.py || die "cmsRun ${test}no_dependency_TryToComplete_cfg.py" $? + cmsRun ${CFG_DIR}/${test}read_found_events.py || die "cmsRun ${test}read_found_events.py for TryToComplete" $? + +exit 0 diff --git a/FWIO/RNTupleTempTests/test/testConcurrentIOVsAndRunsRead_cfg.py b/FWIO/RNTupleTempTests/test/testConcurrentIOVsAndRunsRead_cfg.py new file mode 100644 index 0000000000000..a9151dbdf2c8a --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testConcurrentIOVsAndRunsRead_cfg.py @@ -0,0 +1,48 @@ +# Tests concurrent runs along with concurrent IOVs +# Note that 3 concurrent runs implies at least 7 +# concurrent IOVs are needed and we configure +# 8 concurrent IOVs so that concurrent runs are +# really the limiting factor for the test. +# Note 7 includes 1 for the first run and then 3 +# for each subsequent concurrent run which includes +# an IOV for end run, begin run, and begin lumi necessary +# to get to the next event. In this test every lumi is +# only valid for one transition (see internals of +# RunLumiESSource). This test checks that correct +# EventSetup info is retrieved in all the transitions. +# Manual examination of the times in the log output should +# show 3 events in 3 different runs being processed +# concurrently. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READ") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testConcurrentIOVsAndRuns.root') +) + +process.options = cms.untracked.PSet( + numberOfThreads = cms.untracked.uint32(8), + numberOfStreams = cms.untracked.uint32(8), + numberOfConcurrentRuns = cms.untracked.uint32(3), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(8), + eventSetup = cms.untracked.PSet( + numberOfConcurrentIOVs = cms.untracked.uint32(8) + ) +) + +process.runLumiESSource = cms.ESSource("RunLumiESSource") + +process.test = cms.EDAnalyzer("RunLumiESAnalyzer") + +process.busy1 = cms.EDProducer("BusyWaitIntProducer",ivalue = cms.int32(1), iterations = cms.uint32(40*1000*1000)) + +process.p1 = cms.Path(process.busy1 * process.test) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testConcurrentIOVsAndRunsRead.root') +) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testConcurrentIOVsAndRuns_cfg.py b/FWIO/RNTupleTempTests/test/testConcurrentIOVsAndRuns_cfg.py new file mode 100644 index 0000000000000..7f4bf7fc71162 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testConcurrentIOVsAndRuns_cfg.py @@ -0,0 +1,53 @@ +# Tests concurrent runs along with concurrent IOVs +# Note that 3 concurrent runs implies at least 7 +# concurrent IOVs are needed and we configure +# 8 concurrent IOVs so that concurrent runs are +# really the limiting factor for the test. +# Note 7 includes 1 for the first run and then 3 +# for each subsequent concurrent run which includes +# an IOV for end run, begin run, and begin lumi necessary +# to get to the next event. In this test every lumi is +# only valid for one transition (see internals of +# RunLumiESSource). This test checks that correct +# EventSetup info is retrieved in all the transitions. +# Manual examination of the times in the log output should +# show 3 events in 3 different runs being processed +# concurrently. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") + +process.source = cms.Source("EmptySource", + firstRun = cms.untracked.uint32(1), + firstLuminosityBlock = cms.untracked.uint32(1), + firstEvent = cms.untracked.uint32(1), + numberEventsInLuminosityBlock = cms.untracked.uint32(1), + numberEventsInRun = cms.untracked.uint32(1) +) + +process.maxEvents.input = 30 + +process.options = dict( + numberOfThreads = 8, + numberOfStreams = 8, + numberOfConcurrentRuns = 3, + numberOfConcurrentLuminosityBlocks = 8, + eventSetup = dict( + numberOfConcurrentIOVs = 8 + ) +) + +process.runLumiESSource = cms.ESSource("RunLumiESSource") + +process.test = cms.EDAnalyzer("RunLumiESAnalyzer") + +process.busy1 = cms.EDProducer("BusyWaitIntProducer",ivalue = cms.int32(1), iterations = cms.uint32(40*1000*1000)) + +process.p1 = cms.Path(process.busy1 * process.test) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testConcurrentIOVsAndRuns.root') +) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testDropOnInput1_1_cfg.py b/FWIO/RNTupleTempTests/test/testDropOnInput1_1_cfg.py new file mode 100644 index 0000000000000..3afa193277c7b --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testDropOnInput1_1_cfg.py @@ -0,0 +1,100 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST1") + +process.source = cms.Source("EmptySource", + firstEvent = cms.untracked.uint32(1) +) + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(1) +) + +process.prod1 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag() +) + +process.prod2 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("prod1") +) + +process.prod3 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("prod2") +) + +process.prodA = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag() +) + +process.prodB = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("prodA"), + onlyGetOnEvent = cms.untracked.uint32(1) +) + +process.prodC = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("prodB"), + onlyGetOnEvent = cms.untracked.uint32(2) +) + +process.prodF = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag() +) + +process.prodG = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("prodF"), + onlyGetOnEvent = cms.untracked.uint32(2) +) + +process.K100 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag() +) + +process.NK101 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("K100") +) + +process.A101 = cms.EDAlias(NK101 = cms.VPSet( cms.PSet(type=cms.string('edmtestIntProduct') ) ) ) + +process.K102 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("NK101") +) + +process.K104 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("A101") +) + +process.K200 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag() +) + +process.K201 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("K200") +) + +process.A201 = cms.EDAlias(K201 = cms.VPSet( cms.PSet(type=cms.string('edmtestIntProduct') ) ) ) + +process.K202 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("K201") +) + +process.K204 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("A201") +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testDropOnInput1_1.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_NK101_*_*', + 'drop *_A201_*_*', + 'drop *_prod2_*_*' + ) +) + +process.path1 = cms.Path(process.prod1 + process.prod2 + process.prod3) +process.path2 = cms.Path(process.prodA + process.prodB + process.prodC) +process.path3 = cms.Path(process.prodF + process.prodG) +process.path4 = cms.Path(process.K100 + process.NK101 + process.K102 + process.K104) +process.path5 = cms.Path(process.K200 + process.K201 + process.K202 + process.K204) + +process.endpath = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testDropOnInput1_2_cfg.py b/FWIO/RNTupleTempTests/test/testDropOnInput1_2_cfg.py new file mode 100644 index 0000000000000..578585ef413c1 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testDropOnInput1_2_cfg.py @@ -0,0 +1,100 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST1") + +process.source = cms.Source("EmptySource", + firstEvent = cms.untracked.uint32(2) +) + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(1) +) + +process.prod1 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag() +) + +process.prod2 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("prod1") +) + +process.prod3 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("prod2") +) + +process.prodA = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag() +) + +process.prodB = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("prodA"), + onlyGetOnEvent = cms.untracked.uint32(1) +) + +process.prodC = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("prodB"), + onlyGetOnEvent = cms.untracked.uint32(2) +) + +process.prodF = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag() +) + +process.prodG = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("prodF"), + onlyGetOnEvent = cms.untracked.uint32(2) +) + +process.K100 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag() +) + +process.NK101 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("K100") +) + +process.A101 = cms.EDAlias(NK101 = cms.VPSet( cms.PSet(type=cms.string('edmtestIntProduct') ) ) ) + +process.K102 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("NK101") +) + +process.K104 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("A101") +) + +process.K200 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag() +) + +process.K201 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("K200") +) + +process.A201 = cms.EDAlias(K201 = cms.VPSet( cms.PSet(type=cms.string('edmtestIntProduct') ) ) ) + +process.K202 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("K201") +) + +process.K204 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("A201") +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testDropOnInput1_2.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_NK101_*_*', + 'drop *_A201_*_*', + 'drop *_prod2_*_*' + ) +) + +process.path1 = cms.Path(process.prod1 + process.prod2 + process.prod3) +process.path2 = cms.Path(process.prodA + process.prodB + process.prodC) +process.path3 = cms.Path(process.prodF + process.prodG) +process.path4 = cms.Path(process.K100 + process.NK101 + process.K102 + process.K104) +process.path5 = cms.Path(process.K200 + process.K201 + process.K202 + process.K204) + +process.endpath = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testDropOnInput2_cfg.py b/FWIO/RNTupleTempTests/test/testDropOnInput2_cfg.py new file mode 100644 index 0000000000000..f0f82615642b0 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testDropOnInput2_cfg.py @@ -0,0 +1,59 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST2") + +process.options = cms.untracked.PSet( + fileMode = cms.untracked.string('NOMERGE') +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testDropOnInput1_1.root', + 'file:testDropOnInput1_2.root' + ), + inputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_prod1_*_*' + ), + dropDescendantsOfDroppedBranches = cms.untracked.bool(True) +) + +process.prodD = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("prodC"), + onlyGetOnEvent = cms.untracked.uint32(1) +) + +process.prodE = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("prodD"), + onlyGetOnEvent = cms.untracked.uint32(2) +) + +process.K103 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("A101") +) + +process.K203 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("K201") +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testDropOnInput2.root') +) + +process.a1 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( + cms.InputTag("prodA"), + cms.InputTag("prodB"), + cms.InputTag("prodC") + ), + inputTagsNotFound = cms.untracked.VInputTag( + cms.InputTag("prod1"), + cms.InputTag("prod2"), + cms.InputTag("prod3") + ) +) + +process.path = cms.Path(process.prodD + process.prodE + process.a1) +process.path4 = cms.Path(process.K103 + process.K203) + +process.endpath = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testDropOnInput3_cfg.py b/FWIO/RNTupleTempTests/test/testDropOnInput3_cfg.py new file mode 100644 index 0000000000000..150c0b429b528 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testDropOnInput3_cfg.py @@ -0,0 +1,143 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST2") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testDropOnInput1_1.root', + 'file:testDropOnInput1_2.root' + ), + inputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_prod1_*_*' + ), + dropDescendantsOfDroppedBranches = cms.untracked.bool(False) +) + +process.prodD = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("prodC"), + onlyGetOnEvent = cms.untracked.uint32(1) +) + +process.prodE = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("prodD"), + onlyGetOnEvent = cms.untracked.uint32(2) +) + +process.K1 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag() +) + +process.K2 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag() +) + +process.NK1 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("K1", "K2") +) + +process.NK2 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("NK1") +) + +process.NK3 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("NK2") +) + +process.K3 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag() +) + +process.K4 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag() +) + +process.NK4 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("K3", "K4") +) + +process.NK5 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("NK4") +) + +process.K5 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("NK5") +) + +process.NK6 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("K4") +) + +process.NK7 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("NK6") +) + +process.NK8 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("NK6") +) + +process.K103 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("A101") +) + +process.K203 = cms.EDProducer("AddIntsProducer", + labels = cms.VInputTag("K201") +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testDropOnInput3.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_NK*_*_*' + ) +) + +process.a1 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( + cms.InputTag("prod3"), + cms.InputTag("prodA"), + cms.InputTag("prodB"), + cms.InputTag("prodC"), + cms.InputTag("prodD"), + cms.InputTag("prodE") + ), + inputTagsNotFound = cms.untracked.VInputTag( + cms.InputTag("prod1"), + cms.InputTag("prod2") + ) +) + +process.test1 = cms.EDAnalyzer("TestParentage", + inputTag = cms.InputTag("A101"), + expectedAncestors = cms.vstring("K100") +) + +process.test2 = cms.EDAnalyzer("TestParentage", + inputTag = cms.InputTag("K100"), + expectedAncestors = cms.vstring() +) + +process.test3 = cms.EDAnalyzer("TestParentage", + inputTag = cms.InputTag("K102"), + expectedAncestors = cms.vstring("K100", "NK101") +) + +process.test4 = cms.EDAnalyzer("TestParentage", + inputTag = cms.InputTag("K103"), + expectedAncestors = cms.vstring("K100", "NK101") +) + +process.test5 = cms.EDAnalyzer("TestParentage", + inputTag = cms.InputTag("K104"), + expectedAncestors = cms.vstring("K100", "NK101") +) + +process.path1 = cms.Path(process.prodD + process.prodE + process.a1) +process.path2 = cms.Path(process.K1 + process.K2 + process.NK1 + process.NK2 + process.NK3) +process.path3 = cms.Path(process.K3 + process.K4 + process.NK4 + process.NK5 + + process.K5 + process.NK6 + process.NK7 + + process.NK8) +process.path4 = cms.Path(process.K103 + process.K203) + +process.endpath = cms.EndPath(process.out * process.test1 * process.test2 * + process.test3 * process.test4 * process.test5) diff --git a/FWIO/RNTupleTempTests/test/testDropOnInputRead2001_cfg.py b/FWIO/RNTupleTempTests/test/testDropOnInputRead2001_cfg.py new file mode 100644 index 0000000000000..cc2e34b1d473d --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testDropOnInputRead2001_cfg.py @@ -0,0 +1,63 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST3") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testDropOnInput2001.root' + ), + inputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_prodA_*_*', + 'drop *_prodD_*_*', + 'drop *_prodF_*_*' + ), + dropDescendantsOfDroppedBranches = cms.untracked.bool(True) +) + +process.a1 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( + cms.InputTag("prodC"), + cms.InputTag("prodB") + ), + inputTagsNotFound = cms.untracked.VInputTag( + cms.InputTag("prodA"), + cms.InputTag("prodD"), + cms.InputTag("prodE"), + cms.InputTag("prodF"), + cms.InputTag("prodG") + ) +) + +process.test1 = cms.EDAnalyzer("TestParentage", + inputTag = cms.InputTag("A101"), + expectedAncestors = cms.vstring("K100") +) + +process.test2 = cms.EDAnalyzer("TestParentage", + inputTag = cms.InputTag("K100"), + expectedAncestors = cms.vstring() +) + +process.test3 = cms.EDAnalyzer("TestParentage", + inputTag = cms.InputTag("K102"), + expectedAncestors = cms.vstring("K100", "NK101") +) + +process.test4 = cms.EDAnalyzer("TestParentage", + inputTag = cms.InputTag("K103"), + expectedAncestors = cms.vstring("K100", "NK101") +) + +process.test5 = cms.EDAnalyzer("TestParentage", + inputTag = cms.InputTag("K104"), + expectedAncestors = cms.vstring("K100", "NK101") +) + +process.path = cms.Path(process.a1 * + process.test1 * + process.test2 * + process.test3 * + process.test4 * + process.test5 +) diff --git a/FWIO/RNTupleTempTests/test/testDropOnInputRead2_cfg.py b/FWIO/RNTupleTempTests/test/testDropOnInputRead2_cfg.py new file mode 100644 index 0000000000000..99da47fd6cce9 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testDropOnInputRead2_cfg.py @@ -0,0 +1,46 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST3") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testDropOnInput2.root' + ), + inputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_prodA_*_*', + 'drop *_prodD_*_*', + 'drop *_prodF_*_*', + 'drop *_A101_*_*', + 'drop *_K201_*_*' + ), + dropDescendantsOfDroppedBranches = cms.untracked.bool(True) +) + +process.a1 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( + cms.InputTag("prodC"), + cms.InputTag("prodE"), + cms.InputTag("prodG"), + cms.InputTag("K100"), + cms.InputTag("K200") + ), + inputTagsNotFound = cms.untracked.VInputTag( + cms.InputTag("prodA"), + cms.InputTag("prodD"), + cms.InputTag("prodB"), + cms.InputTag("prodF"), + cms.InputTag("NK101"), + cms.InputTag("A101"), + cms.InputTag("K102"), + cms.InputTag("K103"), + cms.InputTag("K104"), + cms.InputTag("K201"), + cms.InputTag("A201"), + cms.InputTag("K202"), + cms.InputTag("K203"), + cms.InputTag("K204") + ) +) + +process.path = cms.Path(process.a1) diff --git a/FWIO/RNTupleTempTests/test/testDropOnInputRead3_cfg.py b/FWIO/RNTupleTempTests/test/testDropOnInputRead3_cfg.py new file mode 100644 index 0000000000000..8df8180e743b8 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testDropOnInputRead3_cfg.py @@ -0,0 +1,60 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST3") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testDropOnInput3.root' + ), + inputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_prodA_*_*', + 'drop *_K100_*_*', + 'drop *_K200_*_*' + ), + dropDescendantsOfDroppedBranches = cms.untracked.bool(True) +) + +process.a1 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( + cms.InputTag("prod3") + ), + + inputTagsNotFound = cms.untracked.VInputTag( + cms.InputTag("prod1"), + cms.InputTag("prod2"), + cms.InputTag("prodA"), + cms.InputTag("prodB"), + cms.InputTag("prodC"), + cms.InputTag("prodD"), + cms.InputTag("prodE"), + cms.InputTag("K100"), + cms.InputTag("NK101"), + cms.InputTag("A101"), + cms.InputTag("K102"), + cms.InputTag("K103"), + cms.InputTag("K104"), + cms.InputTag("K200"), + cms.InputTag("K201"), + cms.InputTag("A201"), + cms.InputTag("K202"), + cms.InputTag("K203"), + cms.InputTag("K204") + ) +) + +process.test1 = cms.EDAnalyzer("TestParentage", + inputTag = cms.InputTag("K5"), + expectedAncestors = cms.vstring("K3", "K4", "NK4", "NK5") +) + +process.test2 = cms.EDAnalyzer("TestParentage", + inputTag = cms.InputTag("prod3"), + # prod1 gets converted to an empty string in the TestParentage module + # because drop on input removes it from the ProductRegistry + # completely, but its BranchID still appears in the Parentage. + expectedAncestors = cms.vstring("prod2", ""), + callGetProvenance = cms.untracked.bool(False) +) + +process.path = cms.Path(process.a1 * process.test1 * process.test2) diff --git a/FWIO/RNTupleTempTests/test/testDuplicateProcess_cfg.py b/FWIO/RNTupleTempTests/test/testDuplicateProcess_cfg.py new file mode 100644 index 0000000000000..8f9c1dec3e2ed --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testDuplicateProcess_cfg.py @@ -0,0 +1,15 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD1") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testGetBy1.root' + ) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRepeatProcess.root') +) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testEDAliasAnalyze_cfg.py b/FWIO/RNTupleTempTests/test/testEDAliasAnalyze_cfg.py new file mode 100644 index 0000000000000..a1c1abe67bbd2 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testEDAliasAnalyze_cfg.py @@ -0,0 +1,15 @@ +import FWCore.ParameterSet.Config as cms +import sys + +process = cms.Process("ANA1") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring("file:"+sys.argv[-1]) +) + +process.analyzer = cms.EDAnalyzer("IntTestAnalyzer", + moduleLabel = cms.untracked.InputTag("intProducer"), + valueMustMatch = cms.untracked.int32(1) +) + +process.p = cms.Path(process.analyzer) diff --git a/FWIO/RNTupleTempTests/test/testEDAliasTask_cfg.py b/FWIO/RNTupleTempTests/test/testEDAliasTask_cfg.py new file mode 100644 index 0000000000000..565b2b66c91fc --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testEDAliasTask_cfg.py @@ -0,0 +1,35 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD1") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("EmptySource") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testEDAliasTask.root'), + outputCommands = cms.untracked.vstring( + 'keep *_intProducer_*_*', + ) +) + +process.intProducerOrig = cms.EDProducer("ManyIntProducer", ivalue = cms.int32(1)) + +process.intAlias = cms.EDAlias( + intProducerOrig = cms.VPSet(cms.PSet(type = cms.string("edmtestIntProduct"))) +) + +process.intProducer = cms.EDProducer("ManyIntWhenRegisteredProducer", src = cms.string("intAlias")) + +process.t = cms.Task(process.intProducer, process.intProducerOrig) +process.p = cms.Path(process.t) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testExistingDictionaryCheckingRead_cfg.py b/FWIO/RNTupleTempTests/test/testExistingDictionaryCheckingRead_cfg.py new file mode 100644 index 0000000000000..c4b0851d76ec7 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testExistingDictionaryCheckingRead_cfg.py @@ -0,0 +1,16 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READ") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring("file:testExistingDictionaryChecking.root") +) +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) +process.read = cms.EDAnalyzer("ExistingDictionaryTestAnalyzer", + src = cms.InputTag("prod"), + testVecUniqInt = cms.bool(False) # reading in plain vector> does not work yet +) + +process.p = cms.Path(process.read) diff --git a/FWIO/RNTupleTempTests/test/testExistingDictionaryChecking_cfg.py b/FWIO/RNTupleTempTests/test/testExistingDictionaryChecking_cfg.py new file mode 100644 index 0000000000000..ddb13137d20c2 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testExistingDictionaryChecking_cfg.py @@ -0,0 +1,22 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD") + +process.source = cms.Source("EmptySource") +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(1) +) +process.prod = cms.EDProducer("ExistingDictionaryTestProducer") +process.read = cms.EDAnalyzer("ExistingDictionaryTestAnalyzer", + src = cms.InputTag("prod") +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testExistingDictionaryChecking.root'), + outputCommands = cms.untracked.vstring( + 'keep *_prod_*_*', + ) +) + +process.p = cms.Path(process.prod+process.read) +process.ep = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testGetBy1Mod_cfg.py b/FWIO/RNTupleTempTests/test/testGetBy1Mod_cfg.py new file mode 100644 index 0000000000000..1deb7efac8c46 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testGetBy1Mod_cfg.py @@ -0,0 +1,30 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD1") + +process.source = cms.Source("EmptySource", + firstRun = cms.untracked.uint32(2) +) + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testGetBy1Mod.root'), + outputCommands = cms.untracked.vstring( + 'keep *' + ) +) + +process.intProducer = cms.EDProducer("IntProducer", ivalue = cms.int32(2)) + +process.intProducerBeginProcessBlock = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(10000)) + +process.intProducerEndProcessBlock = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(100000)) + +process.t = cms.Task(process.intProducer, + process.intProducerBeginProcessBlock, + process.intProducerEndProcessBlock ) + +process.e = cms.EndPath(process.out, process.t) diff --git a/FWIO/RNTupleTempTests/test/testGetBy1_cfg.py b/FWIO/RNTupleTempTests/test/testGetBy1_cfg.py new file mode 100644 index 0000000000000..033eb6b39cf94 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testGetBy1_cfg.py @@ -0,0 +1,111 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD1") + +process.Tracer = cms.Service('Tracer', + dumpContextForLabels = cms.untracked.vstring('intProducerA', 'a1'), + dumpNonModuleContext = cms.untracked.bool(True) +) + +process.MessageLogger = cms.Service("MessageLogger", + cout = cms.untracked.PSet( + Tracer = cms.untracked.PSet( + limit = cms.untracked.int32(100000000) + ), + default = cms.untracked.PSet( + limit = cms.untracked.int32(0) + ), + enable = cms.untracked.bool(True) + ) +) + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("IntSource") +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testGetBy1.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerA_*_*' + ) +) + +process.a1 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("source") ), + expectedSum = cms.untracked.int32(530021), + inputTagsNotFound = cms.untracked.VInputTag( + cms.InputTag("source", processName=cms.InputTag.skipCurrentProcess()), + cms.InputTag("intProducer", processName=cms.InputTag.skipCurrentProcess()), + cms.InputTag("intProducerU", processName=cms.InputTag.skipCurrentProcess()) + ), + inputTagsBeginProcessBlock = cms.untracked.VInputTag( + cms.InputTag("intProducerBeginProcessBlock"), + ), + inputTagsEndProcessBlock = cms.untracked.VInputTag( + cms.InputTag("intProducerEndProcessBlock"), + ), + inputTagsEndProcessBlock2 = cms.untracked.VInputTag( + cms.InputTag("intProducerEndProcessBlock", "two"), + ), + inputTagsEndProcessBlock3 = cms.untracked.VInputTag( + cms.InputTag("intProducerEndProcessBlock", "three"), + ), + inputTagsEndProcessBlock4 = cms.untracked.VInputTag( + cms.InputTag("intProducerEndProcessBlock", "four"), + ), + testGetterOfProducts = cms.untracked.bool(True) +) + +process.a2 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("intProducerA") ), + expectedSum = cms.untracked.int32(300) +) + +process.a3 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("aliasForInt") ), + expectedSum = cms.untracked.int32(300) +) + +process.intProducer = cms.EDProducer("IntProducer", ivalue = cms.int32(1)) + +process.intProducerU = cms.EDProducer("IntProducer", ivalue = cms.int32(10)) + +process.intProducerA = cms.EDProducer("IntProducer", ivalue = cms.int32(100)) + +process.aliasForInt = cms.EDAlias( + intProducerA = cms.VPSet( + cms.PSet(type = cms.string('edmtestIntProduct') + ) + ) +) + +process.intVectorProducer = cms.EDProducer("IntVectorProducer", + count = cms.int32(9), + ivalue = cms.int32(11) +) + +process.intProducerB = cms.EDProducer("IntProducer", ivalue = cms.int32(1000)) + +process.intProducerBeginProcessBlock = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(10000)) + +process.intProducerEndProcessBlock = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(100000)) + +process.t = cms.Task(process.intProducerU, + process.intProducerA, + process.intProducerB, + process.intVectorProducer, + process.intProducerBeginProcessBlock, + process.intProducerEndProcessBlock +) + +process.p = cms.Path(process.intProducer * process.a1 * process.a2 * process.a3, process.t) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testGetBy2_cfg.py b/FWIO/RNTupleTempTests/test/testGetBy2_cfg.py new file mode 100644 index 0000000000000..8fbde24d5760d --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testGetBy2_cfg.py @@ -0,0 +1,51 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD2") + +process.Tracer = cms.Service('Tracer', + dumpContextForLabels = cms.untracked.vstring('intProducer'), + dumpNonModuleContext = cms.untracked.bool(True) +) + +process.MessageLogger = cms.Service("MessageLogger", + cout = cms.untracked.PSet( + Tracer = cms.untracked.PSet( + limit = cms.untracked.int32(100000000) + ), + default = cms.untracked.PSet( + limit = cms.untracked.int32(0) + ), + enable = cms.untracked.bool(True) + ) +) + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testGetBy1.root' + ) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testGetBy2.root') +) + +process.intProducer = cms.EDProducer("IntProducer", ivalue = cms.int32(2)) + +process.intProducerU = cms.EDProducer("IntProducer", ivalue = cms.int32(20)) + +process.intVectorProducer = cms.EDProducer("IntVectorProducer", + count = cms.int32(9), + ivalue = cms.int32(21) +) + +process.t = cms.Task(process.intProducerU, process.intVectorProducer) + +process.p = cms.Path(process.intProducer, process.t) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testGetBy3_cfg.py b/FWIO/RNTupleTempTests/test/testGetBy3_cfg.py new file mode 100644 index 0000000000000..0f8a722441e61 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testGetBy3_cfg.py @@ -0,0 +1,213 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD3") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testGetBy2.root' + ) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testGetBy3.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerA_*_*' + ) +) + +process.intProducer = cms.EDProducer("IntProducer", ivalue = cms.int32(3)) + +process.intProducerU = cms.EDProducer("IntProducer", ivalue = cms.int32(30)) + +process.intProducerA = cms.EDProducer("IntProducer", ivalue = cms.int32(200)) + +process.aliasForInt = cms.EDAlias( + intProducerA = cms.VPSet( + cms.PSet(type = cms.string('edmtestIntProduct') + ) + ) +) + +process.nonProducer = cms.EDProducer("NonProducer") + +process.intVectorSetProducer = cms.EDProducer("IntVectorSetProducer") + +process.intVectorProducer = cms.EDProducer("IntVectorProducer", + count = cms.int32(9), + ivalue = cms.int32(31) +) + +process.a1 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("intProducer") ), + expectedSum = cms.untracked.int32(9) +) + +process.a2 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("intProducer", processName=cms.InputTag.skipCurrentProcess()) ), + expectedSum = cms.untracked.int32(6) +) + +process.a3 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("intProducer", "", "PROD1") ), + expectedSum = cms.untracked.int32(3) +) + +process.a4 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("intProducer", "", "PROD2") ), + expectedSum = cms.untracked.int32(6) +) + +process.a5 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("intProducer", "", "PROD3") ), + expectedSum = cms.untracked.int32(9) +) + +process.a6 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("intProducer", "", processName=cms.InputTag.currentProcess()) ), + expectedSum = cms.untracked.int32(9) +) + +process.a10 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("intProducerU") ), + expectedSum = cms.untracked.int32(90) +) + +process.a20 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("intProducerU", processName=cms.InputTag.skipCurrentProcess()) ), + expectedSum = cms.untracked.int32(60) +) + +process.a30 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("intProducerU", "", "PROD1") ), + expectedSum = cms.untracked.int32(30) +) + +process.a40 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("intProducerU", "", "PROD2") ), + expectedSum = cms.untracked.int32(60) +) + +process.a50 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("intProducerU", "", "PROD3") ), + expectedSum = cms.untracked.int32(90) +) + +process.a60 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("intProducerU", "", processName=cms.InputTag.currentProcess()) ), + expectedSum = cms.untracked.int32(90), + inputTagsNotFound = cms.untracked.VInputTag( + cms.InputTag("intProducerB", processName=cms.InputTag.currentProcess()) + ) +) + +process.a70 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("intProducerB", processName=cms.InputTag.skipCurrentProcess()) ), + expectedSum = cms.untracked.int32(3000) +) + +process.a100 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("source", processName=cms.InputTag.skipCurrentProcess()) ), + expectedSum = cms.untracked.int32(12) +) + +process.a200 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("intProducerA") ), + expectedSum = cms.untracked.int32(600) +) + +process.a300 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("aliasForInt") ), + expectedSum = cms.untracked.int32(600) +) + +process.a400 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("aliasForInt", processName=cms.InputTag.skipCurrentProcess()) ), + expectedSum = cms.untracked.int32(300), + inputTagsNotFound = cms.untracked.VInputTag( + cms.InputTag("intProducerA", processName=cms.InputTag.skipCurrentProcess()) + ) +) + +process.a1000 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("doesNotExist") ), + expectedSum = cms.untracked.int32(1), + getByTokenFirst = cms.untracked.bool(True) +) + +process.a1001 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("nonProducer") ), + expectedSum = cms.untracked.int32(1), + getByTokenFirst = cms.untracked.bool(True) +) + +process.a1002 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag(), + inputTagsView = cms.untracked.VInputTag( cms.InputTag("intVectorSetProducer", "", "PROD3") ), + expectedSum = cms.untracked.int32(1), + getByTokenFirst = cms.untracked.bool(True) +) + +process.a1003 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag(), + inputTagsView = cms.untracked.VInputTag( cms.InputTag("intVectorSetProducer") ), + expectedSum = cms.untracked.int32(1), + getByTokenFirst = cms.untracked.bool(True) +) + +process.a1004 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag(), + inputTagsView = cms.untracked.VInputTag( cms.InputTag("intVectorProducer") ), + expectedSum = cms.untracked.int32(93) +) + +process.a1005 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag(), + inputTagsView = cms.untracked.VInputTag( cms.InputTag("intVectorProducer", processName=cms.InputTag.skipCurrentProcess()) ), + expectedSum = cms.untracked.int32(63) +) + +process.a1006 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag(), + inputTagsView = cms.untracked.VInputTag( cms.InputTag("intVectorProducer", processName=cms.InputTag.currentProcess()) ), + expectedSum = cms.untracked.int32(93) +) + +process.a2000 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag(), + inputTagsInputProcessBlock = cms.untracked.VInputTag( cms.InputTag("intProducerBeginProcessBlock"), cms.InputTag("intProducerEndProcessBlock")), + expectedSum = cms.untracked.int32(110000), + expectedCache = cms.untracked.int32(110000) +) + +process.p = cms.Path(process.intProducer * process.a1 * process.a2 * process.a3 * process.a4 * process.a5 * process.a6) + +process.p0 = cms.Path(process.a10 * process.a20 * process.a30 * process.a40 * process.a50 * process.a60 * process.a70) + +process.p00 = cms.Path(process.a100 * process.a200 * process.a300 * process.a400) + +# Cause exception with a module label that does not exist +#process.p1000 = cms.Path(process.a1000) + +# Cause exception where product branch exists but +# product was not put into the event +#process.p1001 = cms.Path(process.a1001) + +# Cause exception where product request is ambiguous +# and process is specified +#process.p1002 = cms.Path(process.a1002) + +# Cause exception where product request is ambiguous +# and process is not specified +#process.p1003 = cms.Path(process.a1003) + +process.p1004 = cms.Path(process.a1004) + +process.p1005 = cms.Path(process.a1005 * process.a1006) + +process.p2000 = cms.Path(process.a2000) + +process.t = cms.Task(process.intProducerU, process.intProducerA, process.nonProducer, + process.intVectorSetProducer, process.intVectorProducer) + +process.e = cms.EndPath(process.out, process.t) diff --git a/FWIO/RNTupleTempTests/test/testGetByLabelStep1_cfg.py b/FWIO/RNTupleTempTests/test/testGetByLabelStep1_cfg.py new file mode 100644 index 0000000000000..24b8d22b5f5b8 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testGetByLabelStep1_cfg.py @@ -0,0 +1,15 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TESTPROD") +process.maxEvents.input = 3 + +process.source = cms.Source("EmptySource") + +process.intProduct = cms.EDProducer("IntProducer", ivalue = cms.int32(42)) + +process.output = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('getbylabel_step1.root') +) + +process.p = cms.Path(process.intProduct) +process.ep = cms.EndPath(process.output) diff --git a/FWIO/RNTupleTempTests/test/testGetByLabelStep2_cfg.py b/FWIO/RNTupleTempTests/test/testGetByLabelStep2_cfg.py new file mode 100644 index 0000000000000..6381979f49adc --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testGetByLabelStep2_cfg.py @@ -0,0 +1,52 @@ +import FWCore.ParameterSet.Config as cms + +import sys +import argparse + +parser = argparse.ArgumentParser(prog=sys.argv[0], description='Test ProcessAccelerator.') + +parser.add_argument("--noConsumes", help="Do not call consumes", action="store_true") +parser.add_argument("--thing", help="Add producer and consumer for Thing", action="store_true") +parser.add_argument("--otherInt", help="Add another producer and consumer for int", action="store_true") + +args = parser.parse_args() + +process = cms.Process("TESTANA") +process.maxEvents.input = -1 + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring("file:getbylabel_step1.root") +) + +process.intAnalyzer = cms.EDAnalyzer("edmtest::TestGetByLabelIntAnalyzer", + src = cms.untracked.InputTag("intProduct"), + consumes = cms.untracked.bool(True) +) + +process.p = cms.Path( + process.intAnalyzer +) + +if args.thing: + process.thingProduct = cms.EDProducer("ThingProducer") + process.thingAnalyzer = cms.EDAnalyzer("edmtest::TestGetByLabelThingAnalyzer", + src = cms.untracked.InputTag("thingProduct"), + consumes = cms.untracked.bool(True) + ) + process.p += (process.thingProduct+process.thingAnalyzer) + +if args.otherInt: + process.otherIntProduct = cms.EDProducer("IntProducer", ivalue = cms.int32(314)) + process.otherIntAnalyzer = cms.EDAnalyzer("edmtest::TestGetByLabelIntAnalyzer", + src = cms.untracked.InputTag("otherIntProduct"), + consumes = cms.untracked.bool(True) + ) + process.p += (process.otherIntProduct+process.otherIntAnalyzer) + +if args.noConsumes: + process.intAnalyzer.consumes = False + process.intAnalyzer.getExceptionCategory = cms.untracked.string("GetByLabelWithoutRegistration") + + if args.thing: + process.thingAnalyzer.consumes = False + process.thingAnalyzer.getExceptionCategory = cms.untracked.string("GetByLabelWithoutRegistration") diff --git a/FWIO/RNTupleTempTests/test/testGetByMerge_cfg.py b/FWIO/RNTupleTempTests/test/testGetByMerge_cfg.py new file mode 100644 index 0000000000000..c815e8c4801de --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testGetByMerge_cfg.py @@ -0,0 +1,33 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testGetBy1.root', + 'file:testGetBy1Mod.root' + ) +) + +process.a1 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("intProducer"), + cms.InputTag("intProducer", processName=cms.InputTag.skipCurrentProcess()) + ), + expectedSum = cms.untracked.int32(27), + runProducerParameterCheck = cms.untracked.bool(True) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testGetByMerge.root'), + outputCommands = cms.untracked.vstring( + 'keep *' + ) +) + +process.intProducer = cms.EDProducer("IntProducer", ivalue = cms.int32(3)) + +process.t = cms.Task(process.intProducer) + +process.p = cms.Path(process.a1, process.t) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testGetByRunsLumisMode_cfg.py b/FWIO/RNTupleTempTests/test/testGetByRunsLumisMode_cfg.py new file mode 100644 index 0000000000000..99b7a401df2af --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testGetByRunsLumisMode_cfg.py @@ -0,0 +1,41 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD2") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testGetBy1.root' + ), + processingMode=cms.untracked.string('RunsAndLumis') +) + +process.intProducerBeginProcessBlock = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(10000)) + +process.intProducerEndProcessBlock = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(100000)) + +process.a1 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag(), + expectedSum = cms.untracked.int32(110000), + inputTagsBeginProcessBlock = cms.untracked.VInputTag( + cms.InputTag("intProducerBeginProcessBlock"), + ), + inputTagsEndProcessBlock = cms.untracked.VInputTag( + cms.InputTag("intProducerEndProcessBlock"), + ), + inputTagsEndProcessBlock2 = cms.untracked.VInputTag( + cms.InputTag("intProducerEndProcessBlock", "two"), + ), + inputTagsEndProcessBlock3 = cms.untracked.VInputTag( + cms.InputTag("intProducerEndProcessBlock", "three"), + ), + inputTagsEndProcessBlock4 = cms.untracked.VInputTag( + cms.InputTag("intProducerEndProcessBlock", "four"), + ) +) + + +process.p1 = cms.Path(process.intProducerBeginProcessBlock + + process.intProducerEndProcessBlock + + process.a1 +) + diff --git a/FWIO/RNTupleTempTests/test/testGetByRunsMode_cfg.py b/FWIO/RNTupleTempTests/test/testGetByRunsMode_cfg.py new file mode 100644 index 0000000000000..2fcec736d593a --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testGetByRunsMode_cfg.py @@ -0,0 +1,45 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD2") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testGetBy1.root' + ), + processingMode=cms.untracked.string('Runs') +) + +process.intProducerBeginProcessBlock = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(10000)) + +process.intProducerEndProcessBlock = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(100000)) + +process.a1 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag(), + expectedSum = cms.untracked.int32(110000), + inputTagsBeginProcessBlock = cms.untracked.VInputTag( + cms.InputTag("intProducerBeginProcessBlock"), + ), + inputTagsEndProcessBlock = cms.untracked.VInputTag( + cms.InputTag("intProducerEndProcessBlock"), + ), + inputTagsEndProcessBlock2 = cms.untracked.VInputTag( + cms.InputTag("intProducerEndProcessBlock", "two"), + ), + inputTagsEndProcessBlock3 = cms.untracked.VInputTag( + cms.InputTag("intProducerEndProcessBlock", "three"), + ), + inputTagsEndProcessBlock4 = cms.untracked.VInputTag( + cms.InputTag("intProducerEndProcessBlock", "four"), + ) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testGetByRunsMode.root') +) + +process.p1 = cms.Path(process.intProducerBeginProcessBlock + + process.intProducerEndProcessBlock + + process.a1 +) + +process.e1 = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testGetByWithEmptyRun_cfg.py b/FWIO/RNTupleTempTests/test/testGetByWithEmptyRun_cfg.py new file mode 100644 index 0000000000000..e895e23ed4705 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testGetByWithEmptyRun_cfg.py @@ -0,0 +1,43 @@ +# The purpose of this is to test the case where a +# file ends with an empty run (no lumis) and we +# go into another file and process things. +# This tests a rarely hit code path in processRuns. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD3") + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = [ + 'file:testGetByRunsMode.root', + 'file:testGetBy1.root' + ], + inputCommands = [ + 'keep *', + 'drop *_*_*_PROD2' + ] +) + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.out = RNTupleTempOutputModule(fileName = 'testGetByWithEmptyRun.root') + +from FWCore.Framework.modules import RunLumiEventAnalyzer +process.test = RunLumiEventAnalyzer( + verbose = True, + expectedRunLumiEvents = [ +1, 0, 0, +1, 0, 0, +1, 0, 0, +1, 1, 0, +1, 1, 1, +1, 1, 2, +1, 1, 3, +1, 1, 0, +1, 0, 0 +] +) + +process.p1 = cms.Path(process.test) + +process.e1 = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testLooperEventNavigation.txt b/FWIO/RNTupleTempTests/test/testLooperEventNavigation.txt new file mode 100644 index 0000000000000..18bd904f0abf8 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testLooperEventNavigation.txt @@ -0,0 +1,36 @@ +0 +2 +1 +1 +15 +1 +0 +2 +2 +1 +5 +2 +2 +1 +3 +0 +1 +q +2 +q +1 +q +1 +q +17 +2 +1000 +1000 +1000 +3 +2 +1 +1 +20 +0 +4 diff --git a/FWIO/RNTupleTempTests/test/testLooperEventNavigation1_cfg.py b/FWIO/RNTupleTempTests/test/testLooperEventNavigation1_cfg.py new file mode 100644 index 0000000000000..647e16a8fa2f1 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testLooperEventNavigation1_cfg.py @@ -0,0 +1,76 @@ +# The only difference between this configuration and +# the one in testLoopEventNavigation_cfg.py is that +# the noEventSort parameter is True instead of False. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = [ + 'file:testRunMerge1.root', + 'file:testRunMerge2.root' + ] + #, processingMode = 'RunsAndLumis' + #, duplicateCheckMode = 'checkEachRealDataFile' + , noEventSort = True + , inputCommands = [ + 'keep *', + 'drop edmtestThingWithMerge_makeThingToBeDropped1_*_*' + ] +) + +from FWCore.Framework.modules import RunLumiEventAnalyzer +process.test = RunLumiEventAnalyzer( + verbose = True + , expectedRunLumiEvents = [ + 1, 0, 0, + 1, 1, 0, + 1, 1, 11, + 1, 1, 12, + 1, 1, 15, + 1, 1, 14, + 1, 1, 15, + 1, 1, 0, + 1, 0, 0, + 2, 0, 0, + 2, 1, 0, + 2, 1, 5, + 2, 1, 3, + 2, 1, 4, + 2, 1, 3, + 2, 1, 0, + 2, 0, 0, + 1, 0, 0, + 1, 1, 0, + 1, 1, 17, + 1, 1, 0, + 1, 0, 0, + 1, 0, 0, + 1, 1, 0, + 1, 1, 11, + 1, 1, 20, + 1, 1, 21, + 1, 1, 0, + 1, 0, 0 + ] +) + +process.looper = cms.Looper("NavigateEventsLooper") + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.out = RNTupleTempOutputModule( + fileName = 'testLooperEventNavigation.root', + fastCloning = False +) + +process.path1 = cms.Path(process.test) +process.endpath1 = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testLooperEventNavigation2.txt b/FWIO/RNTupleTempTests/test/testLooperEventNavigation2.txt new file mode 100644 index 0000000000000..77448094fdfcb --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testLooperEventNavigation2.txt @@ -0,0 +1,24 @@ +0 +0 +0 +2 +3 +1 +2 +0 +0 +0 +0 +2 +1 +1 +1 +2 +3 +1 +1 +2 +2 +1 +2 +4 diff --git a/FWIO/RNTupleTempTests/test/testLooperEventNavigation2_cfg.py b/FWIO/RNTupleTempTests/test/testLooperEventNavigation2_cfg.py new file mode 100644 index 0000000000000..a73d28176e46c --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testLooperEventNavigation2_cfg.py @@ -0,0 +1,41 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMerge.root', + 'file:testProcessBlockMerge2.root' + ) +) + +# 57 transitions = 12 events + 15 access input ProcessBlock transitions + 30 fill calls +# sum 16442 = 2 x (3300 + 4400 + 444) + 3 x (11 + 22 + 44) = 16288 + 231 = +process.readProcessBlocksOneAnalyzer = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(57), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22, 3300, 4400), + expectedSum = cms.int32(16519) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedProcessNamesAtWrite = cms.untracked.vstring('PROD1', 'PROD1', 'MERGE', 'PROD1', 'PROD1', 'MERGE', 'PROD1', 'PROD1', 'MERGE', 'PROD1', 'PROD1', 'MERGE', 'PROD1', 'PROD1', 'MERGE', 'TEST'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(16), + testTTreesInFileBlock = cms.untracked.bool(True), + expectedCacheIndexSize = cms.untracked.vuint32(2, 2, 2, 4, 4, 4, 6, 6, 6, 8, 8, 8, 10, 10, 10) +) + +process.looper = cms.Looper("NavigateEventsLooper") + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('file:testLooperEventNavigation2.root'), + fastCloning = cms.untracked.bool(False) +) + +process.path1 = cms.Path(process.readProcessBlocksOneAnalyzer) +process.endpath1 = cms.EndPath(process.out * process.testOneOutput) diff --git a/FWIO/RNTupleTempTests/test/testLooperEventNavigation3.txt b/FWIO/RNTupleTempTests/test/testLooperEventNavigation3.txt new file mode 100644 index 0000000000000..4567460864833 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testLooperEventNavigation3.txt @@ -0,0 +1,6 @@ +1 +1 +1 +1 +1 +4 diff --git a/FWIO/RNTupleTempTests/test/testLooperEventNavigation3_cfg.py b/FWIO/RNTupleTempTests/test/testLooperEventNavigation3_cfg.py new file mode 100644 index 0000000000000..2e4166abad1a5 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testLooperEventNavigation3_cfg.py @@ -0,0 +1,42 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMerge.root', + 'file:testProcessBlockMerge2.root' + ) + , skipEvents = cms.untracked.uint32(7) +) + +# 24 transitions = 6 events + 6 access input ProcessBlock transitions + 12 fill calls +# sum 8221 = 3300 + 4400 + 444 + 11 + 22 + 44 = 7700 + 444 + 77 = +process.readProcessBlocksOneAnalyzer = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(24), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22, 3300, 4400), + expectedSum = cms.int32(8221) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedProcessNamesAtWrite = cms.untracked.vstring('PROD1', 'PROD1', 'MERGE', 'PROD1', 'PROD1', 'MERGE', 'TEST'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(7), + testTTreesInFileBlock = cms.untracked.bool(True), + expectedCacheIndexSize = cms.untracked.vuint32(2, 2, 2, 4, 4, 4, 4) +) + +process.looper = cms.Looper("NavigateEventsLooper") + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('file:testLooperEventNavigation2.root'), + fastCloning = cms.untracked.bool(False) +) + +process.path1 = cms.Path(process.readProcessBlocksOneAnalyzer) +process.endpath1 = cms.EndPath(process.out * process.testOneOutput) diff --git a/FWIO/RNTupleTempTests/test/testLooperEventNavigation_cfg.py b/FWIO/RNTupleTempTests/test/testLooperEventNavigation_cfg.py new file mode 100644 index 0000000000000..bbab316fa67ee --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testLooperEventNavigation_cfg.py @@ -0,0 +1,67 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = [ + 'file:testRunMerge1.root', + 'file:testRunMerge2.root' + ] + #, processingMode = 'RunsAndLumis' + #, duplicateCheckMode = 'checkEachRealDataFile' + , noEventSort = False + , inputCommands = [ + 'keep *', + 'drop edmtestThingWithMerge_makeThingToBeDropped1_*_*' + ] +) + +from FWCore.Framework.modules import RunLumiEventAnalyzer +process.test = RunLumiEventAnalyzer( + verbose = True + , expectedRunLumiEvents = [ + 1, 0, 0, + 1, 1, 0, + 1, 1, 11, + 1, 1, 12, + 1, 1, 15, + 1, 1, 14, + 1, 1, 15, + 1, 1, 0, + 1, 0, 0, + 2, 0, 0, + 2, 1, 0, + 2, 1, 5, + 2, 1, 3, + 2, 1, 4, + 2, 1, 3, + 2, 1, 0, + 2, 0, 0, + 1, 0, 0, + 1, 1, 0, + 1, 1, 17, + 1, 1, 0, + 1, 0, 0, + 1, 0, 0, + 1, 1, 0, + 1, 1, 11, + 1, 1, 20, + 1, 1, 21, + 1, 1, 0, + 1, 0, 0 + ] +) + +process.looper = cms.Looper("NavigateEventsLooper") + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.out = RNTupleTempOutputModule( + fileName = 'testLooperEventNavigation.root', + fastCloning = False +) + +process.path1 = cms.Path(process.test) +process.endpath1 = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testMaxEventsOutput_cfg.py b/FWIO/RNTupleTempTests/test/testMaxEventsOutput_cfg.py new file mode 100644 index 0000000000000..637f55f243aa1 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testMaxEventsOutput_cfg.py @@ -0,0 +1,41 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") + +process.source = cms.Source("EmptySource", + firstRun = cms.untracked.uint32(1), + firstLuminosityBlock = cms.untracked.uint32(1), + firstEvent = cms.untracked.uint32(1), + numberEventsInLuminosityBlock = cms.untracked.uint32(1), + numberEventsInRun = cms.untracked.uint32(100) +) + +# set to 6 output events. Actual number could be +# anywhere from 6 to 9 because after 6 events written +# it will finish the other events already running concurrently. +# In this config there are 3 other streams possible. +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(20), + output = cms.untracked.int32(6) +) + +process.options = cms.untracked.PSet( + numberOfThreads = cms.untracked.uint32(4), + numberOfStreams = cms.untracked.uint32(4), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(4) +) + +process.busy1 = cms.EDProducer("BusyWaitIntProducer", + ivalue = cms.int32(1), + iterations = cms.uint32(10*1000*1000), + lumiNumberToThrow = cms.uint32(0) +) + +process.p1 = cms.Path(process.busy1) +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testMaxEventsOutput.root') +) + +process.e = cms.EndPath(process.out) + diff --git a/FWIO/RNTupleTempTests/test/testMaybeUninitializedIntProductPart1_cfg.py b/FWIO/RNTupleTempTests/test/testMaybeUninitializedIntProductPart1_cfg.py new file mode 100644 index 0000000000000..592a3bc4fc7e6 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testMaybeUninitializedIntProductPart1_cfg.py @@ -0,0 +1,29 @@ +#! /usr/bin/env cmsRun + +# This configuration is designed to be run as the first in a series of two cmsRun processes. +# It produces a collection of edmtest::MaybeUninitializedIntProduct and stores them to a ROOT file, +# to test that edm::Wrapper can be used to read and write objects without a default constructor. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("Part1") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1 + +process.source = cms.Source("EmptySource") +process.maxEvents.input = 10 + +process.prod = cms.EDProducer("edmtest::MaybeUninitializedIntProducer", + value = cms.int32(42) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testMaybeUninitializedIntProduct.root'), + outputCommands = cms.untracked.vstring( + 'keep *_prod_*_*' + ) +) + +process.path = cms.Path(process.prod) +process.endp = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testMaybeUninitializedIntProductPart2_cfg.py b/FWIO/RNTupleTempTests/test/testMaybeUninitializedIntProductPart2_cfg.py new file mode 100644 index 0000000000000..8e19413b3e7c2 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testMaybeUninitializedIntProductPart2_cfg.py @@ -0,0 +1,23 @@ +#! /usr/bin/env cmsRun + +# This configuration is designed to be run as the second in a series of two cmsRun processes. +# It reads a collection of edmtest::MaybeUninitializedIntProduct from a ROOT file and check their values, +# to test that edm::Wrapper can be used to read and write objects without a default constructor. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("Part2") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1 + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testMaybeUninitializedIntProduct.root') +) + +process.read = cms.EDAnalyzer("edmtest::MaybeUninitializedIntAnalyzer", + source = cms.InputTag("prod"), + value = cms.int32(42) +) + +process.path = cms.Path(process.read) diff --git a/FWIO/RNTupleTempTests/test/testMissingDictionaryChecking_cfg.py b/FWIO/RNTupleTempTests/test/testMissingDictionaryChecking_cfg.py new file mode 100644 index 0000000000000..dc671ed5dec34 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testMissingDictionaryChecking_cfg.py @@ -0,0 +1,34 @@ +# This configuration file tests the code that checks +# for missing ROOT dictionaries. It is intentional +# that the cmsRun job fails with an exception. + +# Note that this only tests one simple case, +# but one could use this as a starting point +# to test other cases by editing the classes_def.xml +# to comment out dictionary definitions or the editing +# the consumes and produces calls in +# MissingDictionaryTestProducer to test the many other +# possible cases in this part of the code. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD") + +process.source = cms.Source("EmptySource") +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testMissingDictionaries.root') +) + +process.a3 = cms.EDProducer("TestMod") + +process.a1 = cms.EDProducer("MissingDictionaryTestProducer", + inputTag = cms.InputTag("a2") +) + +process.p = cms.Path(process.a3 * process.a1) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testNoProcessFallback1_cfg.py b/FWIO/RNTupleTempTests/test/testNoProcessFallback1_cfg.py new file mode 100644 index 0000000000000..588b4400b62b6 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testNoProcessFallback1_cfg.py @@ -0,0 +1,26 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("FALLBACK1") +from FWCore.Modules.modules import EmptySource +process.source = EmptySource( + numberEventsInRun = 8, + numberEventsInLuminosityBlock = 8, +) + +process.maxEvents.input = 8 + +from FWCore.Modules.modules import timestudy_SleepingProducer +process.intProducer = timestudy_SleepingProducer( ivalue = 1, eventTimes = [0] ) + +from FWCore.Modules.modules import EventIDFilter +process.keeper = EventIDFilter( + eventsToPass = [cms.EventID(1,1,2), cms.EventID(1,1,4), cms.EventID(1,1,6), cms.EventID(1,1,8)] +) + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.out = RNTupleTempOutputModule( + fileName = 'testNoProcessFallback1.root', +) + +process.p = cms.Path(process.keeper + process.intProducer ) +process.e = cms.EndPath(process.out) \ No newline at end of file diff --git a/FWIO/RNTupleTempTests/test/testNoProcessFallback2_cfg.py b/FWIO/RNTupleTempTests/test/testNoProcessFallback2_cfg.py new file mode 100644 index 0000000000000..74294d7dd4629 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testNoProcessFallback2_cfg.py @@ -0,0 +1,25 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("FALLBACK2") +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = cms.untracked.vstring( + 'file:testNoProcessFallback1.root', + ) +) + +from FWCore.Modules.modules import timestudy_SleepingProducer +process.intProducer = timestudy_SleepingProducer( ivalue = 2, eventTimes = [0] ) + +from FWCore.Modules.modules import EventIDFilter +process.keeper = EventIDFilter( + eventsToPass = [cms.EventID(1,1,3), cms.EventID(1,1,4), cms.EventID(1,1,7), cms.EventID(1,1,8)] +) + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.out = RNTupleTempOutputModule( + fileName = 'testNoProcessFallback2.root', +) + +process.p = cms.Path(process.keeper + process.intProducer ) +process.e = cms.EndPath(process.out) \ No newline at end of file diff --git a/FWIO/RNTupleTempTests/test/testNoProcessFallback3_cfg.py b/FWIO/RNTupleTempTests/test/testNoProcessFallback3_cfg.py new file mode 100644 index 0000000000000..a05f6d9282fce --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testNoProcessFallback3_cfg.py @@ -0,0 +1,131 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("FALLBACK3") +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = cms.untracked.vstring( + 'file:testNoProcessFallback2.root', + ) +) + +from FWCore.Modules.modules import timestudy_SleepingProducer +process.intProducer = timestudy_SleepingProducer( ivalue = 3, eventTimes = [0] ) + +from FWCore.Modules.modules import EventIDFilter +process.keeper = EventIDFilter( + eventsToPass = [cms.EventID(1,1,5), cms.EventID(1,1,6), cms.EventID(1,1,7), cms.EventID(1,1,8)] +) + +# The matrix of passed Events for the three processes (X is pass, - is fail) +# EventID FALLBACK1 FALLBACK2 FALLBACK3 +# (1,1,1) - - - +# (1,1,2) X - - +# (1,1,3) - X - +# (1,1,4) X X - +# (1,1,5) - - X +# (1,1,6) X - X +# (1,1,7) - X X +# (1,1,8) X X X + + + +process.p = cms.Path(process.keeper + process.intProducer ) + +process.testerNoProcessMissing = cms.EDAnalyzer("BuiltinIntTestAnalyzer", + moduleLabel = cms.untracked.InputTag("intProducer"), + valueMustBeMissing = cms.untracked.bool(True), + valueMustMatch = cms.untracked.int32(1) # Ignored if valueMustBeMissing is true +) +process.testerSkipCurrentProcessMissing = cms.EDAnalyzer("BuiltinIntTestAnalyzer", + moduleLabel = cms.untracked.InputTag("intProducer", processName=cms.InputTag.skipCurrentProcess()), + valueMustBeMissing = cms.untracked.bool(True), + valueMustMatch = cms.untracked.int32(1) # Ignored if valueMustBeMissing is true +) + +process.testerCurrentProcessMissing = cms.EDAnalyzer("BuiltinIntTestAnalyzer", + moduleLabel = cms.untracked.InputTag("intProducer", processName=cms.InputTag.currentProcess()), + valueMustBeMissing = cms.untracked.bool(True), + valueMustMatch = cms.untracked.int32(1) # Ignored if valueMustBeMissing is true +) + + +process.keeper1 = EventIDFilter( + eventsToPass = [cms.EventID(1,1,1)] +) +process.p1 = cms.Path(process.keeper1 + process.testerNoProcessMissing + process.testerSkipCurrentProcessMissing + process.testerCurrentProcessMissing) + +process.testerNoProcess1 = cms.EDAnalyzer("BuiltinIntTestAnalyzer", + moduleLabel = cms.untracked.InputTag("intProducer"), + valueMustMatch = cms.untracked.int32(1), + valueMustBeMissing = cms.untracked.bool(False) +) + +process.testerSkipCurrentProcess1 = cms.EDAnalyzer("BuiltinIntTestAnalyzer", + moduleLabel = cms.untracked.InputTag("intProducer", processName=cms.InputTag.skipCurrentProcess()), + valueMustMatch = cms.untracked.int32(1), + valueMustBeMissing = cms.untracked.bool(False) +) + +process.keeper2 = EventIDFilter( + eventsToPass = [cms.EventID(1,1,2)] +) +process.p2 = cms.Path(process.keeper2 + process.testerNoProcessMissing + process.testerSkipCurrentProcessMissing + process.testerCurrentProcessMissing) + +process.keeper3 = EventIDFilter( + eventsToPass = [cms.EventID(1,1,3)] +) + +process.testerNoProcess2 = cms.EDAnalyzer("BuiltinIntTestAnalyzer", + moduleLabel = cms.untracked.InputTag("intProducer"), + valueMustMatch = cms.untracked.int32(2), + valueMustBeMissing = cms.untracked.bool(False) +) + +process.testerSkipCurrentProcess2 = cms.EDAnalyzer("BuiltinIntTestAnalyzer", + moduleLabel = cms.untracked.InputTag("intProducer", processName=cms.InputTag.skipCurrentProcess()), + valueMustMatch = cms.untracked.int32(2), + valueMustBeMissing = cms.untracked.bool(False) +) + +process.p3 = cms.Path(process.keeper3 + process.testerNoProcessMissing + process.testerSkipCurrentProcess2 + process.testerCurrentProcessMissing) + +process.keeper4 = EventIDFilter( + eventsToPass = [cms.EventID(1,1,4)] +) +process.p4 = cms.Path(process.keeper4 + process.testerNoProcessMissing + process.testerSkipCurrentProcess2 + process.testerCurrentProcessMissing) + +process.testerNoProcess3 = cms.EDAnalyzer("BuiltinIntTestAnalyzer", + moduleLabel = cms.untracked.InputTag("intProducer"), + valueMustMatch = cms.untracked.int32(3), + valueMustBeMissing = cms.untracked.bool(False) +) +process.testerCurrentProcess3 = cms.EDAnalyzer("BuiltinIntTestAnalyzer", + moduleLabel = cms.untracked.InputTag("intProducer", processName=cms.InputTag.currentProcess()), + valueMustBeMissing = cms.untracked.bool(False), + valueMustMatch = cms.untracked.int32(3) +) + + +process.keeper5 = EventIDFilter( + eventsToPass = [cms.EventID(1,1,5)] +) +process.p5 = cms.Path(process.keeper5 + process.testerNoProcess3 + process.testerSkipCurrentProcessMissing + process.testerCurrentProcess3) + +process.keeper6 = EventIDFilter( + eventsToPass = [cms.EventID(1,1,6)] +) +process.p6 = cms.Path(process.keeper6 + process.testerNoProcess3 + process.testerSkipCurrentProcessMissing + process.testerCurrentProcess3) + +process.keeper7 = EventIDFilter( + eventsToPass = [cms.EventID(1,1,7)] +) +process.p7 = cms.Path(process.keeper7 + process.testerNoProcess3 + process.testerSkipCurrentProcess2 + process.testerCurrentProcess3) + +process.keeper8 = EventIDFilter( + eventsToPass = [cms.EventID(1,1,8)] +) +process.p8 = cms.Path(process.keeper8 + process.testerNoProcess3 + process.testerSkipCurrentProcess2 + process.testerCurrentProcess3) + +#from FWCore.Modules.modules import EventContentAnalyzer +#process.dump = EventContentAnalyzer() +#process.dumpPath = cms.Path(process.dump) \ No newline at end of file diff --git a/FWIO/RNTupleTempTests/test/testNoProcessFallbackNoCurrent_cfg.py b/FWIO/RNTupleTempTests/test/testNoProcessFallbackNoCurrent_cfg.py new file mode 100644 index 0000000000000..7eb30940e3f83 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testNoProcessFallbackNoCurrent_cfg.py @@ -0,0 +1,90 @@ +#test fallback logic when current process does not produce the product +import FWCore.ParameterSet.Config as cms + +process = cms.Process("FALLBACK4") +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = cms.untracked.vstring( + 'file:testNoProcessFallback2.root', + ) +) + +process.maxEvents.input = 4 + +# The matrix of passed Events for the three processes (X is pass, - is fail) +# EventID FALLBACK1 FALLBACK2 FALLBACK4 +# (1,1,1) - - - +# (1,1,2) X - - +# (1,1,3) - X - +# (1,1,4) X X - + + + + +process.testerNoProcessMissing = cms.EDAnalyzer("BuiltinIntTestAnalyzer", + moduleLabel = cms.untracked.InputTag("intProducer"), + valueMustBeMissing = cms.untracked.bool(True), + valueMustMatch = cms.untracked.int32(1) # Ignored if valueMustBeMissing is true +) +process.testerSkipCurrentProcessMissing = cms.EDAnalyzer("BuiltinIntTestAnalyzer", + moduleLabel = cms.untracked.InputTag("intProducer", processName=cms.InputTag.skipCurrentProcess()), + valueMustBeMissing = cms.untracked.bool(True), + valueMustMatch = cms.untracked.int32(1) # Ignored if valueMustBeMissing is true +) + +process.testerCurrentProcessMissing = cms.EDAnalyzer("BuiltinIntTestAnalyzer", + moduleLabel = cms.untracked.InputTag("intProducer", processName=cms.InputTag.currentProcess()), + valueMustBeMissing = cms.untracked.bool(True), + valueMustMatch = cms.untracked.int32(1) # Ignored if valueMustBeMissing is true +) + +from FWCore.Modules.modules import EventIDFilter +process.keeper1 = EventIDFilter( + eventsToPass = [cms.EventID(1,1,1)] +) +process.p1 = cms.Path(process.keeper1 + process.testerNoProcessMissing + process.testerSkipCurrentProcessMissing + process.testerCurrentProcessMissing) + +process.testerNoProcess1 = cms.EDAnalyzer("BuiltinIntTestAnalyzer", + moduleLabel = cms.untracked.InputTag("intProducer"), + valueMustMatch = cms.untracked.int32(1), + valueMustBeMissing = cms.untracked.bool(False) +) + +process.testerSkipCurrentProcess1 = cms.EDAnalyzer("BuiltinIntTestAnalyzer", + moduleLabel = cms.untracked.InputTag("intProducer", processName=cms.InputTag.skipCurrentProcess()), + valueMustMatch = cms.untracked.int32(1), + valueMustBeMissing = cms.untracked.bool(False) +) + +process.keeper2 = EventIDFilter( + eventsToPass = [cms.EventID(1,1,2)] +) +process.p2 = cms.Path(process.keeper2 + process.testerNoProcessMissing + process.testerSkipCurrentProcessMissing + process.testerCurrentProcessMissing) + +process.keeper3 = EventIDFilter( + eventsToPass = [cms.EventID(1,1,3)] +) + +process.testerNoProcess2 = cms.EDAnalyzer("BuiltinIntTestAnalyzer", + moduleLabel = cms.untracked.InputTag("intProducer"), + valueMustMatch = cms.untracked.int32(2), + valueMustBeMissing = cms.untracked.bool(False) +) + +process.testerSkipCurrentProcess2 = cms.EDAnalyzer("BuiltinIntTestAnalyzer", + moduleLabel = cms.untracked.InputTag("intProducer", processName=cms.InputTag.skipCurrentProcess()), + valueMustMatch = cms.untracked.int32(2), + valueMustBeMissing = cms.untracked.bool(False) +) + +process.p3 = cms.Path(process.keeper3 + process.testerNoProcess2 + process.testerSkipCurrentProcess2 + process.testerCurrentProcessMissing) + +process.keeper4 = EventIDFilter( + eventsToPass = [cms.EventID(1,1,4)] +) +process.p4 = cms.Path(process.keeper4 + process.testerNoProcess2 + process.testerSkipCurrentProcess2 + process.testerCurrentProcessMissing) + + +#from FWCore.Modules.modules import EventContentAnalyzer +#process.dump = EventContentAnalyzer() +#process.dumpPath = cms.Path(process.dump) \ No newline at end of file diff --git a/FWIO/RNTupleTempTests/test/testOutput1_cfg.py b/FWIO/RNTupleTempTests/test/testOutput1_cfg.py new file mode 100644 index 0000000000000..01eb6074a28c1 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testOutput1_cfg.py @@ -0,0 +1,43 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD1") + +process.source = cms.Source("EmptySource") +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.intProducerA = cms.EDProducer("IntProducer", ivalue = cms.int32(100)) + +process.aliasForInt = cms.EDAlias( + intProducerA = cms.VPSet( + cms.PSet(type = cms.string('edmtestIntProduct') + ) + ) +) + +process.testout = cms.OutputModule("TestGlobalOutput", + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_intProducerA_*_*" + ) +) + +process.testoutlimited = cms.OutputModule("TestLimitedOutput", + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_intProducerA_*_*" + ) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testOutput1.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerA_*_*' + ) +) + +process.path = cms.Path(process.intProducerA) + +process.endpath = cms.EndPath(process.testout + process.testoutlimited + process.out) diff --git a/FWIO/RNTupleTempTests/test/testOutput2_cfg.py b/FWIO/RNTupleTempTests/test/testOutput2_cfg.py new file mode 100644 index 0000000000000..4a16c52d161e2 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testOutput2_cfg.py @@ -0,0 +1,45 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD2") + + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testOutput1.root' + ) +) + +process.intProducerA = cms.EDProducer("IntProducer", ivalue = cms.int32(100)) + +process.aliasForInt = cms.EDAlias( + intProducerA = cms.VPSet( + cms.PSet(type = cms.string('edmtestIntProduct') + ) + ) +) + +process.testout = cms.OutputModule("TestGlobalOutput", + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_intProducerA_*_*" + ) +) + +process.testoutlimited = cms.OutputModule("TestLimitedOutput", + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_intProducerA_*_*" + ) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testOutput2.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerA_*_*' + ) +) + +process.path = cms.Path(process.intProducerA) + +process.endpath = cms.EndPath(process.testout + process.testoutlimited + process.out) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlock1_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlock1_cfg.py new file mode 100644 index 0000000000000..8b8fd6f2f73ba --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlock1_cfg.py @@ -0,0 +1,98 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD1") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("EmptySource") +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlock1.root'), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_testEDAliasOriginal_*_*" + ) +) + +process.testGlobalOutput = cms.OutputModule("TestGlobalOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(1), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_testEDAliasOriginal_*_*" + ) +) + +process.testLimitedOutput = cms.OutputModule("TestLimitedOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(1), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_testEDAliasOriginal_*_*" + ) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(1), + requireNullTTreesInFileBlock = cms.untracked.bool(True), + expectedProductsFromInputKept = cms.untracked.bool(False), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_testEDAliasOriginal_*_*" + ) +) + +process.intProducerBeginProcessBlock = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(1)) + +process.intProducerEndProcessBlock = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(10)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(5)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(50)) + +process.testEDAliasOriginal = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(111)) + +process.testEDAliasAlias = cms.EDAlias(testEDAliasOriginal = cms.VPSet( cms.PSet(type=cms.string('edmtestIntProduct') ) ) ) + +process.a1 = cms.EDAnalyzer("TestFindProduct", + expectedSum = cms.untracked.int32(111), + inputTags = cms.untracked.VInputTag(), + inputTagsBeginProcessBlock = cms.untracked.VInputTag( + cms.InputTag("testEDAliasOriginal") + ) +) + +process.a2 = cms.EDAnalyzer("TestFindProduct", + expectedSum = cms.untracked.int32(111), + inputTags = cms.untracked.VInputTag(), + inputTagsBeginProcessBlock = cms.untracked.VInputTag( + cms.InputTag("testEDAliasAlias") + ) +) + +process.p = cms.Path(process.intProducerBeginProcessBlock * + process.intProducerEndProcessBlock * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB * + process.testEDAliasOriginal * + process.a1 * + process.a2 +) + +process.e = cms.EndPath(process.out * + process.testGlobalOutput * + process.testLimitedOutput * + process.testOneOutput +) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlock2Dropped_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlock2Dropped_cfg.py new file mode 100644 index 0000000000000..7ae62c7425cb9 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlock2Dropped_cfg.py @@ -0,0 +1,74 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD1") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("EmptySource", + firstRun = cms.untracked.uint32(2), + firstLuminosityBlock = cms.untracked.uint32(1), + firstEvent = cms.untracked.uint32(1), + numberEventsInLuminosityBlock = cms.untracked.uint32(1), + numberEventsInRun = cms.untracked.uint32(100) +) + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlock2Dropped.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlock_*_*', + 'drop *_intProducerEndProcessBlock_*_*' + ) +) + +process.testGlobalOutput = cms.OutputModule("TestGlobalOutput", + verbose = cms.untracked.bool(False), + expectedWriteProcessBlockTransitions = cms.untracked.int32(1), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlock_*_*', + 'drop *_intProducerEndProcessBlock_*_*' + ) +) + +process.testLimitedOutput = cms.OutputModule("TestLimitedOutput", + verbose = cms.untracked.bool(False), + expectedWriteProcessBlockTransitions = cms.untracked.int32(1), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlock_*_*', + 'drop *_intProducerEndProcessBlock_*_*' + ) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedWriteProcessBlockTransitions = cms.untracked.int32(1), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlock_*_*', + 'drop *_intProducerEndProcessBlock_*_*' + ), + expectedProductsFromInputKept = cms.untracked.bool(False) +) + +process.intProducerBeginProcessBlock = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(1)) + +process.intProducerEndProcessBlock = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(10)) + +process.p = cms.Path(process.intProducerBeginProcessBlock * + process.intProducerEndProcessBlock) + +process.e = cms.EndPath(process.out * + process.testGlobalOutput * + process.testLimitedOutput * + process.testOneOutput +) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlock2_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlock2_cfg.py new file mode 100644 index 0000000000000..3a8d7d2972bd7 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlock2_cfg.py @@ -0,0 +1,45 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD1") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("EmptySource", + firstRun = cms.untracked.uint32(2) +) +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlock2.root'), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_testEDAliasOriginal_*_*" + ) +) + +process.intProducerBeginProcessBlock = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(2)) + +process.intProducerEndProcessBlock = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(20)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(7)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(70)) + +process.testEDAliasOriginal = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(111)) + +process.testEDAliasAlias = cms.EDAlias(testEDAliasOriginal = cms.VPSet( cms.PSet(type=cms.string('edmtestIntProduct') ) ) ) + +process.p = cms.Path(process.intProducerBeginProcessBlock * + process.intProducerEndProcessBlock * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB * + process.testEDAliasOriginal +) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlock3_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlock3_cfg.py new file mode 100644 index 0000000000000..04f6bbc42058d --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlock3_cfg.py @@ -0,0 +1,36 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD1") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("EmptySource", + firstRun = cms.untracked.uint32(3) +) +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlock3.root') +) + +process.intProducerBeginProcessBlock = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(300)) + +process.intProducerEndProcessBlock = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(3000)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(30000)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(300000)) + +process.p = cms.Path(process.intProducerBeginProcessBlock * + process.intProducerEndProcessBlock * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB +) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlock4_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlock4_cfg.py new file mode 100644 index 0000000000000..8ed203d6e33a3 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlock4_cfg.py @@ -0,0 +1,36 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD1") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("EmptySource", + firstRun = cms.untracked.uint32(4) +) +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlock4.root') +) + +process.intProducerBeginProcessBlock = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(400)) + +process.intProducerEndProcessBlock = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(4000)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(40000)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(400000)) + +process.p = cms.Path(process.intProducerBeginProcessBlock * + process.intProducerEndProcessBlock * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB +) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlock5_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlock5_cfg.py new file mode 100644 index 0000000000000..058ef30b5f100 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlock5_cfg.py @@ -0,0 +1,37 @@ +import FWCore.ParameterSet.Config as cms + +# Intentionally reversing the order of process names in this series +process = cms.Process("PROD1") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("EmptySource", + firstRun = cms.untracked.uint32(5) +) +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlock5.root') +) + +process.intProducerBeginProcessBlock = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(707)) + +process.intProducerEndProcessBlock = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(7000)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(40000)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(400000)) + +process.p = cms.Path(process.intProducerBeginProcessBlock * + process.intProducerEndProcessBlock * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB +) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockDropOnInput_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockDropOnInput_cfg.py new file mode 100644 index 0000000000000..69321d69f008c --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockDropOnInput_cfg.py @@ -0,0 +1,82 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READ") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMergeOfMergedFiles.root', + 'file:testProcessBlockMergeOfMergedFiles2.root' + ), + inputCommands=cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlockB_*_*', + 'drop *_intProducerEndProcessBlockB_*_*', + 'drop *_intProducerBeginProcessBlockM_*_*', + 'drop *_intProducerEndProcessBlockM_*_*' + ) +) + +process.readProcessBlocksOneAnalyzer1 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(37), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22, 3300, 4400, 7707), + expectedSum = cms.int32(15440), + consumesProcessBlockNotFound1 = cms.InputTag("intProducerBeginProcessBlockB"), + consumesProcessBlockNotFound2 = cms.InputTag("intProducerEndProcessBlockB"), + consumesProcessBlockNotFound3 = cms.InputTag("intProducerBeginProcessBlockM"), + consumesProcessBlockNotFound4 = cms.InputTag("intProducerEndProcessBlockM") +) + +process.readProcessBlocksOneAnalyzer2 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(28), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockMM", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(644, 644, 644, 644, 844), + expectedSum = cms.int32(0), + consumesProcessBlockNotFound1 = cms.InputTag("intProducerBeginProcessBlockB"), + consumesProcessBlockNotFound2 = cms.InputTag("intProducerEndProcessBlockB"), + consumesProcessBlockNotFound3 = cms.InputTag("intProducerBeginProcessBlockM"), + consumesProcessBlockNotFound4 = cms.InputTag("intProducerEndProcessBlockM") +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockDropOnInput.root'), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_*_READ" + ) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGEOFMERGED'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGEOFMERGED'), + expectedTopAddedProcesses = cms.untracked.vstring(), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 4, 1, 4, 2, 4, 3, 4), + expectedTopCacheIndices1 = cms.untracked.vuint32(0, 4, 1, 4, 2, 4, 3, 4, 5, 6), + expectedWriteProcessBlockTransitions = cms.untracked.int32(8), + expectedProcessesInFirstFile = cms.untracked.uint32(2), + expectedCacheIndexVectorsPerFile = cms.untracked.vuint32(4), + expectedNEntries0 = cms.untracked.vuint32(4, 1), + expectedCacheEntriesPerFile0 = cms.untracked.vuint32(5) +) + + +process.p = cms.Path(process.readProcessBlocksOneAnalyzer1 * process.readProcessBlocksOneAnalyzer2) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockDropOnOutput2_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockDropOnOutput2_cfg.py new file mode 100644 index 0000000000000..eca85c75e5288 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockDropOnOutput2_cfg.py @@ -0,0 +1,92 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("DROPONOUTPUT") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMergeOfMergedFiles.root', + 'file:testProcessBlockMergeOfMergedFiles2.root' + ), +) + +process.intProducerBeginProcessBlockN = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(5)) + +process.intProducerEndProcessBlockN = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(50)) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockDropOnOutput2.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlockB_*_*', + 'drop *_intProducerEndProcessBlockB_*_*', + 'drop *_intProducerBeginProcessBlockM_*_*', + 'drop *_intProducerEndProcessBlockM_*_*' + ) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlockB_*_*', + 'drop *_intProducerEndProcessBlockB_*_*', + 'drop *_intProducerBeginProcessBlockM_*_*', + 'drop *_intProducerEndProcessBlockM_*_*' + ), + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGEOFMERGED', 'DROPONOUTPUT'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED', 'DROPONOUTPUT'), + expectedTranslateFromStoredIndex = cms.untracked.vuint32(0, 2, 3), + expectedNAddedProcesses = cms.untracked.uint32(1), + expectedProductsFromInputKept = cms.untracked.bool(True) +) + +process.out2 = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockDropOnOutput2_2.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlock_*_*', + 'drop *_intProducerEndProcessBlock_*_*', + 'drop *_intProducerBeginProcessBlockB_*_*', + 'drop *_intProducerEndProcessBlockB_*_*', + 'drop *_intProducerBeginProcessBlockM_*_*', + 'drop *_intProducerEndProcessBlockM_*_*', + 'drop *_intProducerBeginProcessBlockMM_*_*', + 'drop *_intProducerEndProcessBlockMM_*_*' + ) +) + +process.testOneOutput2 = cms.OutputModule("TestOneOutput", + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlock_*_*', + 'drop *_intProducerEndProcessBlock_*_*', + 'drop *_intProducerBeginProcessBlockB_*_*', + 'drop *_intProducerEndProcessBlockB_*_*', + 'drop *_intProducerBeginProcessBlockM_*_*', + 'drop *_intProducerEndProcessBlockM_*_*', + 'drop *_intProducerBeginProcessBlockMM_*_*', + 'drop *_intProducerEndProcessBlockMM_*_*' + ), + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('DROPONOUTPUT'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED', 'DROPONOUTPUT'), + expectedTranslateFromStoredIndex = cms.untracked.vuint32(3), + expectedNAddedProcesses = cms.untracked.uint32(1), + expectedProductsFromInputKept = cms.untracked.bool(False) +) + +process.p = cms.Path(process.intProducerBeginProcessBlockN * process.intProducerEndProcessBlockN) + +process.e = cms.EndPath( + process.out * + process.testOneOutput * + process.out2 * + process.testOneOutput2 +) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockDropOnOutput_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockDropOnOutput_cfg.py new file mode 100644 index 0000000000000..feafc571df6bc --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockDropOnOutput_cfg.py @@ -0,0 +1,53 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("DROPONOUTPUT") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMergeOfMergedFiles.root', + 'file:testProcessBlockMergeOfMergedFiles2.root' + ), +) + + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockDropOnOutput.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlockB_*_*', + 'drop *_intProducerEndProcessBlockB_*_*', + 'drop *_intProducerBeginProcessBlockM_*_*', + 'drop *_intProducerEndProcessBlockM_*_*' + ) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlockB_*_*', + 'drop *_intProducerEndProcessBlockB_*_*', + 'drop *_intProducerBeginProcessBlockM_*_*', + 'drop *_intProducerEndProcessBlockM_*_*' + ), + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGEOFMERGED'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTranslateFromStoredIndex = cms.untracked.vuint32(0, 2), + expectedNAddedProcesses = cms.untracked.uint32(0), + expectedProductsFromInputKept = cms.untracked.bool(True), + expectedTopAddedProcesses = cms.untracked.vstring(), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6), + expectedTopCacheIndices1 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6, 7, 8, 9) +) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockDummy_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockDummy_cfg.py new file mode 100644 index 0000000000000..79fb51debf33d --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockDummy_cfg.py @@ -0,0 +1,27 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("DUMMY") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlock5.root' + ) +) + +process.dummy = cms.EDProducer("IntProducer", ivalue = cms.int32(1)) + +process.p = cms.Path(process.dummy) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockDummy.root'), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_*_DUMMY") +) +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockFailMerge_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockFailMerge_cfg.py new file mode 100644 index 0000000000000..6df1d9a1ca689 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockFailMerge_cfg.py @@ -0,0 +1,32 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +# The second file has ProcessBlock products dropped that +# were in the first file so this should fail the strict +# merge requirements on ProcessBlock products. +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlock1.root', + 'file:testProcessBlock2Dropped.root' + ) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockFailMerge.root') +) + +process.intProducerBeginProcessBlock = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(2)) + +process.intProducerEndProcessBlock = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(20)) + +process.p = cms.Path(process.intProducerBeginProcessBlock * + process.intProducerEndProcessBlock) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockLooper_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockLooper_cfg.py new file mode 100644 index 0000000000000..fccac126a2e86 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockLooper_cfg.py @@ -0,0 +1,75 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") + +process.essource = cms.ESSource("EmptyESSource", + recordName = cms.string('DummyRecord'), + iovIsRunNotTime = cms.bool(True), + firstValid = cms.vuint32(1) +) +process.add_(cms.ESProducer("LoadableDummyProvider", + value = cms.untracked.int32(5))) + +process.looper = cms.Looper("IntTestLooper", + expectESValue = cms.untracked.int32(5) +) + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMergeOfMergedFiles.root' + ) +) + +process.intProducerBeginProcessBlockT = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(4000)) + +process.intProducerEndProcessBlockT = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(40000)) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockLooperTest.root'), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED', 'TEST'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(24), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +process.eventIntProducer = cms.EDProducer("IntProducer", ivalue = cms.int32(1)) + +process.transientIntProducerEndProcessBlock = cms.EDProducer("TransientIntProducerEndProcessBlock", + ivalue = cms.int32(90) +) + +process.nonEventIntProducer = cms.EDProducer("NonEventIntProducer", + ivalue = cms.int32(1) +) + +process.p = cms.Path( + process.eventIntProducer * + process.transientIntProducerEndProcessBlock * + process.nonEventIntProducer * + process.intProducerBeginProcessBlockT * + process.intProducerEndProcessBlockT +) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockMerge2_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockMerge2_cfg.py new file mode 100644 index 0000000000000..cf1747a165db4 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockMerge2_cfg.py @@ -0,0 +1,59 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) + #fileMode = cms.untracked.string('NOMERGE') +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlock3.root', + 'file:testProcessBlock4.root' + ) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockMerge2.root') +) + +process.testGlobalOutput = cms.OutputModule("TestGlobalOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(3) +) + +process.testLimitedOutput = cms.OutputModule("TestLimitedOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(3) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(3) +) + +process.intProducerBeginProcessBlockM = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(204)) + +process.intProducerEndProcessBlockM = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(240)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(208)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(280)) + +process.p = cms.Path(process.intProducerBeginProcessBlockM * + process.intProducerEndProcessBlockM * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB +) + +process.e = cms.EndPath(process.out * + process.testGlobalOutput * + process.testLimitedOutput * + process.testOneOutput +) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockMerge3_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockMerge3_cfg.py new file mode 100644 index 0000000000000..87feaa40c250c --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockMerge3_cfg.py @@ -0,0 +1,43 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockDummy.root' + ) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockMerge3.root') +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(2) +) + +process.intProducerBeginProcessBlockM = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(4)) + +process.intProducerEndProcessBlockM = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(40)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(8)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(80)) + +process.p = cms.Path(process.intProducerBeginProcessBlockM * + process.intProducerEndProcessBlockM * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB +) + +process.e = cms.EndPath(process.out * + process.testOneOutput +) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockMergeOfMergedFiles2_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockMergeOfMergedFiles2_cfg.py new file mode 100644 index 0000000000000..ee692c39e8a6a --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockMergeOfMergedFiles2_cfg.py @@ -0,0 +1,37 @@ +import FWCore.ParameterSet.Config as cms + +# Intentionally reversing the order of process names in this series +process = cms.Process("MERGEOFMERGED") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMerge3.root' + ) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockMergeOfMergedFiles2.root') +) + +#NEW +process.intProducerBeginProcessBlockMM = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(404)) + +process.intProducerEndProcessBlockMM = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(440)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(308)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(380)) + +process.p = cms.Path(process.intProducerBeginProcessBlockMM * + process.intProducerEndProcessBlockMM * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB +) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockMergeOfMergedFiles_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockMergeOfMergedFiles_cfg.py new file mode 100644 index 0000000000000..c4fde0d1b53d6 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockMergeOfMergedFiles_cfg.py @@ -0,0 +1,95 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGEOFMERGED") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMerge.root', + 'file:testProcessBlockMerge2.root' + ) +) + +# In the next module, the expectedSum is a sum of values from previous +# processes as follow: +# +# testProcessBlock1_cfg.py intProducerBeginProcessBlock 1 +# intProducerEndProcessBlock 10 +# testProcessBlock2_cfg.py intProducerBeginProcessBlock 2 +# intProducerEndProcessBlock 20 +# testProcessBlock3_cfg.py intProducerBeginProcessBlock 300 +# intProducerEndProcessBlock 3000 +# testProcessBlock4_cfg.py intProducerBeginProcessBlock 400 +# intProducerEndProcessBlock 4000 +# testProcessBlockMerge_cfg.py intProducerBeginProcessBlockM 4 +# intProducerBeginProcessBlockM 40 +# testProcessBlockMerge2_cfg.py intProducerBeginProcessBlockM 204 +# intProducerBeginProcessBlockM 240 +# +# TOTAL 8221 + +process.readProcessBlocksOneAnalyzer = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(30), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22, 3300, 4400), + expectedSum = cms.int32(8221) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockMergeOfMergedFiles.root') +) + +process.testGlobalOutput = cms.OutputModule("TestGlobalOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(7) +) + +process.testLimitedOutput = cms.OutputModule("TestLimitedOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(7) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopAddedProcesses = cms.untracked.vstring('MERGEOFMERGED'), + expectedProcessNamesAtWrite = cms.untracked.vstring('PROD1', 'PROD1', 'MERGE', 'PROD1', 'PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(7), + testTTreesInFileBlock = cms.untracked.bool(True), + expectedCacheIndexSize = cms.untracked.vuint32(2, 2, 2, 4, 4, 4, 4), + expectedNAddedProcesses = cms.untracked.uint32(1), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 2, 1, 2), + expectedTopCacheIndices1 = cms.untracked.vuint32(0, 2, 1, 2, 3, 5, 4, 5) +) + +process.intProducerBeginProcessBlockMM = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(304)) + +process.intProducerEndProcessBlockMM = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(340)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(308)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(380)) + +process.p = cms.Path(process.intProducerBeginProcessBlockMM * + process.intProducerEndProcessBlockMM * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB * + process.readProcessBlocksOneAnalyzer +) + +process.e = cms.EndPath(process.out * + process.testGlobalOutput * + process.testLimitedOutput * + process.testOneOutput +) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockMerge_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockMerge_cfg.py new file mode 100644 index 0000000000000..52ba3b1ddc284 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockMerge_cfg.py @@ -0,0 +1,87 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlock1.root', + 'file:testProcessBlock2.root' + ) +) + +# transitions 14 = 6 events + 2 InputProcessBlock transitions + 3 x 2 cache filling calls +process.readProcessBlocksOneAnalyzer1 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(14), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22, 3300, 4400, 7700), + expectedSum = cms.int32(33) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockMerge.root'), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_testEDAliasAlias_*_*" + ) +) + +process.testGlobalOutput = cms.OutputModule("TestGlobalOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(3) +) + +process.testLimitedOutput = cms.OutputModule("TestLimitedOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(3) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedTopAddedProcesses = cms.untracked.vstring('MERGE'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(3), + expectedNAddedProcesses = cms.untracked.uint32(1), + expectedTopCacheIndices0 = cms.untracked.vuint32(0), + expectedTopCacheIndices1 = cms.untracked.vuint32(0, 1) +) + +process.intProducerBeginProcessBlockM = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(4)) + +process.intProducerEndProcessBlockM = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(40)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(8)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(80)) + +process.a2000 = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag(), + inputTagsInputProcessBlock = cms.untracked.VInputTag( cms.InputTag("testEDAliasAlias")), + expectedSum = cms.untracked.int32(222), + expectedCache = cms.untracked.int32(111) +) + +process.p = cms.Path(process.intProducerBeginProcessBlockM * + process.intProducerEndProcessBlockM * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB * + process.readProcessBlocksOneAnalyzer1 * + process.a2000 +) + +process.e = cms.EndPath(process.out * + process.testGlobalOutput * + process.testLimitedOutput * + process.testOneOutput +) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockNOMergeOfMergedFiles_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockNOMergeOfMergedFiles_cfg.py new file mode 100644 index 0000000000000..9275ed557964e --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockNOMergeOfMergedFiles_cfg.py @@ -0,0 +1,93 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGEOFMERGED") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1), + fileMode = cms.untracked.string('NOMERGE') +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMerge.root', + 'file:testProcessBlockMerge2.root' + ) +) + +# In the next module, the expectedSum is a sum of values from previous +# processes as follow: +# +# testProcessBlock1_cfg.py intProducerBeginProcessBlock 1 +# intProducerEndProcessBlock 10 +# testProcessBlock2_cfg.py intProducerBeginProcessBlock 2 +# intProducerEndProcessBlock 20 +# testProcessBlock3_cfg.py intProducerBeginProcessBlock 300 +# intProducerEndProcessBlock 3000 +# testProcessBlock4_cfg.py intProducerBeginProcessBlock 400 +# intProducerEndProcessBlock 4000 +# testProcessBlockMerge_cfg.py intProducerBeginProcessBlockM 4 +# intProducerBeginProcessBlockM 40 +# testProcessBlockMerge2_cfg.py intProducerBeginProcessBlockM 204 +# intProducerBeginProcessBlockM 240 +# +# TOTAL 8221 + +#must use skipCurrentProcess as all ProcessBlocks use the same lookups and +# the current process contains a module with the same label that makes the +# same product as is found in the 'MERGE' process block +process.readProcessBlocksOneAnalyzer = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(30), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", "", cms.InputTag.skipCurrentProcess()), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", "", cms.InputTag.skipCurrentProcess()), + expectedByRun = cms.vint32(11, 22, 3300, 4400), + expectedSum = cms.int32(8221) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockNOMergeOfMergedFiles.root') +) + +process.testGlobalOutput = cms.OutputModule("TestGlobalOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(8) +) + +process.testLimitedOutput = cms.OutputModule("TestLimitedOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(8) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(8), + testTTreesInFileBlock = cms.untracked.bool(True), + expectedCacheIndexSize = cms.untracked.vuint32(2, 2, 2, 2, 2, 2, 2, 2) +) + +process.intProducerBeginProcessBlockM = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(304)) + +process.intProducerEndProcessBlockM = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(340)) + +process.intProducerBeginProcessBlockB = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(308)) + +process.intProducerEndProcessBlockB = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(380)) + +process.p = cms.Path(process.intProducerBeginProcessBlockM * + process.intProducerEndProcessBlockM * + process.intProducerBeginProcessBlockB * + process.intProducerEndProcessBlockB * + process.readProcessBlocksOneAnalyzer +) + +process.e = cms.EndPath(process.out * + process.testGlobalOutput * + process.testLimitedOutput * + process.testOneOutput +) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockNonStrict2_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockNonStrict2_cfg.py new file mode 100644 index 0000000000000..77edad8099829 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockNonStrict2_cfg.py @@ -0,0 +1,83 @@ +# This is a test that things will run OK if we disable the +# the strict merging requirement for ProcessBlock products +# in the ProductRegistry merging function. This +# requirement is currently always enforced and this configuration +# will fail. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("NONSTRICTTEST2") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMergeOfMergedFiles.root', + 'file:testProcessBlockMergeOfMergedFiles2.root', + 'file:testProcessBlockNonStrict.root' + ), + duplicateCheckMode = cms.untracked.string("noDuplicateCheck"), + inputCommands=cms.untracked.vstring( + 'keep *', + 'drop *_*_*_NONSTRICTTEST' + ) +) + +# 117 transitions = 45 events + 27 InputProcessBlock transitions + (3 x 15) Cache filling transitions +# sum = (11 + 22 + 3300 + 4400 + 7707) x 3 + (44 + 44 + 444) x 2 = 47384 +process.readProcessBlocksOneAnalyzer1 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(117), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22, 3300, 4400, 7707), + expectedSum = cms.int32(47384) +) + +# 90 transitions = 45 events + 27 InputProcessBlock transitions + (3 x 6) Cache filling transitions +# sum = (44 + 44 + 444) x 2 = 1064 +process.readProcessBlocksOneAnalyzer2 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(90), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockMM", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(644, 644, 644, 644, 844), + expectedSum = cms.int32(1064) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockNonStrict2.root') +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6), + expectedTopCacheIndices1 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6, 7, 8, 9), + expectedTopCacheIndices2 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6, 7, 8, 9, 10, 20, 23, 11, 20, 23, 12, 21, 23, 13, 21, 23, 14, 22, 24, 15, 4294967295, 25, 16, 4294967295, 25, 17, 4294967295, 25, 18, 4294967295, 25, 19, 4294967295, 26), + expectedWriteProcessBlockTransitions = cms.untracked.int32(28), + expectedProcessesInFirstFile = cms.untracked.uint32(3), + expectedCacheIndexVectorsPerFile = cms.untracked.vuint32(4, 1, 10), + expectedNEntries0 = cms.untracked.vuint32(4, 2, 1), + expectedNEntries1 = cms.untracked.vuint32(1, 1, 1), + expectedNEntries2 = cms.untracked.vuint32(10, 3, 4), + expectedCacheEntriesPerFile0 = cms.untracked.vuint32(7), + expectedCacheEntriesPerFile1 = cms.untracked.vuint32(7, 3), + expectedCacheEntriesPerFile2 = cms.untracked.vuint32(7, 3, 17), + expectedOuterOffset = cms.untracked.vuint32(0, 4, 5) +) + +process.p = cms.Path(process.readProcessBlocksOneAnalyzer1 * process.readProcessBlocksOneAnalyzer2) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockNonStrict3_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockNonStrict3_cfg.py new file mode 100644 index 0000000000000..371bc0590b1cf --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockNonStrict3_cfg.py @@ -0,0 +1,86 @@ +# This is a test that things will run OK if we disable the +# the strict merging requirement for ProcessBlock products +# in the ProductRegistry merging function. This +# requirement is currently always enforced and this configuration +# will fail. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("NONSTRICTTEST") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMergeOfMergedFiles.root', + 'file:testProcessBlockMergeOfMergedFiles2.root', + 'file:testProcessBlockDropOnInput.root' + ), + duplicateCheckMode = cms.untracked.string("noDuplicateCheck"), + inputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_intProducerBeginProcessBlock_*_*', + 'drop *_intProducerEndProcessBlock_*_*', + 'drop *_intProducerBeginProcessBlockMM_*_*', + 'drop *_intProducerEndProcessBlockMM_*_*' + ) +) + +# 40 transitions = 30 events + 10 InputProcessBlock transitions + (3 x 0) Cache filling transitions +# sum = 44 + 444 + 44 = 532 +process.readProcessBlocksOneAnalyzer1 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(40), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + #expectedByRun = cms.vint32(11, 22, 3300, 4400, 7707), + expectedSum = cms.int32(532) +) + +# 40 transitions = 30 events + 10 InputProcessBlock transitions + (3 x 0) Cache filling transitions +# sum = 44 + 444 + 44 = 532 +process.readProcessBlocksOneAnalyzer2 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(40), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockMM", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + #expectedByRun = cms.vint32(644, 644, 644, 644, 844), + expectedSum = cms.int32(532) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockNonStrict3.root') +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6), + expectedTopCacheIndices1 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6, 7, 8, 9), + expectedTopCacheIndices2 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6, 7, 8, 9, 4294967295, 4294967295, 4294967295), + expectedWriteProcessBlockTransitions = cms.untracked.int32(11), + expectedProcessesInFirstFile = cms.untracked.uint32(3), + expectedCacheIndexVectorsPerFile = cms.untracked.vuint32(4, 1, 1), + expectedNEntries0 = cms.untracked.vuint32(4, 2, 1), + expectedNEntries1 = cms.untracked.vuint32(1, 1, 1), + expectedNEntries2 = cms.untracked.vuint32(0, 0, 0), + expectedCacheEntriesPerFile0 = cms.untracked.vuint32(7), + expectedCacheEntriesPerFile1 = cms.untracked.vuint32(7, 3), + expectedCacheEntriesPerFile2 = cms.untracked.vuint32(7, 3, 0), + expectedOuterOffset = cms.untracked.vuint32(0, 4, 5) +) + +process.p = cms.Path(process.readProcessBlocksOneAnalyzer1 * process.readProcessBlocksOneAnalyzer2) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockNonStrict_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockNonStrict_cfg.py new file mode 100644 index 0000000000000..305bd153d4889 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockNonStrict_cfg.py @@ -0,0 +1,79 @@ +# This is a test that things will run OK if we disable the +# the strict merging requirement for ProcessBlock products +# in the ProductRegistry merging function. This +# requirement is currently always enforced and this configuration +# will fail. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("NONSTRICTTEST") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMergeOfMergedFiles.root', + 'file:testProcessBlockMergeOfMergedFiles2.root', + 'file:testProcessBlockDropOnInput.root' + ), + duplicateCheckMode = cms.untracked.string("noDuplicateCheck") +) + +# 77 transitions = 30 events + 17 InputProcessBlock transitions + (3 x 10) Cache filling transitions +# sum = (11 + 22 + 3300 + 4400 + 7707) x 2 + 44 + 444 + 44 = 31412 +process.readProcessBlocksOneAnalyzer1 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(77), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22, 3300, 4400, 7707), + expectedSum = cms.int32(31412) +) + +# 59 transitions = 30 events + 17 InputProcessBlock transitions + (3 x 4) Cache filling transitions +# sum = 44 + 444 + 44 = 532 +process.readProcessBlocksOneAnalyzer2 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(59), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockMM", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(644, 644, 644, 644, 844), + expectedSum = cms.int32(532) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockNonStrict.root') +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6), + expectedTopCacheIndices1 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6, 7, 8, 9), + expectedTopCacheIndices2 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6, 7, 8, 9, 10, 4294967295, 15, 11, 4294967295, 15, 12, 4294967295, 15, 13, 4294967295, 15, 14, 4294967295, 16), + expectedWriteProcessBlockTransitions = cms.untracked.int32(18), + expectedProcessesInFirstFile = cms.untracked.uint32(3), + expectedCacheIndexVectorsPerFile = cms.untracked.vuint32(4, 1, 5), + expectedNEntries0 = cms.untracked.vuint32(4, 2, 1), + expectedNEntries1 = cms.untracked.vuint32(1, 1, 1), + expectedNEntries2 = cms.untracked.vuint32(5, 0, 2), + expectedCacheEntriesPerFile0 = cms.untracked.vuint32(7), + expectedCacheEntriesPerFile1 = cms.untracked.vuint32(7, 3), + expectedCacheEntriesPerFile2 = cms.untracked.vuint32(7, 3, 7), + expectedOuterOffset = cms.untracked.vuint32(0, 4, 5) +) + +process.p = cms.Path(process.readProcessBlocksOneAnalyzer1 * process.readProcessBlocksOneAnalyzer2) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockRead2_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockRead2_cfg.py new file mode 100644 index 0000000000000..8764ad7dde981 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockRead2_cfg.py @@ -0,0 +1,51 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READ") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMergeOfMergedFiles.root' + ) +) + +process.readProcessBlocksOneAnalyzer = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(31), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22, 3300, 4400), + expectedSum = cms.int32(8221) +) + +process.transientIntProducerEndProcessBlock = cms.EDProducer("TransientIntProducerEndProcessBlock", + ivalue = cms.int32(90) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockRead2.root') +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopAddedProcesses = cms.untracked.vstring(), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6), + expectedWriteProcessBlockTransitions = cms.untracked.int32(8) +) + + +process.p = cms.Path(process.transientIntProducerEndProcessBlock * process.readProcessBlocksOneAnalyzer) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockReadDropOnOutput2_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockReadDropOnOutput2_cfg.py new file mode 100644 index 0000000000000..1a4cb8f72e3f9 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockReadDropOnOutput2_cfg.py @@ -0,0 +1,42 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READ") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockDropOnOutput2.root' + ) +) + +# 38 = 15 events + 8 access input ProcessBlock transitions + 15 fillCache functor calls +# sum 15440 = 11 + 22 + 3300 + 4400 + 7707 +process.readProcessBlocksOneAnalyzer = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(38), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22, 3300, 4400, 7707), + expectedSum = cms.int32(15440) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockReadDropOnOutput2.root') +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGEOFMERGED', 'DROPONOUTPUT'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGEOFMERGED', 'DROPONOUTPUT'), + expectedTopAddedProcesses = cms.untracked.vstring(), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 5, 7, 1, 5, 7, 2, 5, 7, 3, 5, 7, 4, 6, 7), + expectedWriteProcessBlockTransitions = cms.untracked.int32(9) +) + + +process.p = cms.Path(process.readProcessBlocksOneAnalyzer) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockReadDropOnOutput_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockReadDropOnOutput_cfg.py new file mode 100644 index 0000000000000..aa0505bc3765a --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockReadDropOnOutput_cfg.py @@ -0,0 +1,42 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READ") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockDropOnOutput.root' + ) +) + +# 37 = 15 events + 7 access input ProcessBlock transitions + 15 fillCache functor calls +# sum 15440 = 11 + 22 + 3300 + 4400 + 7707 +process.readProcessBlocksOneAnalyzer = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(37), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22, 3300, 4400, 7707), + expectedSum = cms.int32(15440) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockReadDropOnOutput.root') +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGEOFMERGED'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGEOFMERGED'), + expectedTopAddedProcesses = cms.untracked.vstring(), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 5, 1, 5, 2, 5, 3, 5, 4, 6), + expectedWriteProcessBlockTransitions = cms.untracked.int32(8) +) + + +process.p = cms.Path(process.readProcessBlocksOneAnalyzer) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockReadThreeFileInput_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockReadThreeFileInput_cfg.py new file mode 100644 index 0000000000000..d47adddcf57b3 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockReadThreeFileInput_cfg.py @@ -0,0 +1,55 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READ") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockThreeFileInput.root' + ), + duplicateCheckMode = cms.untracked.string("noDuplicateCheck") +) + +process.readProcessBlocksOneAnalyzer1 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(71), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22, 3300, 4400, 7707), + expectedSum = cms.int32(24193) +) + +process.readProcessBlocksOneAnalyzer2 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(53), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockMM", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(644, 644, 644, 644, 844), + expectedSum = cms.int32(1020) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockReadThreeFileInput.root') +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopAddedProcesses = cms.untracked.vstring(), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 9, 14, 1, 9, 14, 2, 10, 14, 3, 10, 14, 4, 11, 15, 5, 12, 16, 6, 12, 16, 7, 13, 16, 8, 13, 16), + expectedWriteProcessBlockTransitions = cms.untracked.int32(18), + expectedProcessesInFirstFile = cms.untracked.uint32(3), + expectedCacheIndexVectorsPerFile = cms.untracked.vuint32(9), + expectedNEntries0 = cms.untracked.vuint32(9, 5, 3), + expectedCacheEntriesPerFile0 = cms.untracked.vuint32(17), + expectedOuterOffset = cms.untracked.vuint32(0) +) + +process.p = cms.Path(process.readProcessBlocksOneAnalyzer1 * process.readProcessBlocksOneAnalyzer2) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockRead_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockRead_cfg.py new file mode 100644 index 0000000000000..b54cb5fa3febd --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockRead_cfg.py @@ -0,0 +1,277 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READ") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockTest.root' + ) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockRead.root') +) + +process.testGlobalOutput = cms.OutputModule("TestGlobalOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'READ'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(4) +) + +process.testLimitedOutput = cms.OutputModule("TestLimitedOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'READ'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(4) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'READ'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(4) +) + +process.intProducerBeginProcessBlockR = cms.EDProducer("IntProducerBeginProcessBlock", ivalue = cms.int32(100)) + +process.intProducerEndProcessBlockR = cms.EDProducer("IntProducerEndProcessBlock", ivalue = cms.int32(1000)) + +process.readProcessBlocks = cms.EDAnalyzer("edmtest::stream::InputProcessBlockIntAnalyzer", + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + expectedByRun = cms.vint32(11, 22), + sleepTime = cms.uint32(10000) +) + +process.readProcessBlocksG = cms.EDAnalyzer("edmtest::stream::InputProcessBlockIntAnalyzerG", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22), + expectedSum = cms.int32(77), + sleepTime = cms.uint32(10000) +) + +process.readProcessBlocksNS = cms.EDAnalyzer("edmtest::stream::InputProcessBlockIntAnalyzerNS") +process.readProcessBlocksGNS = cms.EDAnalyzer("edmtest::stream::InputProcessBlockIntAnalyzerGNS") + +process.readProcessBlocksStreamFilter = cms.EDFilter("edmtest::stream::InputProcessBlockIntFilter", + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + expectedByRun = cms.vint32(11, 22), + sleepTime = cms.uint32(10000) +) + +process.readProcessBlocksGStreamFilter = cms.EDFilter("edmtest::stream::InputProcessBlockIntFilterG", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22), + expectedSum = cms.int32(77), + sleepTime = cms.uint32(10000) +) + +process.readProcessBlocksStreamProducer = cms.EDProducer("edmtest::stream::InputProcessBlockIntProducer", + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + expectedByRun = cms.vint32(11, 22), + sleepTime = cms.uint32(10000) +) + +process.readProcessBlocksGStreamProducer = cms.EDProducer("edmtest::stream::InputProcessBlockIntProducerG", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22), + expectedSum = cms.int32(77), + sleepTime = cms.uint32(10000) +) + +process.readProcessBlocksOneAnalyzer = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22), + expectedSum = cms.int32(77), + consumesBeginProcessBlockNotFound = cms.InputTag("intProducerBeginProcessBlock"), + consumesEndProcessBlockNotFound = cms.InputTag("intProducerEndProcessBlock") +) + +process.readProcessBlocksOneFilter = cms.EDFilter("edmtest::one::InputProcessBlockIntFilter", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22), + expectedSum = cms.int32(77) +) + +process.readProcessBlocksOneProducer = cms.EDProducer("edmtest::one::InputProcessBlockIntProducer", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22), + expectedSum = cms.int32(77) +) + +process.readProcessBlocksGlobalAnalyzer = cms.EDAnalyzer("edmtest::global::InputProcessBlockIntAnalyzer", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22), + expectedSum = cms.int32(77) +) + +process.readProcessBlocksGlobalFilter = cms.EDFilter("edmtest::global::InputProcessBlockIntFilter", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22), + expectedSum = cms.int32(77) +) + +process.readProcessBlocksGlobalProducer = cms.EDProducer("edmtest::global::InputProcessBlockIntProducer", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22), + expectedSum = cms.int32(77) +) + +process.readProcessBlocksLimitedAnalyzer = cms.EDAnalyzer("edmtest::limited::InputProcessBlockIntAnalyzer", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22), + expectedSum = cms.int32(77), + concurrencyLimit = cms.untracked.uint32(4) +) + +process.readProcessBlocksLimitedFilter = cms.EDFilter("edmtest::limited::InputProcessBlockIntFilter", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22), + expectedSum = cms.int32(77), + concurrencyLimit = cms.untracked.uint32(4) +) + +process.readProcessBlocksLimitedProducer = cms.EDProducer("edmtest::limited::InputProcessBlockIntProducer", + transitions = cms.int32(15), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22), + expectedSum = cms.int32(77), + concurrencyLimit = cms.untracked.uint32(4) +) + +process.readProcessBlocksGlobalAnalyzerNoRegistration = cms.EDAnalyzer("edmtest::global::InputProcessBlockIntAnalyzerNoRegistration", + transitions = cms.int32(6), +) + +process.readProcessBlocksDoesNotExist = cms.EDAnalyzer("edmtest::global::InputProcessBlockAnalyzerThreeTags", + transitions = cms.int32(9), + consumesBeginProcessBlock0 = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock0 = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlock1 = cms.InputTag("doesNotExist", ""), + consumesEndProcessBlock1 = cms.InputTag("doesNotExist", ""), + consumesBeginProcessBlock2 = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlock2 = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun0 = cms.vint32(11, 22), + expectedByRun1 = cms.vint32(), + expectedByRun2 = cms.vint32(44, 44) +) + +process.readProcessBlocksExplicitProcess = cms.EDAnalyzer("edmtest::global::InputProcessBlockAnalyzerThreeTags", + transitions = cms.int32(10), + consumesBeginProcessBlock0 = cms.InputTag("intProducerBeginProcessBlockB", ""), + consumesEndProcessBlock0 = cms.InputTag("intProducerEndProcessBlockB", ""), + consumesBeginProcessBlock1 = cms.InputTag("intProducerBeginProcessBlockB", "", "PROD1"), + consumesEndProcessBlock1 = cms.InputTag("intProducerEndProcessBlockB", "", "PROD1"), + consumesBeginProcessBlock2 = cms.InputTag("intProducerBeginProcessBlockB", "", "MERGE"), + consumesEndProcessBlock2 = cms.InputTag("intProducerEndProcessBlockB", "", "MERGE"), + expectedByRun0 = cms.vint32(88, 88), + expectedByRun1 = cms.vint32(55, 77), + expectedByRun2 = cms.vint32(88, 88) +) + +process.readProcessBlocksExplicitProcess2 = cms.EDAnalyzer("edmtest::global::InputProcessBlockAnalyzerThreeTags", + transitions = cms.int32(11), + consumesBeginProcessBlock0 = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock0 = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlock1 = cms.InputTag("intProducerBeginProcessBlock", "", "PROD1"), + consumesEndProcessBlock1 = cms.InputTag("intProducerEndProcessBlock", "", "PROD1"), + consumesBeginProcessBlock2 = cms.InputTag("intProducerBeginProcessBlockM", "", "MERGE"), + consumesEndProcessBlock2 = cms.InputTag("intProducerEndProcessBlockM", "", "MERGE"), + expectedByRun0 = cms.vint32(11, 22), + expectedByRun1 = cms.vint32(11, 22), + expectedByRun2 = cms.vint32(44, 44) +) + +process.readProcessBlocksReuseCache = cms.EDAnalyzer("edmtest::global::InputProcessBlockAnalyzerReuseCache", + transitions = cms.int32(8), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + expectedByRun = cms.vint32(11, 11) +) + +process.p = cms.Path(process.intProducerBeginProcessBlockR * + process.intProducerEndProcessBlockR * + process.readProcessBlocks * + process.readProcessBlocksG * + process.readProcessBlocksNS * + process.readProcessBlocksGNS * + process.readProcessBlocksStreamFilter * + process.readProcessBlocksGStreamFilter * + process.readProcessBlocksStreamProducer * + process.readProcessBlocksGStreamProducer * + process.readProcessBlocksOneAnalyzer * + process.readProcessBlocksOneFilter * + process.readProcessBlocksOneProducer * + process.readProcessBlocksGlobalAnalyzer * + process.readProcessBlocksGlobalFilter * + process.readProcessBlocksGlobalProducer * + process.readProcessBlocksLimitedAnalyzer * + process.readProcessBlocksLimitedFilter * + process.readProcessBlocksLimitedProducer * + process.readProcessBlocksDoesNotExist * + process.readProcessBlocksGlobalAnalyzerNoRegistration * + process.readProcessBlocksExplicitProcess * + process.readProcessBlocksExplicitProcess2 * + process.readProcessBlocksReuseCache +) + +process.e = cms.EndPath( + process.out * + process.testGlobalOutput * + process.testLimitedOutput * + process.testOneOutput +) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockTEST_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockTEST_cfg.py new file mode 100644 index 0000000000000..c65f21cbce2fe --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockTEST_cfg.py @@ -0,0 +1,81 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMerge.root' + ) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockTest.root'), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +process.testGlobalOutput = cms.OutputModule("TestGlobalOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(4), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +process.testLimitedOutput = cms.OutputModule("TestLimitedOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(4), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), + expectedWriteProcessBlockTransitions = cms.untracked.int32(4), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_*_beginProcessBlock_*", + "drop *_*_endProcessBlock_*" + ) +) + +process.eventIntProducer = cms.EDProducer("IntProducer", ivalue = cms.int32(1)) + +process.transientIntProducerEndProcessBlock = cms.EDProducer("TransientIntProducerEndProcessBlock", + ivalue = cms.int32(90) +) + +process.nonEventIntProducer = cms.EDProducer("NonEventIntProducer", + ivalue = cms.int32(1) +) + +process.p = cms.Path( + process.eventIntProducer * + process.transientIntProducerEndProcessBlock * + process.nonEventIntProducer +) + +process.e = cms.EndPath( + process.out * + process.testGlobalOutput * + process.testLimitedOutput * + process.testOneOutput +) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockThreeFileInput_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockThreeFileInput_cfg.py new file mode 100644 index 0000000000000..beba728ec8275 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testProcessBlockThreeFileInput_cfg.py @@ -0,0 +1,70 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("THREETEST") + +process.options = cms.untracked.PSet( + numberOfStreams = cms.untracked.uint32(1), + numberOfThreads = cms.untracked.uint32(1), + numberOfConcurrentRuns = cms.untracked.uint32(1), + numberOfConcurrentLuminosityBlocks = cms.untracked.uint32(1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testProcessBlockMergeOfMergedFiles.root', + 'file:testProcessBlockMergeOfMergedFiles2.root', + 'file:testProcessBlockMergeOfMergedFiles.root', + ), + duplicateCheckMode = cms.untracked.string("noDuplicateCheck") +) + +process.readProcessBlocksOneAnalyzer1 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(71), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlock", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlock", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(11, 22, 3300, 4400, 7707), + expectedSum = cms.int32(24193) +) + +process.readProcessBlocksOneAnalyzer2 = cms.EDAnalyzer("edmtest::one::InputProcessBlockIntAnalyzer", + transitions = cms.int32(53), + consumesBeginProcessBlock = cms.InputTag("intProducerBeginProcessBlockMM", ""), + consumesEndProcessBlock = cms.InputTag("intProducerEndProcessBlockMM", ""), + consumesBeginProcessBlockM = cms.InputTag("intProducerBeginProcessBlockM", ""), + consumesEndProcessBlockM = cms.InputTag("intProducerEndProcessBlockM", ""), + expectedByRun = cms.vint32(644, 644, 644, 644, 844), + expectedSum = cms.int32(1020) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testProcessBlockThreeFileInput.root') +) + +process.testOneOutput = cms.OutputModule("TestOneOutput", + verbose = cms.untracked.bool(False), + expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), + expectedTopAddedProcesses = cms.untracked.vstring(), + expectedTopCacheIndices0 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6), + expectedTopCacheIndices1 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6, 7, 8, 9), + expectedTopCacheIndices2 = cms.untracked.vuint32(0, 4, 6, 1, 4, 6, 2, 5, 6, 3, 5, 6, 7, 8, 9, 10, 14, 16, 11, 14, 16, 12, 15, 16, 13, 15, 16), + expectedWriteProcessBlockTransitions = cms.untracked.int32(18), + expectedProcessesInFirstFile = cms.untracked.uint32(3), + expectedCacheIndexVectorsPerFile = cms.untracked.vuint32(4, 1, 4), + expectedNEntries0 = cms.untracked.vuint32(4, 2, 1), + expectedNEntries1 = cms.untracked.vuint32(1, 1, 1), + expectedNEntries2 = cms.untracked.vuint32(4, 2, 1), + expectedCacheEntriesPerFile0 = cms.untracked.vuint32(7), + expectedCacheEntriesPerFile1 = cms.untracked.vuint32(7, 3), + expectedCacheEntriesPerFile2 = cms.untracked.vuint32(7, 3, 7), + expectedOuterOffset = cms.untracked.vuint32(0, 4, 5) +) + +process.p = cms.Path(process.readProcessBlocksOneAnalyzer1 * process.readProcessBlocksOneAnalyzer2) + +process.e = cms.EndPath( + process.out * + process.testOneOutput +) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeCOPY100_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeCOPY100_cfg.py new file mode 100644 index 0000000000000..686ec9675b5c9 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeCOPY100_cfg.py @@ -0,0 +1,56 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("COPY") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testRunMergeTEST100.root' + ), + duplicateCheckMode = cms.untracked.string('noDuplicateCheck') +) + +process.test = cms.EDAnalyzer("TestMergeResults", + + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + + expectedBeginRunProd = cms.untracked.vint32( + 10001, 40008, 10003, + 10001, 20004, 10003 + ), + + expectedEndRunProd = cms.untracked.vint32( + 100001, 400008, 100003, + 100001, 200004, 100003 + ), + + expectedEndRunProdImproperlyMerged = cms.untracked.vint32( + 0, 1, 0, + 0, 0, 0 + ) +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") +process.task = cms.Task(process.thingWithMergeProducer) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMergeCOPY100.root') +) + +process.e = cms.EndPath(process.test * process.out, process.task) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeCOPY101_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeCOPY101_cfg.py new file mode 100644 index 0000000000000..dc55a04dc6bf9 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeCOPY101_cfg.py @@ -0,0 +1,56 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("COPY") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testRunMergeTEST101.root' + ), + duplicateCheckMode = cms.untracked.string('noDuplicateCheck') +) + +process.test = cms.EDAnalyzer("TestMergeResults", + + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + + expectedBeginRunProd = cms.untracked.vint32( + 10001, 20004, 10003, + 10001, 10002, 10003 + ), + + expectedEndRunProd = cms.untracked.vint32( + 100001, 200004, 100003, + 100001, 100002, 100003 + ), + + expectedEndRunProdImproperlyMerged = cms.untracked.vint32( + 0, 0, 0, + 0, 0, 0 + ) +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") +process.task = cms.Task(process.thingWithMergeProducer) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMergeCOPY101.root') +) + +process.e = cms.EndPath(process.test * process.out, process.task) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeCOPY1_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeCOPY1_cfg.py new file mode 100644 index 0000000000000..81d98c6202cdf --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeCOPY1_cfg.py @@ -0,0 +1,210 @@ + +# Make sure we can read and write the end result of +# all the merge and secondary file input testing + +# Also test the event duplicate checking code and the +# skip event code in RNTupleTempSource + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("COPY") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + fileMode = cms.untracked.string('FULLMERGE'), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = [ + 'file:testRunMergeRecombined.root', + 'file:testRunMergeRecombined.root' + ] + , duplicateCheckMode = 'checkEachFile' + , skipEvents = 3 + , noEventSort = False +) + +from FWCore.Framework.modules import RunLumiEventAnalyzer +process.test = RunLumiEventAnalyzer( + verbose = True, + expectedRunLumiEvents = [ +1, 0, 0, +1, 1, 0, +1, 1, 4, +1, 1, 5, +1, 1, 6, +1, 1, 7, +1, 1, 8, +1, 1, 9, +1, 1, 10, +1, 1, 11, +1, 1, 12, +1, 1, 13, +1, 1, 14, +1, 1, 15, +1, 1, 16, +1, 1, 17, +1, 1, 18, +1, 1, 19, +1, 1, 20, +1, 1, 21, +1, 1, 22, +1, 1, 23, +1, 1, 24, +1, 1, 25, +1, 1, 0, +1, 0, 0, +2, 0, 0, +2, 1, 0, +2, 1, 1, +2, 1, 2, +2, 1, 3, +2, 1, 4, +2, 1, 5, +2, 1, 0, +2, 0, 0, +11, 0, 0, +11, 1, 0, +11, 1, 1, +11, 1, 0, +11, 2, 0, +11, 2, 1, +11, 2, 0, +11, 0, 0, +100, 0, 0, +100, 100, 0, +100, 100, 100, +100, 100, 0, +100, 0, 0, +1, 0, 0, +1, 1, 0, +1, 1, 21, +1, 1, 22, +1, 1, 23, +1, 1, 24, +1, 1, 25, +1, 1, 0, +1, 0, 0, +2, 0, 0, +2, 1, 0, +2, 1, 1, +2, 1, 2, +2, 1, 3, +2, 1, 4, +2, 1, 5, +2, 1, 0, +2, 0, 0, +1, 0, 0, +1, 1, 0, +1, 1, 1, +1, 1, 2, +1, 1, 3, +1, 1, 4, +1, 1, 5, +1, 1, 6, +1, 1, 7, +1, 1, 8, +1, 1, 9, +1, 1, 10, +1, 1, 0, +1, 0, 0 +] +) +# At this point the first input file is done and +# we start with the second input file here. +process.test.expectedRunLumiEvents.extend([ +1, 0, 0, +1, 1, 0, +1, 1, 1, +1, 1, 2, +1, 1, 3, +1, 1, 4, +1, 1, 5, +1, 1, 6, +1, 1, 7, +1, 1, 8, +1, 1, 9, +1, 1, 10, +1, 1, 11, +1, 1, 12, +1, 1, 13, +1, 1, 14, +1, 1, 15, +1, 1, 16, +1, 1, 17, +1, 1, 18, +1, 1, 19, +1, 1, 20, +1, 1, 21, +1, 1, 22, +1, 1, 23, +1, 1, 24, +1, 1, 25, +1, 1, 0, +1, 0, 0, +2, 0, 0, +2, 1, 0, +2, 1, 1, +2, 1, 2, +2, 1, 3, +2, 1, 4, +2, 1, 5, +2, 1, 0, +2, 0, 0, +11, 0, 0, +11, 1, 0, +11, 1, 1, +11, 1, 0, +11, 2, 0, +11, 2, 1, +11, 2, 0, +11, 0, 0, +100, 0, 0, +100, 100, 0, +100, 100, 100, +100, 100, 0, +100, 0, 0, +1, 0, 0, +1, 1, 0, +1, 1, 21, +1, 1, 22, +1, 1, 23, +1, 1, 24, +1, 1, 25, +1, 1, 0, +1, 0, 0, +2, 0, 0, +2, 1, 0, +2, 1, 1, +2, 1, 2, +2, 1, 3, +2, 1, 4, +2, 1, 5, +2, 1, 0, +2, 0, 0, +1, 0, 0, +1, 1, 0, +1, 1, 1, +1, 1, 2, +1, 1, 3, +1, 1, 4, +1, 1, 5, +1, 1, 6, +1, 1, 7, +1, 1, 8, +1, 1, 9, +1, 1, 10, +1, 1, 0, +1, 0, 0 +]) + +process.path1 = cms.Path(process.test) + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.out = RNTupleTempOutputModule(fileName = 'testRunMergeRecombinedCopied1.root') + +process.endpath1 = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeCOPY_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeCOPY_cfg.py new file mode 100644 index 0000000000000..9f66368bfc214 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeCOPY_cfg.py @@ -0,0 +1,146 @@ + +# Make sure we can read and write the end result of +# all the merge and secondary file input testing + +# Also test the event duplicate checking code and the +# skip event code in RNTupleTempSource + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("COPY") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + fileMode = cms.untracked.string('FULLMERGE'), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = [ + 'file:testRunMergeRecombined.root', + 'file:testRunMergeRecombined.root' + ] + , duplicateCheckMode = 'checkAllFilesOpened' + , skipEvents = 14 + , noEventSort = False +) + +from FWCore.Framework.modules import RunLumiEventAnalyzer +process.test = RunLumiEventAnalyzer( + verbose = True, + expectedRunLumiEvents = [ +1, 0, 0, +1, 1, 0, +1, 1, 15, +1, 1, 16, +1, 1, 17, +1, 1, 18, +1, 1, 19, +1, 1, 20, +1, 1, 21, +1, 1, 22, +1, 1, 23, +1, 1, 24, +1, 1, 25, +1, 1, 0, +1, 0, 0, +2, 0, 0, +2, 1, 0, +2, 1, 1, +2, 1, 2, +2, 1, 3, +2, 1, 4, +2, 1, 5, +2, 1, 0, +2, 0, 0, +11, 0, 0, +11, 1, 0, +11, 1, 1, +11, 1, 0, +11, 2, 0, +11, 2, 1, +11, 2, 0, +11, 0, 0, +100, 0, 0, +100, 100, 0, +100, 100, 100, +100, 100, 0, +100, 0, 0, +1, 0, 0, +1, 1, 0, +1, 1, 21, +1, 1, 22, +1, 1, 23, +1, 1, 24, +1, 1, 25, +1, 1, 0, +1, 0, 0, +2, 0, 0, +2, 1, 0, +2, 1, 1, +2, 1, 2, +2, 1, 3, +2, 1, 4, +2, 1, 5, +2, 1, 0, +2, 0, 0, +1, 0, 0, +1, 1, 0, +1, 1, 1, +1, 1, 2, +1, 1, 3, +1, 1, 4, +1, 1, 5, +1, 1, 6, +1, 1, 7, +1, 1, 8, +1, 1, 9, +1, 1, 10, +1, 1, 0, +1, 0, 0 +] +) +# At this point the first input file is done and +# we start with the second input file here. +process.test.expectedRunLumiEvents.extend([ +1, 0, 0, +1, 1, 0, +1, 1, 0, +1, 0, 0, +2, 0, 0, +2, 1, 0, +2, 1, 0, +2, 0, 0, +11, 0, 0, +11, 1, 0, +11, 1, 0, +11, 2, 0, +11, 2, 0, +11, 0, 0, +100, 0, 0, +100, 100, 0, +100, 100, 0, +100, 0, 0, +1, 0, 0, +1, 1, 0, +1, 1, 0, +1, 0, 0, +2, 0, 0, +2, 1, 0, +2, 1, 0, +2, 0, 0, +1, 0, 0, +1, 1, 0, +1, 1, 0, +1, 0, 0 +]) + +process.path1 = cms.Path(process.test) + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.out = RNTupleTempOutputModule(fileName = 'testRunMergeRecombinedCopied.root') + +process.endpath1 = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeFastCloning_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeFastCloning_cfg.py new file mode 100644 index 0000000000000..5e9c91393f553 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeFastCloning_cfg.py @@ -0,0 +1,45 @@ +# This is intended to test whether fast cloning is +# actually occurring when it should. It uses a trick +# to check. With this configuration, fast cloning should +# occur. An exception unrelated to fast cloning is forced +# on the second event, and after that there should be a +# secondary exception as the Framework attempts to write out +# and close the root files. This secondary exception occurs +# because the Event TTree should be imbalanced. This imbalance +# is caused by fast cloning. The TTree would not be imbalanced +# if fast cloning was not occurring. The shell script that +# invokes cmsRun with this configuration then uses grep to +# look for the expected inbalance error and this verifies +# that fast cloning was occurring. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") + +process.load("FWCore.MessageService.MessageLogger_cfi") + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testRunMerge6.root' + ) + , duplicateCheckMode = cms.untracked.string('checkAllFilesOpened') +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('test.root') + #, fastCloning = cms.untracked.bool(False) +) + +process.testThrow = cms.EDAnalyzer("TestFailuresAnalyzer", + whichFailure = cms.int32(5), + eventToThrow = cms.untracked.uint64(2) +) + +process.p = cms.Path(process.testThrow) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeMERGE100_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeMERGE100_cfg.py new file mode 100644 index 0000000000000..8f788b4f8414f --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeMERGE100_cfg.py @@ -0,0 +1,54 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testRunMergeSPLIT100.root', + 'file:testRunMergeSPLIT101.root' + ), + lumisToProcess = cms.untracked.VLuminosityBlockRange('41:6-41:15'), +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") +process.task = cms.Task(process.thingWithMergeProducer) + +process.test = cms.EDAnalyzer("TestMergeResults", + + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + + expectedBeginRunProd = cms.untracked.vint32( + 10001, 20004, 10003 + ), + + expectedEndRunProd = cms.untracked.vint32( + 100001, 200004, 100003 + ), + + expectedEndRunProdImproperlyMerged = cms.untracked.vint32( + 0, 0, 0 + ) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMergeMERGE100.root') +) + +process.e = cms.EndPath(process.test * process.out, process.task) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeMERGE101_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeMERGE101_cfg.py new file mode 100644 index 0000000000000..a43a6ecf0f0b2 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeMERGE101_cfg.py @@ -0,0 +1,57 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testRunMergeSPLIT103.root', + 'file:testRunMergeSPLIT102.root' + ), + lumisToProcess = cms.untracked.VLuminosityBlockRange('41:16-41:25', '42:16-42:20') +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") +process.task = cms.Task(process.thingWithMergeProducer) + +process.test = cms.EDAnalyzer("TestMergeResults", + + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + + expectedBeginRunProd = cms.untracked.vint32( + 10001, 20004, 10003, + 10001, 10002, 10003 + ), + + expectedEndRunProd = cms.untracked.vint32( + 100001, 200004, 100003, + 100001, 100002, 100003 + ), + + expectedEndRunProdImproperlyMerged = cms.untracked.vint32( + 0, 0, 0, + 0, 0, 0 + ) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMergeMERGE101.root') +) + +process.e = cms.EndPath(process.test * process.out, process.task) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeMERGE102_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeMERGE102_cfg.py new file mode 100644 index 0000000000000..07bb03981aea4 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeMERGE102_cfg.py @@ -0,0 +1,54 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testRunMergeSPLIT100.root', + 'file:testRunMergeSPLIT101.root' + ), + lumisToProcess = cms.untracked.VLuminosityBlockRange('42:6-42:15') +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") +process.task = cms.Task(process.thingWithMergeProducer) + +process.test = cms.EDAnalyzer("TestMergeResults", + + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + + expectedBeginRunProd = cms.untracked.vint32( + 10001, 20004, 10003 + ), + + expectedEndRunProd = cms.untracked.vint32( + 100001, 200004, 100003 + ), + + expectedEndRunProdImproperlyMerged = cms.untracked.vint32( + 0, 0, 0 + ) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMergeMERGE102.root') +) + +process.e = cms.EndPath(process.test * process.out, process.task) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeMERGE1_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeMERGE1_cfg.py new file mode 100644 index 0000000000000..24f4ca67422fd --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeMERGE1_cfg.py @@ -0,0 +1,116 @@ + +# This configuration tests the lumisToSkip, firstRun, +# firstLuminosityBlock, and firstEvent parameters of +# the RNTupleTempSource. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + fileMode = cms.untracked.string('FULLMERGE'), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = [ + 'file:testRunMerge1.root', + 'file:testRunMerge2.root', + 'file:testRunMerge3.root', + 'file:testRunMerge4.root', + 'file:testRunMerge5.root' + ] + , firstRun = 17 + , firstLuminosityBlock = 3 + , firstEvent = 6 + , lumisToSkip = [ + '18:3', + '19:2', + '21:4', + '16:2' + ] + , duplicateCheckMode = 'checkEachRealDataFile' +) + +from FWCore.Integration.modules import ThingWithMergeProducer +process.thingWithMergeProducer = ThingWithMergeProducer() + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.out = RNTupleTempOutputModule(fileName = 'testRunMerge_a.root') + +from FWCore.Framework.modules import RunLumiEventAnalyzer +process.test = RunLumiEventAnalyzer( + verbose = True, + expectedRunLumiEvents = [ +17, 0, 0, +17, 3, 0, +17, 3, 6, +17, 3, 0, +17, 4, 0, +17, 4, 7, +17, 4, 8, +17, 4, 9, +17, 4, 0, +17, 0, 0, +18, 0, 0, +18, 2, 0, +18, 2, 1, +18, 2, 2, +18, 2, 3, +18, 2, 0, +18, 4, 0, +18, 4, 7, +18, 4, 8, +18, 4, 9, +18, 4, 0, +18, 0, 0, +19, 0, 0, +19, 3, 0, +19, 3, 4, +19, 3, 5, +19, 3, 6, +19, 3, 0, +19, 4, 0, +19, 4, 7, +19, 4, 8, +19, 4, 9, +19, 4, 0, +19, 0, 0, +20, 0, 0, +20, 2, 0, +20, 2, 1, +20, 2, 2, +20, 2, 3, +20, 2, 0, +20, 3, 0, +20, 3, 4, +20, 3, 5, +20, 3, 6, +20, 3, 0, +20, 4, 0, +20, 4, 7, +20, 4, 8, +20, 4, 9, +20, 4, 0, +20, 0, 0, +21, 0, 0, +21, 2, 0, +21, 2, 1, +21, 2, 2, +21, 2, 3, +21, 2, 0, +21, 3, 0, +21, 3, 4, +21, 3, 5, +21, 3, 6, +21, 3, 0, +21, 0, 0 +] +) + +process.path1 = cms.Path(process.thingWithMergeProducer + process.test) +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeMERGE2_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeMERGE2_cfg.py new file mode 100644 index 0000000000000..46336cc79d194 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeMERGE2_cfg.py @@ -0,0 +1,141 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + fileMode = cms.untracked.string('FULLMERGE'), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(33) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testRunMerge0.root', + 'file:testRunMerge1.root', + 'file:testRunMerge2extra.root', + 'file:testRunMerge3extra.root', + 'file:testRunMerge4.root', + 'file:testRunMerge5.root' + ), + inputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_C_*_*', + 'drop *_*_*_EXTRA', + 'drop edmtestThingWithMerge_makeThingToBeDropped1_*_*' + ) + , duplicateCheckMode = cms.untracked.string('checkEachRealDataFile') +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") + +process.test = cms.EDAnalyzer("TestMergeResults", + + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + # 0's are just placeholders, if the value is a "0" the check is not made. + + expectedBeginRunProd = cms.untracked.vint32( + 10001, 10002, 10003, # end run 100 + 10001, 10002, 10003, # end run 1 + 10001, 10002, 10003, # end run 1, no merge of runs because ProcessHistoryID different + 10001, 10002, 10003, # end run 2 + 10001, 10002, 10004 # end run 1 + ), + + expectedEndRunProd = cms.untracked.vint32( + 100001, 100002, 100003, # end run 100 + 100001, 100002, 100003, # end run 1 + 100001, 100002, 100003, # end run 1, no merge of runs because ProcessHistoryID different + 100001, 100002, 100003, # end run 2 + 100001, 100002, 100004 # end run 1 + ), + + expectedBeginLumiProd = cms.untracked.vint32( + 101, 102, 103, # end run 100 lumi 100 + 101, 102, 103, # end run 1 lumi 1 + 101, 102, 103, # end run 1 lumi 1, no merge of runs because ProcessHistoryID different + 101, 102, 103, # end run 2 lumi 1 + 101, 102, 104 # end run 1 lumi 1 + ), + + expectedEndLumiProd = cms.untracked.vint32( + 1001, 1002, 1003, # end run 100 lumi 100 + 1001, 1002, 1003, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 1 lumi 1, no merge of runs because ProcessHistoryID different + 1001, 1002, 1003, # end run 2 lumi 1 + 1001, 1002, 1004 # end run 1 lumi 1 + ), + + expectedBeginRunNew = cms.untracked.vint32( + 10001, 10002, 10003, # end run 100 + 10001, 10002, 10003, # end run 1 + 10001, 10002, 10003, # end run 1 + 10001, 10002, 10003, # end run 2 + 10001, 10002, 10003 # end run 1 + ), + + expectedEndRunNew = cms.untracked.vint32( + 100001, 100002, 100003, # end run 100 + 100001, 100002, 100003, # end run 1 + 100001, 100002, 100003, # end run 1 + 100001, 100002, 100003, # end run 2 + 100001, 100002, 100003 # end run 1 + ), + + expectedBeginLumiNew = cms.untracked.vint32( + 101, 102, 103, # end run 100 lumi 100 + 101, 102, 103, # end run 1 lumi 1 + 101, 102, 103, # end run 1 lumi 1 + 101, 102, 103, # end run 2 lumi 1 + 101, 102, 103 # end run 1 lumi 1 + ), + + expectedEndLumiNew = cms.untracked.vint32( + 1001, 1002, 1003, # end run 100 lumi 100 + 1001, 1002, 1003, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 2 lumi 1 + 1001, 1002, 1003 # end run 1 lumi 1 + ), + + expectedDroppedEvent1 = cms.untracked.vint32(13, 13, -1, -1, -1, 13), + expectedDroppedEvent1NEvents = cms.untracked.vint32(1,10,10,10,1,99), + + verbose = cms.untracked.bool(False), + testAlias = cms.untracked.bool(True) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMergeMERGE2.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_makeThingToBeDropped_*_*', + 'drop *_aliasForThingToBeDropped2_*_*', + 'drop *_B_*_*', + 'drop *_G_*_*', + 'drop *_H_*_*', + 'drop *_I_*_*', + 'drop *_J_*_*' + ) +) + +process.checker = cms.OutputModule("GetProductCheckerOutputModule") + +process.path1 = cms.Path(process.thingWithMergeProducer + process.test) +process.e = cms.EndPath(process.out * process.checker) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeMERGE3_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeMERGE3_cfg.py new file mode 100644 index 0000000000000..9d278f67b51b9 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeMERGE3_cfg.py @@ -0,0 +1,103 @@ + +# This configuration tests the lumisToProcess and eventsToSkip +# parameters of the RNTupleTempSource. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = [ + 'file:testRunMerge1.root', + 'file:testRunMerge2.root', + 'file:testRunMerge3.root', + 'file:testRunMerge4.root', + 'file:testRunMerge5.root' + ] + , lumisToProcess = [ + '11:2', + '15:2-15:8', + '19:2-20:2', + '21:4' + ] + , eventsToSkip = [ + '19:6-19:8', + '21:8' + ] + , duplicateCheckMode = 'checkEachRealDataFile' +) + +from FWCore.Integration.modules import ThingWithMergeProducer +process.thingWithMergeProducer = ThingWithMergeProducer() + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.out = RNTupleTempOutputModule(fileName = 'testRunMergeMERGE3.root') + +from FWCore.Framework.modules import RunLumiEventAnalyzer +process.test = RunLumiEventAnalyzer( + verbose = True, + expectedRunLumiEvents = [ +11, 0, 0, +11, 2, 0, +11, 2, 1, +11, 2, 2, +11, 2, 3, +11, 2, 0, +11, 0, 0, +15, 0, 0, +15, 2, 0, +15, 2, 1, +15, 2, 2, +15, 2, 3, +15, 2, 0, +15, 3, 0, +15, 3, 4, +15, 3, 5, +15, 3, 6, +15, 3, 0, +15, 4, 0, +15, 4, 7, +15, 4, 8, +15, 4, 9, +15, 4, 0, +15, 0, 0, +19, 0, 0, +19, 2, 0, +19, 2, 1, +19, 2, 2, +19, 2, 3, +19, 2, 0, +19, 3, 0, +19, 3, 4, +19, 3, 5, +19, 3, 0, +19, 4, 0, +19, 4, 9, +19, 4, 0, +19, 0, 0, +20, 0, 0, +20, 2, 0, +20, 2, 1, +20, 2, 2, +20, 2, 3, +20, 2, 0, +20, 0, 0, +21, 0, 0, +21, 4, 0, +21, 4, 7, +21, 4, 9, +21, 4, 0, +21, 0, 0 +] +) + +process.path1 = cms.Path(process.thingWithMergeProducer + process.test) +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeMERGE3x_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeMERGE3x_cfg.py new file mode 100644 index 0000000000000..38325e79435c3 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeMERGE3x_cfg.py @@ -0,0 +1,101 @@ + +# This configuration tests the lumisToProcess and eventsToSkip +# parameters of the RNTupleTempSource. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = [ + 'file:testRunMerge1.root', + 'file:testRunMerge2.root', + 'file:testRunMerge3.root', + 'file:testRunMerge4.root', + 'file:testRunMerge5.root' + ] + , lumisToProcess = [ + '11:2', + '15:2-15:8', + '19:2-20:2', + '21:4' + ] + , eventsToSkip = [ + '19:4:6-19:4:8', + '21:3:8' + ] + , duplicateCheckMode = 'checkEachRealDataFile' +) + +from FWCore.Integration.modules import ThingWithMergeProducer +process.thingWithMergeProducer = ThingWithMergeProducer() + +from FWCore.Framework.modules import RunLumiEventAnalyzer +process.test = RunLumiEventAnalyzer( + verbose = True, + expectedRunLumiEvents = [ +11, 0, 0, +11, 2, 0, +11, 2, 1, +11, 2, 2, +11, 2, 3, +11, 2, 0, +11, 0, 0, +15, 0, 0, +15, 2, 0, +15, 2, 1, +15, 2, 2, +15, 2, 3, +15, 2, 0, +15, 3, 0, +15, 3, 4, +15, 3, 5, +15, 3, 6, +15, 3, 0, +15, 4, 0, +15, 4, 7, +15, 4, 8, +15, 4, 9, +15, 4, 0, +15, 0, 0, +19, 0, 0, +19, 2, 0, +19, 2, 1, +19, 2, 2, +19, 2, 3, +19, 2, 0, +19, 3, 0, +19, 3, 4, +19, 3, 5, +19, 3, 6, +19, 3, 0, +19, 4, 0, +19, 4, 9, +19, 4, 0, +19, 0, 0, +20, 0, 0, +20, 2, 0, +20, 2, 1, +20, 2, 2, +20, 2, 3, +20, 2, 0, +20, 0, 0, +21, 0, 0, +21, 4, 0, +21, 4, 7, +21, 4, 8, +21, 4, 9, +21, 4, 0, +21, 0, 0 +] +) + +process.path1 = cms.Path(process.thingWithMergeProducer + process.test) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeMERGE4_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeMERGE4_cfg.py new file mode 100644 index 0000000000000..7e87834e5dece --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeMERGE4_cfg.py @@ -0,0 +1,121 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + fileMode = cms.untracked.string('FULLMERGE'), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testRunMerge1.root', + 'file:testRunMerge2.root', + 'file:testRunMerge3.root', + 'file:testRunMerge4.root', + 'file:testRunMerge7.root' + ), + inputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_C_*_*', + 'drop edmtestThingWithMerge_makeThingToBeDropped1_*_*' + ) + , duplicateCheckMode = cms.untracked.string('checkEachRealDataFile') +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") + +process.test = cms.EDAnalyzer("TestMergeResults", + + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + # 0's are just placeholders, if the value is a "0" the check is not made. + + expectedBeginRunProd = cms.untracked.vint32( + 10001, 20004, 10003, # File boundary before this causing merge + 10001, 10002, 10003, + 10001, 10002, 10004 + ), + + expectedEndRunProd = cms.untracked.vint32( + 100001, 200004, 100003, # File boundary before this causing merge + 100001, 100002, 100003, + 100001, 100002, 100004 + ), + + expectedBeginLumiProd = cms.untracked.vint32( + 101, 204, 103, # File boundary before this causing merge + 101, 102, 103, + 101, 102, 104 + ), + + expectedEndLumiProd = cms.untracked.vint32( + 1001, 2004, 1003, # File boundary before this causing merge + 1001, 1002, 1003, + 1001, 1002, 1004 + ), + + expectedBeginRunNew = cms.untracked.vint32( + 10001, 10002, 10003, + 10001, 10002, 10003, + 10001, 10002, 10003 + ), + + expectedEndRunNew = cms.untracked.vint32( + 100001, 100002, 100003, + 100001, 100002, 100003, + 100001, 100002, 100003 + ), + + expectedBeginLumiNew = cms.untracked.vint32( + 101, 102, 103, + 101, 102, 103, + 101, 102, 103 + ), + + expectedEndLumiNew = cms.untracked.vint32( + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003 + ), + + expectedDroppedEvent1 = cms.untracked.vint32(13, -1, -1, -1, 13), + expectedDroppedEvent1NEvents = cms.untracked.vint32(10,10,10,1,1), + + verbose = cms.untracked.bool(False) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMergeMERGE4.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_makeThingToBeDropped_*_*', + 'drop *_B_*_*', + 'drop *_G_*_*', + 'drop *_H_*_*', + 'drop *_I_*_*', + 'drop *_J_*_*' + ) +) + +process.checker = cms.OutputModule("GetProductCheckerOutputModule") + +process.path1 = cms.Path(process.thingWithMergeProducer + process.test) +process.e = cms.EndPath(process.out * process.checker) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeMERGE5_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeMERGE5_cfg.py new file mode 100644 index 0000000000000..9c7025f833554 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeMERGE5_cfg.py @@ -0,0 +1,23 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE5") + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testRunMerge2extra.root' + ) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMergeMERGE5.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_aliasForThingToBeDropped2_*_*' + ) +) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeMERGE6_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeMERGE6_cfg.py new file mode 100644 index 0000000000000..c30362b4a6ed5 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeMERGE6_cfg.py @@ -0,0 +1,106 @@ +# The purpose of this configuration is to prepare +# input for a test of the noRunLumiSort configuration +# parameter that has non-contiguous sequences of +# events from the same run (and the same lumi). + +# It is expected there are 6 warnings that +# print out while this runs related to merging. +# The test should pass with these warnings. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testRunMerge1.root', + 'file:testRunMerge2.root', + 'file:testRunMerge3.root' + ), + inputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_A_*_*', + 'drop *_B_*_*', + 'drop *_C_*_*', + 'drop *_D_*_*', + 'drop *_E_*_*', + 'drop *_F_*_*', + 'drop *_G_*_*', + 'drop *_H_*_*', + 'drop *_I_*_*', + 'drop *_J_*_*', + 'drop *_K_*_*', + 'drop *_L_*_*', + 'drop *_tryNoPut_*_*', + 'drop *_aliasForThingToBeDropped2_*_*', + 'drop *_dependsOnThingToBeDropped1_*_*', + 'drop *_makeThingToBeDropped_*_*', + 'drop edmtestThingWithMerge_makeThingToBeDropped1_*_*', + 'drop edmtestThing_*_*_*' + ) +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") + +process.test = cms.EDAnalyzer("TestMergeResults", + + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + # 0's are just placeholders, if the value is a "0" the check is not made. + + expectedBeginRunProd = cms.untracked.vint32( + 0, 20004, 10003, # File boundary before this causing merge + 0, 10002, 10003, + 0, 10002, 10004 + ), + expectedEndRunProd = cms.untracked.vint32( + 0, 200004, 100003, # File boundary before this causing merge + 0, 100002, 100003, + 0, 100002, 100004 + ), + expectedBeginLumiProd = cms.untracked.vint32( + 0, 204, 103, # File boundary before this causing merge + 0, 102, 103, + 0, 102, 104 + ), + expectedEndLumiProd = cms.untracked.vint32( + 0, 2004, 1003, # File boundary before this causing merge + 0, 1002, 1003, + 0, 1002, 1004 + ), + expectedBeginRunNew = cms.untracked.vint32( + 0, 10002, 10003, + 0, 10002, 10003, + 0, 10002, 10003 + ), + expectedEndRunNew = cms.untracked.vint32( + 0, 100002, 100003, + 0, 100002, 100003, + 0, 100002, 100003 + ), + expectedBeginLumiNew = cms.untracked.vint32( + 0, 102, 103, + 0, 102, 103, + 0, 102, 103 + ), + expectedEndLumiNew = cms.untracked.vint32( + 0, 1002, 1003, + 0, 1002, 1003, + 0, 1002, 1003 + ) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMergeMERGE6.root') +) + +process.path1 = cms.Path(process.thingWithMergeProducer + process.test) +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeMERGE_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeMERGE_cfg.py new file mode 100644 index 0000000000000..1b8b919c219e3 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeMERGE_cfg.py @@ -0,0 +1,123 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("MERGE") + +process.load("FWCore.MessageService.MessageLogger_cfi") +#process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +#process.MessageLogger.cerr.threshold = 'ERROR' + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + fileMode = cms.untracked.string('FULLMERGE'), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(32) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testRunMerge1.root', + 'file:testRunMerge2.root', + 'file:testRunMerge3.root', + 'file:testRunMerge4.root', + 'file:testRunMerge5.root' + ), + inputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_C_*_*', + 'drop edmtestThingWithMerge_makeThingToBeDropped1_*_*' + ) + , duplicateCheckMode = cms.untracked.string('checkEachRealDataFile') +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") + +process.test = cms.EDAnalyzer("TestMergeResults", + + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + # 0's are just placeholders, if the value is a "0" the check is not made. + + expectedBeginRunProd = cms.untracked.vint32( + 10001, 20004, 10003, # File boundary before this causing merge + 10001, 10002, 10003, + 10001, 10002, 10004 + ), + + expectedEndRunProd = cms.untracked.vint32( + 100001, 200004, 100003, # File boundary before this causing merge + 100001, 100002, 100003, + 100001, 100002, 100004 + ), + + expectedBeginLumiProd = cms.untracked.vint32( + 101, 204, 103, # File boundary before this causing merge + 101, 102, 103, + 101, 102, 104 + ), + + expectedEndLumiProd = cms.untracked.vint32( + 1001, 2004, 1003, # File boundary before this causing merge + 1001, 1002, 1003, + 1001, 1002, 1004 + ), + + expectedBeginRunNew = cms.untracked.vint32( + 10001, 10002, 10003, + 10001, 10002, 10003, + 10001, 10002, 10003 + ), + + expectedEndRunNew = cms.untracked.vint32( + 100001, 100002, 100003, + 100001, 100002, 100003, + 100001, 100002, 100003 + ), + + expectedBeginLumiNew = cms.untracked.vint32( + 101, 102, 103, + 101, 102, 103, + 101, 102, 103 + ), + + expectedEndLumiNew = cms.untracked.vint32( + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003 + ), + + expectedDroppedEvent1 = cms.untracked.vint32(13, -1, -1, -1, 13), + expectedDroppedEvent1NEvents = cms.untracked.vint32(10,10,10,1,99), + + verbose = cms.untracked.bool(False), + testAlias = cms.untracked.bool(True) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMerge.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_makeThingToBeDropped_*_*', + 'drop *_aliasForThingToBeDropped2_*_*', + 'drop *_B_*_*', + 'drop *_G_*_*', + 'drop *_H_*_*', + 'drop *_I_*_*', + 'drop *_J_*_*' + ) +) + +process.checker = cms.OutputModule("GetProductCheckerOutputModule") + +process.path1 = cms.Path(process.thingWithMergeProducer + process.test) +process.e = cms.EndPath(process.out * process.checker) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeNoRunLumiSort_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeNoRunLumiSort_cfg.py new file mode 100644 index 0000000000000..dde0217fb2e20 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeNoRunLumiSort_cfg.py @@ -0,0 +1,117 @@ +# A test of the noRunLumiSort configuration parameter +# where the input has non-contiguous event sequences +# from the same run. + +# It is expected there are 8 warnings and 8 errors that +# print out while this runs related to merging. +# The test should pass with these errors and warnings. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("NORUNLUMISORT") + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = [ + 'file:testRunMergeMERGE6.root', + 'file:testRunMergeMERGE6.root' + ], + duplicateCheckMode = 'checkEachRealDataFile', + noRunLumiSort = True +) + +from FWCore.Framework.modules import RunLumiEventAnalyzer +process.test2 = RunLumiEventAnalyzer( + verbose = True, + expectedRunLumiEvents = [ +1, 0, 0, +1, 1, 0, +1, 1, 11, +1, 1, 12, +1, 1, 13, +1, 1, 14, +1, 1, 15, +1, 1, 16, +1, 1, 17, +1, 1, 18, +1, 1, 19, +1, 1, 20, +1, 1, 21, +1, 1, 22, +1, 1, 23, +1, 1, 24, +1, 1, 25, +1, 1, 0, +1, 0, 0, +2, 0, 0, +2, 1, 0, +2, 1, 1, +2, 1, 2, +2, 1, 3, +2, 1, 4, +2, 1, 5, +2, 1, 0, +2, 0, 0, +1, 0, 0, +1, 1, 0, +1, 1, 1, +1, 1, 2, +1, 1, 3, +1, 1, 4, +1, 1, 5, +1, 1, 6, +1, 1, 7, +1, 1, 8, +1, 1, 9, +1, 1, 10 +] +) +process.test2.expectedRunLumiEvents.extend([ +1, 1, 11, +1, 1, 12, +1, 1, 13, +1, 1, 14, +1, 1, 15, +1, 1, 16, +1, 1, 17, +1, 1, 18, +1, 1, 19, +1, 1, 20, +1, 1, 21, +1, 1, 22, +1, 1, 23, +1, 1, 24, +1, 1, 25, +1, 1, 0, +1, 0, 0, +2, 0, 0, +2, 1, 0, +2, 1, 1, +2, 1, 2, +2, 1, 3, +2, 1, 4, +2, 1, 5, +2, 1, 0, +2, 0, 0, +1, 0, 0, +1, 1, 0, +1, 1, 1, +1, 1, 2, +1, 1, 3, +1, 1, 4, +1, 1, 5, +1, 1, 6, +1, 1, 7, +1, 1, 8, +1, 1, 9, +1, 1, 10, +1, 1, 0, +1, 0, 0, +]) + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.out = RNTupleTempOutputModule(fileName = 'testRunMergeNoRunLumiSort.root') + +process.path1 = cms.Path(process.test2) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergePROD0_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergePROD0_cfg.py new file mode 100644 index 0000000000000..11a4879e86a59 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergePROD0_cfg.py @@ -0,0 +1,178 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +import FWCore.Framework.test.cmsExceptionsFatal_cff +process.options = FWCore.Framework.test.cmsExceptionsFatal_cff.options + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(1) +) + +process.source = cms.Source("EmptySource", + firstLuminosityBlock = cms.untracked.uint32(100), + numberEventsInLuminosityBlock = cms.untracked.uint32(100), + firstEvent = cms.untracked.uint32(100), + firstRun = cms.untracked.uint32(100), + numberEventsInRun = cms.untracked.uint32(100) +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('m1') +) + +# These are here only for tests of parentage merging +process.m1 = cms.EDProducer("ThingWithMergeProducer") +process.m2 = cms.EDProducer("ThingWithMergeProducer") +process.m3 = cms.EDProducer("ThingWithMergeProducer") + +process.tryNoPut = cms.EDProducer("ThingWithMergeProducer", + noPut = cms.untracked.bool(True) +) + +# This one tests products dropped and then restored by secondary file +# input +process.makeThingToBeDropped = cms.EDProducer("ThingWithMergeProducer") + +process.makeThingToBeDropped2 = cms.EDProducer("ThingWithMergeProducer") + +process.aliasForThingToBeDropped2 = cms.EDAlias( + makeThingToBeDropped2 = cms.VPSet( + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('event'), + toProductInstance = cms.string('instance2')), + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('endLumi'), + toProductInstance = cms.string('endLumi2')), + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('endRun'), + toProductInstance = cms.string('endRun2')) + ) +) + +# This product will be produced in configuration PROD1 and PROD5 +# In PROD2 it will be produced and dropped and there will be another +# product whose provenance includes it as a parent. In PROD3 it will +# be produced and dropped and there will not be a product that includes +# it as a parent. In PROD4 it will never be produced at all. +process.makeThingToBeDropped1 = cms.EDProducer("ThingWithMergeProducer") +process.dependsOnThingToBeDropped1 = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('makeThingToBeDropped1') +) + +process.test = cms.EDAnalyzer("TestMergeResults", + + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + # 0's are just placeholders, if the value is a "0" the check is not made. + expectedBeginRunProd = cms.untracked.vint32( + 10001, 10002, 10003 + ), + + expectedEndRunProd = cms.untracked.vint32( + 100001, 100002, 100003 + ), + + expectedBeginLumiProd = cms.untracked.vint32( + 101, 102, 103 + ), + + expectedEndLumiProd = cms.untracked.vint32( + 1001, 1002, 1003 + ), + + verbose = cms.untracked.bool(False), + + expectedParents = cms.untracked.vstring( + 'm1'), + testAlias = cms.untracked.bool(True) +) + +process.A = cms.EDProducer("ThingWithMergeProducer") + +process.B = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.C = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.D = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('B') +) + +process.E = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('B', 'C') +) + +process.F = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('C') +) + +process.G = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.H = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('G') +) + +process.I = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.J = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('I') +) + +process.K = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('I') +) + +process.L = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('F') +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMerge0.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_makeThingToBeDropped2_*_*' + ) +) + +process.p1 = cms.Path((process.m1 + process.m2 + process.m3) * + process.thingWithMergeProducer * + process.makeThingToBeDropped2 * + process.test * + process.tryNoPut * + process.makeThingToBeDropped * + process.makeThingToBeDropped1 * + process.dependsOnThingToBeDropped1) + +process.p2 = cms.Path(process.A * + process.B * + process.C * + process.D * + process.E * + process.F * + process.G * + process.H * + process.I * + process.J * + process.K * + process.L) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergePROD100_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergePROD100_cfg.py new file mode 100644 index 0000000000000..8e9fd5afd3dae --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergePROD100_cfg.py @@ -0,0 +1,31 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(60) +) + +# This will generate: +# run 41 lumis 1-10 events 1-30 +# run 42 lumis 1-10 events 1-30 +process.source = cms.Source("EmptySource", + firstRun = cms.untracked.uint32(41), + firstLuminosityBlock = cms.untracked.uint32(1), + firstEvent = cms.untracked.uint32(1), + numberEventsInLuminosityBlock = cms.untracked.uint32(3), + numberEventsInRun = cms.untracked.uint32(30) +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMerge100.root') +) + +process.task = cms.Task(process.thingWithMergeProducer) + +process.e = cms.EndPath(process.out, process.task) diff --git a/FWIO/RNTupleTempTests/test/testRunMergePROD101_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergePROD101_cfg.py new file mode 100644 index 0000000000000..b135d58bf3c14 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergePROD101_cfg.py @@ -0,0 +1,31 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(60) +) + +# This will generate: +# run 41 lumis 11-20 events 1-30 +# run 42 lumis 11-20 events 1-30 +process.source = cms.Source("EmptySource", + firstRun = cms.untracked.uint32(41), + firstLuminosityBlock = cms.untracked.uint32(11), + firstEvent = cms.untracked.uint32(1), + numberEventsInLuminosityBlock = cms.untracked.uint32(3), + numberEventsInRun = cms.untracked.uint32(30) +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMerge101.root') +) + +process.task = cms.Task(process.thingWithMergeProducer) + +process.e = cms.EndPath(process.out, process.task) diff --git a/FWIO/RNTupleTempTests/test/testRunMergePROD102_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergePROD102_cfg.py new file mode 100644 index 0000000000000..39b052d49183b --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergePROD102_cfg.py @@ -0,0 +1,31 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(60) +) + +# This will generate: +# run 41 lumis 21-30 events 1-30 +# run 42 lumis 21-30 events 1-30 +process.source = cms.Source("EmptySource", + firstRun = cms.untracked.uint32(41), + firstLuminosityBlock = cms.untracked.uint32(21), + firstEvent = cms.untracked.uint32(1), + numberEventsInLuminosityBlock = cms.untracked.uint32(3), + numberEventsInRun = cms.untracked.uint32(30) +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMerge102.root') +) + +process.task = cms.Task(process.thingWithMergeProducer) + +process.e = cms.EndPath(process.out, process.task) diff --git a/FWIO/RNTupleTempTests/test/testRunMergePROD11_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergePROD11_cfg.py new file mode 100644 index 0000000000000..aae80d20777df --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergePROD11_cfg.py @@ -0,0 +1,225 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +import FWCore.Framework.test.cmsExceptionsFatal_cff +process.options = FWCore.Framework.test.cmsExceptionsFatal_cff.options + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(10) +) + +process.source = cms.Source("EmptySource", + firstLuminosityBlock = cms.untracked.uint32(1), + numberEventsInLuminosityBlock = cms.untracked.uint32(100), + firstEvent = cms.untracked.uint32(11), + firstRun = cms.untracked.uint32(1), + numberEventsInRun = cms.untracked.uint32(100) +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('m1') +) + +# These are here only for tests of parentage merging +process.m1 = cms.EDProducer("ThingWithMergeProducer") +process.m2 = cms.EDProducer("ThingWithMergeProducer") +process.m3 = cms.EDProducer("ThingWithMergeProducer") + +process.tryNoPut = cms.EDProducer("ThingWithMergeProducer", + noPut = cms.untracked.bool(True) +) + +# This one tests products dropped and then restored by secondary file +# input +process.makeThingToBeDropped = cms.EDProducer("ThingWithMergeProducer") + +# This product will be produced in configuration PROD1 and PROD5 +# In PROD2 it will be produced and dropped and there will be another +# product whose provenance includes it as a parent. In PROD3 it will +# be produced and dropped and there will not be a product that includes +# it as a parent. In PROD4 it will never be produced at all. +process.makeThingToBeDropped1 = cms.EDProducer("ThingWithMergeProducer") +process.dependsOnThingToBeDropped1 = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('makeThingToBeDropped1') +) + +process.test = cms.EDAnalyzer("TestMergeResults", + + verbose = cms.untracked.bool(False), + + expectedParents = cms.untracked.vstring( + 'm1', 'm1', 'm1', 'm1', 'm1', + 'm1', 'm1', 'm1', 'm1', 'm1') +) + +process.A = cms.EDProducer("ThingWithMergeProducer") + +process.B = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.C = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.D = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('B') +) + +process.E = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('B', 'C') +) + +process.F = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('C') +) + +process.G = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.H = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('G') +) + +process.I = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.J = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('I') +) + +process.K = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('I') +) + +process.L = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('F') +) + +process.testGetterOfProducts = cms.EDFilter("TestGetterOfProducts", + processName = cms.string('PROD'), + expectedInputTagLabels = cms.vstring('A','B','C','D','E','F', + 'G', 'H', 'I', 'J', 'K', 'L', + 'dependsOnThingToBeDropped1', + 'm1', 'm2', 'm3', + 'makeThingToBeDropped', + 'makeThingToBeDropped1', + 'thingWithMergeProducer', + 'tryNoPut' ), + expectedLabelsAfterGet = cms.vstring('A','B','C','D','E','F', + 'G', 'H', 'I', 'J', 'K', 'L', + 'dependsOnThingToBeDropped1', + 'm1', 'm2', 'm3', + 'makeThingToBeDropped', + 'makeThingToBeDropped1', + 'thingWithMergeProducer' + + ) +) + +process.testGetterOfProductsA = cms.EDAnalyzer("TestGetterOfProductsA", + processName = cms.string('PROD'), + branchType = cms.int32(0), + expectedInputTagLabels = cms.vstring('A','B','C','D','E','F', + 'G', 'H', 'I', 'J', 'K', 'L', + 'dependsOnThingToBeDropped1', + 'm1', 'm2', 'm3', + 'makeThingToBeDropped', + 'makeThingToBeDropped1', + 'thingWithMergeProducer', + 'tryNoPut' + ), + expectedLabelsAfterGet = cms.vstring('A','B','C','D','E','F', + 'G', 'H', 'I', 'J', 'K', 'L', + 'dependsOnThingToBeDropped1', + 'm1', 'm2', 'm3', + 'makeThingToBeDropped', + 'makeThingToBeDropped1', + 'thingWithMergeProducer' + ), + expectedNumberOfThingsWithLabelA = cms.uint32(1) +) + +process.testGetterOfProductsARun = cms.EDAnalyzer("TestGetterOfProductsA", + processName = cms.string('PROD'), + branchType = cms.int32(2), + expectedInputTagLabels = cms.vstring('A','A','B','B','C','C','D','D','E','E','F','F', + 'G', 'G', 'H', 'H', 'I', 'I', 'J', 'J', 'K', 'K', 'L', 'L', + 'dependsOnThingToBeDropped1','dependsOnThingToBeDropped1', + 'm1', 'm1', 'm2', 'm2', 'm3','m3', + 'makeThingToBeDropped','makeThingToBeDropped', + 'makeThingToBeDropped1','makeThingToBeDropped1', + 'thingWithMergeProducer','thingWithMergeProducer', + 'tryNoPut','tryNoPut' + ), + expectedLabelsAfterGet = cms.vstring('A','A','B','B','C','C','D','D','E','E','F','F', + 'G', 'G', 'H', 'H', 'I', 'I', 'J', 'J', 'K', 'K', 'L', 'L', + 'dependsOnThingToBeDropped1', 'dependsOnThingToBeDropped1', + 'm1', 'm1', 'm2', 'm2', 'm3', 'm3', + 'makeThingToBeDropped', 'makeThingToBeDropped', + 'makeThingToBeDropped1', 'makeThingToBeDropped1', + 'thingWithMergeProducer', 'thingWithMergeProducer' + ), + expectedNumberOfThingsWithLabelA = cms.uint32(2) +) + +process.testGetterOfProductsALumi = cms.EDAnalyzer("TestGetterOfProductsA", + processName = cms.string('PROD'), + branchType = cms.int32(1), + expectedInputTagLabels = cms.vstring('A','A','B','B','C','C','D','D','E','E','F','F', + 'G', 'G', 'H', 'H', 'I', 'I', 'J', 'J', 'K', 'K', 'L', 'L', + 'dependsOnThingToBeDropped1','dependsOnThingToBeDropped1', + 'm1', 'm1', 'm2', 'm2', 'm3','m3', + 'makeThingToBeDropped','makeThingToBeDropped', + 'makeThingToBeDropped1','makeThingToBeDropped1', + 'thingWithMergeProducer','thingWithMergeProducer', + 'tryNoPut','tryNoPut' + ), + expectedLabelsAfterGet = cms.vstring('A','A','B','B','C','C','D','D','E','E','F','F', + 'G', 'G', 'H', 'H', 'I', 'I', 'J', 'J', 'K', 'K', 'L', 'L', + 'dependsOnThingToBeDropped1', 'dependsOnThingToBeDropped1', + 'm1', 'm1', 'm2', 'm2', 'm3', 'm3', + 'makeThingToBeDropped', 'makeThingToBeDropped', + 'makeThingToBeDropped1', 'makeThingToBeDropped1', + 'thingWithMergeProducer', 'thingWithMergeProducer' + ), + expectedNumberOfThingsWithLabelA = cms.uint32(2) +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMerge11.root') +) + +process.p1 = cms.Path((process.m1 + process.m2 + process.m3) * + process.thingWithMergeProducer * + process.test * + process.tryNoPut * + process.makeThingToBeDropped * + process.makeThingToBeDropped1 * + process.dependsOnThingToBeDropped1) + +process.p2 = cms.Path(process.A * + process.B * + process.C * + process.D * + process.E * + process.F * + process.G * + process.H * + process.I * + process.J * + process.K * + process.L * + process.testGetterOfProducts * + process.testGetterOfProductsA * + process.testGetterOfProductsARun * + process.testGetterOfProductsALumi +) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergePROD1_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergePROD1_cfg.py new file mode 100644 index 0000000000000..46b19aa238bca --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergePROD1_cfg.py @@ -0,0 +1,179 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +import FWCore.Framework.test.cmsExceptionsFatal_cff +process.options = FWCore.Framework.test.cmsExceptionsFatal_cff.options + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(10) +) + +process.source = cms.Source("EmptySource", + firstLuminosityBlock = cms.untracked.uint32(1), + numberEventsInLuminosityBlock = cms.untracked.uint32(100), + firstEvent = cms.untracked.uint32(11), + firstRun = cms.untracked.uint32(1), + numberEventsInRun = cms.untracked.uint32(100) +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('m1') +) + +# These are here only for tests of parentage merging +process.m1 = cms.EDProducer("ThingWithMergeProducer") +process.m2 = cms.EDProducer("ThingWithMergeProducer") +process.m3 = cms.EDProducer("ThingWithMergeProducer") + +process.tryNoPut = cms.EDProducer("ThingWithMergeProducer", + noPut = cms.untracked.bool(True) +) + +# This one tests products dropped and then restored by secondary file +# input +process.makeThingToBeDropped = cms.EDProducer("ThingWithMergeProducer") + +process.makeThingToBeDropped2 = cms.EDProducer("ThingWithMergeProducer") + +process.aliasForThingToBeDropped2 = cms.EDAlias( + makeThingToBeDropped2 = cms.VPSet( + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('event'), + toProductInstance = cms.string('instance2')), + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('endLumi'), + toProductInstance = cms.string('endLumi2')), + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('endRun'), + toProductInstance = cms.string('endRun2')) + ) +) + +# This product will be produced in configuration PROD1 and PROD5 +# In PROD2 it will be produced and dropped and there will be another +# product whose provenance includes it as a parent. In PROD3 it will +# be produced and dropped and there will not be a product that includes +# it as a parent. In PROD4 it will never be produced at all. +process.makeThingToBeDropped1 = cms.EDProducer("ThingWithMergeProducer") +process.dependsOnThingToBeDropped1 = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('makeThingToBeDropped1') +) + +process.test = cms.EDAnalyzer("TestMergeResults", + + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + # 0's are just placeholders, if the value is a "0" the check is not made. + expectedBeginRunProd = cms.untracked.vint32( + 10001, 10002, 10003 + ), + + expectedEndRunProd = cms.untracked.vint32( + 100001, 100002, 100003 + ), + + expectedBeginLumiProd = cms.untracked.vint32( + 101, 102, 103 + ), + + expectedEndLumiProd = cms.untracked.vint32( + 1001, 1002, 1003 + ), + + verbose = cms.untracked.bool(False), + + expectedParents = cms.untracked.vstring( + 'm1', 'm1', 'm1', 'm1', 'm1', + 'm1', 'm1', 'm1', 'm1', 'm1'), + testAlias = cms.untracked.bool(True) +) + +process.A = cms.EDProducer("ThingWithMergeProducer") + +process.B = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.C = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.D = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('B') +) + +process.E = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('B', 'C') +) + +process.F = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('C') +) + +process.G = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.H = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('G') +) + +process.I = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.J = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('I') +) + +process.K = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('I') +) + +process.L = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('F') +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMerge1.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_makeThingToBeDropped2_*_*' + ) +) + +process.p1 = cms.Path((process.m1 + process.m2 + process.m3) * + process.thingWithMergeProducer * + process.makeThingToBeDropped2 * + process.test * + process.tryNoPut * + process.makeThingToBeDropped * + process.makeThingToBeDropped1 * + process.dependsOnThingToBeDropped1) + +process.p2 = cms.Path(process.A * + process.B * + process.C * + process.D * + process.E * + process.F * + process.G * + process.H * + process.I * + process.J * + process.K * + process.L) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergePROD2EXTRA_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergePROD2EXTRA_cfg.py new file mode 100644 index 0000000000000..9a799848e9917 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergePROD2EXTRA_cfg.py @@ -0,0 +1,31 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("EXTRA") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + fileMode = cms.untracked.string('FULLMERGE'), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testRunMerge2.root') +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") +process.dependsOnThingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('thingWithMergeProducer') +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMerge2extra.root') +) + +process.path1 = cms.Path(process.thingWithMergeProducer * + process.dependsOnThingWithMergeProducer) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergePROD2_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergePROD2_cfg.py new file mode 100644 index 0000000000000..b4dd8523dcdb7 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergePROD2_cfg.py @@ -0,0 +1,184 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +import FWCore.Framework.test.cmsExceptionsFatal_cff +process.options = FWCore.Framework.test.cmsExceptionsFatal_cff.options + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(10) +) + +process.source = cms.Source("EmptySource", + firstLuminosityBlock = cms.untracked.uint32(1), + numberEventsInLuminosityBlock = cms.untracked.uint32(5), + firstEvent = cms.untracked.uint32(21), + firstRun = cms.untracked.uint32(1), + numberEventsInRun = cms.untracked.uint32(5) +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('m2') +) + +# These are here only for tests of parentage merging +process.m1 = cms.EDProducer("ThingWithMergeProducer") +process.m2 = cms.EDProducer("ThingWithMergeProducer") +process.m3 = cms.EDProducer("ThingWithMergeProducer") + +process.tryNoPut = cms.EDProducer("ThingWithMergeProducer", + noPut = cms.untracked.bool(True) +) + +# This one tests products dropped and then restored by secondary file +# input +process.makeThingToBeDropped = cms.EDProducer("ThingWithMergeProducer") + +process.makeThingToBeDropped2 = cms.EDProducer("ThingWithMergeProducer") + +process.aliasForThingToBeDropped2 = cms.EDAlias( + makeThingToBeDropped2 = cms.VPSet( + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('event'), + toProductInstance = cms.string('instance2')), + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('endLumi'), + toProductInstance = cms.string('endLumi2')), + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('endRun'), + toProductInstance = cms.string('endRun2')) + ) +) + +# This product will be produced in configuration PROD1 and PROD5 +# In PROD2 it will be produced and dropped and there will be another +# product whose provenance includes it as a parent. In PROD3 it will +# be produced and dropped and there will not be a product that includes +# it as a parent. In PROD4 it will never be produced at all. +process.makeThingToBeDropped1 = cms.EDProducer("ThingWithMergeProducer") +process.dependsOnThingToBeDropped1 = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('makeThingToBeDropped1') +) + +process.test = cms.EDAnalyzer("TestMergeResults", + + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + # 0's are just placeholders, if the value is a "0" the check is not made. + expectedBeginRunProd = cms.untracked.vint32( + 10001, 10002, 10003, + 10001, 10002, 10003 + ), + + expectedEndRunProd = cms.untracked.vint32( + 100001, 100002, 100003, + 100001, 100002, 100003 + ), + + expectedBeginLumiProd = cms.untracked.vint32( + 101, 102, 103, + 101, 102, 103 + ), + + expectedEndLumiProd = cms.untracked.vint32( + 1001, 1002, 1003, + 1001, 1002, 1003 + ), + + verbose = cms.untracked.bool(False), + + expectedParents = cms.untracked.vstring( + 'm2', 'm2', 'm2', 'm2', 'm2', + 'm2', 'm2', 'm2', 'm2', 'm2'), + testAlias = cms.untracked.bool(True) +) + +process.A = cms.EDProducer("ThingWithMergeProducer") + +process.B = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.C = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.D = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('B') +) + +process.E = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('B', 'C') +) + +process.F = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('C') +) + +process.G = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.H = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('G') +) + +process.I = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.J = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('I') +) + +process.K = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('I') +) + +process.L = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('F') +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMerge2.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_makeThingToBeDropped1_*_*', + 'drop *_makeThingToBeDropped2_*_*' + ) +) + +process.p1 = cms.Path((process.m1 + process.m2 + process.m3) * + process.thingWithMergeProducer * + process.makeThingToBeDropped2 * + process.test * + process.tryNoPut * + process.makeThingToBeDropped * + process.makeThingToBeDropped1 * + process.dependsOnThingToBeDropped1) + +process.p2 = cms.Path(process.A * + process.B * + process.C * + process.D * + process.E * + process.F * + process.G * + process.H * + process.I * + process.J * + process.K * + process.L) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergePROD3EXTRA_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergePROD3EXTRA_cfg.py new file mode 100644 index 0000000000000..cb2fbe88a3748 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergePROD3EXTRA_cfg.py @@ -0,0 +1,27 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("EXTRA") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + fileMode = cms.untracked.string('FULLMERGE'), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testRunMerge3.root') +) + +process.thingWithMergeProducer3extra = cms.EDProducer("ThingWithMergeProducer") + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMerge3extra.root') +) + +process.path1 = cms.Path(process.thingWithMergeProducer3extra) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergePROD3_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergePROD3_cfg.py new file mode 100644 index 0000000000000..57ec559d35b39 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergePROD3_cfg.py @@ -0,0 +1,179 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +import FWCore.Framework.test.cmsExceptionsFatal_cff +process.options = FWCore.Framework.test.cmsExceptionsFatal_cff.options + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(10) +) + +process.source = cms.Source("EmptySource", + firstLuminosityBlock = cms.untracked.uint32(1), + numberEventsInLuminosityBlock = cms.untracked.uint32(100), + firstEvent = cms.untracked.uint32(1), + firstRun = cms.untracked.uint32(1), + numberEventsInRun = cms.untracked.uint32(100) +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer", + changeIsEqualValue = cms.untracked.bool(True), + labelsToGet = cms.untracked.vstring('m3') +) + +# These are here only for tests of parentage merging +process.m1 = cms.EDProducer("ThingWithMergeProducer") +process.m2 = cms.EDProducer("ThingWithMergeProducer") +process.m3 = cms.EDProducer("ThingWithMergeProducer") + +process.tryNoPut = cms.EDProducer("ThingWithMergeProducer", + noPut = cms.untracked.bool(True) +) + +# This one tests products dropped and then restored by secondary file +# input +process.makeThingToBeDropped = cms.EDProducer("ThingWithMergeProducer") + +process.makeThingToBeDropped2 = cms.EDProducer("ThingWithMergeProducer") + +process.aliasForThingToBeDropped2 = cms.EDAlias( + makeThingToBeDropped2 = cms.VPSet( + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('event'), + toProductInstance = cms.string('instance2')), + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('endLumi'), + toProductInstance = cms.string('endLumi2')), + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('endRun'), + toProductInstance = cms.string('endRun2')) + ) +) + +# This product will be produced in configuration PROD1 and PROD5 +# In PROD2 it will be produced and dropped and there will be another +# product whose provenance includes it as a parent. In PROD3 it will +# be produced and dropped and there will not be a product that includes +# it as a parent. In PROD4 it will never be produced at all. +process.makeThingToBeDropped1 = cms.EDProducer("ThingWithMergeProducer") +process.dependsOnThingToBeDropped1 = cms.EDProducer("ThingWithMergeProducer") + +process.test = cms.EDAnalyzer("TestMergeResults", + + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + # 0's are just placeholders, if the value is a "0" the check is not made. + expectedBeginRunProd = cms.untracked.vint32( + 10001, 10002, 10004 + ), + + expectedEndRunProd = cms.untracked.vint32( + 100001, 100002, 100004 + ), + + expectedBeginLumiProd = cms.untracked.vint32( + 101, 102, 104 + ), + + expectedEndLumiProd = cms.untracked.vint32( + 1001, 1002, 1004 + ), + + verbose = cms.untracked.bool(False), + + expectedParents = cms.untracked.vstring( + 'm3', 'm3', 'm3', 'm3', 'm3', + 'm3', 'm3', 'm3', 'm3', 'm3'), + testAlias = cms.untracked.bool(True) +) + +process.A = cms.EDProducer("ThingWithMergeProducer") + +process.B = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.C = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.D = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('B') +) + +process.E = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('B', 'C') +) + +process.F = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('C') +) + +process.G = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.H = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('G') +) + +process.I = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.J = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('I') +) + +process.K = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('I') +) + +process.L = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('F') +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMerge3.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_makeThingToBeDropped1_*_*', + 'drop *_makeThingToBeDropped2_*_*' + ) +) + +process.p1 = cms.Path((process.m1 + process.m2 + process.m3) * + process.thingWithMergeProducer * + process.makeThingToBeDropped2 * + process.test * + process.tryNoPut * + process.makeThingToBeDropped * + process.makeThingToBeDropped1 * + process.dependsOnThingToBeDropped1) + +process.p2 = cms.Path(process.A * + process.B * + process.C * + process.D * + process.E * + process.F * + process.G * + process.H * + process.I * + process.J * + process.K * + process.L) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergePROD4_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergePROD4_cfg.py new file mode 100644 index 0000000000000..7487c1fe0a77a --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergePROD4_cfg.py @@ -0,0 +1,145 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +import FWCore.Framework.test.cmsExceptionsFatal_cff +process.options = FWCore.Framework.test.cmsExceptionsFatal_cff.options + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(1) +) + +process.source = cms.Source("EmptySource", + firstLuminosityBlock = cms.untracked.uint32(1), + numberEventsInLuminosityBlock = cms.untracked.uint32(100), + firstEvent = cms.untracked.uint32(1), + firstRun = cms.untracked.uint32(11), + numberEventsInRun = cms.untracked.uint32(100) +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('m1') +) + +# These are here only for tests of parentage merging +process.m1 = cms.EDProducer("ThingWithMergeProducer") +process.m2 = cms.EDProducer("ThingWithMergeProducer") +process.m3 = cms.EDProducer("ThingWithMergeProducer") + +process.tryNoPut = cms.EDProducer("ThingWithMergeProducer", + noPut = cms.untracked.bool(True) +) + +# This one tests products dropped and then restored by secondary file +# input +process.makeThingToBeDropped = cms.EDProducer("ThingWithMergeProducer") + +process.makeThingToBeDropped2 = cms.EDProducer("ThingWithMergeProducer") + +process.aliasForThingToBeDropped2 = cms.EDAlias( + makeThingToBeDropped2 = cms.VPSet( + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('event'), + toProductInstance = cms.string('instance2')), + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('endLumi'), + toProductInstance = cms.string('endLumi2')), + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('endRun'), + toProductInstance = cms.string('endRun2')) + ) +) + +# This product will be produced in configuration PROD1 and PROD5 +# In PROD2 it will be produced and dropped and there will be another +# product whose provenance includes it as a parent. In PROD3 it will +# be produced and dropped and there will not be a product that includes +# it as a parent. In PROD4 it will never be produced at all. +process.makeThingToBeDropped1 = cms.EDProducer("ThingWithMergeProducer", + noPut = cms.untracked.bool(True) +) +process.dependsOnThingToBeDropped1 = cms.EDProducer("ThingWithMergeProducer") + +process.test = cms.EDAnalyzer("TestMergeResults", + testAlias = cms.untracked.bool(True)) + +process.A = cms.EDProducer("ThingWithMergeProducer") + +process.B = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.C = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.D = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('B') +) + +process.E = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('B', 'C') +) + +process.F = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('C') +) + +process.G = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.H = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('G') +) + +process.I = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.J = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('I') +) + +process.K = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('I') +) + +process.L = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('F') +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMerge4.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_makeThingToBeDropped2_*_*' + ) +) + +process.p1 = cms.Path((process.m1 + process.m2 + process.m3) * + process.thingWithMergeProducer * + process.makeThingToBeDropped2 * + process.test * + process.tryNoPut * + process.makeThingToBeDropped * + process.makeThingToBeDropped1 * + process.dependsOnThingToBeDropped1) + +process.p2 = cms.Path(process.A * + process.B * + process.C * + process.D * + process.E * + process.F * + process.G * + process.H * + process.I * + process.J * + process.K * + process.L) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergePROD5_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergePROD5_cfg.py new file mode 100644 index 0000000000000..18e7e1e76903c --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergePROD5_cfg.py @@ -0,0 +1,144 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +import FWCore.Framework.test.cmsExceptionsFatal_cff +process.options = FWCore.Framework.test.cmsExceptionsFatal_cff.options + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(99) +) + +process.source = cms.Source("EmptySource", + firstLuminosityBlock = cms.untracked.uint32(2), + numberEventsInLuminosityBlock = cms.untracked.uint32(3), + firstEvent = cms.untracked.uint32(1), + firstRun = cms.untracked.uint32(11), + numberEventsInRun = cms.untracked.uint32(9) +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('m1') +) + +# These are here only for tests of parentage merging +process.m1 = cms.EDProducer("ThingWithMergeProducer") +process.m2 = cms.EDProducer("ThingWithMergeProducer") +process.m3 = cms.EDProducer("ThingWithMergeProducer") + +process.tryNoPut = cms.EDProducer("ThingWithMergeProducer", + noPut = cms.untracked.bool(True) +) + +# This one tests products dropped and then restored by secondary file +# input +process.makeThingToBeDropped = cms.EDProducer("ThingWithMergeProducer") + +process.makeThingToBeDropped2 = cms.EDProducer("ThingWithMergeProducer") + +process.aliasForThingToBeDropped2 = cms.EDAlias( + makeThingToBeDropped2 = cms.VPSet( + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('event'), + toProductInstance = cms.string('instance2')), + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('endLumi'), + toProductInstance = cms.string('endLumi2')), + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('endRun'), + toProductInstance = cms.string('endRun2')) + ) +) + +# This product will be produced in configuration PROD1 and PROD5 +# In PROD2 it will be produced and dropped and there will be another +# product whose provenance includes it as a parent. In PROD3 it will +# be produced and dropped and there will not be a product that includes +# it as a parent. In PROD4 it will never be produced at all. +process.makeThingToBeDropped1 = cms.EDProducer("ThingWithMergeProducer") +process.dependsOnThingToBeDropped1 = cms.EDProducer("ThingWithMergeProducer") + +process.test = cms.EDAnalyzer("TestMergeResults", + testAlias = cms.untracked.bool(True) +) + +process.A = cms.EDProducer("ThingWithMergeProducer") + +process.B = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.C = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.D = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('B') +) + +process.E = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('B', 'C') +) + +process.F = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('C') +) + +process.G = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.H = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('G') +) + +process.I = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.J = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('I') +) + +process.K = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('I') +) + +process.L = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('F') +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMerge5.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_makeThingToBeDropped2_*_*' + ) +) + +process.p1 = cms.Path((process.m1 + process.m2 + process.m3) * + process.thingWithMergeProducer * + process.makeThingToBeDropped2 * + process.test * + process.tryNoPut * + process.makeThingToBeDropped * + process.makeThingToBeDropped1 * + process.dependsOnThingToBeDropped1) + +process.p2 = cms.Path(process.A * + process.B * + process.C * + process.D * + process.E * + process.F * + process.G * + process.H * + process.I * + process.J * + process.K * + process.L) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergePROD6_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergePROD6_cfg.py new file mode 100644 index 0000000000000..20e9e4ac7c287 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergePROD6_cfg.py @@ -0,0 +1,146 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +import FWCore.Framework.test.cmsExceptionsFatal_cff +process.options = FWCore.Framework.test.cmsExceptionsFatal_cff.options + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(10) +) + +process.source = cms.Source("EmptySource", + firstLuminosityBlock = cms.untracked.uint32(2), + numberEventsInLuminosityBlock = cms.untracked.uint32(3), + firstEvent = cms.untracked.uint32(2), + firstRun = cms.untracked.uint32(11), + numberEventsInRun = cms.untracked.uint32(9) +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('m1') +) + +# These are here only for tests of parentage merging +process.m1 = cms.EDProducer("ThingWithMergeProducer") +process.m2 = cms.EDProducer("ThingWithMergeProducer") +process.m3 = cms.EDProducer("ThingWithMergeProducer") + +process.tryNoPut = cms.EDProducer("ThingWithMergeProducer", + noPut = cms.untracked.bool(True) +) + +# This one tests products dropped and then restored by secondary file +# input +process.makeThingToBeDropped = cms.EDProducer("ThingWithMergeProducer") + +process.makeThingToBeDropped2 = cms.EDProducer("ThingWithMergeProducer") + +process.aliasForThingToBeDropped2 = cms.EDAlias( + makeThingToBeDropped2 = cms.VPSet( + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('event'), + toProductInstance = cms.string('instance2')), + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('endLumi'), + toProductInstance = cms.string('endLumi2')), + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('endRun'), + toProductInstance = cms.string('endRun2')) + ) +) + +# This product will be produced in configuration PROD1 and PROD5 +# In PROD2 it will be produced and dropped and there will be another +# product whose provenance includes it as a parent. In PROD3 it will +# be produced and dropped and there will not be a product that includes +# it as a parent. In PROD6 it will never be produced at all. +process.makeThingToBeDropped1 = cms.EDProducer("ThingWithMergeProducer", + noPut = cms.untracked.bool(True) +) +process.dependsOnThingToBeDropped1 = cms.EDProducer("ThingWithMergeProducer") + +process.test = cms.EDAnalyzer("TestMergeResults", + testAlias = cms.untracked.bool(True) +) + +process.A = cms.EDProducer("ThingWithMergeProducer") + +process.B = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.C = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.D = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('B') +) + +process.E = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('B', 'C') +) + +process.F = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('C') +) + +process.G = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.H = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('G') +) + +process.I = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.J = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('I') +) + +process.K = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('I') +) + +process.L = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('F') +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMerge6.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_makeThingToBeDropped2_*_*' + ) +) + +process.p1 = cms.Path((process.m1 + process.m2 + process.m3) * + process.thingWithMergeProducer * + process.makeThingToBeDropped2 * + process.test * + process.tryNoPut * + process.makeThingToBeDropped * + process.makeThingToBeDropped1 * + process.dependsOnThingToBeDropped1) + +process.p2 = cms.Path(process.A * + process.B * + process.C * + process.D * + process.E * + process.F * + process.G * + process.H * + process.I * + process.J * + process.K * + process.L) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergePROD7_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergePROD7_cfg.py new file mode 100644 index 0000000000000..001a78cd6819b --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergePROD7_cfg.py @@ -0,0 +1,150 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +import FWCore.Framework.test.cmsExceptionsFatal_cff +process.options = FWCore.Framework.test.cmsExceptionsFatal_cff.options + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(1) +) + +process.source = cms.Source("EmptySource", + firstLuminosityBlock = cms.untracked.uint32(1), + numberEventsInLuminosityBlock = cms.untracked.uint32(100), + firstEvent = cms.untracked.uint32(26), + firstRun = cms.untracked.uint32(1), + numberEventsInRun = cms.untracked.uint32(100) +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('m1') +) + +# These are here only for tests of parentage merging +process.m1 = cms.EDProducer("ThingWithMergeProducer") +process.m2 = cms.EDProducer("ThingWithMergeProducer") +process.m3 = cms.EDProducer("ThingWithMergeProducer") + +process.tryNoPut = cms.EDProducer("ThingWithMergeProducer", + noPut = cms.untracked.bool(True) +) + +# This one tests products dropped and then restored by secondary file +# input +process.makeThingToBeDropped = cms.EDProducer("ThingWithMergeProducer") + +process.makeThingToBeDropped2 = cms.EDProducer("ThingWithMergeProducer") + +process.aliasForThingToBeDropped2 = cms.EDAlias( + makeThingToBeDropped2 = cms.VPSet( + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('event'), + toProductInstance = cms.string('instance2')), + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('endLumi'), + toProductInstance = cms.string('endLumi2')), + cms.PSet(type = cms.string('edmtestThing'), + fromProductInstance = cms.string('endRun'), + toProductInstance = cms.string('endRun2')) + ) +) + +# This product will be produced in configuration PROD1 and PROD5 +# In PROD2 it will be produced and dropped and there will be another +# product whose provenance includes it as a parent. In PROD3 it will +# be produced and dropped and there will not be a product that includes +# it as a parent. In PROD4 it will never be produced at all. +process.makeThingToBeDropped1 = cms.EDProducer("ThingWithMergeProducer") +process.dependsOnThingToBeDropped1 = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('makeThingToBeDropped1') +) + +process.test = cms.EDAnalyzer("TestMergeResults", + + verbose = cms.untracked.bool(False), + + expectedParents = cms.untracked.vstring('m1'), + testAlias = cms.untracked.bool(True) +) + +process.A = cms.EDProducer("ThingWithMergeProducer") + +process.B = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.C = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.D = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('B') +) + +process.E = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('B', 'C') +) + +process.F = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('C') +) + +process.G = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.H = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('G') +) + +process.I = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('A') +) + +process.J = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('I') +) + +process.K = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('I') +) + +process.L = cms.EDProducer("ThingWithMergeProducer", + labelsToGet = cms.untracked.vstring('F') +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMerge7.root'), + outputCommands = cms.untracked.vstring( + 'keep *', + 'drop *_makeThingToBeDropped2_*_*' + ) +) + +process.p1 = cms.Path((process.m1 + process.m2 + process.m3) * + process.thingWithMergeProducer * + process.makeThingToBeDropped2 * + process.test * + process.tryNoPut * + process.makeThingToBeDropped * + process.makeThingToBeDropped1 * + process.dependsOnThingToBeDropped1) + +process.p2 = cms.Path(process.A * + process.B * + process.C * + process.D * + process.E * + process.F * + process.G * + process.H * + process.I * + process.J * + process.K * + process.L) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergePickEvents_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergePickEvents_cfg.py new file mode 100644 index 0000000000000..bae42690ea139 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergePickEvents_cfg.py @@ -0,0 +1,51 @@ + +# Test the RNTupleTempSource parameter that one can use +# to request specific events. This interacts with +# the duplicate checking and skip event feature so +# I set those parameters also and test that. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("COPY") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + fileMode = cms.untracked.string('FULLMERGE'), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = [ + 'file:testRunMergeRecombined.root', + 'file:testRunMergeRecombined.root' + ] + #, duplicateCheckMode = 'checkAllFilesOpened' + , skipEvents = 3 + , eventsToProcess = ['1:2-1:7'] +) + +from FWCore.Framework.modules import RunLumiEventAnalyzer +process.test = RunLumiEventAnalyzer( + verbose = True, + expectedRunLumiEvents = [ +1, 0, 0, +1, 1, 0, +1, 1, 5, +1, 1, 6, +1, 1, 7, +1, 1, 0, +1, 0, 0, +] +) + +process.path1 = cms.Path(process.test) + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.out = RNTupleTempOutputModule( + fileName = 'testRunMergePickEvent.root' +) + +process.endpath1 = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergePickEventsx_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergePickEventsx_cfg.py new file mode 100644 index 0000000000000..fa24739ab347a --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergePickEventsx_cfg.py @@ -0,0 +1,43 @@ + +# Test the RNTupleTempSource parameter that one can use +# to request specific events. This interacts with +# the duplicate checking and skip event feature so +# I set those parameters also and test that. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("COPY") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + fileMode = cms.untracked.string('FULLMERGE'), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = [ + 'file:testRunMergeRecombined.root' + ] + #, duplicateCheckMode = 'checkAllFilesOpened' + , skipEvents = 3 + , eventsToProcess = ['1:1:2-1:1:7'] +) + +from FWCore.Framework.modules import RunLumiEventAnalyzer +process.test = RunLumiEventAnalyzer( + verbose = True, + expectedRunLumiEvents = [ +1, 0, 0, +1, 1, 0, +1, 1, 5, +1, 1, 6, +1, 1, 7, +1, 1, 0, +1, 0, 0, +] +) + +process.path1 = cms.Path(process.test) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeSPLIT100_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeSPLIT100_cfg.py new file mode 100644 index 0000000000000..327969ab0f5b2 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeSPLIT100_cfg.py @@ -0,0 +1,27 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("SPLIT") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testRunMerge100.root' + ), + lumisToProcess = cms.untracked.VLuminosityBlockRange('41:6-41:10', '42:6-42:10'), +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") +process.task = cms.Task(process.thingWithMergeProducer) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMergeSPLIT100.root') +) + +process.e = cms.EndPath(process.out, process.task) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeSPLIT101_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeSPLIT101_cfg.py new file mode 100644 index 0000000000000..cfbab4e2f8741 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeSPLIT101_cfg.py @@ -0,0 +1,27 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("SPLIT") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testRunMerge101.root' + ), + lumisToProcess = cms.untracked.VLuminosityBlockRange('41:11-41:15', '42:11-42:15'), +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") +process.task = cms.Task(process.thingWithMergeProducer) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMergeSPLIT101.root') +) + +process.e = cms.EndPath(process.out, process.task) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeSPLIT102_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeSPLIT102_cfg.py new file mode 100644 index 0000000000000..3d3166af7da90 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeSPLIT102_cfg.py @@ -0,0 +1,27 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("SPLIT") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testRunMerge101.root' + ), + lumisToProcess = cms.untracked.VLuminosityBlockRange('41:16-41:20', '42:16-42:20'), +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") +process.task = cms.Task(process.thingWithMergeProducer) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMergeSPLIT102.root') +) + +process.e = cms.EndPath(process.out, process.task) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeSPLIT103_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeSPLIT103_cfg.py new file mode 100644 index 0000000000000..ec12d9a62fb65 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeSPLIT103_cfg.py @@ -0,0 +1,27 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("SPLIT") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testRunMerge102.root' + ), + lumisToProcess = cms.untracked.VLuminosityBlockRange('41:21-41:25', '42:21-42:25'), +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") +process.task = cms.Task(process.thingWithMergeProducer) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMergeSPLIT103.root') +) + +process.e = cms.EndPath(process.out, process.task) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeTEST100_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeTEST100_cfg.py new file mode 100644 index 0000000000000..2bcd85a274f32 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeTEST100_cfg.py @@ -0,0 +1,137 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testRunMergeMERGE100.root', + 'file:testRunMergeMERGE101.root', + 'file:testRunMergeMERGE102.root', + 'file:testRunMergeMERGE100.root', + 'file:testRunMergeMERGE101.root', + 'file:testRunMergeMERGE102.root' + ), + duplicateCheckMode = cms.untracked.string('noDuplicateCheck') +) + +process.test = cms.EDAnalyzer("TestMergeResults", + + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + + expectedBeginRunProd = cms.untracked.vint32( + 10001, 40008, 10003, + 10001, 20004, 10003, + 10001, 40008, 10003, + 10001, 20004, 10003 + ), + + expectedEndRunProd = cms.untracked.vint32( + 100001, 400008, 100003, + 100001, 200004, 100003, + 100001, 400008, 100003, + 100001, 200004, 100003 + ), + + expectedEndRunProdImproperlyMerged = cms.untracked.vint32( + 0, 1, 0, + 0, 0, 0, + 0, 1, 0, + 0, 0, 0 + ), + + expectedBeginLumiProd = cms.untracked.vint32( + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103 + ), + + expectedEndLumiProd = cms.untracked.vint32( + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003 + ), + + expectedEndLumiProdImproperlyMerged = cms.untracked.vint32( + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0 + ), + verbose = cms.untracked.bool(False) +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") +process.task = cms.Task(process.thingWithMergeProducer) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMergeTEST100.root') +) + +process.e = cms.EndPath(process.test * process.out, process.task) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeTEST101_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeTEST101_cfg.py new file mode 100644 index 0000000000000..9e1c6bd5266cc --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeTEST101_cfg.py @@ -0,0 +1,109 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(-1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testRunMergeSPLIT101.root', + 'file:testRunMergeSPLIT102.root', + 'file:testRunMergeSPLIT103.root', + 'file:testRunMergeSPLIT101.root', + 'file:testRunMergeSPLIT102.root', + 'file:testRunMergeSPLIT103.root' + ), + lumisToProcess = cms.untracked.VLuminosityBlockRange('41:11-41:20', '42:21-42:25'), + duplicateCheckMode = cms.untracked.string('noDuplicateCheck') +) + +process.test = cms.EDAnalyzer("TestMergeResults", + + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + + expectedBeginRunProd = cms.untracked.vint32( + 10001, 10002, 10003, + 10001, 10002, 10003, + 10001, 10002, 10003, + 10001, 10002, 10003 + ), + + expectedEndRunProd = cms.untracked.vint32( + 100001, 100002, 100003, + 100001, 100002, 100003, + 100001, 100002, 100003, + 100001, 100002, 100003 + ), + + expectedEndRunProdImproperlyMerged = cms.untracked.vint32( + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0 + ), + + expectedBeginLumiProd = cms.untracked.vint32( + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103, + 101, 102, 103 + ), + + expectedEndLumiProd = cms.untracked.vint32( + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003, + 1001, 1002, 1003 + ), + + expectedEndLumiProdImproperlyMerged = cms.untracked.vint32( + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0, + 0, 0, 0 + ), + + verbose = cms.untracked.bool(False) +) + +process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer") +process.task = cms.Task(process.thingWithMergeProducer) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMergeTEST101.root') +) + +process.e = cms.EndPath(process.test * process.out, process.task) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeTEST11_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeTEST11_cfg.py new file mode 100644 index 0000000000000..c9f3a13df06be --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeTEST11_cfg.py @@ -0,0 +1,122 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST11") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(1) +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + # CAUTION if you recreate the PROD files then you must recreate BOTH + # of these files otherwise you will get exceptions because the GUIDs + # used to check the match of the event in the secondary files will + # not be the same. + 'file:testRunMerge.root', + 'file:testRunMergeMERGE2.root' + ), + secondaryFileNames = cms.untracked.vstring( + 'file:testRunMerge0.root', + 'file:testRunMerge1.root', + 'file:testRunMerge2.root', + 'file:testRunMerge3.root', + 'file:testRunMerge4.root', + 'file:testRunMerge5.root' + ) + , duplicateCheckMode = cms.untracked.string('checkEachRealDataFile') + , noEventSort = cms.untracked.bool(False) +) + +process.testGetterOfProducts = cms.EDFilter("TestGetterOfProducts", + processName = cms.string('PROD'), + expectedInputTagLabels = cms.vstring('A','B','C','D','E','F', 'G', 'H', 'I', 'J', 'K', 'L', + 'aliasForThingToBeDropped2', + 'dependsOnThingToBeDropped1', + 'm1', 'm2', 'm3', + 'makeThingToBeDropped', + 'makeThingToBeDropped1', + 'thingWithMergeProducer', + 'tryNoPut' + ), + expectedLabelsAfterGet = cms.vstring('A','B','C','D','E','F', 'G', 'H', 'I', 'J', 'K', 'L', + 'aliasForThingToBeDropped2', + 'dependsOnThingToBeDropped1', + 'm1', 'm2', 'm3', + 'makeThingToBeDropped', + 'thingWithMergeProducer' + ) +) + +process.testGetterOfProductsA = cms.EDAnalyzer("TestGetterOfProductsA", + processName = cms.string('PROD'), + branchType = cms.int32(0), + expectedInputTagLabels = cms.vstring('A','B','C','D','E','F', 'G', 'H', 'I', 'J', 'K', 'L', + 'aliasForThingToBeDropped2', + 'dependsOnThingToBeDropped1', + 'm1', 'm2', 'm3', + 'makeThingToBeDropped', + 'makeThingToBeDropped1', + 'thingWithMergeProducer', + 'tryNoPut' + ), + expectedLabelsAfterGet = cms.vstring('A','B','C','D','E','F', 'G', 'H', 'I', 'J', 'K', 'L', + 'aliasForThingToBeDropped2', + 'dependsOnThingToBeDropped1', + 'm1', 'm2', 'm3', + 'makeThingToBeDropped', + 'thingWithMergeProducer' + ), + expectedNumberOfThingsWithLabelA = cms.uint32(1) +) + +process.testGetterOfProductsALumi = cms.EDAnalyzer("TestGetterOfProductsA", + processName = cms.string('PROD'), + branchType = cms.int32(1), + expectedInputTagLabels = cms.vstring('A','A','B','B','C','C','D','D','E','E','F','F', 'G','G', 'H','H', 'I','I', 'J','J', 'K','K', 'L','L', + 'aliasForThingToBeDropped2', + 'dependsOnThingToBeDropped1','dependsOnThingToBeDropped1', + 'm1','m1', 'm2','m2', 'm3','m3', + 'makeThingToBeDropped','makeThingToBeDropped', + 'makeThingToBeDropped1','makeThingToBeDropped1', + 'thingWithMergeProducer','thingWithMergeProducer', + 'tryNoPut','tryNoPut' + ), + expectedLabelsAfterGet = cms.vstring('A','A','B','B','C','C','D','D','E','E','F','F', 'G','G', 'H','H', 'I','I', 'J','J', 'K','K', 'L','L', + 'aliasForThingToBeDropped2', + 'dependsOnThingToBeDropped1','dependsOnThingToBeDropped1', + 'm1','m1', 'm2','m2', 'm3','m3', + 'makeThingToBeDropped','makeThingToBeDropped', + 'makeThingToBeDropped1','makeThingToBeDropped1', + 'thingWithMergeProducer','thingWithMergeProducer' + ), + expectedNumberOfThingsWithLabelA = cms.uint32(2) +) + +process.testGetterOfProductsARun = cms.EDAnalyzer("TestGetterOfProductsA", + processName = cms.string('PROD'), + branchType = cms.int32(2), + expectedInputTagLabels = cms.vstring('A','A','B','B','C','C','D','D','E','E','F','F', 'G','G', 'H','H', 'I','I', 'J','J', 'K','K', 'L','L', + 'aliasForThingToBeDropped2', + 'dependsOnThingToBeDropped1','dependsOnThingToBeDropped1', + 'm1','m1', 'm2','m2', 'm3','m3', + 'makeThingToBeDropped','makeThingToBeDropped', + 'makeThingToBeDropped1','makeThingToBeDropped1', + 'thingWithMergeProducer','thingWithMergeProducer', + 'tryNoPut','tryNoPut' + ), + expectedLabelsAfterGet = cms.vstring('A','A','B','B','C','C','D','D','E','E','F','F', 'G','G', 'H','H', 'I','I', 'J','J', 'K','K', 'L','L', + 'aliasForThingToBeDropped2', + 'dependsOnThingToBeDropped1','dependsOnThingToBeDropped1', + 'm1','m1', 'm2','m2', 'm3','m3', + 'makeThingToBeDropped','makeThingToBeDropped', + 'makeThingToBeDropped1','makeThingToBeDropped1', + 'thingWithMergeProducer','thingWithMergeProducer' + ), + expectedNumberOfThingsWithLabelA = cms.uint32(2) +) + +process.path1 = cms.Path(process.testGetterOfProducts*process.testGetterOfProductsA*process.testGetterOfProductsALumi*process.testGetterOfProductsARun) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeTEST1_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeTEST1_cfg.py new file mode 100644 index 0000000000000..36c3738dc2b09 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeTEST1_cfg.py @@ -0,0 +1,248 @@ + +# This is the same as testRunMergeTEST except the noEventSort +# option is set in the RNTupleTempSource, which changes the order +# events are processed. Within a LuminosityBlock, they are +# in entry order instead of event number order. The RunLumiEventAnalyzer +# module checks this. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + fileMode = cms.untracked.string('FULLMERGE'), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = [ + 'file:testRunMerge.root', + 'file:testRunMerge.root' + ], + secondaryFileNames = [ + 'file:testRunMerge1.root', + 'file:testRunMerge2.root', + 'file:testRunMerge3.root', + 'file:testRunMerge4.root', + 'file:testRunMerge5.root' + + ], + noEventSort = True + , duplicateCheckMode = 'checkEachRealDataFile' +) + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.out = RNTupleTempOutputModule(fileName = 'testRunMergeRecombined1.root') + +from FWCore.Framework.modules import TestMergeResults, RunLumiEventAnalyzer +process.test = TestMergeResults( + + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + # 0's are just placeholders, if the value is a "0" the check is not made. + + expectedBeginRunProd = [ + 10001, 30006, 10003, # end run 1 + 10001, 10002, 10003, # end run 2 + 10001, 20004, 10003, # end run 11 + 10001, 30006, 10003, # end run 1 + 10001, 10002, 10003, # end run 2 + 10001, 20004, 10003 # end run 11 + ], + + expectedEndRunProd = [ + 100001, 300006, 100003, # end run 1 + 100001, 100002, 100003, # end run 2 + 100001, 200004, 100003, # end run 11 + 100001, 300006, 100003, # end run 1 + 100001, 100002, 100003, # end run 2 + 100001, 200004, 100003 # end run 11 + ], + + expectedBeginLumiProd = [ + 101, 306, 103, # end run 1 lumi 1 + 101, 102, 103, # end run 2 lumi 1 + 101, 102, 103, # end run 11 lumi 1 + 101, 102, 103, # end run 11 lumi 2 + 101, 306, 103, # end run 1 lumi 1 + 101, 102, 103, # end run 2 lumi 1 + 101, 102, 103, # end run 11 lumi 1 + 101, 102, 103 # end run 11 lumi 2 + ], + + expectedEndLumiProd = [ + 1001, 3006, 1003, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 2 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 2 + 1001, 3006, 1003, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 2 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 1 + 1001, 1002, 1003 # end run 11 lumi 2 + ], + + expectedBeginRunNew = [ + 10001, 20004, 10003, # end run 1 + 10001, 10002, 10003, # end run 2 + 10001, 10002, 10003, # end run 11 + 10001, 20004, 10003, # end run 1 + 10001, 10002, 10003, # end run 2 + 10001, 10002, 10003 # end run 11 + ], + + expectedEndRunNew = [ + 100001, 200004, 100003, # end run 1 + 100001, 100002, 100003, # end run 2 + 100001, 100002, 100003, # end run 11 + 100001, 200004, 100003, # end run 1 + 100001, 100002, 100003, # end run 2 + 100001, 100002, 100003 # end run 11 + ], + + expectedBeginLumiNew = [ + 101, 204, 103, # end run 1 lumi 1 + 101, 102, 103, # end run 2 lumi 1 + 101, 102, 103, # end run 11 lumi 1 + 101, 102, 103, # end run 11 lumi 2 + 101, 204, 103, # end run 1 lumi 1 + 101, 102, 103, # end run 2 lumi 1 + 101, 102, 103, # end run 11 lumi 1 + 101, 102, 103 # end run 11 lumi 2 + ], + + expectedEndLumiNew = [ + 1001, 2004, 1003, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 2 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 2 + 1001, 2004, 1003, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 2 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 1 + 1001, 1002, 1003 # end run 11 lumi 2 + ], + + expectedDroppedEvent = [13, 10003, 100003, 103, 1003], + verbose = False, + + expectedParents = [ + 'm1', 'm1', 'm1', 'm1', 'm1', + 'm1', 'm1', 'm1', 'm1', 'm1', + 'm2', 'm2', 'm2', 'm2', 'm2', + 'm3', 'm3', 'm3', 'm3', 'm3', + 'm3', 'm3', 'm3', 'm3', 'm3', + 'm2', 'm2', 'm2', 'm2', 'm2', + 'm1', 'm1', + 'm1', 'm1', 'm1', 'm1', 'm1', + 'm1', 'm1', 'm1', 'm1', 'm1', + 'm2', 'm2', 'm2', 'm2', 'm2', + 'm3', 'm3', 'm3', 'm3', 'm3', + 'm3', 'm3', 'm3', 'm3', 'm3', + 'm2', 'm2', 'm2', 'm2', 'm2', + 'm1', 'm1' + ] +) + +process.test2 = RunLumiEventAnalyzer( + verbose = True, + expectedRunLumiEvents = [ +1, 0, 0, +1, 1, 0, +1, 1, 11, +1, 1, 12, +1, 1, 13, +1, 1, 14, +1, 1, 15, +1, 1, 16, +1, 1, 17, +1, 1, 18, +1, 1, 19, +1, 1, 20, +1, 1, 21, +1, 1, 22, +1, 1, 23, +1, 1, 24, +1, 1, 25, +1, 1, 1, +1, 1, 2, +1, 1, 3, +1, 1, 4, +1, 1, 5, +1, 1, 6, +1, 1, 7, +1, 1, 8, +1, 1, 9, +1, 1, 10, +1, 1, 0, +1, 0, 0, +2, 0, 0, +2, 1, 0, +2, 1, 1, +2, 1, 2, +2, 1, 3, +2, 1, 4, +2, 1, 5, +2, 1, 0, +2, 0, 0, +11, 0, 0, +11, 1, 0, +11, 1, 1, +11, 1, 0, +11, 2, 0, +11, 2, 1, +11, 2, 0, +11, 0, 0, +1, 0, 0, +1, 1, 0, +1, 1, 11, +1, 1, 12, +1, 1, 13, +1, 1, 14, +1, 1, 15, +1, 1, 16, +1, 1, 17, +1, 1, 18, +1, 1, 19, +1, 1, 20, +1, 1, 21, +1, 1, 22, +1, 1, 23, +1, 1, 24, +1, 1, 25, +1, 1, 1, +1, 1, 2, +1, 1, 3, +1, 1, 4, +1, 1, 5, +1, 1, 6, +1, 1, 7, +1, 1, 8, +1, 1, 9, +1, 1, 10, +1, 1, 0, +1, 0, 0, +2, 0, 0, +2, 1, 0, +2, 1, 1, +2, 1, 2, +2, 1, 3, +2, 1, 4, +2, 1, 5, +2, 1, 0, +2, 0, 0 +] +) + +process.path1 = cms.Path(process.test + process.test2) +process.endpath1 = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeTEST2_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeTEST2_cfg.py new file mode 100644 index 0000000000000..e9f93e583ce9c --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeTEST2_cfg.py @@ -0,0 +1,87 @@ +# +# Test the most standard case where we read the same run +# from different files but each file has its own lumi blocks +# + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + fileMode = cms.untracked.string('FULLMERGE'), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +process.maxEvents.input = 10 + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource(fileNames = ['file:testRunMerge4.root', 'file:testRunMerge6.root']) + +from FWCore.Framework.modules import TestMergeResults, RunLumiEventAnalyzer +process.test = TestMergeResults( + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + # 0's are just placeholders, if the value is a "0" the check is not made. + + expectedBeginRunProd = [ + 10001, 20004, 10003 # end run 11 + ], + + expectedEndRunProd = [ + 100001, 200004, 100003 #end run 11 + ], + + expectedBeginLumiProd = [ + 101, 102, 103, # end run 11 lumi 1 + 101, 102, 103, # end run 11 lumi 2 + 101, 102, 103 # end run 11 lumi 3 + ], + + expectedEndLumiProd = [ + 1001, 1002, 1003, # end run 11 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 2 + 1001, 1002, 1003 # end run 11 lumi 3 + ], + + verbose = False +) + +process.test2 = RunLumiEventAnalyzer( + verbose = True, + expectedRunLumiEvents = [ +11, 0, 0, +11, 1, 0, +11, 1, 1, +11, 1, 0, +11, 2, 0, +11, 2, 2, +11, 2, 3, +11, 2, 4, +11, 2, 0, +11, 3, 0, +11, 3, 5, +11, 3, 6, +11, 3, 7, +11, 3, 0, +11, 4, 0, +11, 4, 8, +11, 4, 9, +11, 4, 10, +11, 4, 0, +11, 0, 0 +] +) + +process.path1 = cms.Path(process.test + process.test2) + diff --git a/FWIO/RNTupleTempTests/test/testRunMergeTEST3_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeTEST3_cfg.py new file mode 100644 index 0000000000000..b983e060f1713 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeTEST3_cfg.py @@ -0,0 +1,293 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + fileMode = cms.untracked.string('FULLMERGE'), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = [ + # CAUTION if you recreate the PROD files then you must recreate BOTH + # of these files otherwise you will get exceptions because the GUIDs + # used to check the match of the event in the secondary files will + # not be the same. + 'file:testRunMergeMERGE2.root', + 'file:testRunMerge.root' + ], + secondaryFileNames = [ + 'file:testRunMerge0.root', + 'file:testRunMerge1.root', + 'file:testRunMerge2.root', + 'file:testRunMerge3.root', + 'file:testRunMerge4.root', + 'file:testRunMerge5.root' + ] + , duplicateCheckMode = 'checkEachRealDataFile' + , noEventSort = True +) + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.out = RNTupleTempOutputModule(fileName = 'testRunMergeRecombined2.root') + +from FWCore.Framework.modules import TestMergeResults, RunLumiEventAnalyzer +process.test = TestMergeResults( + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + # 0's are just placeholders, if the value is a "0" the check is not made. + + expectedBeginRunProd = [ + 10001, 10002, 10003, # end run 100 + 10001, 10002, 10003, # end run 1 + 10001, 10002, 10003, # end run 1 + 10001, 10002, 10003, # end run 2 + 10001, 10002, 10004, # end run 1 + 10001, 20004, 10003, # end run 11 + 10001, 30006, 10003, # end run 1 + 10001, 10002, 10003, # end run 2 + 10001, 20004, 10003 # end run 11 + ], + + expectedEndRunProd = [ + 100001, 100002, 100003, # end run 100 + 100001, 100002, 100003, # end run 1 + 100001, 100002, 100003, # end run 1 + 100001, 100002, 100003, # end run 2 + 100001, 100002, 100004, # end run 1 + 100001, 200004, 100003, # end run 11 + 100001, 300006, 100003, # end run 1 + 100001, 100002, 100003, # end run 2 + 100001, 200004, 100003 # end run 11 + ], + + expectedBeginLumiProd = [ + 101, 102, 103, # end run 100 lumi 100 + 101, 102, 103, # end run 1 lumi 1 + 101, 102, 103, # end run 1 lumi 1 + 101, 102, 103, # end run 2 lumi 1 + 101, 102, 104, # end run 1 lumi 1 + 101, 102, 103, # end run 11 lumi 1 + 101, 102, 103, # end run 11 lumi 2 + 101, 306, 103, # end run 1 lumi 1 + 101, 102, 103, # end run 2 lumi 1 + 101, 102, 103, # end run 11 lumi 1 + 101, 102, 103 # end run 11 lumi 2 + ], + + expectedEndLumiProd = [ + 1001, 1002, 1003, # end run 100 lumi 100 + 1001, 1002, 1003, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 2 lumi 1 + 1001, 1002, 1004, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 2 + 1001, 3006, 1003, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 2 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 1 + 1001, 1002, 1003 # end run 11 lumi 2 + ], + + expectedBeginRunNew = [ + 10001, 10002, 10003, # end run 100 + 10001, 10002, 10003, # end run 1 + 10001, 10002, 10003, # end run 1 + 10001, 10002, 10003, # end run 2 + 10001, 10002, 10003, # end run 1 + 10001, 10002, 10003, # end run 11 + 10001, 20004, 10003, # end run 1 + 10001, 10002, 10003, # end run 2 + 10001, 10002, 10003 # end run 11 + ], + + expectedEndRunNew = [ + 100001, 100002, 100003, # end run 100 + 100001, 100002, 100003, # end run 1 + 100001, 100002, 100003, # end run 1 + 100001, 100002, 100003, # end run 2 + 100001, 100002, 100003, # end run 1 + 100001, 100002, 100003, # end run 11 + 100001, 200004, 100003, # end run 1 + 100001, 100002, 100003, # end run 2 + 100001, 100002, 100003 # end run 11 + ], + + expectedBeginLumiNew = [ + 101, 102, 103, # end run 100 lumi 100 + 101, 102, 103, # end run 1 lumi 1 + 101, 102, 103, # end run 1 lumi 1 + 101, 102, 103, # end run 2 lumi 1 + 101, 102, 103, # end run 1 lumi 1 + 101, 102, 103, # end run 11 lumi 1 + 101, 102, 103, # end run 11 lumi 2 + 101, 204, 103, # end run 1 lumi 1 + 101, 102, 103, # end run 2 lumi 1 + 101, 102, 103, # end run 11 lumi 1 + 101, 102, 103 # end run 11 lumi 2 + ], + + expectedEndLumiNew = [ + 1001, 1002, 1003, # end run 100 lumi 100 + 1001, 1002, 1003, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 2 lumi 1 + 1001, 1002, 1003, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 2 + 1001, 2004, 1003, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 2 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 1 + 1001, 1002, 1003 # end run 11 lumi 2 + ], + + expectedDroppedEvent = [13, 10003, 100003, 103, 1003], + verbose = True, + + expectedParents = [ + 'm1', #(100,100,100) + 'm1', 'm1', 'm1', 'm1', 'm1', + 'm1', 'm1', 'm1', 'm1', 'm1', + 'm2', 'm2', 'm2', 'm2', 'm2', + 'm2', 'm2', 'm2', 'm2', 'm2', + 'm3', 'm3', 'm3', 'm3', 'm3', + 'm3', 'm3', 'm3', 'm3', 'm3', + 'm1', 'm1', #(11,...) + + 'm1', 'm1', 'm1', 'm1', 'm1', + 'm1', 'm1', 'm1', 'm1', 'm1', + 'm2', 'm2', 'm2', 'm2', 'm2', + 'm3', 'm3', 'm3', 'm3', 'm3', + 'm3', 'm3', 'm3', 'm3', 'm3', + 'm2', 'm2', 'm2', 'm2', 'm2', + 'm1', 'm1' + ] +) + +process.test2 = RunLumiEventAnalyzer( + verbose = True, + expectedRunLumiEvents = [ + 100, 0, 0, + 100, 100, 0, + 100, 100, 100, + 100, 100, 0, + 100, 0, 0, + 1, 0, 0, + 1, 1, 0, + 1, 1, 11, + 1, 1, 12, + 1, 1, 13, + 1, 1, 14, + 1, 1, 15, + 1, 1, 16, + 1, 1, 17, + 1, 1, 18, + 1, 1, 19, + 1, 1, 20, + 1, 1, 0, + 1, 0, 0, + 1, 0, 0, #new process history ID + 1, 1, 0, + 1, 1, 21, + 1, 1, 22, + 1, 1, 23, + 1, 1, 24, + 1, 1, 25, + 1, 1, 0, + 1, 0, 0, + 2, 0, 0, + 2, 1, 0, + 2, 1, 1, + 2, 1, 2, + 2, 1, 3, + 2, 1, 4, + 2, 1, 5, + 2, 1, 0, + 2, 0, 0, + 1, 0, 0, + 1, 1, 0, + 1, 1, 1, + 1, 1, 2, + 1, 1, 3, + 1, 1, 4, + 1, 1, 5, + 1, 1, 6, + 1, 1, 7, + 1, 1, 8, + 1, 1, 9, + 1, 1, 10, + 1, 1, 0, + 1, 0, 0, + 11, 0, 0, + 11, 1, 0, + 11, 1, 1, + 11, 1, 0, + 11, 2, 0, + 11, 2, 1, + 11, 2, 0, + 11, 0, 0 +] +) +process.test2.expectedRunLumiEvents.extend([ +1, 0, 0, +1, 1, 0, +1, 1, 11, +1, 1, 12, +1, 1, 13, +1, 1, 14, +1, 1, 15, +1, 1, 16, +1, 1, 17, +1, 1, 18, +1, 1, 19, +1, 1, 20, +1, 1, 21, +1, 1, 22, +1, 1, 23, +1, 1, 24, +1, 1, 25, +1, 1, 1, +1, 1, 2, +1, 1, 3, +1, 1, 4, +1, 1, 5, +1, 1, 6, +1, 1, 7, +1, 1, 8, +1, 1, 9, +1, 1, 10, +1, 1, 0, +1, 0, 0, +2, 0, 0, +2, 1, 0, +2, 1, 1, +2, 1, 2, +2, 1, 3, +2, 1, 4, +2, 1, 5, +2, 1, 0, +2, 0, 0, +11, 0, 0, +11, 1, 0, +11, 1, 1, +11, 1, 0, +11, 2, 0, +11, 2, 1, +11, 2, 0, +11, 0, 0, +]) + +process.path1 = cms.Path(process.test + process.test2) +process.endpath1 = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeTEST4_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeTEST4_cfg.py new file mode 100644 index 0000000000000..00237489f8c84 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeTEST4_cfg.py @@ -0,0 +1,139 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + fileMode = cms.untracked.string('FULLMERGE'), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = [ + 'file:testRunMergeMERGE4.root', + 'file:testRunMergeMERGE4.root' + ] + , duplicateCheckMode = 'checkEachRealDataFile' + , noEventSort = True + , lumisToProcess = ['1:1'] +) + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.out = RNTupleTempOutputModule(fileName = 'testRunMergeRecombined4.root') + +from FWCore.Framework.modules import TestMergeResults, RunLumiEventAnalyzer +process.test = TestMergeResults( + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + # 0's are just placeholders, if the value is a "0" the check is not made. + + expectedBeginRunProd = [ + 10001, 80016, 10003 # end run 1 + ], + + expectedEndRunProd = [ + 100001, 800016, 100003 # end run 1 + ], + + expectedBeginLumiProd = [ + 101, 816, 103 # end run 1 lumi 1 + ], + + expectedEndLumiProd = [ + 1001, 8016, 1003 # end run 1 lumi 1 + ], + + expectedBeginRunNew = [ + 10001, 60012, 10003 # end run 1 + ], + + expectedEndRunNew = [ + 100001, 600012, 100003 # end run 1 + ], + + expectedBeginLumiNew = [ + 101, 612, 103 # end run 1 lumi 1 + ], + + expectedEndLumiNew = [ + 1001, 6012, 1003 # end run 1 lumi 1 + ], + + verbose = True +) + +process.test2 = RunLumiEventAnalyzer( + verbose = True, + expectedRunLumiEvents = [ +1, 0, 0, +1, 1, 0, +1, 1, 11, +1, 1, 12, +1, 1, 13, +1, 1, 14, +1, 1, 15, +1, 1, 16, +1, 1, 17, +1, 1, 18, +1, 1, 19, +1, 1, 20, +1, 1, 21, +1, 1, 22, +1, 1, 23, +1, 1, 24, +1, 1, 25, +1, 1, 1, +1, 1, 2, +1, 1, 3, +1, 1, 4, +1, 1, 5, +1, 1, 6, +1, 1, 7, +1, 1, 8, +1, 1, 9, +1, 1, 10, +1, 1, 26, +1, 1, 11, +1, 1, 12, +1, 1, 13, +1, 1, 14, +1, 1, 15, +1, 1, 16, +1, 1, 17, +1, 1, 18, +1, 1, 19, +1, 1, 20, +1, 1, 21, +1, 1, 22, +1, 1, 23, +1, 1, 24, +1, 1, 25, +1, 1, 1, +1, 1, 2, +1, 1, 3, +1, 1, 4, +1, 1, 5, +1, 1, 6, +1, 1, 7, +1, 1, 8, +1, 1, 9, +1, 1, 10, +1, 1, 26, +1, 1, 0, +1, 0, 0 +] +) + +process.path1 = cms.Path(process.test + process.test2) +process.endpath1 = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeTEST5_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeTEST5_cfg.py new file mode 100644 index 0000000000000..602e89222a409 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeTEST5_cfg.py @@ -0,0 +1,24 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST5") + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testRunMergeMERGE5.root' + ), + secondaryFileNames = cms.untracked.vstring( + 'file:testRunMerge2extra.root' + ) +) + +process.test = cms.EDAnalyzer("TestMergeResults", + testAlias = cms.untracked.bool(True) +) + +process.path1 = cms.Path(process.test) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testRunMergeTEST5.root') +) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeTEST6_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeTEST6_cfg.py new file mode 100644 index 0000000000000..2d7d93ab23fb1 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeTEST6_cfg.py @@ -0,0 +1,157 @@ +# A test of the noRunLumiSort configuration parameter +# where the input has non-contiguous event sequences +# from the same run. This configuration reads a file +# created using that parameter and checks that the +# run product and lumi product merging that occurred +# was done properly. + +# It is expected there are 8 warnings that +# print out while this runs related to merging. +# The test should pass with these warnings. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = 'file:testRunMergeNoRunLumiSort.root', + duplicateCheckMode = 'noDuplicateCheck' +) + +from FWCore.Framework.modules import TestMergeResults, RunLumiEventAnalyzer +process.test = TestMergeResults( + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + # 0's are just placeholders, if the value is a "0" the check is not made. + + expectedBeginRunProd = [ + 0, 60012, 10003, + 0, 20004, 10003 + ], + + expectedEndRunProd = [ + 0, 600012, 100003, + 0, 200004, 100003 + ], + + expectedBeginLumiProd = [ + 0, 612, 103, + 0, 204, 103 + ], + + expectedEndLumiProd = [ + 0, 6012, 1003, + 0, 2004, 1003 + ], + + expectedBeginRunNew = [ + 10001, 40008, 10003, + 10001, 20004, 10003 + ], + + expectedEndRunNew = [ + 100001, 400008, 100003, + 100001, 200004, 100003 + ], + + expectedBeginLumiNew = [ + 101, 408, 103, + 101, 204, 103 + ], + + expectedEndLumiNew = [ + 1001, 4008, 1003, + 1001, 2004, 1003 + ] +) + +process.test2 = RunLumiEventAnalyzer( + verbose = True, + expectedRunLumiEvents = [ +1, 0, 0, +1, 1, 0, +1, 1, 11, +1, 1, 12, +1, 1, 13, +1, 1, 14, +1, 1, 15, +1, 1, 16, +1, 1, 17, +1, 1, 18, +1, 1, 19, +1, 1, 20, +1, 1, 21, +1, 1, 22, +1, 1, 23, +1, 1, 24, +1, 1, 25, +1, 1, 1, +1, 1, 2, +1, 1, 3, +1, 1, 4, +1, 1, 5, +1, 1, 6, +1, 1, 7, +1, 1, 8, +1, 1, 9, +1, 1, 10, +1, 1, 11, +1, 1, 12, +1, 1, 13, +1, 1, 14, +1, 1, 15, +1, 1, 16, +1, 1, 17, +1, 1, 18, +1, 1, 19, +1, 1, 20, +1, 1, 21, +1, 1, 22, +1, 1, 23, +1, 1, 24, +1, 1, 25, +1, 1, 1, +1, 1, 2, +1, 1, 3, +1, 1, 4, +1, 1, 5, +1, 1, 6, +1, 1, 7, +1, 1, 8, +1, 1, 9, +1, 1, 10, +1, 1, 0, +1, 0, 0 +] +) +process.test2.expectedRunLumiEvents.extend([ +2, 0, 0, +2, 1, 0, +2, 1, 1, +2, 1, 2, +2, 1, 3, +2, 1, 4, +2, 1, 5, +2, 1, 1, +2, 1, 2, +2, 1, 3, +2, 1, 4, +2, 1, 5, +2, 1, 0, +2, 0, 0 +]) + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.out = RNTupleTempOutputModule(fileName = 'testRunMergeTEST6.root') + +process.path1 = cms.Path(process.test * process.test2) +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeTESTFAIL_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeTESTFAIL_cfg.py new file mode 100644 index 0000000000000..b0ac44b507936 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeTESTFAIL_cfg.py @@ -0,0 +1,30 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + fileMode = cms.untracked.string('FULLMERGE'), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + + +#this should fail +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring( + 'file:testRunMerge.root', + 'file:testRunMerge.root' + ), + needSecondaryFileNames = cms.untracked.bool(True), + duplicateCheckMode = cms.untracked.string('checkEachRealDataFile') +) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('file:testRunMergeRecombinedFail.root') +) + +process.endpath1 = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testRunMergeTEST_cfg.py b/FWIO/RNTupleTempTests/test/testRunMergeTEST_cfg.py new file mode 100644 index 0000000000000..263602ddc0247 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testRunMergeTEST_cfg.py @@ -0,0 +1,293 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 +process.MessageLogger.cerr.threshold = 'ERROR' + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + fileMode = cms.untracked.string('FULLMERGE'), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +from FWIO.RNTupleTempInput.modules import RNTupleTempSource +process.source = RNTupleTempSource( + fileNames = [ + # CAUTION if you recreate the PROD files then you must recreate BOTH + # of these files otherwise you will get exceptions because the GUIDs + # used to check the match of the event in the secondary files will + # not be the same. + 'file:testRunMerge.root', + 'file:testRunMergeMERGE2.root' + ], + secondaryFileNames = [ + 'file:testRunMerge0.root', + 'file:testRunMerge1.root', + 'file:testRunMerge2.root', + 'file:testRunMerge3.root', + 'file:testRunMerge4.root', + 'file:testRunMerge5.root' + ] + , duplicateCheckMode = 'checkEachRealDataFile' + , noEventSort = False +) + +from FWIO.RNTupleTempOutput.modules import RNTupleTempOutputModule +process.out = RNTupleTempOutputModule(fileName = 'testRunMergeRecombined.root') + +from FWCore.Framework.modules import TestMergeResults, RunLumiEventAnalyzer +process.test = TestMergeResults( + # Check to see that the value we read matches what we know + # was written. Expected values listed below come in sets of three + # value expected in Thing + # value expected in ThingWithMerge + # value expected in ThingWithIsEqual + # Each set of 3 is tested at endRun for the expected + # run values or at endLuminosityBlock for the expected + # lumi values. And then the next set of three values + # is tested at the next endRun or endLuminosityBlock. + # When the sequence of parameter values is exhausted it stops checking + # 0's are just placeholders, if the value is a "0" the check is not made. + + expectedBeginRunProd = [ + 10001, 30006, 10003, # end run 1, merged run entries within a file + 10001, 10002, 10003, # end run 2 + 10001, 20004, 10003, # end run 11 + 10001, 10002, 10003, # end run 1, second input file starts here + 10001, 20004, 10003, # end run 11 + 10001, 10002, 10003, # end run 100 + 10001, 10002, 10003, # end run 1, not merged different ProcessHistoryID + 10001, 10002, 10003, # end run 2 + 10001, 10002, 10004 # end run 1 + ], + + expectedEndRunProd = [ + 100001, 300006, 100003, # end run 1 + 100001, 100002, 100003, # end run 2 + 100001, 200004, 100003, # end run 11 + 100001, 100002, 100003, # end run 1 + 100001, 200004, 100003, # end run 11 + 100001, 100002, 100003, # end run 100 + 100001, 100002, 100003, # end run 1 + 100001, 100002, 100003, # end run 2 + 100001, 100002, 100004 # end run 1 + ], + + expectedBeginLumiProd = [ + 101, 306, 103, # end run 1 lumi 1 + 101, 102, 103, # end run 2 lumi 1 + 101, 102, 103, # end run 11 lumi 1 + 101, 102, 103, # end run 11 lumi 2 + 101, 102, 103, # end run 1 lumi 1 + 101, 102, 103, # end run 11 lumi 1 + 101, 102, 103, # end run 11 lumi 2 + 101, 102, 103, # end run 100 lumi 100 + 101, 102, 103, # end run 1 lumi 1 + 101, 102, 103, # end run 2 lumi 1 + 101, 102, 104 # end run 1 lumi 1 + ], + + expectedEndLumiProd = [ + 1001, 3006, 1003, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 2 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 2 + 1001, 1002, 1003, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 2 + 1001, 1002, 1003, # end run 100 lumi 100 + 1001, 1002, 1003, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 2 lumi 1 + 1001, 1002, 1004 # end run 1 lumi 1 + ], + + expectedBeginRunNew = [ + 10001, 20004, 10003, # end run 1 + 10001, 10002, 10003, # end run 2 + 10001, 10002, 10003, # end run 11 + 10001, 10002, 10003, # end run 1 + 10001, 10002, 10003, # end run 11 + 10001, 10002, 10003, # end run 100 + 10001, 10002, 10003, # end run 1 + 10001, 10002, 10003, # end run 2 + 10001, 10002, 10003 # end run 1 + ], + + expectedEndRunNew = [ + 100001, 200004, 100003, # end run 1 + 100001, 100002, 100003, # end run 2 + 100001, 100002, 100003, # end run 11 + 100001, 100002, 100003, # end run 1 + 100001, 100002, 100003, # end run 11 + 100001, 100002, 100003, # end run 100 + 100001, 100002, 100003, # end run 1 + 100001, 100002, 100003, # end run 2 + 100001, 100002, 100003 # end run 1 + ], + + expectedBeginLumiNew = [ + 101, 204, 103, # end run 1 lumi 1 + 101, 102, 103, # end run 2 lumi 1 + 101, 102, 103, # end run 11 lumi 1 + 101, 102, 103, # end run 11 lumi 2 + 101, 102, 103, # end run 1 lumi 1 + 101, 102, 103, # end run 11 lumi 1 + 101, 102, 103, # end run 11 lumi 2 + 101, 102, 103, # end run 100 lumi 100 + 101, 102, 103, # end run 1 lumi 1 + 101, 102, 103, # end run 2 lumi 1 + 101, 102, 103 # end run 1 lumi 1 + ], + + expectedEndLumiNew = [ + 1001, 2004, 1003, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 2 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 2 + 1001, 1002, 1003, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 1 + 1001, 1002, 1003, # end run 11 lumi 2 + 1001, 1002, 1003, # end run 100 lumi 100 + 1001, 1002, 1003, # end run 1 lumi 1 + 1001, 1002, 1003, # end run 2 lumi 1 + 1001, 1002, 1003 # end run 1 lumi 1 + ], + + expectedDroppedEvent = [13, 10003, 100003, 103, 1003], + verbose = True, + + expectedParents = [ + 'm3', 'm3', 'm3', 'm3', 'm3', + 'm3', 'm3', 'm3', 'm3', 'm3', + 'm1', 'm1', 'm1', 'm1', 'm1', + 'm1', 'm1', 'm1', 'm1', 'm1', + 'm2', 'm2', 'm2', 'm2', 'm2', + 'm2', 'm2', 'm2', 'm2', 'm2', + 'm1', 'm1', + 'm1', 'm1', 'm1', 'm1', 'm1', + 'm1', 'm1', 'm1', 'm1', 'm1', + 'm1', 'm1', + 'm1', + 'm2', 'm2', 'm2', 'm2', 'm2', + 'm2', 'm2', 'm2', 'm2', 'm2', + 'm3', 'm3', 'm3', 'm3', 'm3', + 'm3', 'm3', 'm3', 'm3', 'm3' + ], + testAlias = True +) + +process.test2 = RunLumiEventAnalyzer( + verbose = True, + expectedRunLumiEvents = [ +1, 0, 0, +1, 1, 0, +1, 1, 1, +1, 1, 2, +1, 1, 3, +1, 1, 4, +1, 1, 5, +1, 1, 6, +1, 1, 7, +1, 1, 8, +1, 1, 9, +1, 1, 10, +1, 1, 11, +1, 1, 12, +1, 1, 13, +1, 1, 14, +1, 1, 15, +1, 1, 16, +1, 1, 17, +1, 1, 18, +1, 1, 19, +1, 1, 20, +1, 1, 21, +1, 1, 22, +1, 1, 23, +1, 1, 24, +1, 1, 25, +1, 1, 0, +1, 0, 0, +2, 0, 0, +2, 1, 0, +2, 1, 1, +2, 1, 2, +2, 1, 3, +2, 1, 4, +2, 1, 5, +2, 1, 0, +2, 0, 0, +11, 0, 0, +11, 1, 0, +11, 1, 1, +11, 1, 0, +11, 2, 0, +11, 2, 1, +11, 2, 0, +11, 0, 0, +1, 0, 0, +1, 1, 0, +1, 1, 11, +1, 1, 12, +1, 1, 13, +1, 1, 14, +1, 1, 15, +1, 1, 16, +1, 1, 17, +1, 1, 18, +1, 1, 19, +1, 1, 20, +1, 1, 0, +1, 0, 0, +11, 0, 0, +11, 1, 0, +11, 1, 1, +11, 1, 0, +11, 2, 0, +11, 2, 1, +11, 2, 0, +11, 0, 0, +100, 0, 0, +100, 100, 0, +100, 100, 100, +100, 100, 0, +100, 0, 0 +] +) +process.test2.expectedRunLumiEvents.extend([ +1, 0, 0, +1, 1, 0, +1, 1, 21, +1, 1, 22, +1, 1, 23, +1, 1, 24, +1, 1, 25, +1, 1, 0, +1, 0, 0, +2, 0, 0, +2, 1, 0, +2, 1, 1, +2, 1, 2, +2, 1, 3, +2, 1, 4, +2, 1, 5, +2, 1, 0, +2, 0, 0, +1, 0, 0, +1, 1, 0, +1, 1, 1, +1, 1, 2, +1, 1, 3, +1, 1, 4, +1, 1, 5, +1, 1, 6, +1, 1, 7, +1, 1, 8, +1, 1, 9, +1, 1, 10, +1, 1, 0, +1, 0, 0 +]) + +process.path1 = cms.Path(process.test + process.test2) +process.endpath1 = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testSeriesOfProcessesHLT_cfg.py b/FWIO/RNTupleTempTests/test/testSeriesOfProcessesHLT_cfg.py new file mode 100644 index 0000000000000..de5f14a760c8a --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testSeriesOfProcessesHLT_cfg.py @@ -0,0 +1,115 @@ + +# This configuration is designed to be run as the first +# in a series of cmsRun processes. Several things get +# tested independently in this series of processes. +# This first process will create a streamer file. + +# For event selection tests several paths are run: +# 99 events are generated +# path p01 events 1:98 pass +# path p02 events 21:30 pass +# path p03 event 71 only passes +# path p04 all fail + +# Checks the path names returned by the TriggerNames +# service. + +# Multiple products are put in the event for use +# in subsequent processes. Some products faking +# raw data products and some faking hlt products. +# They are not used here, just created for later +# use. + +# Creates multiple luminosity blocks for a later +# test of the maxLuminosityBlock parameter + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("HLT") + +process.source = cms.Source("EmptySource", + firstLuminosityBlock = cms.untracked.uint32(1), + numberEventsInLuminosityBlock = cms.untracked.uint32(5), + firstEvent = cms.untracked.uint32(1), + firstRun = cms.untracked.uint32(1), + numberEventsInRun = cms.untracked.uint32(1000) +) + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(99) +) + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( +# wantSummary = cms.untracked.bool(True), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +#import FWCore.Framework.test.cmsExceptionsFatal_cff +#process.options = FWCore.Framework.test.cmsExceptionsFatal_cff.options + +process.f1 = cms.EDFilter("TestFilterModule", + acceptValue = cms.untracked.int32(98), + onlyOne = cms.untracked.bool(False) +) + +process.f2a = cms.EDFilter("TestFilterModule", + acceptValue = cms.untracked.int32(20), + onlyOne = cms.untracked.bool(False) +) + +process.f2b = cms.EDFilter("TestFilterModule", + acceptValue = cms.untracked.int32(10), + onlyOne = cms.untracked.bool(False) +) + +process.f3 = cms.EDFilter("TestFilterModule", + acceptValue = cms.untracked.int32(71), + onlyOne = cms.untracked.bool(True) +) + +process.f4 = cms.EDFilter("TestFilterModule", + acceptValue = cms.untracked.int32(101), + onlyOne = cms.untracked.bool(True) +) + +process.a = cms.EDAnalyzer( + "TestTriggerNames", + trigPaths = cms.untracked.vstring( + 'p01', + 'p02', + 'p03', + 'p04' + ), + endPaths = cms.untracked.vstring('e'), + dumpPSetRegistry = cms.untracked.bool(False) +) + +process.fakeRaw = cms.EDProducer( + "IntProducer", + ivalue = cms.int32(10) +) + +process.fakeHLTDebug = cms.EDProducer( + "IntProducer", + ivalue = cms.int32(1000) +) + +process.out = cms.OutputModule("EventStreamFileWriter", + fileName = cms.untracked.string('testSeriesOfProcessesHLT.dat'), + compression_level = cms.untracked.int32(1), + use_compression = cms.untracked.bool(True), + max_event_size = cms.untracked.int32(7000000) +) + +process.p01 = cms.Path(process.f1) +process.p02 = cms.Path(~process.f2a*process.f2b) +process.p03 = cms.Path(process.f3) +process.p04 = cms.Path(process.a * + process.fakeRaw * process.fakeHLTDebug * + process.f4) + +process.e = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testSeriesOfProcessesPROD2TEST_cfg.py b/FWIO/RNTupleTempTests/test/testSeriesOfProcessesPROD2TEST_cfg.py new file mode 100644 index 0000000000000..2d8b0d73b3d13 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testSeriesOfProcessesPROD2TEST_cfg.py @@ -0,0 +1,64 @@ + +# This configuration is designed to be run as the last +# in a series of cmsRun processes. + +# Tests the maxLuminosityBlocks parameter + +# checks to see that the process level fakeRaw overrides the file based one + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD2TEST") + +process.maxLuminosityBlocks = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( +# wantSummary = cms.untracked.bool(True), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSeriesOfProcessesTEST.root'), +) + +# Override the fakeRaw found in the test file +process.fakeRaw = cms.EDProducer( + "IntProducer", + ivalue = cms.int32(20) +) + + +# This module tests to see if the products put in at the first step +# (the fake HLT step) survived through to the last file. At the PROD +# stage the products were split into two files so this test secondary +# file input. +process.a = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("fakeRaw"), + cms.InputTag("fakeHLTDebug") ), + + # Test the maxLuminosityBlock parameter + # 3 luminosity blocks contain 15 events + # Each event contains one product with a value of 20 and + # one product with a value of 1000 + # If the maxLuminosityBlock parameter is working correctly the + # following should be the sum of all the values. + # The product values are hard coded into the fake + # HLT configuration (the first one in this series). + expectedSum = cms.untracked.int32(15300) +) + +process.test1 = cms.Path(process.fakeRaw*process.a) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSeriesOfProcessesPROD2TEST.root'), + outputCommands = cms.untracked.vstring( + "keep *" + ) +) +process.o = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testSeriesOfProcessesPROD2TEST_unscheduled_cfg.py b/FWIO/RNTupleTempTests/test/testSeriesOfProcessesPROD2TEST_unscheduled_cfg.py new file mode 100644 index 0000000000000..34922af78162d --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testSeriesOfProcessesPROD2TEST_unscheduled_cfg.py @@ -0,0 +1,66 @@ + +# This configuration is designed to be run as the last +# in a series of cmsRun processes. + +# Tests the maxLuminosityBlocks parameter + +# checks to see that the process level fakeRaw overrides the file based one + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD2TEST") + +process.maxLuminosityBlocks = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSeriesOfProcessesTEST.root'), +) + +# Override the fakeRaw found in the test file +process.fakeRaw = cms.EDProducer( + "IntProducer", + ivalue = cms.int32(20) +) + + +# This module tests to see if the products put in at the first step +# (the fake HLT step) survived through to the last file. At the PROD +# stage the products were split into two files so this test secondary +# file input. +process.a = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("fakeRaw"), + cms.InputTag("fakeHLTDebug") ), + + # Test the maxLuminosityBlock parameter + # 3 luminosity blocks contain 15 events + # Each event contains one product with a value of 20 and + # one product with a value of 1000 + # If the maxLuminosityBlock parameter is working correctly the + # following should be the sum of all the values. + # The product values are hard coded into the fake + # HLT configuration (the first one in this series). + expectedSum = cms.untracked.int32(15300) +) + +process.t = cms.Task(process.fakeRaw) + +process.test1 = cms.Path(process.a, process.t) + +process.out = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSeriesOfProcessesPROD2TEST_u.root'), + outputCommands = cms.untracked.vstring( + "keep *_fakeRaw_*_*" + ) +) +process.o = cms.EndPath(process.out) diff --git a/FWIO/RNTupleTempTests/test/testSeriesOfProcessesPROD3TEST_cfg.py b/FWIO/RNTupleTempTests/test/testSeriesOfProcessesPROD3TEST_cfg.py new file mode 100644 index 0000000000000..9e173a4c5aa1d --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testSeriesOfProcessesPROD3TEST_cfg.py @@ -0,0 +1,49 @@ + +# This configuration is designed to be run as the last +# in a series of cmsRun processes. + +# Tests the maxLuminosityBlocks parameter + +# checks to see that the process level fakeRaw overrides the file based one + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD3TEST") + +process.maxLuminosityBlocks = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( +# wantSummary = cms.untracked.bool(True), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSeriesOfProcessesPROD2TEST.root'), +) + +# This module tests to see if the products put in at the second step +# (the fake PROD step) survived through to the last file. At the PROD +# stage the products were split into two files so this test secondary +# file input. +process.a = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("fakeRaw"), + cms.InputTag("fakeHLTDebug") ), + + # Test the maxLuminosityBlock parameter + # 3 luminosity blocks contain 15 events + # Each event contains one product with a value of 20 and + # one product with a value of 1000 + # If the maxLuminosityBlock parameter is working correctly the + # following should be the sum of all the values. + # The product values are hard coded into the fake + # HLT configuration (the first one in this series). + expectedSum = cms.untracked.int32(15300) +) + +process.test1 = cms.Path(process.a) diff --git a/FWIO/RNTupleTempTests/test/testSeriesOfProcessesPROD_cfg.py b/FWIO/RNTupleTempTests/test/testSeriesOfProcessesPROD_cfg.py new file mode 100644 index 0000000000000..f50a93f883930 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testSeriesOfProcessesPROD_cfg.py @@ -0,0 +1,96 @@ + +# This configuration is designed to be run as the second +# in a series of cmsRun processes. The process it configures +# will read a file in streamer format and produces two root +# files. + +# For later event selection tests these paths are run: +# path p1 1:25 pass +# path p2 pass 51:60 + +# Checks the path names returned by the TriggerNames +# service. + +# Multiple products are put in the event for use +# in subsequent processes. + +# Two output files are created, one contains some +# fake raw data, the other contains some fake +# HLTDebug data (actual just dummy products containing +# an int, just for test purposes) + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("PROD") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( +# wantSummary = cms.untracked.bool(True), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +process.source = cms.Source("NewEventStreamFileReader", + fileNames = cms.untracked.vstring('file:testSeriesOfProcessesHLT.dat') +) + +process.f1 = cms.EDFilter("TestFilterModule", + acceptValue = cms.untracked.int32(25), + onlyOne = cms.untracked.bool(False) +) + +process.f2a = cms.EDFilter("TestFilterModule", + acceptValue = cms.untracked.int32(50), + onlyOne = cms.untracked.bool(False) +) + +process.f2b = cms.EDFilter("TestFilterModule", + acceptValue = cms.untracked.int32(10), + onlyOne = cms.untracked.bool(False) +) + +process.a = cms.EDAnalyzer( + "TestTriggerNames", + trigPathsPrevious = cms.untracked.vstring( + 'p01', + 'p02', + 'p03', + 'p04' + ), + streamerSource = cms.untracked.bool(True), + trigPaths = cms.untracked.vstring('p1', 'p2'), + dumpPSetRegistry = cms.untracked.bool(False) +) + +# This puts products in the lumi's and run's. One failure +# mode of the maxLuminosityBlock parameter is tested by their +# mere existence. +process.makeRunLumiProducts = cms.EDProducer("ThingWithMergeProducer") + +# In the next process we want to test input from a secondary input +# file so we split the products over 2 output files. + +process.out1 = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSeriesOfProcessesPROD1.root'), + outputCommands = cms.untracked.vstring( + "drop *", + "keep *_fakeRaw_*_*" + ) +) + +process.out2 = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSeriesOfProcessesPROD2.root'), + outputCommands = cms.untracked.vstring( + "keep *", + "drop *_fakeRaw_*_*" + ) +) + +process.pathanalysis = cms.EDAnalyzer("PathAnalyzer") + +process.p1 = cms.Path(process.f1 * process.makeRunLumiProducts) +process.p2 = cms.Path(~process.f2a * process.f2b) + +process.e = cms.EndPath(process.a * process.pathanalysis * process.out1 * process.out2) diff --git a/FWIO/RNTupleTempTests/test/testSeriesOfProcessesTEST1_cfg.py b/FWIO/RNTupleTempTests/test/testSeriesOfProcessesTEST1_cfg.py new file mode 100644 index 0000000000000..6ce990d483e25 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testSeriesOfProcessesTEST1_cfg.py @@ -0,0 +1,50 @@ + +# This configuration is designed to be run as the last +# in a series of cmsRun processes. + +# Tests the maxLuminosityBlocks parameter + +# checks to see that both the fakeRaw and fakeHLTDEBUG +# products are in the input file. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST1") + +process.maxLuminosityBlocks = cms.untracked.PSet( + input = cms.untracked.int32(3) +) + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( +# wantSummary = cms.untracked.bool(True), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSeriesOfProcessesTEST.root'), +) + +# This module tests to see if the products put in at the first step +# (the fake HLT step) survived through to the last file. At the PROD +# stage the products were split into two files so this test secondary +# file input. +process.a = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag( cms.InputTag("fakeRaw"), + cms.InputTag("fakeHLTDebug") ), + + # Test the maxLuminosityBlock parameter + # 3 luminosity blocks contain 15 events + # Each event contains one product with a value of 10 and + # one product with a value of 1000 + # If the maxLuminosityBlock parameter is working correctly the + # following should be the sum of all the values. + # The product values are hard coded into the fake + # HLT configuration (the first one in this series). + expectedSum = cms.untracked.int32(15150) +) + +process.test1 = cms.Path(process.a) diff --git a/FWIO/RNTupleTempTests/test/testSeriesOfProcessesTEST2_cfg.py b/FWIO/RNTupleTempTests/test/testSeriesOfProcessesTEST2_cfg.py new file mode 100644 index 0000000000000..aa22fc209a181 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testSeriesOfProcessesTEST2_cfg.py @@ -0,0 +1,29 @@ + +# Test the case of a duplicate process name being used. +# This should fail with an exception error message that +# indicates a duplicate process name was used. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSeriesOfProcessesTEST.root'), +) + +# Add one module in a path to force the process to be +# added to the process history. This could be any module. +# The module serves no other purpose. +process.a = cms.EDAnalyzer("TestFindProduct", + inputTags = cms.untracked.VInputTag() +) + +process.test1 = cms.Path(process.a) diff --git a/FWIO/RNTupleTempTests/test/testSeriesOfProcessesTEST3_cfg.py b/FWIO/RNTupleTempTests/test/testSeriesOfProcessesTEST3_cfg.py new file mode 100644 index 0000000000000..b6856d8f3f11d --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testSeriesOfProcessesTEST3_cfg.py @@ -0,0 +1,26 @@ + +# This tests that the process history information that should have +# been added by previous process is available. No output is written. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST3") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow + ) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSeriesOfProcessesTEST.root'), + ) + +process.hk = cms.EDAnalyzer("TestHistoryKeeping", + expected_processes = cms.vstring('HLT','PROD','TEST'), + number_of_expected_HLT_processes_for_each_run = cms.int32(1) + ) + +process.hist = cms.Path(process.hk) diff --git a/FWIO/RNTupleTempTests/test/testSeriesOfProcessesTEST_cfg.py b/FWIO/RNTupleTempTests/test/testSeriesOfProcessesTEST_cfg.py new file mode 100644 index 0000000000000..344f7e6993543 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/testSeriesOfProcessesTEST_cfg.py @@ -0,0 +1,127 @@ + +# This configuration is designed to be run as the third +# in a series of cmsRun processes. Several things get +# tested. + +# For event selection tests +# path1 even pass +# path2 1:40 pass + +# Checks the path names returned by the TriggerNames +# service. + +# We read both files previously written and test that +# the secondary input file feature of the RNTupleTempSource +# works even in the case when the products went through +# a streamer file. + +# The SewerModule OutputModule's test the SelectEvents +# feature. If the expected number of events does not +# pass the selection, they abort with an error message. + +# We also test that the lookup of processing history information +# works. + +import FWCore.ParameterSet.Config as cms + +process = cms.Process("TEST") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.MessageLogger.cerr.FwkReport.reportEvery = 1000 + +import FWCore.Framework.test.cmsExceptionsFatalOption_cff +process.options = cms.untracked.PSet( +# wantSummary = cms.untracked.bool(True), + Rethrow = FWCore.Framework.test.cmsExceptionsFatalOption_cff.Rethrow +) + +process.source = cms.Source("RNTupleTempSource", + fileNames = cms.untracked.vstring('file:testSeriesOfProcessesPROD2.root'), + secondaryFileNames = cms.untracked.vstring('file:testSeriesOfProcessesPROD1.root') +) + +process.f1 = cms.EDFilter("TestFilterModule", + acceptValue = cms.untracked.int32(2), + onlyOne = cms.untracked.bool(True) +) + +process.f2 = cms.EDFilter("TestFilterModule", + acceptValue = cms.untracked.int32(40), + onlyOne = cms.untracked.bool(False) +) + +process.a = cms.EDAnalyzer("TestTriggerNames", + trigPathsPrevious = cms.untracked.vstring('p1', 'p2'), + trigPaths = cms.untracked.vstring( + 'path1', + 'path2', + 'path3', + 'path4', + 'path5', + 'path6', + 'path7', + 'path8'), + dumpPSetRegistry = cms.untracked.bool(False), + expectedTriggerResultsHLT = cms.untracked.vuint32( + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0 + ), + expectedTriggerResultsPROD = cms.untracked.vuint32( + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0 + ) +) + +process.out1 = cms.OutputModule("SewerModule", + shouldPass = cms.int32(60), + name = cms.string('out1'), + SelectEvents = cms.untracked.PSet( + SelectEvents = cms.vstring('p02:HLT', + ' p03 : HLT', + 'p2:PROD', + 'path1:TEST') + ) +) + +process.out2 = cms.OutputModule("SewerModule", + shouldPass = cms.int32(98), + name = cms.string('out2'), + SelectEvents = cms.untracked.PSet( + SelectEvents = cms.vstring('*:HLT') + ) +) + +process.out3 = cms.OutputModule("SewerModule", + shouldPass = cms.int32(64), + name = cms.string('out3'), + SelectEvents = cms.untracked.PSet( + SelectEvents = cms.vstring('!*:PROD') + ) +) + +process.out4 = cms.OutputModule("RNTupleTempOutputModule", + fileName = cms.untracked.string('testSeriesOfProcessesTEST.root'), +) + +process.path1 = cms.Path(process.f1) +process.path2 = cms.Path(process.f2) +process.path3 = cms.Path(process.f1) +process.path4 = cms.Path(process.f2) +process.path5 = cms.Path(process.f1) +process.path6 = cms.Path(process.f2) +process.path7 = cms.Path(process.f1) +process.path8 = cms.Path(process.f2) + +process.e = cms.EndPath(process.a+process.out1+process.out2+process.out3+process.out4) diff --git a/FWIO/RNTupleTempTests/test/unit_test_outputs/testGetBy1.log b/FWIO/RNTupleTempTests/test/unit_test_outputs/testGetBy1.log new file mode 100644 index 0000000000000..4b6d957087269 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/unit_test_outputs/testGetBy1.log @@ -0,0 +1,1523 @@ +++ starting: constructing source: IntSource +Module type=IntSource, Module label=source, Parameter Set ID=6f0b3d3a362a6270c801489bb066414a +++ finished: constructing source: IntSource +Module type=IntSource, Module label=source, Parameter Set ID=6f0b3d3a362a6270c801489bb066414a +++++ starting: constructing module with label 'TriggerResults' id = 1 +++++ finished: constructing module with label 'TriggerResults' id = 1 +++++ starting: constructing module with label 'p' id = 2 +++++ finished: constructing module with label 'p' id = 2 +++++ starting: constructing module with label 'intProducer' id = 3 +++++ finished: constructing module with label 'intProducer' id = 3 +++++ starting: constructing module with label 'a1' id = 4 +Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c +++++ finished: constructing module with label 'a1' id = 4 +Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c +++++ starting: constructing module with label 'a2' id = 5 +++++ finished: constructing module with label 'a2' id = 5 +++++ starting: constructing module with label 'a3' id = 6 +++++ finished: constructing module with label 'a3' id = 6 +++++ starting: constructing module with label 'out' id = 7 +++++ finished: constructing module with label 'out' id = 7 +++++ starting: constructing module with label 'intProducerA' id = 8 +Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 +++++ finished: constructing module with label 'intProducerA' id = 8 +Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 +++++ starting: constructing module with label 'intProducerB' id = 9 +++++ finished: constructing module with label 'intProducerB' id = 9 +++++ starting: constructing module with label 'intProducerBeginProcessBlock' id = 10 +++++ finished: constructing module with label 'intProducerBeginProcessBlock' id = 10 +++++ starting: constructing module with label 'intProducerEndProcessBlock' id = 11 +++++ finished: constructing module with label 'intProducerEndProcessBlock' id = 11 +++++ starting: constructing module with label 'intProducerU' id = 12 +++++ finished: constructing module with label 'intProducerU' id = 12 +++++ starting: constructing module with label 'intVectorProducer' id = 13 +++++ finished: constructing module with label 'intVectorProducer' id = 13 +++ preallocate: 1 concurrent runs, 1 concurrent luminosity sections, 1 streams +++ starting: begin job +++++ starting: begin job for module with label 'TriggerResults' id = 1 +++++ finished: begin job for module with label 'TriggerResults' id = 1 +++++ starting: begin job for module with label 'a1' id = 4 +Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c +++++ finished: begin job for module with label 'a1' id = 4 +Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c +++++ starting: begin job for module with label 'a2' id = 5 +++++ finished: begin job for module with label 'a2' id = 5 +++++ starting: begin job for module with label 'a3' id = 6 +++++ finished: begin job for module with label 'a3' id = 6 +++++ starting: begin job for module with label 'intProducer' id = 3 +++++ finished: begin job for module with label 'intProducer' id = 3 +++++ starting: begin job for module with label 'intProducerA' id = 8 +Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 +++++ finished: begin job for module with label 'intProducerA' id = 8 +Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 +++++ starting: begin job for module with label 'intProducerB' id = 9 +++++ finished: begin job for module with label 'intProducerB' id = 9 +++++ starting: begin job for module with label 'intProducerBeginProcessBlock' id = 10 +++++ finished: begin job for module with label 'intProducerBeginProcessBlock' id = 10 +++++ starting: begin job for module with label 'intProducerEndProcessBlock' id = 11 +++++ finished: begin job for module with label 'intProducerEndProcessBlock' id = 11 +++++ starting: begin job for module with label 'intProducerU' id = 12 +++++ finished: begin job for module with label 'intProducerU' id = 12 +++++ starting: begin job for module with label 'intVectorProducer' id = 13 +++++ finished: begin job for module with label 'intVectorProducer' id = 13 +++++ starting: begin job for module with label 'out' id = 7 +++++ finished: begin job for module with label 'out' id = 7 +++++ starting: begin job for module with label 'p' id = 2 +++++ finished: begin job for module with label 'p' id = 2 +++ finished: begin job +++ starting: begin stream 0 +++++ starting: begin stream for module: stream = 0 label = 'TriggerResults' id = 1 +++++ finished: begin stream for module: stream = 0 label = 'TriggerResults' id = 1 +++++ starting: begin stream for module: stream = 0 label = 'a1' id = 4 +StreamContext: StreamID = 0 transition = BeginStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + StreamContext: StreamID = 0 transition = BeginStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ finished: begin stream for module: stream = 0 label = 'a1' id = 4 +StreamContext: StreamID = 0 transition = BeginStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + StreamContext: StreamID = 0 transition = BeginStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ starting: begin stream for module: stream = 0 label = 'a2' id = 5 +++++ finished: begin stream for module: stream = 0 label = 'a2' id = 5 +++++ starting: begin stream for module: stream = 0 label = 'a3' id = 6 +++++ finished: begin stream for module: stream = 0 label = 'a3' id = 6 +++++ starting: begin stream for module: stream = 0 label = 'intProducer' id = 3 +++++ finished: begin stream for module: stream = 0 label = 'intProducer' id = 3 +++++ starting: begin stream for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = BeginStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + StreamContext: StreamID = 0 transition = BeginStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ finished: begin stream for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = BeginStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + StreamContext: StreamID = 0 transition = BeginStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ starting: begin stream for module: stream = 0 label = 'intProducerB' id = 9 +++++ finished: begin stream for module: stream = 0 label = 'intProducerB' id = 9 +++++ starting: begin stream for module: stream = 0 label = 'intProducerBeginProcessBlock' id = 10 +++++ finished: begin stream for module: stream = 0 label = 'intProducerBeginProcessBlock' id = 10 +++++ starting: begin stream for module: stream = 0 label = 'intProducerEndProcessBlock' id = 11 +++++ finished: begin stream for module: stream = 0 label = 'intProducerEndProcessBlock' id = 11 +++++ starting: begin stream for module: stream = 0 label = 'intProducerU' id = 12 +++++ finished: begin stream for module: stream = 0 label = 'intProducerU' id = 12 +++++ starting: begin stream for module: stream = 0 label = 'intVectorProducer' id = 13 +++++ finished: begin stream for module: stream = 0 label = 'intVectorProducer' id = 13 +++++ starting: begin stream for module: stream = 0 label = 'out' id = 7 +++++ finished: begin stream for module: stream = 0 label = 'out' id = 7 +++++ starting: begin stream for module: stream = 0 label = 'p' id = 2 +++++ finished: begin stream for module: stream = 0 label = 'p' id = 2 +++ finished: begin stream 0 +++++ starting: begin process block +GlobalContext: transition = BeginProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: prefetching before processing begin ProcessBlock for module: label = 'a3' id = 6 +++++++++ starting: prefetching before processing begin ProcessBlock for module: label = 'a2' id = 5 +++++++++ starting: prefetching before processing begin ProcessBlock for module: label = 'a1' id = 4 +GlobalContext: transition = BeginProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = BeginProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: prefetching before processing begin ProcessBlock for module: label = 'intProducerEndProcessBlock' id = 11 +++++++++ starting: prefetching before processing begin ProcessBlock for module: label = 'intProducerBeginProcessBlock' id = 10 +++++++++ finished: prefetching before processing begin ProcessBlock for module: label = 'intProducerBeginProcessBlock' id = 10 +++++++ starting: begin process block for module: label = 'intProducerBeginProcessBlock' id = 10 +++++++ finished: begin process block for module: label = 'intProducerBeginProcessBlock' id = 10 +++++++++ finished: prefetching before processing begin ProcessBlock for module: label = 'a1' id = 4 +GlobalContext: transition = BeginProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = BeginProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: begin process block for module: label = 'a1' id = 4 +GlobalContext: transition = BeginProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = BeginProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ finished: begin process block for module: label = 'a1' id = 4 +GlobalContext: transition = BeginProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = BeginProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ finished: prefetching before processing begin ProcessBlock for module: label = 'intProducerEndProcessBlock' id = 11 +++++++ starting: begin process block for module: label = 'intProducerEndProcessBlock' id = 11 +++++++ finished: begin process block for module: label = 'intProducerEndProcessBlock' id = 11 +++++++++ finished: prefetching before processing begin ProcessBlock for module: label = 'a2' id = 5 +++++++ starting: begin process block for module: label = 'a2' id = 5 +++++++ finished: begin process block for module: label = 'a2' id = 5 +++++++++ finished: prefetching before processing begin ProcessBlock for module: label = 'a3' id = 6 +++++++ starting: begin process block for module: label = 'a3' id = 6 +++++++ finished: begin process block for module: label = 'a3' id = 6 +++++ finished: begin process block +GlobalContext: transition = BeginProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ queuing: EventSetup synchronization run: 1 lumi: 0 event: 0 +++++ pre: EventSetup synchronizing run: 1 lumi: 0 event: 0 +++++ post: EventSetup synchronizing run: 1 lumi: 0 event: 0 +++++ starting: source run +++++ finished: source run +++++ starting: global begin run 1 : time = 1 +GlobalContext: transition = BeginRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: prefetching before processing global begin Run for module: label = 'a3' id = 6 +++++++++ starting: prefetching before processing global begin Run for module: label = 'a2' id = 5 +++++++++ starting: prefetching before processing global begin Run for module: label = 'a1' id = 4 +GlobalContext: transition = BeginRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = BeginRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ finished: prefetching before processing global begin Run for module: label = 'a1' id = 4 +GlobalContext: transition = BeginRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = BeginRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: global begin run for module: label = 'a1' id = 4 +GlobalContext: transition = BeginRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = BeginRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ finished: global begin run for module: label = 'a1' id = 4 +GlobalContext: transition = BeginRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = BeginRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ finished: prefetching before processing global begin Run for module: label = 'a2' id = 5 +++++++ starting: global begin run for module: label = 'a2' id = 5 +++++++ finished: global begin run for module: label = 'a2' id = 5 +++++++++ finished: prefetching before processing global begin Run for module: label = 'a3' id = 6 +++++++ starting: global begin run for module: label = 'a3' id = 6 +++++++ finished: global begin run for module: label = 'a3' id = 6 +++++ finished: global begin run 1 : time = 1 +GlobalContext: transition = BeginRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ starting: begin run: stream = 0 run = 1 time = 1 +StreamContext: StreamID = 0 transition = BeginRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: begin run for module: stream = 0 label = 'intProducer' id = 3 +++++++ finished: begin run for module: stream = 0 label = 'intProducer' id = 3 +++++++ starting: begin run for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = BeginRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + StreamContext: StreamID = 0 transition = BeginRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ finished: begin run for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = BeginRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + StreamContext: StreamID = 0 transition = BeginRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: begin run for module: stream = 0 label = 'intProducerB' id = 9 +++++++ finished: begin run for module: stream = 0 label = 'intProducerB' id = 9 +++++++ starting: begin run for module: stream = 0 label = 'intProducerU' id = 12 +++++++ finished: begin run for module: stream = 0 label = 'intProducerU' id = 12 +++++ finished: begin run: stream = 0 run = 1 time = 1 +StreamContext: StreamID = 0 transition = BeginRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ queuing: EventSetup synchronization run: 1 lumi: 1 event: 0 +++++ pre: EventSetup synchronizing run: 1 lumi: 1 event: 0 +++++ post: EventSetup synchronizing run: 1 lumi: 1 event: 0 +++++ starting: source lumi +++++ finished: source lumi +++++ starting: global begin lumi: run = 1 lumi = 1 time = 1 +GlobalContext: transition = BeginLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: prefetching before processing global begin LuminosityBlock for module: label = 'a3' id = 6 +++++++++ starting: prefetching before processing global begin LuminosityBlock for module: label = 'a2' id = 5 +++++++++ starting: prefetching before processing global begin LuminosityBlock for module: label = 'a1' id = 4 +GlobalContext: transition = BeginLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = BeginLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ finished: prefetching before processing global begin LuminosityBlock for module: label = 'a1' id = 4 +GlobalContext: transition = BeginLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = BeginLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: global begin lumi for module: label = 'a1' id = 4 +GlobalContext: transition = BeginLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = BeginLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ finished: global begin lumi for module: label = 'a1' id = 4 +GlobalContext: transition = BeginLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = BeginLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ finished: prefetching before processing global begin LuminosityBlock for module: label = 'a2' id = 5 +++++++ starting: global begin lumi for module: label = 'a2' id = 5 +++++++ finished: global begin lumi for module: label = 'a2' id = 5 +++++++++ finished: prefetching before processing global begin LuminosityBlock for module: label = 'a3' id = 6 +++++++ starting: global begin lumi for module: label = 'a3' id = 6 +++++++ finished: global begin lumi for module: label = 'a3' id = 6 +++++ finished: global begin lumi: run = 1 lumi = 1 time = 1 +GlobalContext: transition = BeginLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ starting: begin lumi: stream = 0 run = 1 lumi = 1 time = 1 +StreamContext: StreamID = 0 transition = BeginLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: begin lumi for module: stream = 0 label = 'intProducer' id = 3 +++++++ finished: begin lumi for module: stream = 0 label = 'intProducer' id = 3 +++++++ starting: begin lumi for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = BeginLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + StreamContext: StreamID = 0 transition = BeginLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ finished: begin lumi for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = BeginLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + StreamContext: StreamID = 0 transition = BeginLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: begin lumi for module: stream = 0 label = 'intProducerB' id = 9 +++++++ finished: begin lumi for module: stream = 0 label = 'intProducerB' id = 9 +++++++ starting: begin lumi for module: stream = 0 label = 'intProducerU' id = 12 +++++++ finished: begin lumi for module: stream = 0 label = 'intProducerU' id = 12 +++++ finished: begin lumi: stream = 0 run = 1 lumi = 1 time = 1 +StreamContext: StreamID = 0 transition = BeginLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ starting: source event +++++ finished: source event +++++ starting: processing event : stream = 0 run = 1 lumi = 1 event = 1 time = 5000001 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: processing path 'e' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +PathContext: pathName = e pathID = 0 (EndPath) + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: processing path 'p' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'a3' id = 6 +++++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a3, Parameter Set ID=9d7fece45bd24c3cee0db87f017e5a55 + PlaceInPathContext 3 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'a2' id = 5 +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'a1' id = 4 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + PlaceInPathContext 1 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intProducer' id = 3 +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intProducer' id = 3 +++++++++ starting: processing event for module: stream = 0 label = 'intProducer' id = 3 +++++++++ finished: processing event for module: stream = 0 label = 'intProducer' id = 3 +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'a1' id = 4 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + PlaceInPathContext 1 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: processing event for module: stream = 0 label = 'a1' id = 4 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + PlaceInPathContext 1 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ finished: processing event for module: stream = 0 label = 'a1' id = 4 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + PlaceInPathContext 1 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a3, Parameter Set ID=9d7fece45bd24c3cee0db87f017e5a55 + PlaceInPathContext 3 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++++ starting: processing event for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a3, Parameter Set ID=9d7fece45bd24c3cee0db87f017e5a55 + PlaceInPathContext 3 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++++ finished: processing event for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a3, Parameter Set ID=9d7fece45bd24c3cee0db87f017e5a55 + PlaceInPathContext 3 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'a3' id = 6 +++++++++ starting: processing event for module: stream = 0 label = 'a3' id = 6 +++++++++ finished: processing event for module: stream = 0 label = 'a3' id = 6 +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'a2' id = 5 +++++++++ starting: processing event for module: stream = 0 label = 'a2' id = 5 +++++++++ finished: processing event for module: stream = 0 label = 'a2' id = 5 +++++++++ starting: processing event for module: stream = 0 label = 'p' id = 2 +++++++++ finished: processing event for module: stream = 0 label = 'p' id = 2 +++++++ finished: processing path 'p' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: processing event for module: stream = 0 label = 'TriggerResults' id = 1 +++++++++ finished: processing event for module: stream = 0 label = 'TriggerResults' id = 1 +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'out' id = 7 +++++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intProducerB' id = 9 +++++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intProducerU' id = 12 +++++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intVectorProducer' id = 13 +++++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intVectorProducer' id = 13 +++++++++++ starting: processing event for module: stream = 0 label = 'intVectorProducer' id = 13 +++++++++++ finished: processing event for module: stream = 0 label = 'intVectorProducer' id = 13 +++++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intProducerU' id = 12 +++++++++++ starting: processing event for module: stream = 0 label = 'intProducerU' id = 12 +++++++++++ finished: processing event for module: stream = 0 label = 'intProducerU' id = 12 +++++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intProducerB' id = 9 +++++++++++ starting: processing event for module: stream = 0 label = 'intProducerB' id = 9 +++++++++++ finished: processing event for module: stream = 0 label = 'intProducerB' id = 9 +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'out' id = 7 +++++++++ starting: processing event for module: stream = 0 label = 'out' id = 7 +++++++++ finished: processing event for module: stream = 0 label = 'out' id = 7 +++++++ finished: processing path 'e' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +PathContext: pathName = e pathID = 0 (EndPath) + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ finished: processing event : stream = 0 run = 1 lumi = 1 event = 1 time = 5000001 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ starting: source event +++++ finished: source event +++++ starting: processing event : stream = 0 run = 1 lumi = 1 event = 2 time = 10000001 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: processing path 'e' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +PathContext: pathName = e pathID = 0 (EndPath) + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: processing path 'p' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'a3' id = 6 +++++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a3, Parameter Set ID=9d7fece45bd24c3cee0db87f017e5a55 + PlaceInPathContext 3 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'a2' id = 5 +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'a1' id = 4 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + PlaceInPathContext 1 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intProducer' id = 3 +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intProducer' id = 3 +++++++++ starting: processing event for module: stream = 0 label = 'intProducer' id = 3 +++++++++ finished: processing event for module: stream = 0 label = 'intProducer' id = 3 +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'a1' id = 4 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + PlaceInPathContext 1 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: processing event for module: stream = 0 label = 'a1' id = 4 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + PlaceInPathContext 1 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ finished: processing event for module: stream = 0 label = 'a1' id = 4 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + PlaceInPathContext 1 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a3, Parameter Set ID=9d7fece45bd24c3cee0db87f017e5a55 + PlaceInPathContext 3 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++++ starting: processing event for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a3, Parameter Set ID=9d7fece45bd24c3cee0db87f017e5a55 + PlaceInPathContext 3 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++++ finished: processing event for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a3, Parameter Set ID=9d7fece45bd24c3cee0db87f017e5a55 + PlaceInPathContext 3 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'a3' id = 6 +++++++++ starting: processing event for module: stream = 0 label = 'a3' id = 6 +++++++++ finished: processing event for module: stream = 0 label = 'a3' id = 6 +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'a2' id = 5 +++++++++ starting: processing event for module: stream = 0 label = 'a2' id = 5 +++++++++ finished: processing event for module: stream = 0 label = 'a2' id = 5 +++++++++ starting: processing event for module: stream = 0 label = 'p' id = 2 +++++++++ finished: processing event for module: stream = 0 label = 'p' id = 2 +++++++ finished: processing path 'p' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: processing event for module: stream = 0 label = 'TriggerResults' id = 1 +++++++++ finished: processing event for module: stream = 0 label = 'TriggerResults' id = 1 +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'out' id = 7 +++++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intProducerB' id = 9 +++++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intProducerU' id = 12 +++++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intVectorProducer' id = 13 +++++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intVectorProducer' id = 13 +++++++++++ starting: processing event for module: stream = 0 label = 'intVectorProducer' id = 13 +++++++++++ finished: processing event for module: stream = 0 label = 'intVectorProducer' id = 13 +++++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intProducerU' id = 12 +++++++++++ starting: processing event for module: stream = 0 label = 'intProducerU' id = 12 +++++++++++ finished: processing event for module: stream = 0 label = 'intProducerU' id = 12 +++++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intProducerB' id = 9 +++++++++++ starting: processing event for module: stream = 0 label = 'intProducerB' id = 9 +++++++++++ finished: processing event for module: stream = 0 label = 'intProducerB' id = 9 +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'out' id = 7 +++++++++ starting: processing event for module: stream = 0 label = 'out' id = 7 +++++++++ finished: processing event for module: stream = 0 label = 'out' id = 7 +++++++ finished: processing path 'e' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +PathContext: pathName = e pathID = 0 (EndPath) + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ finished: processing event : stream = 0 run = 1 lumi = 1 event = 2 time = 10000001 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ starting: source event +++++ finished: source event +++++ starting: processing event : stream = 0 run = 1 lumi = 1 event = 3 time = 15000001 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: processing path 'e' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +PathContext: pathName = e pathID = 0 (EndPath) + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: processing path 'p' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'a3' id = 6 +++++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a3, Parameter Set ID=9d7fece45bd24c3cee0db87f017e5a55 + PlaceInPathContext 3 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'a2' id = 5 +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'a1' id = 4 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + PlaceInPathContext 1 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intProducer' id = 3 +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intProducer' id = 3 +++++++++ starting: processing event for module: stream = 0 label = 'intProducer' id = 3 +++++++++ finished: processing event for module: stream = 0 label = 'intProducer' id = 3 +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'a1' id = 4 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + PlaceInPathContext 1 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: processing event for module: stream = 0 label = 'a1' id = 4 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + PlaceInPathContext 1 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ finished: processing event for module: stream = 0 label = 'a1' id = 4 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + PlaceInPathContext 1 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a3, Parameter Set ID=9d7fece45bd24c3cee0db87f017e5a55 + PlaceInPathContext 3 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++++ starting: processing event for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a3, Parameter Set ID=9d7fece45bd24c3cee0db87f017e5a55 + PlaceInPathContext 3 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++++ finished: processing event for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a3, Parameter Set ID=9d7fece45bd24c3cee0db87f017e5a55 + PlaceInPathContext 3 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'a3' id = 6 +++++++++ starting: processing event for module: stream = 0 label = 'a3' id = 6 +++++++++ finished: processing event for module: stream = 0 label = 'a3' id = 6 +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'a2' id = 5 +++++++++ starting: processing event for module: stream = 0 label = 'a2' id = 5 +++++++++ finished: processing event for module: stream = 0 label = 'a2' id = 5 +++++++++ starting: processing event for module: stream = 0 label = 'p' id = 2 +++++++++ finished: processing event for module: stream = 0 label = 'p' id = 2 +++++++ finished: processing path 'p' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: processing event for module: stream = 0 label = 'TriggerResults' id = 1 +++++++++ finished: processing event for module: stream = 0 label = 'TriggerResults' id = 1 +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'out' id = 7 +++++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intProducerB' id = 9 +++++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intProducerU' id = 12 +++++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intVectorProducer' id = 13 +++++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intVectorProducer' id = 13 +++++++++++ starting: processing event for module: stream = 0 label = 'intVectorProducer' id = 13 +++++++++++ finished: processing event for module: stream = 0 label = 'intVectorProducer' id = 13 +++++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intProducerU' id = 12 +++++++++++ starting: processing event for module: stream = 0 label = 'intProducerU' id = 12 +++++++++++ finished: processing event for module: stream = 0 label = 'intProducerU' id = 12 +++++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intProducerB' id = 9 +++++++++++ starting: processing event for module: stream = 0 label = 'intProducerB' id = 9 +++++++++++ finished: processing event for module: stream = 0 label = 'intProducerB' id = 9 +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'out' id = 7 +++++++++ starting: processing event for module: stream = 0 label = 'out' id = 7 +++++++++ finished: processing event for module: stream = 0 label = 'out' id = 7 +++++++ finished: processing path 'e' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +PathContext: pathName = e pathID = 0 (EndPath) + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ finished: processing event : stream = 0 run = 1 lumi = 1 event = 3 time = 15000001 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ queuing: EventSetup synchronization run: 1 lumi: 4294967295 event: 18446744073709551615 +++++ pre: EventSetup synchronizing run: 1 lumi: 4294967295 event: 18446744073709551615 +++++ post: EventSetup synchronizing run: 1 lumi: 4294967295 event: 18446744073709551615 +++++ starting: end lumi: stream = 0 run = 1 lumi = 1 time = 15000001 +StreamContext: StreamID = 0 transition = EndLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: end lumi for module: stream = 0 label = 'intProducer' id = 3 +++++++ finished: end lumi for module: stream = 0 label = 'intProducer' id = 3 +++++++ starting: end lumi for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = EndLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + StreamContext: StreamID = 0 transition = EndLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ finished: end lumi for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = EndLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + StreamContext: StreamID = 0 transition = EndLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: end lumi for module: stream = 0 label = 'intProducerB' id = 9 +++++++ finished: end lumi for module: stream = 0 label = 'intProducerB' id = 9 +++++++ starting: end lumi for module: stream = 0 label = 'intProducerU' id = 12 +++++++ finished: end lumi for module: stream = 0 label = 'intProducerU' id = 12 +++++ finished: end lumi: stream = 0 run = 1 lumi = 1 time = 15000001 +StreamContext: StreamID = 0 transition = EndLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ starting: global end lumi: run = 1 lumi = 1 time = 1 +GlobalContext: transition = EndLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: prefetching before processing global end LuminosityBlock for module: label = 'a3' id = 6 +++++++++ starting: prefetching before processing global end LuminosityBlock for module: label = 'a2' id = 5 +++++++++ starting: prefetching before processing global end LuminosityBlock for module: label = 'a1' id = 4 +GlobalContext: transition = EndLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = EndLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ finished: prefetching before processing global end LuminosityBlock for module: label = 'a1' id = 4 +GlobalContext: transition = EndLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = EndLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: global end lumi for module: label = 'a1' id = 4 +GlobalContext: transition = EndLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = EndLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ finished: global end lumi for module: label = 'a1' id = 4 +GlobalContext: transition = EndLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = EndLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ finished: prefetching before processing global end LuminosityBlock for module: label = 'a2' id = 5 +++++++ starting: global end lumi for module: label = 'a2' id = 5 +++++++ finished: global end lumi for module: label = 'a2' id = 5 +++++++++ finished: prefetching before processing global end LuminosityBlock for module: label = 'a3' id = 6 +++++++ starting: global end lumi for module: label = 'a3' id = 6 +++++++ finished: global end lumi for module: label = 'a3' id = 6 +++++ finished: global end lumi: run = 1 lumi = 1 time = 1 +GlobalContext: transition = EndLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ starting: global write lumi: run = 1 lumi = 1 time = 1 +GlobalContext: transition = WriteLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: write lumi for module: label = 'out' id = 7 +++++++ finished: write lumi for module: label = 'out' id = 7 +++++ finished: global write lumi: run = 1 lumi = 1 time = 1 +GlobalContext: transition = WriteLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ starting: end run: stream = 0 run = 1 time = 15000001 +StreamContext: StreamID = 0 transition = EndRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: end run for module: stream = 0 label = 'intProducer' id = 3 +++++++ finished: end run for module: stream = 0 label = 'intProducer' id = 3 +++++++ starting: end run for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = EndRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + StreamContext: StreamID = 0 transition = EndRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ finished: end run for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = EndRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + StreamContext: StreamID = 0 transition = EndRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: end run for module: stream = 0 label = 'intProducerB' id = 9 +++++++ finished: end run for module: stream = 0 label = 'intProducerB' id = 9 +++++++ starting: end run for module: stream = 0 label = 'intProducerU' id = 12 +++++++ finished: end run for module: stream = 0 label = 'intProducerU' id = 12 +++++ finished: end run: stream = 0 run = 1 time = 15000001 +StreamContext: StreamID = 0 transition = EndRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ starting: global end run 1 : time = 15000001 +GlobalContext: transition = EndRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: prefetching before processing global end Run for module: label = 'a3' id = 6 +++++++++ starting: prefetching before processing global end Run for module: label = 'a2' id = 5 +++++++++ starting: prefetching before processing global end Run for module: label = 'a1' id = 4 +GlobalContext: transition = EndRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = EndRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ finished: prefetching before processing global end Run for module: label = 'a1' id = 4 +GlobalContext: transition = EndRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = EndRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: global end run for module: label = 'a1' id = 4 +GlobalContext: transition = EndRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = EndRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ finished: global end run for module: label = 'a1' id = 4 +GlobalContext: transition = EndRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = EndRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ finished: prefetching before processing global end Run for module: label = 'a2' id = 5 +++++++ starting: global end run for module: label = 'a2' id = 5 +++++++ finished: global end run for module: label = 'a2' id = 5 +++++++++ finished: prefetching before processing global end Run for module: label = 'a3' id = 6 +++++++ starting: global end run for module: label = 'a3' id = 6 +++++++ finished: global end run for module: label = 'a3' id = 6 +++++ finished: global end run 1 : time = 15000001 +GlobalContext: transition = EndRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ starting: global write run 1 : time = 15000001 +GlobalContext: transition = WriteRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: write run for module: label = 'out' id = 7 +++++++ finished: write run for module: label = 'out' id = 7 +++++ finished: global write run 1 : time = 15000001 +GlobalContext: transition = WriteRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ starting: end process block +GlobalContext: transition = EndProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: prefetching before processing end ProcessBlock for module: label = 'a3' id = 6 +++++++++ starting: prefetching before processing end ProcessBlock for module: label = 'a2' id = 5 +++++++++ starting: prefetching before processing end ProcessBlock for module: label = 'a1' id = 4 +GlobalContext: transition = EndProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = EndProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ starting: prefetching before processing end ProcessBlock for module: label = 'intProducerEndProcessBlock' id = 11 +++++++++ starting: prefetching before processing end ProcessBlock for module: label = 'intProducerBeginProcessBlock' id = 10 +++++++++ finished: prefetching before processing end ProcessBlock for module: label = 'intProducerBeginProcessBlock' id = 10 +++++++ starting: end process block for module: label = 'intProducerBeginProcessBlock' id = 10 +++++++ finished: end process block for module: label = 'intProducerBeginProcessBlock' id = 10 +++++++++ finished: prefetching before processing end ProcessBlock for module: label = 'intProducerEndProcessBlock' id = 11 +++++++ starting: end process block for module: label = 'intProducerEndProcessBlock' id = 11 +++++++ finished: end process block for module: label = 'intProducerEndProcessBlock' id = 11 +++++++++ finished: prefetching before processing end ProcessBlock for module: label = 'a1' id = 4 +GlobalContext: transition = EndProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = EndProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: end process block for module: label = 'a1' id = 4 +GlobalContext: transition = EndProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = EndProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ finished: end process block for module: label = 'a1' id = 4 +GlobalContext: transition = EndProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + GlobalContext: transition = EndProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++++ finished: prefetching before processing end ProcessBlock for module: label = 'a2' id = 5 +++++++ starting: end process block for module: label = 'a2' id = 5 +++++++ finished: end process block for module: label = 'a2' id = 5 +++++++++ finished: prefetching before processing end ProcessBlock for module: label = 'a3' id = 6 +++++++ starting: end process block for module: label = 'a3' id = 6 +++++++ finished: end process block for module: label = 'a3' id = 6 +++++ finished: end process block +GlobalContext: transition = EndProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ starting: write process block +GlobalContext: transition = WriteProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++++ starting: write process block for module: label = 'out' id = 7 +++++++ finished: write process block for module: label = 'out' id = 7 +++++ finished: write process block +GlobalContext: transition = WriteProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++ starting: end stream 0 +++++ starting: end stream for module: stream = 0 label = 'TriggerResults' id = 1 +++++ finished: end stream for module: stream = 0 label = 'TriggerResults' id = 1 +++++ starting: end stream for module: stream = 0 label = 'a1' id = 4 +StreamContext: StreamID = 0 transition = EndStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + StreamContext: StreamID = 0 transition = EndStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ finished: end stream for module: stream = 0 label = 'a1' id = 4 +StreamContext: StreamID = 0 transition = EndStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c + StreamContext: StreamID = 0 transition = EndStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ starting: end stream for module: stream = 0 label = 'a2' id = 5 +++++ finished: end stream for module: stream = 0 label = 'a2' id = 5 +++++ starting: end stream for module: stream = 0 label = 'a3' id = 6 +++++ finished: end stream for module: stream = 0 label = 'a3' id = 6 +++++ starting: end stream for module: stream = 0 label = 'intProducer' id = 3 +++++ finished: end stream for module: stream = 0 label = 'intProducer' id = 3 +++++ starting: end stream for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = EndStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + StreamContext: StreamID = 0 transition = EndStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ finished: end stream for module: stream = 0 label = 'intProducerA' id = 8 +StreamContext: StreamID = 0 transition = EndStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 + StreamContext: StreamID = 0 transition = EndStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD1 cf8fb4a5e3c9a108eac33826b2a17969 + +++++ starting: end stream for module: stream = 0 label = 'intProducerB' id = 9 +++++ finished: end stream for module: stream = 0 label = 'intProducerB' id = 9 +++++ starting: end stream for module: stream = 0 label = 'intProducerBeginProcessBlock' id = 10 +++++ finished: end stream for module: stream = 0 label = 'intProducerBeginProcessBlock' id = 10 +++++ starting: end stream for module: stream = 0 label = 'intProducerEndProcessBlock' id = 11 +++++ finished: end stream for module: stream = 0 label = 'intProducerEndProcessBlock' id = 11 +++++ starting: end stream for module: stream = 0 label = 'intProducerU' id = 12 +++++ finished: end stream for module: stream = 0 label = 'intProducerU' id = 12 +++++ starting: end stream for module: stream = 0 label = 'intVectorProducer' id = 13 +++++ finished: end stream for module: stream = 0 label = 'intVectorProducer' id = 13 +++++ starting: end stream for module: stream = 0 label = 'out' id = 7 +++++ finished: end stream for module: stream = 0 label = 'out' id = 7 +++++ starting: end stream for module: stream = 0 label = 'p' id = 2 +++++ finished: end stream for module: stream = 0 label = 'p' id = 2 +++ finished: end stream 0 +++ starting: end job +++++ starting: end job for module with label 'TriggerResults' id = 1 +++++ finished: end job for module with label 'TriggerResults' id = 1 +++++ starting: end job for module with label 'a1' id = 4 +Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c +TestFindProduct sum = 530021 +++++ finished: end job for module with label 'a1' id = 4 +Module type=TestFindProduct, Module label=a1, Parameter Set ID=a7caa43fcf5ef35dee69e4bd85169d4c +++++ starting: end job for module with label 'a2' id = 5 +TestFindProduct sum = 300 +++++ finished: end job for module with label 'a2' id = 5 +++++ starting: end job for module with label 'a3' id = 6 +TestFindProduct sum = 300 +++++ finished: end job for module with label 'a3' id = 6 +++++ starting: end job for module with label 'intProducer' id = 3 +++++ finished: end job for module with label 'intProducer' id = 3 +++++ starting: end job for module with label 'intProducerA' id = 8 +Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 +++++ finished: end job for module with label 'intProducerA' id = 8 +Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb02df4e1c3b945954838318 +++++ starting: end job for module with label 'intProducerB' id = 9 +++++ finished: end job for module with label 'intProducerB' id = 9 +++++ starting: end job for module with label 'intProducerBeginProcessBlock' id = 10 +++++ finished: end job for module with label 'intProducerBeginProcessBlock' id = 10 +++++ starting: end job for module with label 'intProducerEndProcessBlock' id = 11 +++++ finished: end job for module with label 'intProducerEndProcessBlock' id = 11 +++++ starting: end job for module with label 'intProducerU' id = 12 +++++ finished: end job for module with label 'intProducerU' id = 12 +++++ starting: end job for module with label 'intVectorProducer' id = 13 +++++ finished: end job for module with label 'intVectorProducer' id = 13 +++++ starting: end job for module with label 'out' id = 7 +++++ finished: end job for module with label 'out' id = 7 +++++ starting: end job for module with label 'p' id = 2 +++++ finished: end job for module with label 'p' id = 2 +++ finished: end job diff --git a/FWIO/RNTupleTempTests/test/unit_test_outputs/testGetBy2.log b/FWIO/RNTupleTempTests/test/unit_test_outputs/testGetBy2.log new file mode 100644 index 0000000000000..25056eef44cdb --- /dev/null +++ b/FWIO/RNTupleTempTests/test/unit_test_outputs/testGetBy2.log @@ -0,0 +1,872 @@ +++ starting: constructing source: RNTupleTempSource +Module type=RNTupleTempSource, Module label=source, Parameter Set ID=f45c0cea718e3436a3db8abdc232e8c6 +++++ starting: open input file: lfn = file:testGetBy1.root +++++ finished: open input file: lfn = file:testGetBy1.root +++ finished: constructing source: RNTupleTempSource +Module type=RNTupleTempSource, Module label=source, Parameter Set ID=f45c0cea718e3436a3db8abdc232e8c6 +++++ starting: constructing module with label 'TriggerResults' id = 1 +++++ finished: constructing module with label 'TriggerResults' id = 1 +++++ starting: constructing module with label 'p' id = 2 +++++ finished: constructing module with label 'p' id = 2 +++++ starting: constructing module with label 'intProducer' id = 3 +Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 +++++ finished: constructing module with label 'intProducer' id = 3 +Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 +++++ starting: constructing module with label 'out' id = 4 +++++ finished: constructing module with label 'out' id = 4 +++++ starting: constructing module with label 'intProducerU' id = 5 +++++ finished: constructing module with label 'intProducerU' id = 5 +++++ starting: constructing module with label 'intVectorProducer' id = 6 +++++ finished: constructing module with label 'intVectorProducer' id = 6 +++ preallocate: 1 concurrent runs, 1 concurrent luminosity sections, 1 streams +++ starting: begin job +++++ starting: begin job for module with label 'TriggerResults' id = 1 +++++ finished: begin job for module with label 'TriggerResults' id = 1 +++++ starting: begin job for module with label 'intProducer' id = 3 +Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 +++++ finished: begin job for module with label 'intProducer' id = 3 +Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 +++++ starting: begin job for module with label 'intProducerU' id = 5 +++++ finished: begin job for module with label 'intProducerU' id = 5 +++++ starting: begin job for module with label 'intVectorProducer' id = 6 +++++ finished: begin job for module with label 'intVectorProducer' id = 6 +++++ starting: begin job for module with label 'out' id = 4 +++++ finished: begin job for module with label 'out' id = 4 +++++ starting: begin job for module with label 'p' id = 2 +++++ finished: begin job for module with label 'p' id = 2 +++ finished: begin job +++ starting: begin stream 0 +++++ starting: begin stream for module: stream = 0 label = 'TriggerResults' id = 1 +++++ finished: begin stream for module: stream = 0 label = 'TriggerResults' id = 1 +++++ starting: begin stream for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = BeginStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + StreamContext: StreamID = 0 transition = BeginStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ finished: begin stream for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = BeginStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + StreamContext: StreamID = 0 transition = BeginStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ starting: begin stream for module: stream = 0 label = 'intProducerU' id = 5 +++++ finished: begin stream for module: stream = 0 label = 'intProducerU' id = 5 +++++ starting: begin stream for module: stream = 0 label = 'intVectorProducer' id = 6 +++++ finished: begin stream for module: stream = 0 label = 'intVectorProducer' id = 6 +++++ starting: begin stream for module: stream = 0 label = 'out' id = 4 +++++ finished: begin stream for module: stream = 0 label = 'out' id = 4 +++++ starting: begin stream for module: stream = 0 label = 'p' id = 2 +++++ finished: begin stream for module: stream = 0 label = 'p' id = 2 +++ finished: begin stream 0 +++++ starting: begin process block +GlobalContext: transition = BeginProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ finished: begin process block +GlobalContext: transition = BeginProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ starting: source process block +++++ finished: source process block PROD1 +++++ starting: access input process block +GlobalContext: transition = AccessInputProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ finished: access input process block +GlobalContext: transition = AccessInputProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ starting: write process block +GlobalContext: transition = WriteProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ starting: write process block for module: label = 'out' id = 4 +++++++ finished: write process block for module: label = 'out' id = 4 +++++ finished: write process block +GlobalContext: transition = WriteProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ queuing: EventSetup synchronization run: 1 lumi: 0 event: 0 +++++ pre: EventSetup synchronizing run: 1 lumi: 0 event: 0 +++++ post: EventSetup synchronizing run: 1 lumi: 0 event: 0 +++++ starting: source run +++++ finished: source run +++++ starting: global begin run 1 : time = 1 +GlobalContext: transition = BeginRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ finished: global begin run 1 : time = 1 +GlobalContext: transition = BeginRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ starting: begin run: stream = 0 run = 1 time = 1 +StreamContext: StreamID = 0 transition = BeginRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ starting: begin run for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = BeginRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + StreamContext: StreamID = 0 transition = BeginRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ finished: begin run for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = BeginRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + StreamContext: StreamID = 0 transition = BeginRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ starting: begin run for module: stream = 0 label = 'intProducerU' id = 5 +++++++ finished: begin run for module: stream = 0 label = 'intProducerU' id = 5 +++++ finished: begin run: stream = 0 run = 1 time = 1 +StreamContext: StreamID = 0 transition = BeginRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ queuing: EventSetup synchronization run: 1 lumi: 1 event: 0 +++++ pre: EventSetup synchronizing run: 1 lumi: 1 event: 0 +++++ post: EventSetup synchronizing run: 1 lumi: 1 event: 0 +++++ starting: source lumi +++++ finished: source lumi +++++ starting: global begin lumi: run = 1 lumi = 1 time = 1 +GlobalContext: transition = BeginLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ finished: global begin lumi: run = 1 lumi = 1 time = 1 +GlobalContext: transition = BeginLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ starting: begin lumi: stream = 0 run = 1 lumi = 1 time = 1 +StreamContext: StreamID = 0 transition = BeginLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ starting: begin lumi for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = BeginLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + StreamContext: StreamID = 0 transition = BeginLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ finished: begin lumi for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = BeginLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + StreamContext: StreamID = 0 transition = BeginLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ starting: begin lumi for module: stream = 0 label = 'intProducerU' id = 5 +++++++ finished: begin lumi for module: stream = 0 label = 'intProducerU' id = 5 +++++ finished: begin lumi: stream = 0 run = 1 lumi = 1 time = 1 +StreamContext: StreamID = 0 transition = BeginLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ starting: source event +++++ finished: source event +++++ starting: processing event : stream = 0 run = 1 lumi = 1 event = 1 time = 5000001 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ starting: processing path 'e' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +PathContext: pathName = e pathID = 0 (EndPath) + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ starting: processing path 'p' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + PlaceInPathContext 0 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + PlaceInPathContext 0 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++++ starting: processing event for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + PlaceInPathContext 0 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++++ finished: processing event for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + PlaceInPathContext 0 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++++ starting: processing event for module: stream = 0 label = 'p' id = 2 +++++++++ finished: processing event for module: stream = 0 label = 'p' id = 2 +++++++ finished: processing path 'p' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++++ starting: processing event for module: stream = 0 label = 'TriggerResults' id = 1 +++++++++ finished: processing event for module: stream = 0 label = 'TriggerResults' id = 1 +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'out' id = 4 +++++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intProducerU' id = 5 +++++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intVectorProducer' id = 6 +++++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intVectorProducer' id = 6 +++++++++++ starting: processing event for module: stream = 0 label = 'intVectorProducer' id = 6 +++++++++++ finished: processing event for module: stream = 0 label = 'intVectorProducer' id = 6 +++++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intProducerU' id = 5 +++++++++++ starting: processing event for module: stream = 0 label = 'intProducerU' id = 5 +++++++++++ finished: processing event for module: stream = 0 label = 'intProducerU' id = 5 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'out' id = 4 +++++++++ starting: processing event for module: stream = 0 label = 'out' id = 4 +++++++++ finished: processing event for module: stream = 0 label = 'out' id = 4 +++++++ finished: processing path 'e' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +PathContext: pathName = e pathID = 0 (EndPath) + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ finished: processing event : stream = 0 run = 1 lumi = 1 event = 1 time = 5000001 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 5000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ starting: source event +++++ finished: source event +++++ starting: processing event : stream = 0 run = 1 lumi = 1 event = 2 time = 10000001 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ starting: processing path 'e' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +PathContext: pathName = e pathID = 0 (EndPath) + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ starting: processing path 'p' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + PlaceInPathContext 0 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + PlaceInPathContext 0 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++++ starting: processing event for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + PlaceInPathContext 0 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++++ finished: processing event for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + PlaceInPathContext 0 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++++ starting: processing event for module: stream = 0 label = 'p' id = 2 +++++++++ finished: processing event for module: stream = 0 label = 'p' id = 2 +++++++ finished: processing path 'p' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++++ starting: processing event for module: stream = 0 label = 'TriggerResults' id = 1 +++++++++ finished: processing event for module: stream = 0 label = 'TriggerResults' id = 1 +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'out' id = 4 +++++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intProducerU' id = 5 +++++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intVectorProducer' id = 6 +++++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intVectorProducer' id = 6 +++++++++++ starting: processing event for module: stream = 0 label = 'intVectorProducer' id = 6 +++++++++++ finished: processing event for module: stream = 0 label = 'intVectorProducer' id = 6 +++++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intProducerU' id = 5 +++++++++++ starting: processing event for module: stream = 0 label = 'intProducerU' id = 5 +++++++++++ finished: processing event for module: stream = 0 label = 'intProducerU' id = 5 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'out' id = 4 +++++++++ starting: processing event for module: stream = 0 label = 'out' id = 4 +++++++++ finished: processing event for module: stream = 0 label = 'out' id = 4 +++++++ finished: processing path 'e' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +PathContext: pathName = e pathID = 0 (EndPath) + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ finished: processing event : stream = 0 run = 1 lumi = 1 event = 2 time = 10000001 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 2 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 10000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ starting: source event +++++ finished: source event +++++ starting: processing event : stream = 0 run = 1 lumi = 1 event = 3 time = 15000001 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ starting: processing path 'e' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +PathContext: pathName = e pathID = 0 (EndPath) + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ starting: processing path 'p' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + PlaceInPathContext 0 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Prefetching + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + PlaceInPathContext 0 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++++ starting: processing event for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + PlaceInPathContext 0 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++++ finished: processing event for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + PlaceInPathContext 0 + PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++++ starting: processing event for module: stream = 0 label = 'p' id = 2 +++++++++ finished: processing event for module: stream = 0 label = 'p' id = 2 +++++++ finished: processing path 'p' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +PathContext: pathName = p pathID = 0 + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++++ starting: processing event for module: stream = 0 label = 'TriggerResults' id = 1 +++++++++ finished: processing event for module: stream = 0 label = 'TriggerResults' id = 1 +++++++++ starting: prefetching before processing event for module: stream = 0 label = 'out' id = 4 +++++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intProducerU' id = 5 +++++++++++ starting: prefetching before processing event for module: stream = 0 label = 'intVectorProducer' id = 6 +++++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intVectorProducer' id = 6 +++++++++++ starting: processing event for module: stream = 0 label = 'intVectorProducer' id = 6 +++++++++++ finished: processing event for module: stream = 0 label = 'intVectorProducer' id = 6 +++++++++++ finished: prefetching before processing event for module: stream = 0 label = 'intProducerU' id = 5 +++++++++++ starting: processing event for module: stream = 0 label = 'intProducerU' id = 5 +++++++++++ finished: processing event for module: stream = 0 label = 'intProducerU' id = 5 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ starting: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++++ finished: event delayed read from source: stream = 0 label = 'out' id = 4 +++++++++ finished: prefetching before processing event for module: stream = 0 label = 'out' id = 4 +++++++++ starting: processing event for module: stream = 0 label = 'out' id = 4 +++++++++ finished: processing event for module: stream = 0 label = 'out' id = 4 +++++++ finished: processing path 'e' : stream = 0 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +PathContext: pathName = e pathID = 0 (EndPath) + StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ finished: processing event : stream = 0 run = 1 lumi = 1 event = 3 time = 15000001 +StreamContext: StreamID = 0 transition = Event + run: 1 lumi: 1 event: 3 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ queuing: EventSetup synchronization run: 1 lumi: 4294967295 event: 18446744073709551615 +++++ pre: EventSetup synchronizing run: 1 lumi: 4294967295 event: 18446744073709551615 +++++ post: EventSetup synchronizing run: 1 lumi: 4294967295 event: 18446744073709551615 +++++ starting: end lumi: stream = 0 run = 1 lumi = 1 time = 15000001 +StreamContext: StreamID = 0 transition = EndLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ starting: end lumi for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = EndLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + StreamContext: StreamID = 0 transition = EndLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ finished: end lumi for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = EndLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + StreamContext: StreamID = 0 transition = EndLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ starting: end lumi for module: stream = 0 label = 'intProducerU' id = 5 +++++++ finished: end lumi for module: stream = 0 label = 'intProducerU' id = 5 +++++ finished: end lumi: stream = 0 run = 1 lumi = 1 time = 15000001 +StreamContext: StreamID = 0 transition = EndLuminosityBlock + run: 1 lumi: 1 event: 0 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ starting: global end lumi: run = 1 lumi = 1 time = 1 +GlobalContext: transition = EndLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ finished: global end lumi: run = 1 lumi = 1 time = 1 +GlobalContext: transition = EndLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ starting: global write lumi: run = 1 lumi = 1 time = 1 +GlobalContext: transition = WriteLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ starting: write lumi for module: label = 'out' id = 4 +++++++ finished: write lumi for module: label = 'out' id = 4 +++++ finished: global write lumi: run = 1 lumi = 1 time = 1 +GlobalContext: transition = WriteLuminosityBlock + run: 1 luminosityBlock: 1 + runIndex = 0 luminosityBlockIndex = 0 unixTime = 0 microsecondOffset = 1 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ starting: end run: stream = 0 run = 1 time = 15000001 +StreamContext: StreamID = 0 transition = EndRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ starting: end run for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = EndRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + StreamContext: StreamID = 0 transition = EndRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ finished: end run for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = EndRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + StreamContext: StreamID = 0 transition = EndRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ starting: end run for module: stream = 0 label = 'intProducerU' id = 5 +++++++ finished: end run for module: stream = 0 label = 'intProducerU' id = 5 +++++ finished: end run: stream = 0 run = 1 time = 15000001 +StreamContext: StreamID = 0 transition = EndRun + run: 1 lumi: 0 event: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ starting: global end run 1 : time = 15000001 +GlobalContext: transition = EndRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ finished: global end run 1 : time = 15000001 +GlobalContext: transition = EndRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ starting: global write run 1 : time = 15000001 +GlobalContext: transition = WriteRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ starting: write run for module: label = 'out' id = 4 +++++++ finished: write run for module: label = 'out' id = 4 +++++ finished: global write run 1 : time = 15000001 +GlobalContext: transition = WriteRun + run: 1 luminosityBlock: 0 + runIndex = 0 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 15000001 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ starting: close input file: lfn = file:testGetBy1.root +++++ finished: close input file: lfn = file:testGetBy1.root +++++ starting: end process block +GlobalContext: transition = EndProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ finished: end process block +GlobalContext: transition = EndProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ starting: write process block +GlobalContext: transition = WriteProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++++ starting: write process block for module: label = 'out' id = 4 +++++++ finished: write process block for module: label = 'out' id = 4 +++++ finished: write process block +GlobalContext: transition = WriteProcessBlock + run: 0 luminosityBlock: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++ starting: end stream 0 +++++ starting: end stream for module: stream = 0 label = 'TriggerResults' id = 1 +++++ finished: end stream for module: stream = 0 label = 'TriggerResults' id = 1 +++++ starting: end stream for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = EndStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + StreamContext: StreamID = 0 transition = EndStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ finished: end stream for module: stream = 0 label = 'intProducer' id = 3 +StreamContext: StreamID = 0 transition = EndStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 +ModuleCallingContext state = Running + moduleDescription: Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 + StreamContext: StreamID = 0 transition = EndStream + run: 0 lumi: 0 event: 0 + runIndex = 4294967295 luminosityBlockIndex = 4294967295 unixTime = 0 microsecondOffset = 0 + ProcessContext: PROD2 2383b6183d2bd58762047deda96e12f0 + +++++ starting: end stream for module: stream = 0 label = 'intProducerU' id = 5 +++++ finished: end stream for module: stream = 0 label = 'intProducerU' id = 5 +++++ starting: end stream for module: stream = 0 label = 'intVectorProducer' id = 6 +++++ finished: end stream for module: stream = 0 label = 'intVectorProducer' id = 6 +++++ starting: end stream for module: stream = 0 label = 'out' id = 4 +++++ finished: end stream for module: stream = 0 label = 'out' id = 4 +++++ starting: end stream for module: stream = 0 label = 'p' id = 2 +++++ finished: end stream for module: stream = 0 label = 'p' id = 2 +++ finished: end stream 0 +++ starting: end job +++++ starting: end job for module with label 'TriggerResults' id = 1 +++++ finished: end job for module with label 'TriggerResults' id = 1 +++++ starting: end job for module with label 'intProducer' id = 3 +Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 +++++ finished: end job for module with label 'intProducer' id = 3 +Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a3015d748c803aa5c60a25d3 +++++ starting: end job for module with label 'intProducerU' id = 5 +++++ finished: end job for module with label 'intProducerU' id = 5 +++++ starting: end job for module with label 'intVectorProducer' id = 6 +++++ finished: end job for module with label 'intVectorProducer' id = 6 +++++ starting: end job for module with label 'out' id = 4 +++++ finished: end job for module with label 'out' id = 4 +++++ starting: end job for module with label 'p' id = 2 +++++ finished: end job for module with label 'p' id = 2 +++ finished: end job diff --git a/FWIO/RNTupleTempTests/test/unit_test_outputs/testLooperEventNavigationOutput.txt b/FWIO/RNTupleTempTests/test/unit_test_outputs/testLooperEventNavigationOutput.txt new file mode 100644 index 0000000000000..bbf24812ecd6b --- /dev/null +++ b/FWIO/RNTupleTempTests/test/unit_test_outputs/testLooperEventNavigationOutput.txt @@ -0,0 +1,123 @@ + +What should we do next? +(0) process the next event +(1) will stop the loop because this is the first event +(2) process a specific event +(3) stop loop +(4) stop process + +What should we do next? +(0) process the next event +(1) process the previous event +(2) process a specific event +(3) stop loop +(4) stop process +Which run? +Which luminosity block? +Which event? + +What should we do next? +(0) process the next event +(1) process the previous event +(2) process a specific event +(3) stop loop +(4) stop process + +What should we do next? +(0) process the next event +(1) process the previous event +(2) process a specific event +(3) stop loop +(4) stop process + +What should we do next? +(0) process the next event +(1) process the previous event +(2) process a specific event +(3) stop loop +(4) stop process +Which run? +Which luminosity block? +Which event? + +What should we do next? +(0) will stop the loop because this is the last event +(1) process the previous event +(2) process a specific event +(3) stop loop +(4) stop process +Which run? +Which luminosity block? +Which event? + +What should we do next? +(0) process the next event +(1) process the previous event +(2) process a specific event +(3) stop loop +(4) stop process + +What should we do next? +(0) process the next event +(1) process the previous event +(2) process a specific event +(3) stop loop +(4) stop process + +What should we do next? +(0) process the next event +(1) process the previous event +(2) process a specific event +(3) stop loop +(4) stop process +Please enter numeric characters only. The value must be in the range 0 to 4 (inclusive). Please try again. +Which run? +Please enter numeric characters only. Please try again. +Which luminosity block? +Please enter numeric characters only. Please try again. +Which event? +Please enter numeric characters only. Please try again. + +What should we do next? +(0) process the next event +(1) process the previous event +(2) process a specific event +(3) stop loop +(4) stop process +Which run? +Which luminosity block? +Which event? +Event could not be found. Nothing done. Try again. + +What should we do next? +(0) process the next event +(1) process the previous event +(2) process a specific event +(3) stop loop +(4) stop process +Ending loop + +What should we do next? +(0) process the next event +(1) will stop the loop because this is the first event +(2) process a specific event +(3) stop loop +(4) stop process +Which run? +Which luminosity block? +Which event? + +What should we do next? +(0) process the next event if it exists (at last event in the open file. there are more files) +(1) process the previous event +(2) process a specific event +(3) stop loop +(4) stop process + +What should we do next? +(0) process the next event +(1) process the previous event if there are any (at first event in the open file. there are previous files) +(2) process a specific event +(3) stop loop +(4) stop process +Ending loop diff --git a/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_IgnoreCompletely_cfg.py b/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_IgnoreCompletely_cfg.py new file mode 100644 index 0000000000000..7b75190ae3caf --- /dev/null +++ b/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_IgnoreCompletely_cfg.py @@ -0,0 +1,4 @@ +import FWCore.ParameterSet.Config as cms + +from unscheduled_fail_on_output_cfg import process +process.options.IgnoreCompletely = cms.untracked.vstring('NotFound') diff --git a/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_Rethrow_cfg.py b/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_Rethrow_cfg.py new file mode 100644 index 0000000000000..72663bca503e3 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_Rethrow_cfg.py @@ -0,0 +1,4 @@ +import FWCore.ParameterSet.Config as cms + +from unscheduled_fail_on_output_cfg import process +process.options.Rethrow = cms.untracked.vstring('NotFound') diff --git a/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_TryToContinue_cfg.py b/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_TryToContinue_cfg.py new file mode 100644 index 0000000000000..988a863093f81 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_TryToContinue_cfg.py @@ -0,0 +1,4 @@ +import FWCore.ParameterSet.Config as cms + +from unscheduled_fail_on_output_cfg import process +process.options.TryToContinue = cms.untracked.vstring('NotFound') diff --git a/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_cfg.py b/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_cfg.py new file mode 100644 index 0000000000000..4f00f98709226 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_cfg.py @@ -0,0 +1,18 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process('Test') + +process.source = cms.Source('EmptySource') +process.failing = cms.EDProducer('FailingProducer') +process.i = cms.EDProducer('IntProducer', + ivalue = cms.int32(10) ) +process.out = cms.OutputModule('RNTupleTempOutputModule', + fileName = cms.untracked.string('unscheduled_fail_on_output.root')) + +process.t = cms.Task(process.failing) +process.o = cms.EndPath(process.out, process.t) +process.p = cms.Path(process.i) + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(10) + ) diff --git a/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_no_dependency_TryToContinue_cfg.py b/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_no_dependency_TryToContinue_cfg.py new file mode 100644 index 0000000000000..c7e60192662fd --- /dev/null +++ b/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_no_dependency_TryToContinue_cfg.py @@ -0,0 +1,8 @@ +import FWCore.ParameterSet.Config as cms + +from unscheduled_fail_on_output_cfg import process +process.options.TryToContinue = ['NotFound'] + +process.out.outputCommands = cms.untracked.vstring('keep *', 'drop *_failing_*_*') +process.failGet = cms.EDAnalyzer('IntTestAnalyzer', moduleLabel = cms.untracked.InputTag('failing'), valueMustMatch = cms.untracked.int32(0)) +process.failingEnd = cms.EndPath(process.failGet) diff --git a/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_read_found_events.py b/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_read_found_events.py new file mode 100644 index 0000000000000..dc55c4260a87d --- /dev/null +++ b/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_read_found_events.py @@ -0,0 +1,14 @@ +import FWCore.ParameterSet.Config as cms +process = cms.Process("READ") +process.read = cms.EDAnalyzer( + "TestFindProduct", + inputTags =cms.untracked.VInputTag(cms.InputTag("i")), + expectedSum = cms.untracked.int32(100) +) +process.p = cms.Path(process.read) +process.options = cms.untracked.PSet(Rethrow=cms.untracked.vstring("ProductNotFound")) + +process.source = cms.Source( + "RNTupleTempSource", + fileNames = cms.untracked.vstring("file:unscheduled_fail_on_output.root") +) diff --git a/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_read_no_events.py b/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_read_no_events.py new file mode 100644 index 0000000000000..9e9b927151a1a --- /dev/null +++ b/FWIO/RNTupleTempTests/test/unscheduled_fail_on_output_read_no_events.py @@ -0,0 +1,14 @@ +import FWCore.ParameterSet.Config as cms +process = cms.Process("READ") +process.read = cms.EDAnalyzer( + "TestFindProduct", + inputTags =cms.untracked.VInputTag(cms.InputTag("i")), + expectedSum = cms.untracked.int32(0) +) +process.p = cms.Path(process.read) +process.options = cms.untracked.PSet(Rethrow=cms.untracked.vstring("ProductNotFound")) + +process.source = cms.Source( + "RNTupleTempSource", + fileNames = cms.untracked.vstring("file:unscheduled_fail_on_output.root") +) diff --git a/FWIO/RNTupleTempTests/test/useEmptyRootFile.py b/FWIO/RNTupleTempTests/test/useEmptyRootFile.py new file mode 100644 index 0000000000000..459201338ed92 --- /dev/null +++ b/FWIO/RNTupleTempTests/test/useEmptyRootFile.py @@ -0,0 +1,11 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("READ") + +process.source = cms.Source("RNTupleTempSource", fileNames = cms.untracked.vstring("file:empty.root")) + +process.Thing = cms.EDProducer("ThingProducer") + +process.OtherThing = cms.EDProducer("OtherThingProducer") + +process.p = cms.Path(process.Thing * process.OtherThing) From 2cac53f50fe14eb932b9e4a155e3a477c86162e9 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Mon, 29 Sep 2025 10:26:41 -0500 Subject: [PATCH 05/19] Removed cloning code from RNTuple implementation Cloning is not an option for RNTuple. --- .../RNTupleTempInput/src/RNTupleTempSource.cc | 3 - FWIO/RNTupleTempInput/src/RootFile.cc | 56 +--- FWIO/RNTupleTempInput/src/RootFile.h | 3 - .../interface/RNTupleTempOutputModule.h | 2 - .../src/RNTupleTempOutputModule.cc | 11 +- FWIO/RNTupleTempOutput/src/RootOutputFile.cc | 224 +------------ FWIO/RNTupleTempOutput/src/RootOutputFile.h | 2 - .../src/RootOutputRNTuple.cc | 314 +----------------- .../RNTupleTempOutput/src/RootOutputRNTuple.h | 41 +-- FWIO/RNTupleTempTests/test/run_RunMerge.sh | 4 - 10 files changed, 16 insertions(+), 644 deletions(-) diff --git a/FWIO/RNTupleTempInput/src/RNTupleTempSource.cc b/FWIO/RNTupleTempInput/src/RNTupleTempSource.cc index 47e0e2e9ab921..26d2f638ce04f 100644 --- a/FWIO/RNTupleTempInput/src/RNTupleTempSource.cc +++ b/FWIO/RNTupleTempInput/src/RNTupleTempSource.cc @@ -163,9 +163,6 @@ namespace edm::rntuple_temp { std::shared_ptr RNTupleTempSource::readFile_() { std::shared_ptr fb = primaryFileSequence_->readFile_(); - if (secondaryFileSequence_) { - fb->setNotFastClonable(FileBlock::HasSecondaryFileSequence); - } return fb; } diff --git a/FWIO/RNTupleTempInput/src/RootFile.cc b/FWIO/RNTupleTempInput/src/RootFile.cc index 2bcb33aa1276b..40fb2ce55407a 100644 --- a/FWIO/RNTupleTempInput/src/RootFile.cc +++ b/FWIO/RNTupleTempInput/src/RootFile.cc @@ -179,7 +179,6 @@ namespace edm::rntuple_temp { noRunLumiSort_(processingOptions.noRunLumiSort), noEventSort_(processingOptions.noEventSort), enforceGUIDInFileName_(fileOptions.enforceGUIDInFileName), - whyNotFastClonable_(0), hasNewlyDroppedBranch_(), branchListIndexesUnchanged_(false), eventAuxCache_(), @@ -405,11 +404,6 @@ namespace edm::rntuple_temp { // propagate_const has no reset() function provenanceReaderMaker_ = std::unique_ptr(makeProvenanceReaderMaker(inputType).release()); - // Merge into the hashed registries. - if (eventSkipperByID_ && eventSkipperByID_->somethingToSkip()) { - whyNotFastClonable_ += FileBlock::EventsOrLumisSelectedByID; - } - initializeDuplicateChecker(crossFileInfo.indexesIntoFiles, crossFileInfo.currentIndexIntoFile); indexIntoFileIter_ = indexIntoFileBegin_ = indexIntoFile_.begin( processingOptions.noRunLumiSort @@ -528,9 +522,6 @@ namespace edm::rntuple_temp { } } - // Determine if this file is fast clonable. - setIfFastClonable(processingOptions.remainingEvents, processingOptions.remainingLumis); - // We are done with our initial reading of EventAuxiliary. indexIntoFile_.doneFileInitialization(); @@ -630,51 +621,6 @@ namespace edm::rntuple_temp { } } - void RootFile::setIfFastClonable(int remainingEvents, int remainingLumis) { - if (fileFormatVersion().noMetaDataTrees() and !fileFormatVersion().storedProductProvenanceUsed()) { - //we must avoid copying the old branch which stored the per product per event provenance - whyNotFastClonable_ += FileBlock::FileTooOld; - return; - } - if (!fileFormatVersion().splitProductIDs()) { - whyNotFastClonable_ += FileBlock::FileTooOld; - return; - } - if (processingMode_ != InputSource::RunsLumisAndEvents) { - whyNotFastClonable_ += FileBlock::NotProcessingEvents; - return; - } - // Find entry for first event in file - IndexIntoFile::IndexIntoFileItr it = indexIntoFileBegin_; - while (it != indexIntoFileEnd_ && it.getEntryType() != IndexIntoFile::kEvent) { - ++it; - } - if (it == indexIntoFileEnd_) { - whyNotFastClonable_ += FileBlock::NoEventsInFile; - return; - } - - // From here on, record all reasons we can't fast clone. - IndexIntoFile::SortOrder sortOrder = - (noRunLumiSort_ ? IndexIntoFile::entryOrder - : (noEventSort_ ? IndexIntoFile::firstAppearanceOrder : IndexIntoFile::numericalOrder)); - if (!indexIntoFile_.iterationWillBeInEntryOrder(sortOrder)) { - whyNotFastClonable_ += (noEventSort_ ? FileBlock::RunOrLumiNotContiguous : FileBlock::EventsToBeSorted); - } - if (skipAnyEvents_) { - whyNotFastClonable_ += FileBlock::InitialEventsSkipped; - } - if (remainingEvents >= 0 && eventTree_.entries() > remainingEvents) { - whyNotFastClonable_ += FileBlock::MaxEventsTooSmall; - } - if (remainingLumis >= 0 && lumiTree_.entries() > remainingLumis) { - whyNotFastClonable_ += FileBlock::MaxLumisTooSmall; - } - if (duplicateChecker_ && !duplicateChecker_->checkDisabled() && !duplicateChecker_->noDuplicatesInFile()) { - whyNotFastClonable_ += FileBlock::DuplicateEventsRemoved; - } - } - std::shared_ptr RootFile::createFileBlock() { std::vector processBlockTrees; std::vector processesWithProcessBlockTrees; @@ -693,7 +639,7 @@ namespace edm::rntuple_temp { runTree_.metaTree(), std::move(processBlockTrees), std::move(processesWithProcessBlockTrees), - whyNotFastClonable(), + 0, hasNewlyDroppedBranch(), file_, branchListIndexesUnchanged(), diff --git a/FWIO/RNTupleTempInput/src/RootFile.h b/FWIO/RNTupleTempInput/src/RootFile.h index 9ae610a1b61a0..394d44d073894 100644 --- a/FWIO/RNTupleTempInput/src/RootFile.h +++ b/FWIO/RNTupleTempInput/src/RootFile.h @@ -153,7 +153,6 @@ namespace edm { RootRNTuple const& lumiTree() const { return lumiTree_; } RootRNTuple const& runTree() const { return runTree_; } FileFormatVersion fileFormatVersion() const { return fileFormatVersion_; } - int whyNotFastClonable() const { return whyNotFastClonable_; } std::array const& hasNewlyDroppedBranch() const { return hasNewlyDroppedBranch_; } bool branchListIndexesUnchanged() const { return branchListIndexesUnchanged_; } bool modifiedIDs() const { return daqProvenanceHelper_.get() != nullptr; } @@ -214,7 +213,6 @@ namespace edm { InputType inputType, StoredProcessBlockHelper const& storedProcessBlockHelper); bool skipThisEntry(); - void setIfFastClonable(int remainingEvents, int remainingLumis); void validateFile(InputType inputType, bool usingGoToEvent, std::vector& orderedProcessHistoryIDs); @@ -288,7 +286,6 @@ namespace edm { bool noRunLumiSort_; bool noEventSort_; bool enforceGUIDInFileName_; - int whyNotFastClonable_; std::array hasNewlyDroppedBranch_; bool branchListIndexesUnchanged_; EventAuxiliary eventAuxCache_; //Should only be used by fillThisEventAuxiliary() diff --git a/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h b/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h index 6a6d04c77dd64..880d5246a60aa 100644 --- a/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h +++ b/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h @@ -66,7 +66,6 @@ namespace edm::rntuple_temp { std::string const& moduleLabel() const { return moduleLabel_; } unsigned int maxFileSize() const { return maxFileSize_; } int inputFileCount() const { return inputFileCount_; } - int whyNotFastClonable() const { return whyNotFastClonable_; } std::string const& currentFileName() const; @@ -213,7 +212,6 @@ namespace edm::rntuple_temp { int const splitLevel_; std::string basketOrder_; int const treeMaxVirtualSize_; - int whyNotFastClonable_; DropMetaData dropMetaData_; std::string const moduleLabel_; bool initializedFromInput_; diff --git a/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc b/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc index fdd339809448c..527866873a607 100644 --- a/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc +++ b/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc @@ -50,8 +50,6 @@ namespace edm::rntuple_temp { splitLevel_(std::min(pset.getUntrackedParameter("splitLevel") + 1, 99)), basketOrder_(pset.getUntrackedParameter("sortBaskets")), treeMaxVirtualSize_(pset.getUntrackedParameter("treeMaxVirtualSize")), - whyNotFastClonable_(pset.getUntrackedParameter("fastCloning") ? FileBlock::CanFastClone - : FileBlock::DisabledInConfigFile), dropMetaData_(DropNone), moduleLabel_(pset.getParameter("@module_label")), initializedFromInput_(false), @@ -88,10 +86,6 @@ namespace edm::rntuple_temp { << "Legal values are 'NONE', 'DROPPED', 'PRIOR', and 'ALL'.\n"; } - if (!wantAllEvents()) { - whyNotFastClonable_ += FileBlock::EventSelectionUsed; - } - auto const& specialSplit{pset.getUntrackedParameterSetVector("overrideBranchesSplitLevel")}; specialSplitLevelForBranches_.reserve(specialSplit.size()); @@ -509,10 +503,7 @@ namespace edm::rntuple_temp { "Used by ROOT when fast copying. Affects performance."); desc.addUntracked("treeMaxVirtualSize", -1) ->setComment("Size of ROOT TTree TBasket cache. Affects performance."); - desc.addUntracked("fastCloning", true) - ->setComment( - "True: Allow fast copying, if possible.\n" - "False: Disable fast copying."); + desc.addUntracked("fastCloning", false)->setComment("Not used by RNTuple"); desc.addUntracked("mergeJob", false) ->setComment( "If set to true and fast copying is disabled, copy input file compression and basket sizes to the output " diff --git a/FWIO/RNTupleTempOutput/src/RootOutputFile.cc b/FWIO/RNTupleTempOutput/src/RootOutputFile.cc index 72624c1300a47..a31fe988a18b8 100644 --- a/FWIO/RNTupleTempOutput/src/RootOutputFile.cc +++ b/FWIO/RNTupleTempOutput/src/RootOutputFile.cc @@ -90,8 +90,6 @@ namespace edm::rntuple_temp { logicalFile_(logicalFileName), reportToken_(0), om_(om), - whyNotFastClonable_(om_->whyNotFastClonable()), - canFastCloneAux_(false), filePtr_(openTFile(file_.c_str(), om_->compressionLevel())), fid_(), eventEntryNumber_(0LL), @@ -143,7 +141,7 @@ namespace edm::rntuple_temp { } if (om_->compactEventAuxiliary()) { eventTree_.addAuxiliary( - BranchTypeToAuxiliaryBranchName(InEvent), pEventAux_, om_->auxItems()[InEvent].basketSize_, false); + BranchTypeToAuxiliaryBranchName(InEvent), pEventAux_, om_->auxItems()[InEvent].basketSize_); eventTree_.tree()->SetBranchStatus(BranchTypeToAuxiliaryBranchName(InEvent).c_str(), false); // see writeEventAuxiliary } else { @@ -155,7 +153,7 @@ namespace edm::rntuple_temp { pEventEntryInfoVector(), om_->auxItems()[InEvent].basketSize_); eventTree_.addAuxiliary( - poolNames::eventSelectionsBranchName(), pEventSelectionIDs_, om_->auxItems()[InEvent].basketSize_, false); + poolNames::eventSelectionsBranchName(), pEventSelectionIDs_, om_->auxItems()[InEvent].basketSize_); eventTree_.addAuxiliary( poolNames::branchListIndexesBranchName(), pBranchListIndexes_, om_->auxItems()[InEvent].basketSize_); @@ -245,210 +243,7 @@ namespace edm::rntuple_temp { branchNames); // branch names being written } - namespace { - void maybeIssueWarning(int whyNotFastClonable, std::string const& ifileName, std::string const& ofileName) { - // No message if fast cloning was deliberately disabled, or if there are no events to copy anyway. - if ((whyNotFastClonable & (FileBlock::DisabledInConfigFile | FileBlock::NoRootInputSource | - FileBlock::NotProcessingEvents | FileBlock::NoEventsInFile)) != 0) { - return; - } - - // There will be a message stating every reason that fast cloning was not possible. - // If at one or more of the reasons was because of something the user explicitly specified (e.g. event selection, skipping events), - // or if the input file was in an old format, the message will be informational. Otherwise, the message will be a warning. - bool isWarning = true; - std::ostringstream message; - message << "Fast copying of file " << ifileName << " to file " << ofileName << " is disabled because:\n"; - if ((whyNotFastClonable & FileBlock::HasSecondaryFileSequence) != 0) { - message << "a SecondaryFileSequence was specified.\n"; - whyNotFastClonable &= ~(FileBlock::HasSecondaryFileSequence); - isWarning = false; - } - if ((whyNotFastClonable & FileBlock::FileTooOld) != 0) { - message << "the input file is in an old format.\n"; - whyNotFastClonable &= ~(FileBlock::FileTooOld); - isWarning = false; - } - if ((whyNotFastClonable & FileBlock::EventsToBeSorted) != 0) { - message << "events need to be sorted.\n"; - whyNotFastClonable &= ~(FileBlock::EventsToBeSorted); - } - if ((whyNotFastClonable & FileBlock::RunOrLumiNotContiguous) != 0) { - message << "a run or a lumi is not contiguous in the input file.\n"; - whyNotFastClonable &= ~(FileBlock::RunOrLumiNotContiguous); - } - if ((whyNotFastClonable & FileBlock::EventsOrLumisSelectedByID) != 0) { - message << "events or lumis were selected or skipped by ID.\n"; - whyNotFastClonable &= ~(FileBlock::EventsOrLumisSelectedByID); - isWarning = false; - } - if ((whyNotFastClonable & FileBlock::InitialEventsSkipped) != 0) { - message << "initial events, lumis or runs were skipped.\n"; - whyNotFastClonable &= ~(FileBlock::InitialEventsSkipped); - isWarning = false; - } - if ((whyNotFastClonable & FileBlock::DuplicateEventsRemoved) != 0) { - message << "some events were skipped because of duplicate checking.\n"; - whyNotFastClonable &= ~(FileBlock::DuplicateEventsRemoved); - } - if ((whyNotFastClonable & FileBlock::MaxEventsTooSmall) != 0) { - message << "some events were not copied because of maxEvents limit.\n"; - whyNotFastClonable &= ~(FileBlock::MaxEventsTooSmall); - isWarning = false; - } - if ((whyNotFastClonable & FileBlock::MaxLumisTooSmall) != 0) { - message << "some events were not copied because of maxLumis limit.\n"; - whyNotFastClonable &= ~(FileBlock::MaxLumisTooSmall); - isWarning = false; - } - if ((whyNotFastClonable & FileBlock::ParallelProcesses) != 0) { - message << "parallel processing was specified.\n"; - whyNotFastClonable &= ~(FileBlock::ParallelProcesses); - isWarning = false; - } - if ((whyNotFastClonable & FileBlock::EventSelectionUsed) != 0) { - message << "an EventSelector was specified.\n"; - whyNotFastClonable &= ~(FileBlock::EventSelectionUsed); - isWarning = false; - } - if ((whyNotFastClonable & FileBlock::OutputMaxEventsTooSmall) != 0) { - message << "some events were not copied because of maxEvents output limit.\n"; - whyNotFastClonable &= ~(FileBlock::OutputMaxEventsTooSmall); - isWarning = false; - } - if ((whyNotFastClonable & FileBlock::SplitLevelMismatch) != 0) { - message << "the split level or basket size of a branch or branches was modified.\n"; - whyNotFastClonable &= ~(FileBlock::SplitLevelMismatch); - } - if ((whyNotFastClonable & FileBlock::BranchMismatch) != 0) { - message << "The format of a data product has changed.\n"; - whyNotFastClonable &= ~(FileBlock::BranchMismatch); - } - assert(whyNotFastClonable == FileBlock::CanFastClone); - if (isWarning) { - LogWarning("FastCloningDisabled") << message.str(); - } else { - LogInfo("FastCloningDisabled") << message.str(); - } - } - } // namespace - - void RootOutputFile::beginInputFile(FileBlock const& fb, int remainingEvents) { - // Reset per input file information - whyNotFastClonable_ = om_->whyNotFastClonable(); - canFastCloneAux_ = false; - - if (fb.tree() != nullptr) { - whyNotFastClonable_ |= fb.whyNotFastClonable(); - - if (remainingEvents >= 0 && remainingEvents < fb.tree()->GetEntries()) { - whyNotFastClonable_ |= FileBlock::OutputMaxEventsTooSmall; - } - - bool match = eventTree_.checkSplitLevelsAndBasketSizes(fb.tree()); - if (!match) { - if (om_->overrideInputFileSplitLevels()) { - // We may be fast copying. We must disable fast copying if the split levels - // or basket sizes do not match. - whyNotFastClonable_ |= FileBlock::SplitLevelMismatch; - } else { - // We are using the input split levels and basket sizes from the first input file - // for copied output branches. In this case, we throw an exception if any branches - // have different split levels or basket sizes in a subsequent input file. - // If the mismatch is in the first file, there is a bug somewhere, so we assert. - assert(om_->inputFileCount() > 1); - throw Exception(errors::MismatchedInputFiles, "RootOutputFile::beginInputFile()") - << "Merge failure because input file " << file_ << " has different ROOT split levels or basket sizes\n" - << "than previous files. To allow merging in spite of this, use the configuration parameter\n" - << "overrideInputFileSplitLevels=cms.untracked.bool(True)\n" - << "in every RNTupleTempOutputModule.\n"; - } - } - - // Since this check can be time consuming, we do it only if we would otherwise fast clone. - if (whyNotFastClonable_ == FileBlock::CanFastClone) { - if (!eventTree_.checkIfFastClonable(fb.tree())) { - whyNotFastClonable_ |= FileBlock::BranchMismatch; - } - } - - // reasons for whyNotFastClonable that are also inconsistent with a merge job - constexpr auto setSubBranchBasketConditions = - FileBlock::EventsOrLumisSelectedByID | FileBlock::InitialEventsSkipped | FileBlock::MaxEventsTooSmall | - FileBlock::MaxLumisTooSmall | FileBlock::EventSelectionUsed | FileBlock::OutputMaxEventsTooSmall | - FileBlock::SplitLevelMismatch | FileBlock::BranchMismatch; - - if (om_->inputFileCount() == 1) { - if (om_->mergeJob()) { - // for merge jobs always forward the compression mode - auto infile = fb.tree()->GetCurrentFile(); - if (infile != nullptr) { - filePtr_->SetCompressionSettings(infile->GetCompressionSettings()); - } - } - - // if we aren't fast cloning, and the reason why is consistent with a - // merge job or is only because of parallel processes, then forward all - // the sub-branch basket sizes - if (whyNotFastClonable_ != FileBlock::CanFastClone && - ((om_->mergeJob() && (whyNotFastClonable_ & setSubBranchBasketConditions) == 0) || - (whyNotFastClonable_ == FileBlock::ParallelProcesses))) { - eventTree_.setSubBranchBasketSizes(fb.tree()); - } - } - - // We now check if we can fast copy the auxiliary branches. - // We can do so only if we can otherwise fast copy, - // the input file has the current format (these branches are in the Events Tree), - // there are no newly dropped or produced products, - // no metadata has been dropped, - // ID's have not been modified, - // and the branch list indexes do not need modification. - - // Note: Fast copy of the EventProductProvenance branch is unsafe - // unless we can enforce that the parentage information for a fully copied - // output file will be the same as for the input file, with nothing dropped. - // This has never been enforced, and, withthe EDAlias feature, it may no longer - // work by accident. - // So, for now, we do not enable fast cloning of the non-product branches. - /* - canFastCloneAux_ = (whyNotFastClonable_ == FileBlock::CanFastClone) && - fb.fileFormatVersion().noMetaDataTrees() && - !om_->hasNewlyDroppedBranch()[InEvent] && - !fb.hasNewlyDroppedBranch()[InEvent] && - om_->dropMetaData() == RNTupleTempOutputModule::DropNone && - !reg->anyProductProduced() && - !fb.modifiedIDs() && - fb.branchListIndexesUnchanged(); - */ - - // Report the fast copying status. - Service reportSvc; - reportSvc->reportFastCopyingStatus(reportToken_, fb.fileName(), whyNotFastClonable_ == FileBlock::CanFastClone); - } else { - whyNotFastClonable_ |= FileBlock::NoRootInputSource; - } - - eventTree_.maybeFastCloneTree( - whyNotFastClonable_ == FileBlock::CanFastClone, canFastCloneAux_, fb.tree(), om_->basketOrder()); - - // Possibly issue warning or informational message if we haven't fast cloned. - if (fb.tree() != nullptr && whyNotFastClonable_ != FileBlock::CanFastClone) { - maybeIssueWarning(whyNotFastClonable_, fb.fileName(), file_); - } - - if (om_->compactEventAuxiliary() && - (whyNotFastClonable_ & (FileBlock::EventsOrLumisSelectedByID | FileBlock::InitialEventsSkipped | - FileBlock::EventSelectionUsed)) == 0) { - long long int reserve = remainingEvents; - if (fb.tree() != nullptr) { - reserve = reserve > 0 ? std::min(fb.tree()->GetEntries(), reserve) : fb.tree()->GetEntries(); - } - if (reserve > 0) { - compactEventAuxiliary_.reserve(compactEventAuxiliary_.size() + reserve); - } - } - } + void RootOutputFile::beginInputFile(FileBlock const& fb, int remainingEvents) {} void RootOutputFile::respondToCloseInputFile(FileBlock const&) { // We can't do setEntries() on the event tree if the EventAuxiliary branch is empty & disabled @@ -693,15 +488,6 @@ namespace edm::rntuple_temp { } void RootOutputFile::writeIndexIntoFile(ROOT::REntry& rentry) { - if (eventTree_.checkEntriesInReadBranches(eventEntryNumber_) == false) { - Exception ex(errors::OtherCMS); - ex << "The number of entries in at least one output TBranch whose entries\n" - "were copied from the input does not match the number of events\n" - "recorded in IndexIntoFile. This might (or might not) indicate a\n" - "problem related to fast copy."; - ex.addContext("Calling RootOutputFile::writeIndexIntoFile"); - throw ex; - } indexIntoFile_.sortVector_Run_Or_Lumi_Entries(); rentry.BindRawPtr(poolNames::indexIntoFileBranchName(), &indexIntoFile_); } @@ -943,7 +729,6 @@ namespace edm::rntuple_temp { (productProvenanceVecPtr != nullptr) && (om_->dropMetaData() != RNTupleTempOutputModule::DropAll); bool const keepProvenanceForPrior = doProvenance && om_->dropMetaData() != RNTupleTempOutputModule::DropPrior; - bool const fastCloning = (branchType == InEvent) && (whyNotFastClonable_ == FileBlock::CanFastClone); std::set provenanceToKeep; // //If we are dropping some of the meta data we need to know @@ -965,8 +750,7 @@ namespace edm::rntuple_temp { branchesWithStoredHistory_.insert(id); bool produced = item.productDescription()->produced(); - bool getProd = - (produced || !fastCloning || treePointers_[ttreeIndex]->uncloned(item.productDescription()->branchName())); + bool getProd = true; bool keepProvenance = doProvenance && (produced || keepProvenanceForPrior); WrapperBase const* product = nullptr; diff --git a/FWIO/RNTupleTempOutput/src/RootOutputFile.h b/FWIO/RNTupleTempOutput/src/RootOutputFile.h index 63af40fb5b4d5..44dda14c8f153 100644 --- a/FWIO/RNTupleTempOutput/src/RootOutputFile.h +++ b/FWIO/RNTupleTempOutput/src/RootOutputFile.h @@ -130,8 +130,6 @@ namespace edm::rntuple_temp { std::string logicalFile_; JobReport::Token reportToken_; edm::propagate_const om_; - int whyNotFastClonable_; - bool canFastCloneAux_; edm::propagate_const> filePtr_; FileID fid_; IndexIntoFile::EntryNumber_t eventEntryNumber_; diff --git a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc index 4cff2e979c5ac..2eb8669ab66dc 100644 --- a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc +++ b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc @@ -16,7 +16,6 @@ #include "TBranchElement.h" #include "TCollection.h" #include "TFile.h" -#include "TTreeCloner.h" #include "Rtypes.h" #include "RVersion.h" @@ -26,73 +25,6 @@ namespace edm { - /** - * Currently, ROOT doesn't use any latency-hiding optimizations for - * fast-cloning. This causes a significant slowdown when doing fast-cloning - * over a high-latency network (30ms latency makes this multiple factors slower). - * - * Accordingly, we allow sites to provide a separate hint on how to treat fast- - * cloning. The DuplicateTreeSentry allows us to implement it - given a tree - * we are about to clone, with the appropriate configs, this will re-open the - * file with lazy-download and re-open the tree. The new tree is appropriate - * for cloning. When the object is destroyed, the new file and tree are cleaned up. - * - */ - class DuplicateTreeSentry { - public: - DuplicateTreeSentry(TTree* tree) : tree_(tree) { dup(); } - DuplicateTreeSentry(DuplicateTreeSentry const&) = delete; // Disallow copying and moving - DuplicateTreeSentry& operator=(DuplicateTreeSentry const&) = delete; - - TTree* tree() const { return mytree_ ? mytree_.get() : tree_; } - - private: - struct CloseBeforeDelete { - void operator()(TFile* iFile) const { - if (iFile) { - iFile->Close(); - } - delete iFile; - } - }; - - void dup() { - edm::Service pSLC; - if (!pSLC.isAvailable()) { - return; - } - if (pSLC->sourceCacheHint() && *(pSLC->sourceCacheHint()) == "lazy-download") { - return; - } - if (!pSLC->sourceCloneCacheHint() || *(pSLC->sourceCloneCacheHint()) != "lazy-download") { - return; - } - edm::LogWarning("DuplicateTreeSentry") << "Re-opening file for fast-cloning"; - - TFile* file = tree_->GetCurrentFile(); - const TUrl* url = file->GetEndpointUrl(); - if (!url) { - return; - } - file_.reset(TFile::Open(url->GetUrl(), "READWRAP")); // May throw an exception. - if (!file_) { - return; - } - mytree_.reset(dynamic_cast(file_->Get(tree_->GetName()))); - if (!mytree_) { - return; - } - } - - /** - * Note this relies on the implicit delete ordering - mytree_ (if non-null) - * must be deleted before file_. Do not reorder the class members! - */ - std::unique_ptr file_; - TTree* tree_ = nullptr; - std::unique_ptr mytree_ = nullptr; - }; - RootOutputRNTuple::RootOutputRNTuple(std::shared_ptr filePtr, BranchType const& branchType, int splitLevel, @@ -103,12 +35,7 @@ namespace edm { ? makeTTree(filePtr.get(), BranchTypeToProductTreeName(branchType), splitLevel) : makeTTree(filePtr.get(), BranchTypeToProductTreeName(branchType, processName), splitLevel)), producedBranches_(), - readBranches_(), - auxBranches_(), - unclonedReadBranches_(), - clonedReadBranchNames_(), - currentlyFastCloning_(), - fastCloneAuxBranches_(false) { + auxBranches_() { if (treeMaxVirtualSize >= 0) tree_->SetMaxVirtualSize(treeMaxVirtualSize); } @@ -132,192 +59,6 @@ namespace edm { return assignTTree(filePtr, tree); } - bool RootOutputRNTuple::checkSplitLevelsAndBasketSizes(TTree* inputTree) const { - assert(inputTree != nullptr); - - // Do the split level and basket size match in the input and output? - for (auto const& outputBranch : readBranches_) { - if (outputBranch != nullptr) { - TBranch* inputBranch = inputTree->GetBranch(outputBranch->GetName()); - - if (inputBranch != nullptr) { - if (inputBranch->GetSplitLevel() != outputBranch->GetSplitLevel() || - inputBranch->GetBasketSize() != outputBranch->GetBasketSize()) { - return false; - } - } - } - } - return true; - } - - namespace { - bool checkMatchingBranches(TBranchElement* inputBranch, TBranchElement* outputBranch) { - if (inputBranch->GetStreamerType() != outputBranch->GetStreamerType()) { - return false; - } - TObjArray* inputArray = inputBranch->GetListOfBranches(); - TObjArray* outputArray = outputBranch->GetListOfBranches(); - - if (outputArray->GetSize() < inputArray->GetSize()) { - return false; - } - TIter iter(outputArray); - TObject* obj = nullptr; - while ((obj = iter.Next()) != nullptr) { - TBranchElement* outBranch = dynamic_cast(obj); - if (outBranch) { - TBranchElement* inBranch = dynamic_cast(inputArray->FindObject(outBranch->GetName())); - if (!inBranch) { - return false; - } - if (!checkMatchingBranches(inBranch, outBranch)) { - return false; - } - } - } - return true; - } - } // namespace - - bool RootOutputRNTuple::checkIfFastClonable(TTree* inputTree) const { - if (inputTree == nullptr) - return false; - - // Do the sub-branches match in the input and output. Extra sub-branches in the input are OK for fast cloning, but not in the output. - for (auto const& outputBr : readBranches_) { - TBranchElement* outputBranch = dynamic_cast(outputBr); - if (outputBranch != nullptr) { - TBranchElement* inputBranch = dynamic_cast(inputTree->GetBranch(outputBranch->GetName())); - if (inputBranch != nullptr) { - // We have a matching top level branch. Do the recursive check on subbranches. - if (!checkMatchingBranches(inputBranch, outputBranch)) { - LogInfo("FastCloning") << "Fast Cloning disabled because a data member has been added to split branch: " - << inputBranch->GetName() << "\n."; - return false; - } - } - } - } - return true; - } - - namespace { - void setMatchingBranchSizes(TBranchElement* inputBranch, TBranchElement* outputBranch) { - if (inputBranch->GetStreamerType() != outputBranch->GetStreamerType()) { - return; - } - TObjArray* inputArray = inputBranch->GetListOfBranches(); - TObjArray* outputArray = outputBranch->GetListOfBranches(); - - if (outputArray->GetSize() < inputArray->GetSize()) { - return; - } - TIter iter(outputArray); - TObject* obj = nullptr; - while ((obj = iter.Next()) != nullptr) { - TBranchElement* outBranch = dynamic_cast(obj); - if (outBranch) { - TBranchElement* inBranch = dynamic_cast(inputArray->FindObject(outBranch->GetName())); - if (inBranch) { - outBranch->SetBasketSize(inBranch->GetBasketSize()); - setMatchingBranchSizes(inBranch, outBranch); - } - } - } - } - } // namespace - - void RootOutputRNTuple::setSubBranchBasketSizes(TTree* inputTree) const { - if (inputTree == nullptr) - return; - - for (auto const& outputBr : readBranches_) { - TBranchElement* outputBranch = dynamic_cast(outputBr); - if (outputBranch != nullptr) { - TBranchElement* inputBranch = dynamic_cast(inputTree->GetBranch(outputBranch->GetName())); - if (inputBranch != nullptr) { - // We have a matching top level branch. Do the recursion on the subbranches. - setMatchingBranchSizes(inputBranch, outputBranch); - } - } - } - } - - bool RootOutputRNTuple::checkEntriesInReadBranches(Long64_t expectedNumberOfEntries) const { - for (auto const& readBranch : readBranches_) { - if (readBranch->GetEntries() != expectedNumberOfEntries) { - return false; - } - } - return true; - } - - void RootOutputRNTuple::fastCloneTTree(TTree* in, std::string const& option) { - if (in->GetEntries() != 0) { - TObjArray* branches = tree_->GetListOfBranches(); - // If any products were produced (not just event products), the EventAuxiliary will be modified. - // In that case, don't fast copy auxiliary branches. Remove them, and add back after fast copying. - std::map auxIndexes; - bool mustRemoveSomeAuxs = false; - if (!fastCloneAuxBranches_) { - for (auto const& auxBranch : auxBranches_) { - int auxIndex = branches->IndexOf(auxBranch); - assert(auxIndex >= 0); - auxIndexes.insert(std::make_pair(auxIndex, auxBranch)); - branches->RemoveAt(auxIndex); - } - mustRemoveSomeAuxs = true; - } - - //Deal with any aux branches which can never be cloned - for (auto const& auxBranch : unclonedAuxBranches_) { - int auxIndex = branches->IndexOf(auxBranch); - assert(auxIndex >= 0); - auxIndexes.insert(std::make_pair(auxIndex, auxBranch)); - branches->RemoveAt(auxIndex); - mustRemoveSomeAuxs = true; - } - - if (mustRemoveSomeAuxs) { - branches->Compress(); - } - - DuplicateTreeSentry dupTree(in); - TTreeCloner cloner( - dupTree.tree(), tree_, option.c_str(), TTreeCloner::kNoWarnings | TTreeCloner::kIgnoreMissingTopLevel); - - if (!cloner.IsValid()) { - // Let's check why - static const char* okerror = "One of the export branch"; - if (strncmp(cloner.GetWarning(), okerror, strlen(okerror)) == 0) { - // That's fine we will handle it; - } else { - throw edm::Exception(errors::FatalRootError) << "invalid TTreeCloner (" << cloner.GetWarning() << ")\n"; - } - } - tree_->SetEntries(tree_->GetEntries() + in->GetEntries()); - Service rootHandler; - rootHandler->ignoreWarningsWhileDoing([&cloner] { cloner.Exec(); }); - - if (mustRemoveSomeAuxs) { - for (auto const& auxIndex : auxIndexes) { - // Add the auxiliary branches back after fast copying the rest of the tree. - Int_t last = branches->GetLast(); - if (last >= 0) { - branches->AddAtAndExpand(branches->At(last), last + 1); - for (Int_t ind = last - 1; ind >= auxIndex.first; --ind) { - branches->AddAt(branches->At(ind), ind + 1); - }; - branches->AddAt(auxIndex.second, auxIndex.first); - } else { - branches->Add(auxIndex.second); - } - } - } - } - } - void RootOutputRNTuple::writeTTree(TTree* tree) { if (tree->GetNbranches() != 0) { // This is required when Fill is called on individual branches @@ -333,40 +74,10 @@ namespace edm { void RootOutputRNTuple::writeTree() { writeTTree(tree()); } - void RootOutputRNTuple::maybeFastCloneTree(bool canFastClone, - bool canFastCloneAux, - TTree* tree, - std::string const& option) { - unclonedReadBranches_.clear(); - clonedReadBranchNames_.clear(); - currentlyFastCloning_ = canFastClone && !readBranches_.empty(); - if (currentlyFastCloning_) { - fastCloneAuxBranches_ = canFastCloneAux; - fastCloneTTree(tree, option); - for (auto const& branch : readBranches_) { - if (branch->GetEntries() == tree_->GetEntries()) { - clonedReadBranchNames_.insert(std::string(branch->GetName())); - } else { - unclonedReadBranches_.push_back(branch); - } - } - Service reportSvc; - reportSvc->reportFastClonedBranches(clonedReadBranchNames_, tree_->GetEntries()); - } - } - void RootOutputRNTuple::fillTree() { - if (currentlyFastCloning_) { - if (!fastCloneAuxBranches_) - fillTTree(auxBranches_); - fillTTree(unclonedAuxBranches_); - fillTTree(producedBranches_); - fillTTree(unclonedReadBranches_); - } else { - // Isolate the fill operation so that IMT doesn't grab other large tasks - // that could lead to RNTupleTempOutputModule stalling - oneapi::tbb::this_task_arena::isolate([&] { tree_->Fill(); }); - } + // Isolate the fill operation so that IMT doesn't grab other large tasks + // that could lead to RNTupleTempOutputModule stalling + oneapi::tbb::this_task_arena::isolate([&] { tree_->Fill(); }); } void RootOutputRNTuple::addBranch(std::string const& branchName, @@ -379,29 +90,14 @@ namespace edm { assert(basketSize != ProductDescription::invalidBasketSize); TBranch* branch = tree_->Branch(branchName.c_str(), className.c_str(), &pProd, basketSize, splitLevel); assert(branch != nullptr); - /* - if(pProd != nullptr) { - // Delete the product that ROOT has allocated. - WrapperBase const* edp = static_cast(pProd); - delete edp; - pProd = nullptr; - } -*/ - if (produced) { - producedBranches_.push_back(branch); - } else { - readBranches_.push_back(branch); - } + producedBranches_.push_back(branch); } void RootOutputRNTuple::close() { // The TFile was just closed. // Just to play it safe, zero all pointers to quantities in the file. auxBranches_.clear(); - unclonedAuxBranches_.clear(); producedBranches_.clear(); - readBranches_.clear(); - unclonedReadBranches_.clear(); tree_ = nullptr; // propagate_const has no reset() function filePtr_ = nullptr; // propagate_const has no reset() function } diff --git a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h index ad8a77609012f..a09abeb9d395c 100644 --- a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h +++ b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h @@ -35,25 +35,15 @@ namespace edm { RootOutputRNTuple& operator=(RootOutputRNTuple const&) = delete; // Disallow copying and moving template - void addAuxiliary(std::string const& branchName, T const*& pAux, int bufSize, bool allowCloning = true) { - if (allowCloning) { - auxBranches_.push_back(tree_->Branch(branchName.c_str(), &pAux, bufSize, 0)); - } else { - unclonedAuxBranches_.push_back(tree_->Branch(branchName.c_str(), &pAux, bufSize, 0)); - } + void addAuxiliary(std::string const& branchName, T const*& pAux, int bufSize) { + auxBranches_.push_back(tree_->Branch(branchName.c_str(), &pAux, bufSize, 0)); } template - void addAuxiliary(std::string const& branchName, T*& pAux, int bufSize, bool allowCloning = true) { - if (allowCloning) { - auxBranches_.push_back(tree_->Branch(branchName.c_str(), &pAux, bufSize, 0)); - } else { - unclonedAuxBranches_.push_back(tree_->Branch(branchName.c_str(), &pAux, bufSize, 0)); - } + void addAuxiliary(std::string const& branchName, T*& pAux, int bufSize) { + auxBranches_.push_back(tree_->Branch(branchName.c_str(), &pAux, bufSize, 0)); } - void fastCloneTTree(TTree* in, std::string const& option); - static TTree* makeTTree(TFile* filePtr, std::string const& name, int splitLevel); static TTree* assignTTree(TFile* file, TTree* tree); @@ -69,16 +59,6 @@ namespace edm { int basketSize, bool produced); - bool checkSplitLevelsAndBasketSizes(TTree* inputTree) const; - - bool checkIfFastClonable(TTree* inputTree) const; - - void setSubBranchBasketSizes(TTree* inputTree) const; - - bool checkEntriesInReadBranches(Long64_t expectedNumberOfEntries) const; - - void maybeFastCloneTree(bool canFastClone, bool canFastCloneAux, TTree* tree, std::string const& option); - void fillTree(); void writeTree(); @@ -92,10 +72,6 @@ namespace edm { tree_->SetEntries(-1); } - bool uncloned(std::string const& branchName) const { - return clonedReadBranchNames_.find(branchName) == clonedReadBranchNames_.end(); - } - void close(); void optimizeBaskets(ULong64_t size) { tree_->OptimizeBaskets(size); } @@ -110,15 +86,8 @@ namespace edm { edm::propagate_const> filePtr_; edm::propagate_const tree_; - std::vector producedBranches_; // does not include cloned branches - std::vector readBranches_; + std::vector producedBranches_; std::vector auxBranches_; - std::vector unclonedAuxBranches_; - std::vector unclonedReadBranches_; - - std::set clonedReadBranchNames_; - bool currentlyFastCloning_; - bool fastCloneAuxBranches_; }; } // namespace edm #endif diff --git a/FWIO/RNTupleTempTests/test/run_RunMerge.sh b/FWIO/RNTupleTempTests/test/run_RunMerge.sh index abe70fff3c628..f39671300b7dc 100755 --- a/FWIO/RNTupleTempTests/test/run_RunMerge.sh +++ b/FWIO/RNTupleTempTests/test/run_RunMerge.sh @@ -104,10 +104,6 @@ LOCAL_TEST_DIR=${SCRAM_TEST_PATH} echo ${test}PickEventsx------------------------------------------------------------ cmsRun ${LOCAL_TEST_DIR}/${test}PickEventsx_cfg.py || die "cmsRun ${test}PickEventsx_cfg.py" $? - echo ${test}FastCloning------------------------------------------------------------ - cmsRun ${LOCAL_TEST_DIR}/${test}FastCloning_cfg.py 2> testFastCloning.txt - grep "Another exception was caught" testFastCloning.txt || die "cmsRun testRunMergeFastCloning_cfg.py" $? - echo testLooperEventNavigation----------------------------------------------------- cmsRun ${LOCAL_TEST_DIR}/testLooperEventNavigation_cfg.py < ${LOCAL_TEST_DIR}/testLooperEventNavigation.txt > testLooperEventNavigationOutput.txt || die "cmsRun testLooperEventNavigation_cfg.py " $? diff ${LOCAL_TEST_DIR}/unit_test_outputs/testLooperEventNavigationOutput.txt testLooperEventNavigationOutput.txt || die "comparing testLooperEventNavigationOutput.txt" $? From 149ace9b2a949d53d01b69178e8cb2ca1bdeb629 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Tue, 30 Sep 2025 16:15:46 -0500 Subject: [PATCH 06/19] Moved output to use only RNTuple --- .../interface/RNTupleTempOutputModule.h | 17 +- .../src/RNTupleTempOutputModule.cc | 115 +---------- FWIO/RNTupleTempOutput/src/RootOutputFile.cc | 191 +++++------------- FWIO/RNTupleTempOutput/src/RootOutputFile.h | 19 +- .../src/RootOutputRNTuple.cc | 136 ++++++++----- .../RNTupleTempOutput/src/RootOutputRNTuple.h | 69 +++---- 6 files changed, 179 insertions(+), 368 deletions(-) diff --git a/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h b/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h index 880d5246a60aa..fc7a7d36aed6b 100644 --- a/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h +++ b/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h @@ -29,7 +29,6 @@ #include "DataFormats/Provenance/interface/ParentageID.h" #include "DataFormats/Provenance/interface/ProductRegistry.h" -class TTree; namespace edm { class EDGetToken; @@ -83,15 +82,6 @@ namespace edm::rntuple_temp { AuxItemArray const& auxItems() const { return auxItems_; } struct OutputItem { - class Sorter { - public: - explicit Sorter(TTree* tree); - bool operator()(OutputItem const& lh, OutputItem const& rh) const; - - private: - std::shared_ptr> treeMap_; - }; - explicit OutputItem(ProductDescription const* bd, EDGetToken const& token, int splitLevel, int basketSize); BranchID branchID() const { return productDescription_->branchID(); } @@ -103,6 +93,7 @@ namespace edm::rntuple_temp { EDGetToken token() const { return token_; } void const* const product() const { return product_; } void const*& product() { return product_; } + void const** productPtr() { return &product_; } void setProduct(void const* iProduct) { product_ = iProduct; } int splitLevel() const { return splitLevel_; } int basketSize() const { return basketSize_; } @@ -185,13 +176,9 @@ namespace edm::rntuple_temp { void writeParameterSetRegistry(); void writeParentageRegistry(); - void writeEventAuxiliary(); void finishEndFile(); - void fillSelectedItemList(BranchType branchtype, - std::string const& processName, - TTree* theInputTree, - OutputItemList&); + void fillSelectedItemList(BranchType branchtype, std::string const& processName, OutputItemList&); void beginInputFile(FileBlock const& fb); RootServiceChecker rootServiceChecker_; diff --git a/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc b/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc index 527866873a607..a48b171ed8884 100644 --- a/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc +++ b/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc @@ -20,7 +20,6 @@ #include "FWCore/Utilities/interface/TimeOfDay.h" #include "FWCore/Utilities/interface/WrappedClassName.h" -#include "TTree.h" #include "TBranchElement.h" #include "TObjArray.h" #include "RVersion.h" @@ -122,38 +121,6 @@ namespace edm::rntuple_temp { int basketSize) : productDescription_(bd), token_(token), product_(nullptr), splitLevel_(splitLevel), basketSize_(basketSize) {} - RNTupleTempOutputModule::OutputItem::Sorter::Sorter(TTree* tree) : treeMap_(new std::map) { - // Fill a map mapping branch names to an index specifying the order in the tree. - if (tree != nullptr) { - TObjArray* branches = tree->GetListOfBranches(); - for (int i = 0; i < branches->GetEntries(); ++i) { - TBranchElement* br = (TBranchElement*)branches->At(i); - treeMap_->insert(std::make_pair(std::string(br->GetName()), i)); - } - } - } - - bool RNTupleTempOutputModule::OutputItem::Sorter::operator()(OutputItem const& lh, OutputItem const& rh) const { - // Provides a comparison for sorting branches according to the index values in treeMap_. - // Branches not found are always put at the end (i.e. not found > found). - if (treeMap_->empty()) - return lh < rh; - std::string const& lstring = lh.productDescription_->branchName(); - std::string const& rstring = rh.productDescription_->branchName(); - std::map::const_iterator lit = treeMap_->find(lstring); - std::map::const_iterator rit = treeMap_->find(rstring); - bool lfound = (lit != treeMap_->end()); - bool rfound = (rit != treeMap_->end()); - if (lfound && rfound) { - return lit->second < rit->second; - } else if (lfound) { - return true; - } else if (rfound) { - return false; - } - return lh < rh; - } - namespace { std::regex convertBranchExpression(std::string const& iGlobBranchExpression) { std::string tmp(iGlobBranchExpression); @@ -182,28 +149,9 @@ namespace edm::rntuple_temp { void RNTupleTempOutputModule::fillSelectedItemList(BranchType branchType, std::string const& processName, - TTree* theInputTree, OutputItemList& outputItemList) { SelectedProducts const& keptVector = keptProducts()[branchType]; - if (branchType != InProcess) { - AuxItem& auxItem = auxItems_[branchType]; - - auto basketSize = (InEvent == branchType) ? eventAuxBasketSize_ : basketSize_; - - // Fill AuxItem - if (theInputTree != nullptr && !overrideInputFileSplitLevels_) { - TBranch* auxBranch = theInputTree->GetBranch(BranchTypeToAuxiliaryBranchName(branchType).c_str()); - if (auxBranch) { - auxItem.basketSize_ = auxBranch->GetBasketSize(); - } else { - auxItem.basketSize_ = basketSize; - } - } else { - auxItem.basketSize_ = basketSize; - } - } - // Fill outputItemList with an entry for each branch. for (auto const& kept : keptVector) { int splitLevel = ProductDescription::invalidSplitLevel; @@ -213,46 +161,8 @@ namespace edm::rntuple_temp { if (branchType == InProcess && processName != prod.processName()) { continue; } - TBranch* theBranch = ((!prod.produced() && theInputTree != nullptr && !overrideInputFileSplitLevels_) - ? theInputTree->GetBranch(prod.branchName().c_str()) - : nullptr); - - if (theBranch != nullptr) { - splitLevel = theBranch->GetSplitLevel(); - basketSize = theBranch->GetBasketSize(); - } else { - auto wp = prod.wrappedType().getClass()->GetAttributeMap(); - auto wpSplitLevel = ProductDescription::invalidSplitLevel; - if (wp && wp->HasKey("splitLevel")) { - wpSplitLevel = strtol(wp->GetPropertyAsString("splitLevel"), nullptr, 0); - if (wpSplitLevel < 0) { - throw cms::Exception("IllegalSplitLevel") << "' An illegal ROOT split level of " << wpSplitLevel - << " is specified for class " << prod.wrappedName() << ".'\n"; - } - wpSplitLevel += 1; //Compensate for wrapper - } - splitLevel = (wpSplitLevel == ProductDescription::invalidSplitLevel ? splitLevel_ : wpSplitLevel); - for (auto const& b : specialSplitLevelForBranches_) { - if (b.match(prod.branchName())) { - splitLevel = b.splitLevel_; - } - } - auto wpBasketSize = ProductDescription::invalidBasketSize; - if (wp && wp->HasKey("basketSize")) { - wpBasketSize = strtol(wp->GetPropertyAsString("basketSize"), nullptr, 0); - if (wpBasketSize <= 0) { - throw cms::Exception("IllegalBasketSize") << "' An illegal ROOT basket size of " << wpBasketSize - << " is specified for class " << prod.wrappedName() << "'.\n"; - } - } - basketSize = (wpBasketSize == ProductDescription::invalidBasketSize ? basketSize_ : wpBasketSize); - } outputItemList.emplace_back(&prod, kept.second, splitLevel, basketSize); } - - // Sort outputItemList to allow fast copying. - // The branches in outputItemList must be in the same order as in the input tree, with all new branches at the end. - sort_all(outputItemList, OutputItem::Sorter(theInputTree)); } void RNTupleTempOutputModule::beginInputFile(FileBlock const& fb) { @@ -281,24 +191,21 @@ namespace edm::rntuple_temp { std::vector const& processesWithProcessBlockProducts = outputProcessBlockHelper().processesWithProcessBlockProducts(); unsigned int numberOfProcessesWithProcessBlockProducts = processesWithProcessBlockProducts.size(); - unsigned int numberOfTTrees = numberOfRunLumiEventProductTrees + numberOfProcessesWithProcessBlockProducts; - selectedOutputItemList_.resize(numberOfTTrees); + unsigned int numberOfTRNTuples = numberOfRunLumiEventProductTrees + numberOfProcessesWithProcessBlockProducts; + selectedOutputItemList_.resize(numberOfTRNTuples); for (unsigned int i = InEvent; i < NumBranchTypes; ++i) { BranchType branchType = static_cast(i); if (branchType != InProcess) { std::string processName; - TTree* theInputTree = - (branchType == InEvent ? fb.tree() : (branchType == InLumi ? fb.lumiTree() : fb.runTree())); OutputItemList& outputItemList = selectedOutputItemList_[branchType]; - fillSelectedItemList(branchType, processName, theInputTree, outputItemList); + fillSelectedItemList(branchType, processName, outputItemList); } else { // Handle output items in ProcessBlocks - for (unsigned int k = InProcess; k < numberOfTTrees; ++k) { + for (unsigned int k = InProcess; k < numberOfTRNTuples; ++k) { OutputItemList& outputItemList = selectedOutputItemList_[k]; std::string const& processName = processesWithProcessBlockProducts[k - InProcess]; - TTree* theInputTree = fb.processBlockTree(processName); - fillSelectedItemList(branchType, processName, theInputTree, outputItemList); + fillSelectedItemList(branchType, processName, outputItemList); } } } @@ -345,7 +252,6 @@ namespace edm::rntuple_temp { } void RNTupleTempOutputModule::reallyCloseFile() { - writeEventAuxiliary(); fillDependencyGraph(); branchParents_.clear(); startEndFile(); @@ -366,7 +272,6 @@ namespace edm::rntuple_temp { void RNTupleTempOutputModule::writeParameterSetRegistry() { rootOutputFile_->writeParameterSetRegistry(); } void RNTupleTempOutputModule::writeParentageRegistry() { rootOutputFile_->writeParentageRegistry(); } - void RNTupleTempOutputModule::writeEventAuxiliary() { rootOutputFile_->writeEventAuxiliary(); } void RNTupleTempOutputModule::finishEndFile() { rootOutputFile_->finishEndFile(); rootOutputFile_ = nullptr; @@ -490,19 +395,13 @@ namespace edm::rntuple_temp { desc.addUntracked("basketSize", 16384)->setComment("Default ROOT basket size in output file."); desc.addUntracked("eventAuxiliaryBasketSize", 16384) ->setComment("Default ROOT basket size in output file for EventAuxiliary branch."); - desc.addUntracked("eventAutoFlushCompressedSize", 20 * 1024 * 1024) - ->setComment( - "Set ROOT auto flush stored data size (in bytes) for event TTree. The value sets how large the compressed " - "buffer is allowed to get. The uncompressed buffer can be quite a bit larger than this depending on the " - "average compression ratio. The value of -1 just uses ROOT's default value. The value of 0 turns off this " - "feature. A value of -N changes the behavior to flush after every Nth event."); + desc.addUntracked("eventAutoFlushCompressedSize", 20 * 1024 * 1024)->setComment("Not used by RNTuple"); desc.addUntracked("splitLevel", 99)->setComment("Default ROOT branch split level in output file."); desc.addUntracked("sortBaskets", std::string("sortbasketsbyoffset")) ->setComment( "Legal values: 'sortbasketsbyoffset', 'sortbasketsbybranch', 'sortbasketsbyentry'.\n" "Used by ROOT when fast copying. Affects performance."); - desc.addUntracked("treeMaxVirtualSize", -1) - ->setComment("Size of ROOT TTree TBasket cache. Affects performance."); + desc.addUntracked("treeMaxVirtualSize", -1)->setComment("Not used by RNTuple."); desc.addUntracked("fastCloning", false)->setComment("Not used by RNTuple"); desc.addUntracked("mergeJob", false) ->setComment( diff --git a/FWIO/RNTupleTempOutput/src/RootOutputFile.cc b/FWIO/RNTupleTempOutput/src/RootOutputFile.cc index a31fe988a18b8..41da46ae1782a 100644 --- a/FWIO/RNTupleTempOutput/src/RootOutputFile.cc +++ b/FWIO/RNTupleTempOutput/src/RootOutputFile.cc @@ -41,7 +41,6 @@ #include "IOPool/Common/interface/getWrapperBasePtr.h" #include "IOPool/Provenance/interface/CommonProvenanceFiller.h" -#include "TTree.h" #include "TFile.h" #include "TClass.h" #include "Rtypes.h" @@ -79,6 +78,14 @@ namespace edm::rntuple_temp { } return file; } + + std::string fixName(std::string_view iName) { + if (not iName.empty() and iName.back() == '.') { + iName.remove_suffix(1); + } + return std::string(iName); + } + } // namespace RootOutputFile::RootOutputFile(RNTupleTempOutputModule* om, @@ -107,9 +114,9 @@ namespace edm::rntuple_temp { pEventEntryInfoVector_(&eventEntryInfoVector_), pBranchListIndexes_(nullptr), pEventSelectionIDs_(nullptr), - eventTree_(filePtr(), InEvent, om_->splitLevel(), om_->treeMaxVirtualSize()), - lumiTree_(filePtr(), InLumi, om_->splitLevel(), om_->treeMaxVirtualSize()), - runTree_(filePtr(), InRun, om_->splitLevel(), om_->treeMaxVirtualSize()), + eventRNTuple_(filePtr(), InEvent, om_->splitLevel(), om_->treeMaxVirtualSize()), + lumiRNTuple_(filePtr(), InLumi, om_->splitLevel(), om_->treeMaxVirtualSize()), + runRNTuple_(filePtr(), InRun, om_->splitLevel(), om_->treeMaxVirtualSize()), dataTypeReported_(false), processHistoryRegistry_(), parentageIDs_(), @@ -118,7 +125,7 @@ namespace edm::rntuple_temp { std::vector const& processesWithProcessBlockProducts = om_->outputProcessBlockHelper().processesWithProcessBlockProducts(); for (auto const& processName : processesWithProcessBlockProducts) { - processBlockTrees_.emplace_back(std::make_unique( + processBlockRNTuples_.emplace_back(std::make_unique( filePtr(), InProcess, om_->splitLevel(), om_->treeMaxVirtualSize(), processName)); } @@ -137,60 +144,56 @@ namespace edm::rntuple_temp { << "Allowed compression algorithms are ZLIB, LZMA, LZ4, and ZSTD\n"; } if (-1 != om->eventAutoFlushSize()) { - eventTree_.setAutoFlush(-1 * om->eventAutoFlushSize()); - } - if (om_->compactEventAuxiliary()) { - eventTree_.addAuxiliary( - BranchTypeToAuxiliaryBranchName(InEvent), pEventAux_, om_->auxItems()[InEvent].basketSize_); - eventTree_.tree()->SetBranchStatus(BranchTypeToAuxiliaryBranchName(InEvent).c_str(), - false); // see writeEventAuxiliary - } else { - eventTree_.addAuxiliary( - BranchTypeToAuxiliaryBranchName(InEvent), pEventAux_, om_->auxItems()[InEvent].basketSize_); + eventRNTuple_.setAutoFlush(-1 * om->eventAutoFlushSize()); } + eventRNTuple_.addAuxiliary( + BranchTypeToAuxiliaryBranchName(InEvent), &pEventAux_, om_->auxItems()[InEvent].basketSize_); - eventTree_.addAuxiliary(BranchTypeToProductProvenanceBranchName(InEvent), - pEventEntryInfoVector(), - om_->auxItems()[InEvent].basketSize_); - eventTree_.addAuxiliary( - poolNames::eventSelectionsBranchName(), pEventSelectionIDs_, om_->auxItems()[InEvent].basketSize_); - eventTree_.addAuxiliary( - poolNames::branchListIndexesBranchName(), pBranchListIndexes_, om_->auxItems()[InEvent].basketSize_); + eventRNTuple_.addAuxiliary(BranchTypeToProductProvenanceBranchName(InEvent), + &pEventEntryInfoVector_, + om_->auxItems()[InEvent].basketSize_); + eventRNTuple_.addAuxiliary( + poolNames::eventSelectionsBranchName(), &pEventSelectionIDs_, om_->auxItems()[InEvent].basketSize_); + eventRNTuple_.addAuxiliary( + poolNames::branchListIndexesBranchName(), &pBranchListIndexes_, om_->auxItems()[InEvent].basketSize_); if (om_->outputProcessBlockHelper().productsFromInputKept()) { - eventTree_.addAuxiliary(poolNames::eventToProcessBlockIndexesBranchName(), - pEventToProcessBlockIndexes_, - om_->auxItems()[InEvent].basketSize_); + eventRNTuple_.addAuxiliary(poolNames::eventToProcessBlockIndexesBranchName(), + &pEventToProcessBlockIndexes_, + om_->auxItems()[InEvent].basketSize_); } - lumiTree_.addAuxiliary( - BranchTypeToAuxiliaryBranchName(InLumi), pLumiAux_, om_->auxItems()[InLumi].basketSize_); + lumiRNTuple_.addAuxiliary( + BranchTypeToAuxiliaryBranchName(InLumi), &pLumiAux_, om_->auxItems()[InLumi].basketSize_); - runTree_.addAuxiliary( - BranchTypeToAuxiliaryBranchName(InRun), pRunAux_, om_->auxItems()[InRun].basketSize_); + runRNTuple_.addAuxiliary( + BranchTypeToAuxiliaryBranchName(InRun), &pRunAux_, om_->auxItems()[InRun].basketSize_); - treePointers_.emplace_back(&eventTree_); - treePointers_.emplace_back(&lumiTree_); - treePointers_.emplace_back(&runTree_); - for (auto& processBlockTree : processBlockTrees_) { - treePointers_.emplace_back(processBlockTree.get()); + treePointers_.emplace_back(&eventRNTuple_); + treePointers_.emplace_back(&lumiRNTuple_); + treePointers_.emplace_back(&runRNTuple_); + for (auto& processBlockRNTuple : processBlockRNTuples_) { + treePointers_.emplace_back(processBlockRNTuple.get()); } for (unsigned int i = 0; i < treePointers_.size(); ++i) { - RootOutputRNTuple* theTree = treePointers_[i]; + RootOutputRNTuple* theRNTuple = treePointers_[i]; for (auto& item : om_->selectedOutputItemList()[i]) { item.setProduct(nullptr); ProductDescription const& desc = *item.productDescription(); - theTree->addBranch(desc.branchName(), - desc.wrappedName(), - item.product(), - item.splitLevel(), - item.basketSize(), - item.productDescription()->produced()); + theRNTuple->addField(fixName(desc.branchName()), + desc.wrappedName(), + item.productPtr(), + item.splitLevel(), + item.basketSize(), + item.productDescription()->produced()); //make sure we always store product registry info for all branches we create branchesWithStoredHistory_.insert(item.branchID()); } } + for (auto& tree : treePointers_) { + tree->finishInitialization(); + } if (overrideGUID.empty()) { fid_ = FileID(createGlobalIdentifier()); @@ -248,10 +251,10 @@ namespace edm::rntuple_temp { void RootOutputFile::respondToCloseInputFile(FileBlock const&) { // We can't do setEntries() on the event tree if the EventAuxiliary branch is empty & disabled if (not om_->compactEventAuxiliary()) { - eventTree_.setEntries(); + eventRNTuple_.setEntries(); } - lumiTree_.setEntries(); - runTree_.setEntries(); + lumiRNTuple_.setEntries(); + runRNTuple_.setEntries(); } bool RootOutputFile::shouldWeCloseFile() const { @@ -283,7 +286,7 @@ namespace edm::rntuple_temp { ProductProvenanceRetriever const* provRetriever = e.productProvenanceRetrieverPtr(); assert(provRetriever); unsigned int ttreeIndex = InEvent; - fillBranches(InEvent, e, ttreeIndex, pEventEntryInfoVector_, provRetriever); + fillBranches(InEvent, e, ttreeIndex, &eventEntryInfoVector_, provRetriever); // Add the dataType to the job report if it hasn't already been done if (!dataTypeReported_) { @@ -304,10 +307,6 @@ namespace edm::rntuple_temp { reducedPHID, pEventAux_->run(), pEventAux_->luminosityBlock(), pEventAux_->event(), eventEntryNumber_); ++eventEntryNumber_; - if (om_->compactEventAuxiliary()) { - compactEventAuxiliary_.push_back(*pEventAux_); - } - // Report event written Service reportSvc; reportSvc->eventWrittenToFile(reportToken_, e.id().run(), e.id().event()); @@ -329,7 +328,7 @@ namespace edm::rntuple_temp { ++lumiEntryNumber_; unsigned int ttreeIndex = InLumi; fillBranches(InLumi, lb, ttreeIndex); - lumiTree_.optimizeBaskets(10ULL * 1024 * 1024); + lumiRNTuple_.optimizeBaskets(10ULL * 1024 * 1024); Service reportSvc; reportSvc->reportLumiSection(reportToken_, lb.id().run(), lb.id().luminosityBlock(), nEventsInLumi_); @@ -352,7 +351,7 @@ namespace edm::rntuple_temp { ++runEntryNumber_; unsigned int ttreeIndex = InRun; fillBranches(InRun, r, ttreeIndex); - runTree_.optimizeBaskets(10ULL * 1024 * 1024); + runRNTuple_.optimizeBaskets(10ULL * 1024 * 1024); Service reportSvc; reportSvc->reportRunNumber(reportToken_, r.run()); @@ -426,7 +425,7 @@ namespace edm::rntuple_temp { } auto rentry = parentageWriter->CreateEntry(); - //now put them into the TTree in the correct order + //now put them into the RNTuple in the correct order for (auto const& orderedID : orderedIDs) { rentry->BindRawPtr(poolNames::parentageBranchName(), const_cast(ptReg.getMapped(orderedID))); parentageWriter->Fill(*rentry); @@ -580,67 +579,13 @@ namespace edm::rntuple_temp { } } - // For duplicate removal and to determine if fast cloning is possible, the input - // module by default reads the entire EventAuxiliary branch when it opens the - // input files. If EventAuxiliary is written in the usual way, this results - // in many small reads scattered throughout the file, which can have very poor - // performance characteristics on some filesystems. As a workaround, we save - // EventAuxiliary and write it at the end of the file. - - void RootOutputFile::writeEventAuxiliary() { - constexpr std::size_t maxEaBasketSize = 4 * 1024 * 1024; - - if (om_->compactEventAuxiliary()) { - auto tree = eventTree_.tree(); - auto const& bname = BranchTypeToAuxiliaryBranchName(InEvent).c_str(); - - tree->SetBranchStatus(bname, true); - auto basketsize = - std::min(maxEaBasketSize, - compactEventAuxiliary_.size() * (sizeof(EventAuxiliary) + 26)); // 26 is an empirical fudge factor - tree->SetBasketSize(bname, basketsize); - auto b = tree->GetBranch(bname); - - assert(b); - - LogDebug("writeEventAuxiliary") << "EventAuxiliary ratio extras/GUIDs/all = " - << compactEventAuxiliary_.extrasSize() << "/" - << compactEventAuxiliary_.guidsSize() << "/" << compactEventAuxiliary_.size(); - - for (auto const& aux : compactEventAuxiliary_) { - const auto ea = aux.eventAuxiliary(); - pEventAux_ = &ea; - // Fill EventAuxiliary branch - b->Fill(); - } - eventTree_.setEntries(); - } - } - void RootOutputFile::finishEndFile() { std::string_view status = "beginning"; std::string_view value = ""; try { - // Create branch aliases for all the branches in the - // events/lumis/runs/processblock trees. The loop is over - // all types of data products. - status = "writeTree() for "; - for (unsigned int i = 0; i < treePointers_.size(); ++i) { - std::string processName; - BranchType branchType = InProcess; - if (i < InProcess) { - branchType = static_cast(i); - } else { - processName = om_->outputProcessBlockHelper().processesWithProcessBlockProducts()[i - InProcess]; - } - setBranchAliases(treePointers_[i]->tree(), om_->keptProducts()[branchType], processName); - value = treePointers_[i]->tree()->GetName(); - treePointers_[i]->writeTree(); - } - // close the file -- mfp // Just to play it safe, zero all pointers to objects in the TFile to be closed. - status = "closing TTrees"; + status = "closing RNTuples"; value = ""; for (auto& treePointer : treePointers_) { treePointer->close(); @@ -661,36 +606,6 @@ namespace edm::rntuple_temp { } } - void RootOutputFile::setBranchAliases(TTree* tree, - SelectedProducts const& branches, - std::string const& processName) const { - if (tree && tree->GetNbranches() != 0) { - auto const& aliasForBranches = om_->aliasForBranches(); - for (auto const& selection : branches) { - ProductDescription const& pd = *selection.first; - if (pd.branchType() == InProcess && processName != pd.processName()) { - continue; - } - std::string const& full = pd.branchName() + "obj"; - bool matched = false; - for (auto const& matcher : aliasForBranches) { - if (matcher.match(pd.branchName())) { - tree->SetAlias(matcher.alias_.c_str(), full.c_str()); - matched = true; - } - } - if (not matched and pd.branchAliases().empty()) { - std::string const& alias = (pd.productInstanceName().empty() ? pd.moduleLabel() : pd.productInstanceName()); - tree->SetAlias(alias.c_str(), full.c_str()); - } else { - for (auto const& alias : pd.branchAliases()) { - tree->SetAlias(alias.c_str(), full.c_str()); - } - } - } - } - } - void RootOutputFile::insertAncestors(ProductProvenance const& iGetParents, ProductProvenanceRetriever const* iMapper, bool produced, @@ -785,7 +700,7 @@ namespace edm::rntuple_temp { if (doProvenance) productProvenanceVecPtr->assign(provenanceToKeep.begin(), provenanceToKeep.end()); - treePointers_[ttreeIndex]->fillTree(); + treePointers_[ttreeIndex]->fill(); if (doProvenance) productProvenanceVecPtr->clear(); } diff --git a/FWIO/RNTupleTempOutput/src/RootOutputFile.h b/FWIO/RNTupleTempOutput/src/RootOutputFile.h index 44dda14c8f153..fd3f60dc5b0a2 100644 --- a/FWIO/RNTupleTempOutput/src/RootOutputFile.h +++ b/FWIO/RNTupleTempOutput/src/RootOutputFile.h @@ -37,7 +37,6 @@ #include "FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h" #include "FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h" -class TTree; class TFile; class TClass; @@ -68,7 +67,6 @@ namespace edm::rntuple_temp { void writeParameterSetRegistry(); void writeParentageRegistry(); void writeMetaData(ProductRegistry const&); - void writeEventAuxiliary(); void finishEndFile(); void beginInputFile(FileBlock const& fb, int remainingEvents); @@ -100,8 +98,6 @@ namespace edm::rntuple_temp { void writeProductDependencies(ROOT::REntry&); void writeProcessBlockHelper(ROOT::REntry&); - void setBranchAliases(TTree* tree, SelectedProducts const& branches, std::string const& processName) const; - void fillBranches(BranchType const& branchType, OccurrenceForOutput const& occurrence, unsigned int ttreeIndex, @@ -118,10 +114,6 @@ namespace edm::rntuple_temp { std::shared_ptr filePtr() const { return get_underlying_safe(filePtr_); } std::shared_ptr& filePtr() { return get_underlying_safe(filePtr_); } - StoredProductProvenanceVector const* pEventEntryInfoVector() const { - return get_underlying_safe(pEventEntryInfoVector_); - } - StoredProductProvenanceVector*& pEventEntryInfoVector() { return get_underlying_safe(pEventEntryInfoVector_); } //------------------------------- // Member data @@ -144,21 +136,20 @@ namespace edm::rntuple_temp { LuminosityBlockAuxiliary const* pLumiAux_; RunAuxiliary const* pRunAux_; StoredProductProvenanceVector eventEntryInfoVector_; - edm::propagate_const pEventEntryInfoVector_; + StoredProductProvenanceVector const* pEventEntryInfoVector_; BranchListIndexes const* pBranchListIndexes_; EventToProcessBlockIndexes const* pEventToProcessBlockIndexes_; EventSelectionIDVector const* pEventSelectionIDs_; - RootOutputRNTuple eventTree_; - RootOutputRNTuple lumiTree_; - RootOutputRNTuple runTree_; - std::vector>> processBlockTrees_; + RootOutputRNTuple eventRNTuple_; + RootOutputRNTuple lumiRNTuple_; + RootOutputRNTuple runRNTuple_; + std::vector>> processBlockRNTuples_; std::vector> treePointers_; bool dataTypeReported_; ProcessHistoryRegistry processHistoryRegistry_; std::map parentageIDs_; std::set branchesWithStoredHistory_; edm::propagate_const wrapperBaseTClass_; - CompactEventAuxiliaryVector compactEventAuxiliary_; }; } // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc index 2eb8669ab66dc..232f13f5286e7 100644 --- a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc +++ b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc @@ -31,66 +31,94 @@ namespace edm { int treeMaxVirtualSize, std::string const& processName) : filePtr_(filePtr), - tree_(processName.empty() - ? makeTTree(filePtr.get(), BranchTypeToProductTreeName(branchType), splitLevel) - : makeTTree(filePtr.get(), BranchTypeToProductTreeName(branchType, processName), splitLevel)), + name_(processName.empty() ? BranchTypeToProductTreeName(branchType) + : BranchTypeToProductTreeName(branchType, processName)), + model_(ROOT::RNTupleModel::Create()), producedBranches_(), - auxBranches_() { - if (treeMaxVirtualSize >= 0) - tree_->SetMaxVirtualSize(treeMaxVirtualSize); - } - - TTree* RootOutputRNTuple::assignTTree(TFile* filePtr, TTree* tree) { - tree->SetDirectory(filePtr); - // Turn off autosaving because it is such a memory hog and we are not using - // this check-pointing feature anyway. - tree->SetAutoSave(std::numeric_limits::max()); - return tree; - } - - TTree* RootOutputRNTuple::makeTTree(TFile* filePtr, std::string const& name, int splitLevel) { - TTree* tree = new TTree(name.c_str(), "", splitLevel); - if (!tree) - throw edm::Exception(errors::FatalRootError) << "Failed to create the tree: " << name << "\n"; - if (tree->IsZombie()) - throw edm::Exception(errors::FatalRootError) << "Tree: " << name << " is a zombie." - << "\n"; - - return assignTTree(filePtr, tree); - } - - void RootOutputRNTuple::writeTTree(TTree* tree) { - if (tree->GetNbranches() != 0) { - // This is required when Fill is called on individual branches - // in the TTree instead of calling Fill once for the entire TTree. - tree->SetEntries(-1); + auxBranches_() {} + + namespace { + template + struct Zip { + T const& first; + U const& second; + + Zip(T const& t, U const& u) : first(t), second(u) { assert(t.size() == u.size()); } + + struct iterator { + using value_type = std::pair; + using difference_type = std::ptrdiff_t; + using pointer = value_type*; + using reference = value_type&; + using iterator_category = std::input_iterator_tag; + + typename T::const_iterator t_iter; + typename U::const_iterator u_iter; + + iterator(typename T::const_iterator t, typename U::const_iterator u) : t_iter(t), u_iter(u) {} + + value_type operator*() const { return std::make_pair(*t_iter, *u_iter); } + iterator& operator++() { + ++t_iter; + ++u_iter; + return *this; + } + iterator operator++(int) { + iterator tmp = *this; + ++(*this); + return tmp; + } + bool operator==(iterator const& other) const { return t_iter == other.t_iter && u_iter == other.u_iter; } + bool operator!=(iterator const& other) const { return !(*this == other); } + }; + + iterator begin() const { return iterator(first.begin(), second.begin()); } + iterator end() const { return iterator(first.end(), second.end()); } + }; + + template + Zip zip(T const& t, U const& u) { + return Zip(t, u); } - tree->AutoSave("FlushBaskets"); - } - - void RootOutputRNTuple::fillTTree(std::vector const& branches) { - for_all(branches, std::bind(&TBranch::Fill, std::placeholders::_1)); - } - - void RootOutputRNTuple::writeTree() { writeTTree(tree()); } + } // namespace - void RootOutputRNTuple::fillTree() { + void RootOutputRNTuple::fill() { // Isolate the fill operation so that IMT doesn't grab other large tasks // that could lead to RNTupleTempOutputModule stalling - oneapi::tbb::this_task_arena::isolate([&] { tree_->Fill(); }); + std::exception_ptr e; + oneapi::tbb::this_task_arena::isolate([&] { + try { + auto entry = writer_->CreateEntry(); + for (auto z = zip(producedBranchPointers_, producedBranches_); auto prod : z) { + entry->BindRawPtr(prod.second, *prod.first); + } + for (auto z = zip(auxBranchPointers_, auxBranches_); auto aux : z) { + entry->BindRawPtr(aux.second, *aux.first); + } + writer_->Fill(*entry); + } catch (...) { + e = std::current_exception(); + } + }); + if (e) { + std::rethrow_exception(e); + } + } + + void RootOutputRNTuple::finishInitialization() { + writer_ = ROOT::RNTupleWriter::Append(std::move(model_), name_, *filePtr_, ROOT::RNTupleWriteOptions()); } - void RootOutputRNTuple::addBranch(std::string const& branchName, - std::string const& className, - void const*& pProd, - int splitLevel, - int basketSize, - bool produced) { - assert(splitLevel != ProductDescription::invalidSplitLevel); - assert(basketSize != ProductDescription::invalidBasketSize); - TBranch* branch = tree_->Branch(branchName.c_str(), className.c_str(), &pProd, basketSize, splitLevel); - assert(branch != nullptr); - producedBranches_.push_back(branch); + void RootOutputRNTuple::addField(std::string const& branchName, + std::string const& className, + void const** pProd, + int splitLevel, + int basketSize, + bool produced) { + auto field = ROOT::RFieldBase::Create(branchName, className).Unwrap(); + model_->AddField(std::move(field)); + producedBranches_.push_back(model_->GetToken(branchName.c_str())); + producedBranchPointers_.push_back(const_cast(pProd)); } void RootOutputRNTuple::close() { @@ -98,7 +126,7 @@ namespace edm { // Just to play it safe, zero all pointers to quantities in the file. auxBranches_.clear(); producedBranches_.clear(); - tree_ = nullptr; // propagate_const has no reset() function + writer_ = nullptr; // propagate_const has no reset() function filePtr_ = nullptr; // propagate_const has no reset() function } } // namespace edm diff --git a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h index a09abeb9d395c..7f9cb7da77f29 100644 --- a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h +++ b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h @@ -1,5 +1,5 @@ -#ifndef IOPool_Output_RootOutputRNTuple_h -#define IOPool_Output_RootOutputRNTuple_h +#ifndef FWIO_RNTupleTempOutput_RootOutputRNTuple_h +#define FWIO_RNTupleTempOutput_RootOutputRNTuple_h /*---------------------------------------------------------------------- @@ -15,10 +15,10 @@ RootOutputRNTuple.h // used by ROOT output modules #include "FWCore/Utilities/interface/BranchType.h" #include "FWCore/Utilities/interface/propagate_const.h" -#include "TTree.h" +#include "ROOT/RNTuple.hxx" +#include "ROOT/RNTupleWriter.hxx" class TFile; -class TBranch; namespace edm { class RootOutputRNTuple { @@ -35,59 +35,50 @@ namespace edm { RootOutputRNTuple& operator=(RootOutputRNTuple const&) = delete; // Disallow copying and moving template - void addAuxiliary(std::string const& branchName, T const*& pAux, int bufSize) { - auxBranches_.push_back(tree_->Branch(branchName.c_str(), &pAux, bufSize, 0)); + void addAuxiliary(std::string const& branchName, T const** pAux, int bufSize) { + assert(model_); + auto field = std::make_unique>(branchName); + model_->AddField(std::move(field)); + auxBranches_.push_back(model_->GetToken(branchName.c_str())); + auxBranchPointers_.push_back(reinterpret_cast(const_cast(pAux))); } - template - void addAuxiliary(std::string const& branchName, T*& pAux, int bufSize) { - auxBranches_.push_back(tree_->Branch(branchName.c_str(), &pAux, bufSize, 0)); - } - - static TTree* makeTTree(TFile* filePtr, std::string const& name, int splitLevel); - - static TTree* assignTTree(TFile* file, TTree* tree); - - static void writeTTree(TTree* tree); - bool isValid() const; - void addBranch(std::string const& branchName, - std::string const& className, - void const*& pProd, - int splitLevel, - int basketSize, - bool produced); - - void fillTree(); + void addField(std::string const& branchName, + std::string const& className, + void const** pProd, + int splitLevel, + int basketSize, + bool produced); - void writeTree(); + void fill(); - TTree const* tree() const { return tree_.get(); } + void finishInitialization(); - TTree* tree() { return tree_.get(); } + std::string const& name() const { return name_; } - void setEntries() { - if (tree_->GetNbranches() != 0) - tree_->SetEntries(-1); - } + void setEntries() {} void close(); - void optimizeBaskets(ULong64_t size) { tree_->OptimizeBaskets(size); } + void optimizeBaskets(ULong64_t size) {} - void setAutoFlush(Long64_t size) { tree_->SetAutoFlush(size); } + void setAutoFlush(Long64_t size) {} private: - static void fillTTree(std::vector const& branches); // We use bare pointers for pointers to some ROOT entities. // Root owns them and uses bare pointers internally. // Therefore, using smart pointers here will do no good. edm::propagate_const> filePtr_; - edm::propagate_const tree_; - - std::vector producedBranches_; - std::vector auxBranches_; + std::string name_; + std::unique_ptr model_; + edm::propagate_const> writer_; + + std::vector producedBranches_; + std::vector producedBranchPointers_; + std::vector auxBranches_; + std::vector auxBranchPointers_; }; } // namespace edm #endif From b9c4d2c7f9fde4f8fca3d2ff3e0232d01176507c Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Wed, 1 Oct 2025 10:41:23 -0500 Subject: [PATCH 07/19] Simplifying input code for RNTuple migration --- FWIO/RNTupleTempInput/src/InputFile.h | 27 -- .../RNTupleTempInput/src/RootDelayedReader.cc | 2 +- FWIO/RNTupleTempInput/src/RootDelayedReader.h | 8 +- .../src/RootEmbeddedFileSequence.cc | 4 - FWIO/RNTupleTempInput/src/RootFile.cc | 246 +------------ .../src/RootPrimaryFileSequence.cc | 3 - .../src/RootPromptReadDelayedReader.cc | 2 +- .../src/RootPromptReadDelayedReader.h | 8 +- FWIO/RNTupleTempInput/src/RootRNTuple.cc | 340 +----------------- FWIO/RNTupleTempInput/src/RootRNTuple.h | 78 +--- .../src/RootOutputRNTuple.cc | 2 +- .../RNTupleTempOutput/src/RootOutputRNTuple.h | 2 +- 12 files changed, 43 insertions(+), 679 deletions(-) diff --git a/FWIO/RNTupleTempInput/src/InputFile.h b/FWIO/RNTupleTempInput/src/InputFile.h index acbde5f9de918..a421d0b39ce6f 100644 --- a/FWIO/RNTupleTempInput/src/InputFile.h +++ b/FWIO/RNTupleTempInput/src/InputFile.h @@ -11,7 +11,6 @@ Holder for an input TFile. #include "TFile.h" #include "TTree.h" -#include "TTreeCache.h" #include #include @@ -52,33 +51,7 @@ namespace edm::rntuple_temp { T* Get(char const* name) { return file_->Get(name); } - std::unique_ptr createCacheWithSize(TTree& iTree, unsigned int cacheSize) { - iTree.SetCacheSize(static_cast(cacheSize)); - std::unique_ptr newCache(dynamic_cast(file_->GetCacheRead(&iTree))); - file_->SetCacheRead(nullptr, &iTree, TFile::kDoNotDisconnect); - return newCache; - } - - class CacheGuard { - public: - CacheGuard(TFile* file, TObject* tree, TFileCacheRead* tfcr) : file_(file), tree_(tree) { - file_->SetCacheRead(tfcr, tree_, TFile::kDoNotDisconnect); - } - CacheGuard() = delete; - CacheGuard(CacheGuard const&) = delete; - CacheGuard& operator=(CacheGuard const&) = delete; - CacheGuard(CacheGuard&&) = delete; - CacheGuard& operator=(CacheGuard&&) = delete; - ~CacheGuard() { file_->SetCacheRead(nullptr, tree_, TFile::kDoNotDisconnect); } - private: - TFile* file_; - TObject* tree_; - }; - [[nodiscard]] CacheGuard setCacheReadTemporarily(TFileCacheRead* tfcr, TObject* iTree) { - return CacheGuard(file_.get(), iTree, tfcr); - } - void clearCacheRead(TObject* iTree) { file_->SetCacheRead(nullptr, iTree, TFile::kDoNotDisconnect); } void logFileAction(char const* msg, char const* fileName) const; private: diff --git a/FWIO/RNTupleTempInput/src/RootDelayedReader.cc b/FWIO/RNTupleTempInput/src/RootDelayedReader.cc index 68406782e7539..beb3f38c1f755 100644 --- a/FWIO/RNTupleTempInput/src/RootDelayedReader.cc +++ b/FWIO/RNTupleTempInput/src/RootDelayedReader.cc @@ -52,7 +52,7 @@ namespace edm::rntuple_temp { throw; } } - auto branchInfo = getBranchInfo(k); + auto branchInfo = getProductInfo(k); if (not branchInfo) { if (nextReader_) { return nextReader_->getProduct(k, ep); diff --git a/FWIO/RNTupleTempInput/src/RootDelayedReader.h b/FWIO/RNTupleTempInput/src/RootDelayedReader.h index 3f2ec7a57ae4a..67e016135d81a 100644 --- a/FWIO/RNTupleTempInput/src/RootDelayedReader.h +++ b/FWIO/RNTupleTempInput/src/RootDelayedReader.h @@ -34,8 +34,8 @@ namespace edm::rntuple_temp { class RootDelayedReader : public RootDelayedReaderBase { public: - typedef rootrntuple::BranchInfo BranchInfo; - typedef rootrntuple::BranchMap BranchMap; + typedef rootrntuple::ProductInfo ProductInfo; + typedef rootrntuple::ProductMap ProductMap; typedef rootrntuple::EntryNumber EntryNumber; RootDelayedReader(RootRNTuple const& tree, std::shared_ptr filePtr, InputType inputType); @@ -52,8 +52,8 @@ namespace edm::rntuple_temp { void reset_() override { nextReader_ = nullptr; } std::pair sharedResources_() const override; - BranchMap const& branches() const { return tree_.branches(); } - BranchInfo const* getBranchInfo(BranchID const& k) const { return branches().find(k); } + ProductMap const& branches() const { return tree_.branches(); } + ProductInfo const* getProductInfo(BranchID const& k) const { return branches().find(k); } // NOTE: filePtr_ appears to be unused, but is needed to prevent // the file containing the branch from being reclaimed. RootRNTuple const& tree_; diff --git a/FWIO/RNTupleTempInput/src/RootEmbeddedFileSequence.cc b/FWIO/RNTupleTempInput/src/RootEmbeddedFileSequence.cc index 12e4abcf2281c..36651340bbaad 100644 --- a/FWIO/RNTupleTempInput/src/RootEmbeddedFileSequence.cc +++ b/FWIO/RNTupleTempInput/src/RootEmbeddedFileSequence.cc @@ -51,7 +51,6 @@ namespace edm::rntuple_temp { // have defined descriptions, the defaults in the getUntrackedParameterSet function calls can // and should be deleted from the code. initialNumberOfEventsToSkip_(pset.getUntrackedParameter("skipEvents", 0U)), - treeCacheSize_(pset.getUntrackedParameter("cacheSize", rootrntuple::defaultCacheSize)), enablePrefetching_(false), enforceGUIDInFileName_(pset.getUntrackedParameter("enforceGUIDInFileName", false)), maxFileSkips_(pset.getUntrackedParameter("maxFileSkips", std::min(3_uz, numberOfFiles()))) { @@ -63,9 +62,6 @@ namespace edm::rntuple_temp { // The SiteLocalConfig controls the TTreeCache size and the prefetching settings. Service pSLC; if (pSLC.isAvailable()) { - if (treeCacheSize_ != 0U && pSLC->sourceTTreeCacheSize()) { - treeCacheSize_ = *(pSLC->sourceTTreeCacheSize()); - } enablePrefetching_ = pSLC->enablePrefetching(); } diff --git a/FWIO/RNTupleTempInput/src/RootFile.cc b/FWIO/RNTupleTempInput/src/RootFile.cc index 40fb2ce55407a..edb1f043ee2fa 100644 --- a/FWIO/RNTupleTempInput/src/RootFile.cc +++ b/FWIO/RNTupleTempInput/src/RootFile.cc @@ -66,7 +66,6 @@ #include "TClass.h" #include "TString.h" #include "TTree.h" -#include "TTreeCache.h" #include "ROOT/RNTuple.hxx" #include "ROOT/RNTupleReader.hxx" @@ -79,26 +78,6 @@ namespace edm::rntuple_temp { // Algorithm classes for making ProvenanceReader: - class MakeDummyProvenanceReader : public MakeProvenanceReader { - public: - std::unique_ptr makeReader(RootRNTuple& eventTree, - DaqProvenanceHelper const* daqProvenanceHelper) const override; - }; - class MakeOldProvenanceReader : public MakeProvenanceReader { - public: - MakeOldProvenanceReader(std::unique_ptr&& entryDescriptionMap) - : MakeProvenanceReader(), entryDescriptionMap_(std::move(entryDescriptionMap)) {} - std::unique_ptr makeReader(RootRNTuple& eventTree, - DaqProvenanceHelper const* daqProvenanceHelper) const override; - - private: - edm::propagate_const> entryDescriptionMap_; - }; - class MakeFullProvenanceReader : public MakeProvenanceReader { - public: - std::unique_ptr makeReader(RootRNTuple& eventTree, - DaqProvenanceHelper const* daqProvenanceHelper) const override; - }; class MakeReducedProvenanceReader : public MakeProvenanceReader { public: MakeReducedProvenanceReader(std::vector const& parentageIDLookup) @@ -389,8 +368,6 @@ namespace edm::rntuple_temp { processHistoryRegistry.registerProcessHistory(history); } - eventTree_.trainCache(BranchTypeToAuxiliaryBranchName(InEvent).c_str()); - // Update the branch id info. This has to be done before validateFile since // depending on the file format, the branchIDListHelper_ may have its fixBranchListIndexes call made if (inputType == InputType::Primary) { @@ -524,16 +501,6 @@ namespace edm::rntuple_temp { // We are done with our initial reading of EventAuxiliary. indexIntoFile_.doneFileInitialization(); - - // Tell the event tree to begin training at the next read. - eventTree_.resetTraining(); - - // Train the run and lumi trees. - runTree_.trainCache("*"); - lumiTree_.trainCache("*"); - for (auto& processBlockTree : processBlockTrees_) { - processBlockTree->trainCache("*"); - } } RootFile::~RootFile() {} @@ -632,11 +599,11 @@ namespace edm::rntuple_temp { } return std::make_shared(fileFormatVersion(), eventTree_.tree(), - eventTree_.metaTree(), + nullptr, lumiTree_.tree(), - lumiTree_.metaTree(), + nullptr, runTree_.tree(), - runTree_.metaTree(), + nullptr, std::move(processBlockTrees), std::move(processesWithProcessBlockTrees), 0, @@ -657,11 +624,11 @@ namespace edm::rntuple_temp { processesWithProcessBlockTrees.push_back(processBlockTree->processName()); } fileBlock.updateTTreePointers(eventTree_.tree(), - eventTree_.metaTree(), + nullptr, lumiTree_.tree(), - lumiTree_.metaTree(), + nullptr, runTree_.tree(), - runTree_.metaTree(), + nullptr, std::move(processBlockTrees), std::move(processesWithProcessBlockTrees)); } @@ -1988,19 +1955,9 @@ namespace edm::rntuple_temp { } std::unique_ptr RootFile::makeProvenanceReaderMaker(InputType inputType) { - if (fileFormatVersion_.storedProductProvenanceUsed()) { - readParentageTree(inputType); - return std::make_unique(parentageIDLookup_); - } else if (fileFormatVersion_.splitProductIDs()) { - readParentageTree(inputType); - return std::make_unique(); - } else if (fileFormatVersion_.perEventProductIDs()) { - auto entryDescriptionMap = std::make_unique(); - readEntryDescriptionTree(*entryDescriptionMap, inputType); - return std::make_unique(std::move(entryDescriptionMap)); - } else { - return std::make_unique(); - } + //if (fileFormatVersion_.storedProductProvenanceUsed()) { + readParentageTree(inputType); + return std::make_unique(parentageIDLookup_); } std::shared_ptr RootFile::makeProductProvenanceRetriever(unsigned int iStreamID) { @@ -2171,191 +2128,6 @@ namespace edm::rntuple_temp { return retValue; } - class FullProvenanceReader : public ProvenanceReaderBase { - public: - explicit FullProvenanceReader(RootRNTuple* rootTree, DaqProvenanceHelper const* daqProvenanceHelper); - ~FullProvenanceReader() override {} - std::set readProvenance(unsigned int transitionIndex) const override; - - private: - void readProvenanceAsync(WaitingTaskHolder task, - ModuleCallingContext const* moduleCallingContext, - unsigned int transitionIndex, - std::atomic*>& writeTo) const noexcept override; - - RootRNTuple* rootTree_; - ProductProvenanceVector infoVector_; - //All access to a ROOT file is serialized - CMS_SA_ALLOW mutable ProductProvenanceVector* pInfoVector_; - DaqProvenanceHelper const* daqProvenanceHelper_; - std::shared_ptr mutex_; - SharedResourcesAcquirer acquirer_; - }; - - FullProvenanceReader::FullProvenanceReader(RootRNTuple* rootTree, DaqProvenanceHelper const* daqProvenanceHelper) - : ProvenanceReaderBase(), - rootTree_(rootTree), - infoVector_(), - pInfoVector_(&infoVector_), - daqProvenanceHelper_(daqProvenanceHelper), - mutex_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().second), - acquirer_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().first) {} - - void FullProvenanceReader::readProvenanceAsync( - WaitingTaskHolder task, - ModuleCallingContext const* moduleCallingContext, - unsigned int transitionIndex, - std::atomic*>& writeTo) const noexcept { - readProvenanceAsyncImpl(this, - acquirer_.serialQueueChain(), - task, - transitionIndex, - writeTo, - moduleCallingContext, - rootTree_->rootDelayedReader()->preEventReadFromSourceSignal(), - rootTree_->rootDelayedReader()->postEventReadFromSourceSignal()); - } - - std::set FullProvenanceReader::readProvenance(unsigned int transitionIndex) const { - { - std::lock_guard guard(*mutex_); - rootTree_->fillBranchEntryMeta( - rootTree_->branchEntryInfoBranch(), rootTree_->entryNumberForIndex(transitionIndex), pInfoVector_); - } - std::set retValue; - if (daqProvenanceHelper_) { - for (auto const& info : infoVector_) { - retValue.emplace(daqProvenanceHelper_->mapBranchID(info.branchID()), - daqProvenanceHelper_->mapParentageID(info.parentageID())); - } - } else { - for (auto const& info : infoVector_) { - retValue.emplace(info); - } - } - return retValue; - } - - class OldProvenanceReader : public ProvenanceReaderBase { - public: - explicit OldProvenanceReader(RootRNTuple* rootTree, - EntryDescriptionMap const& theMap, - DaqProvenanceHelper const* daqProvenanceHelper); - ~OldProvenanceReader() override {} - std::set readProvenance(unsigned int transitionIndex) const override; - - private: - void readProvenanceAsync(WaitingTaskHolder task, - ModuleCallingContext const* moduleCallingContext, - unsigned int transitionIndex, - std::atomic*>& writeTo) const noexcept override; - - edm::propagate_const rootTree_; - std::vector infoVector_; - //All access to ROOT file are serialized - CMS_SA_ALLOW mutable std::vector* pInfoVector_; - EntryDescriptionMap const& entryDescriptionMap_; - DaqProvenanceHelper const* daqProvenanceHelper_; - std::shared_ptr mutex_; - SharedResourcesAcquirer acquirer_; - }; - - OldProvenanceReader::OldProvenanceReader(RootRNTuple* rootTree, - EntryDescriptionMap const& theMap, - DaqProvenanceHelper const* daqProvenanceHelper) - : ProvenanceReaderBase(), - rootTree_(rootTree), - infoVector_(), - pInfoVector_(&infoVector_), - entryDescriptionMap_(theMap), - daqProvenanceHelper_(daqProvenanceHelper), - mutex_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().second), - acquirer_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().first) {} - - void OldProvenanceReader::readProvenanceAsync( - WaitingTaskHolder task, - ModuleCallingContext const* moduleCallingContext, - unsigned int transitionIndex, - std::atomic*>& writeTo) const noexcept { - readProvenanceAsyncImpl(this, - acquirer_.serialQueueChain(), - task, - transitionIndex, - writeTo, - moduleCallingContext, - rootTree_->rootDelayedReader()->preEventReadFromSourceSignal(), - rootTree_->rootDelayedReader()->postEventReadFromSourceSignal()); - } - - std::set OldProvenanceReader::readProvenance(unsigned int transitionIndex) const { - { - std::lock_guard guard(*mutex_); - rootTree_->branchEntryInfoBranch()->SetAddress(&pInfoVector_); - rootrntuple::getEntry(rootTree_->branchEntryInfoBranch(), rootTree_->entryNumberForIndex(transitionIndex)); - } - std::set retValue; - for (auto const& info : infoVector_) { - EntryDescriptionMap::const_iterator iter = entryDescriptionMap_.find(info.entryDescriptionID()); - assert(iter != entryDescriptionMap_.end()); - Parentage parentage(iter->second.parents()); - if (daqProvenanceHelper_) { - retValue.emplace(daqProvenanceHelper_->mapBranchID(info.branchID()), - daqProvenanceHelper_->mapParentageID(parentage.id())); - } else { - retValue.emplace(info.branchID(), parentage.id()); - } - } - return retValue; - } - - class DummyProvenanceReader : public ProvenanceReaderBase { - public: - DummyProvenanceReader(); - ~DummyProvenanceReader() override {} - - private: - std::set readProvenance(unsigned int) const override; - void readProvenanceAsync(WaitingTaskHolder task, - ModuleCallingContext const* moduleCallingContext, - unsigned int transitionIndex, - std::atomic*>& writeTo) const noexcept override; - }; - - DummyProvenanceReader::DummyProvenanceReader() : ProvenanceReaderBase() {} - - std::set DummyProvenanceReader::readProvenance(unsigned int) const { - // Not providing parentage!!! - return std::set{}; - } - void DummyProvenanceReader::readProvenanceAsync( - WaitingTaskHolder task, - ModuleCallingContext const* moduleCallingContext, - unsigned int transitionIndex, - std::atomic*>& writeTo) const noexcept { - if (nullptr == writeTo.load()) { - auto emptyProv = std::make_unique>(); - const std::set* expected = nullptr; - if (writeTo.compare_exchange_strong(expected, emptyProv.get())) { - emptyProv.release(); - } - } - } - - std::unique_ptr MakeDummyProvenanceReader::makeReader(RootRNTuple&, - DaqProvenanceHelper const*) const { - return std::make_unique(); - } - - std::unique_ptr MakeOldProvenanceReader::makeReader( - RootRNTuple& rootTree, DaqProvenanceHelper const* daqProvenanceHelper) const { - return std::make_unique(&rootTree, *entryDescriptionMap_, daqProvenanceHelper); - } - - std::unique_ptr MakeFullProvenanceReader::makeReader( - RootRNTuple& rootTree, DaqProvenanceHelper const* daqProvenanceHelper) const { - return std::make_unique(&rootTree, daqProvenanceHelper); - } - std::unique_ptr MakeReducedProvenanceReader::makeReader( RootRNTuple& rootTree, DaqProvenanceHelper const* daqProvenanceHelper) const { return std::make_unique(&rootTree, parentageIDLookup_, daqProvenanceHelper); diff --git a/FWIO/RNTupleTempInput/src/RootPrimaryFileSequence.cc b/FWIO/RNTupleTempInput/src/RootPrimaryFileSequence.cc index dcebd1bf67655..0db444c268d08 100644 --- a/FWIO/RNTupleTempInput/src/RootPrimaryFileSequence.cc +++ b/FWIO/RNTupleTempInput/src/RootPrimaryFileSequence.cc @@ -48,9 +48,6 @@ namespace edm::rntuple_temp { // The SiteLocalConfig controls the TTreeCache size and the prefetching settings. Service pSLC; if (pSLC.isAvailable()) { - if (treeCacheSize_ != 0U && pSLC->sourceTTreeCacheSize()) { - treeCacheSize_ = *(pSLC->sourceTTreeCacheSize()); - } enablePrefetching_ = pSLC->enablePrefetching(); } diff --git a/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.cc b/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.cc index 6476433e188db..b0dd7c7cfb7e7 100644 --- a/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.cc +++ b/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.cc @@ -90,7 +90,7 @@ namespace edm::rntuple_temp { } } for (auto& it : cacheMap) { - auto branchInfo = getBranchInfo(it.first); + auto branchInfo = getProductInfo(it.first); if (branchInfo == nullptr || branchInfo->productBranch_ == nullptr) { continue; // Skip if branch info or product branch is not available } diff --git a/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.h b/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.h index cadde91e1f407..57df25bbdecf1 100644 --- a/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.h +++ b/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.h @@ -33,8 +33,8 @@ namespace edm::rntuple_temp { class RootPromptReadDelayedReader : public RootDelayedReaderBase { public: - typedef rootrntuple::BranchInfo BranchInfo; - typedef rootrntuple::BranchMap BranchMap; + typedef rootrntuple::ProductInfo ProductInfo; + typedef rootrntuple::ProductMap ProductMap; typedef rootrntuple::EntryNumber EntryNumber; RootPromptReadDelayedReader(RootRNTuple const& tree, std::shared_ptr filePtr, @@ -61,8 +61,8 @@ namespace edm::rntuple_temp { void reset_() override { nextReader_ = nullptr; } std::pair sharedResources_() const override; - BranchMap const& branches() const { return tree_.branches(); } - BranchInfo const* getBranchInfo(unsigned int k) const { return branches().find(k); } + ProductMap const& branches() const { return tree_.branches(); } + ProductInfo const* getProductInfo(unsigned int k) const { return branches().find(k); } std::vector> cacheMaps_; // NOTE: filePtr_ appears to be unused, but is needed to prevent // the file containing the branch from being reclaimed. diff --git a/FWIO/RNTupleTempInput/src/RootRNTuple.cc b/FWIO/RNTupleTempInput/src/RootRNTuple.cc index 0e0a5e1ae75ca..cbecb00dab7ac 100644 --- a/FWIO/RNTupleTempInput/src/RootRNTuple.cc +++ b/FWIO/RNTupleTempInput/src/RootRNTuple.cc @@ -10,7 +10,6 @@ #include "RootPromptReadDelayedReader.h" #include "TTree.h" -#include "TTreeCache.h" #include "TLeaf.h" #include "oneapi/tbb/task_arena.h" @@ -54,9 +53,7 @@ namespace edm::rntuple_temp { : filePtr_(filePtr), branchType_(branchType), entryNumberForIndex_(std::make_unique>(nIndexes, IndexIntoFile::invalidEntry)), - learningEntries_(learningEntries), enablePrefetching_(enablePrefetching), - enableTriggerCache_(branchType_ == InEvent), promptRead_(promptRead), rootDelayedReader_(makeRootDelayedReader(*this, filePtr, inputType, nIndexes, promptRead)) {} @@ -70,12 +67,7 @@ namespace edm::rntuple_temp { : RootRNTuple( filePtr, branchType, nIndexes, learningEntries, options.enablePrefetching, options.promptReading, inputType) { init(BranchTypeToProductTreeName(branchType), options.treeMaxVirtualSize, options.treeCacheSize); - metaTree_ = dynamic_cast(filePtr_->Get(BranchTypeToMetaDataTreeName(branchType).c_str())); auxBranch_ = getAuxiliaryBranch(tree_, branchType_); - branchEntryInfoBranch_ = - metaTree_ ? getProductProvenanceBranch(metaTree_, branchType_) : getProductProvenanceBranch(tree_, branchType_); - infoTree_ = - dynamic_cast(filePtr->Get(BranchTypeToInfoTreeName(branchType).c_str())); // backward compatibility } // Used for ProcessBlock RootRNTuples @@ -113,16 +105,8 @@ namespace edm::rntuple_temp { } else { treeAutoFlush_ = treeAutoFlush; } - if (treeAutoFlush_ < learningEntries_) { - learningEntries_ = treeAutoFlush_; - } setTreeMaxVirtualSize(maxVirtualSize); setCacheSize(cacheSize); - if (branchType_ == InEvent) { - Int_t branchCount = tree_->GetListOfBranches()->GetEntriesFast(); - trainedSet_.reserve(branchCount); - triggerSet_.reserve(branchCount); - } } RootRNTuple::~RootRNTuple() {} @@ -143,17 +127,7 @@ namespace edm::rntuple_temp { return tree_ != nullptr; } // Run/Lumi/Event - if (metaTree_ == nullptr || metaTree_->GetNbranches() == 0) { - return tree_ != nullptr && auxBranch_ != nullptr; - } - // Backward compatibility for Run/Lumi/Event - if (tree_ != nullptr && auxBranch_ != nullptr && metaTree_ != nullptr) { // backward compatibility - if (branchEntryInfoBranch_ != nullptr || infoTree_ != nullptr) - return true; // backward compatibility - return (entries_ == metaTree_->GetEntries() && - tree_->GetNbranches() <= metaTree_->GetNbranches() + 1); // backward compatibility - } // backward compatibility - return false; + return tree_ != nullptr && auxBranch_ != nullptr; } DelayedReader* RootRNTuple::resetAndGetRootDelayedReader() const { @@ -170,14 +144,14 @@ namespace edm::rntuple_temp { } } - void rootrntuple::BranchInfo::setBranch(TBranch* branch, TClass const* wrapperBaseTClass) { + void rootrntuple::ProductInfo::setBranch(TBranch* branch, TClass const* wrapperBaseTClass) { productBranch_ = branch; if (branch) { classCache_ = TClass::GetClass(productDescription_.wrappedName().c_str()); offsetToWrapperBase_ = classCache_->GetBaseClassOffset(wrapperBaseTClass); } } - std::unique_ptr rootrntuple::BranchInfo::newWrapper() const { + std::unique_ptr rootrntuple::ProductInfo::newWrapper() const { assert(nullptr != classCache_); void* p = classCache_->New(); return getWrapperBasePtr(p, offsetToWrapperBase_); @@ -188,7 +162,7 @@ namespace edm::rntuple_temp { static TClass const* const wrapperBaseTClass = TClass::GetClass("edm::WrapperBase"); //use the translated branch name TBranch* branch = tree_->GetBranch(oldBranchName.c_str()); - rootrntuple::BranchInfo info = rootrntuple::BranchInfo(prod); + rootrntuple::ProductInfo info = rootrntuple::ProductInfo(prod); info.productBranch_ = nullptr; if (prod.present()) { info.setBranch(branch, wrapperBaseTClass); @@ -222,19 +196,9 @@ namespace edm::rntuple_temp { } } - rootrntuple::BranchMap const& RootRNTuple::branches() const { return branches_; } + rootrntuple::ProductMap const& RootRNTuple::branches() const { return branches_; } - std::shared_ptr RootRNTuple::createCacheWithSize(unsigned int cacheSize) const { - return filePtr_->createCacheWithSize(*tree_, cacheSize); - } - - void RootRNTuple::setCacheSize(unsigned int cacheSize) { - cacheSize_ = cacheSize; - treeCache_ = createCacheWithSize(cacheSize); - if (treeCache_) - treeCache_->SetEnablePrefetching(enablePrefetching_); - rawTreeCache_.reset(); - } + void RootRNTuple::setCacheSize(unsigned int cacheSize) {} void RootRNTuple::setTreeMaxVirtualSize(int treeMaxVirtualSize) { if (treeMaxVirtualSize >= 0) @@ -251,199 +215,20 @@ namespace edm::rntuple_temp { void RootRNTuple::setEntryNumber(EntryNumber theEntryNumber) { { - auto guard = filePtr_->setCacheReadTemporarily(treeCache_.get(), tree_); - - // Detect a backward skip. If the skip is sufficiently large, we roll the dice and reset the treeCache. - // This will cause some amount of over-reading: we pre-fetch all the events in some prior cluster. - // However, because reading one event in the cluster is supposed to be equivalent to reading all events in the cluster, - // we're not incurring additional over-reading - we're just doing it more efficiently. - // NOTE: Constructor guarantees treeAutoFlush_ is positive, even if TTree->GetAutoFlush() is negative. - if (theEntryNumber < entryNumber_ and theEntryNumber >= 0) { - //We started reading the file near the end, now we need to correct for the learning length - if (switchOverEntry_ > tree_->GetEntries()) { - switchOverEntry_ = switchOverEntry_ - tree_->GetEntries(); - if (rawTreeCache_) { - rawTreeCache_->SetEntryRange(theEntryNumber, switchOverEntry_); - rawTreeCache_->FillBuffer(); - } - } - if (performedSwitchOver_ and triggerTreeCache_) { - //We are using the triggerTreeCache_ not the rawTriggerTreeCache_. - //The triggerTreeCache was originally told to start from an entry further in the file. - triggerTreeCache_->SetEntryRange(theEntryNumber, tree_->GetEntries()); - } else if (rawTriggerTreeCache_) { - //move the switch point to the end of the cluster holding theEntryNumber - rawTriggerSwitchOverEntry_ = -1; - TTree::TClusterIterator clusterIter = tree_->GetClusterIterator(theEntryNumber); - while ((rawTriggerSwitchOverEntry_ < theEntryNumber) || (rawTriggerSwitchOverEntry_ <= 0)) { - rawTriggerSwitchOverEntry_ = clusterIter(); - } - rawTriggerTreeCache_->SetEntryRange(theEntryNumber, rawTriggerSwitchOverEntry_); - } - } - if ((theEntryNumber < static_cast(entryNumber_ - treeAutoFlush_)) && (treeCache_) && - (!treeCache_->IsLearning()) && (entries_ > 0) && (switchOverEntry_ >= 0)) { - treeCache_->SetEntryRange(theEntryNumber, entries_); - treeCache_->FillBuffer(); - } - entryNumber_ = theEntryNumber; tree_->LoadTree(entryNumber_); //want guard to end here } - if (treeCache_ && trainNow_ && entryNumber_ >= 0) { - startTraining(); - trainNow_ = false; - trainedSet_.clear(); - triggerSet_.clear(); - rawTriggerSwitchOverEntry_ = -1; - } - if (not promptRead_ && treeCache_ && treeCache_->IsLearning() && switchOverEntry_ >= 0 && - entryNumber_ >= switchOverEntry_) { - stopTraining(); - } - } - - // The actual implementation is done below; it's split in this strange - // manner in order to keep a by-definition-rare code path out of the instruction cache. - inline TTreeCache* RootRNTuple::checkTriggerCache(TBranch* branch, EntryNumber entryNumber) const { - if (!treeCache_->IsAsyncReading() && enableTriggerCache_ && (trainedSet_.find(branch) == trainedSet_.end())) { - return checkTriggerCacheImpl(branch, entryNumber); - } else { - return nullptr; - } - } - - // See comments in the header. If this function is called, we already know - // the trigger cache is active and it was a cache miss for the regular cache. - TTreeCache* RootRNTuple::checkTriggerCacheImpl(TBranch* branch, EntryNumber entryNumber) const { - // This branch is not going to be in the cache. - // Assume this is a "trigger pattern". - // Always make sure the branch is added to the trigger set. - if (triggerSet_.find(branch) == triggerSet_.end()) { - triggerSet_.insert(branch); - if (triggerTreeCache_.get()) { - triggerTreeCache_->AddBranch(branch, kTRUE); - } - } - - if (rawTriggerSwitchOverEntry_ < 0) { - // The trigger has never fired before. Take everything not in the - // trainedSet and load it from disk - - // Calculate the end of the next cluster; triggers in the next cluster - // will use the triggerCache, not the rawTriggerCache. - // - // Guarantee that rawTriggerSwitchOverEntry_ is positive (non-zero) after completion - // of this if-block. - TTree::TClusterIterator clusterIter = tree_->GetClusterIterator(entryNumber); - while ((rawTriggerSwitchOverEntry_ < entryNumber) || (rawTriggerSwitchOverEntry_ <= 0)) { - rawTriggerSwitchOverEntry_ = clusterIter(); - } - - // ROOT will automatically expand the cache to fit one cluster; hence, we use - // 5 MB as the cache size below - rawTriggerTreeCache_ = createCacheWithSize(5 * 1024 * 1024); - if (rawTriggerTreeCache_) - rawTriggerTreeCache_->SetEnablePrefetching(false); - TObjArray* branches = tree_->GetListOfBranches(); - int branchCount = branches->GetEntriesFast(); - - // Train the rawTriggerCache to have everything not in the regular cache. - rawTriggerTreeCache_->SetLearnEntries(0); - rawTriggerTreeCache_->SetEntryRange(entryNumber, rawTriggerSwitchOverEntry_); - for (int i = 0; i < branchCount; i++) { - TBranch* tmp_branch = (TBranch*)branches->UncheckedAt(i); - if (trainedSet_.find(tmp_branch) != trainedSet_.end()) { - continue; - } - rawTriggerTreeCache_->AddBranch(tmp_branch, kTRUE); - } - performedSwitchOver_ = false; - rawTriggerTreeCache_->StopLearningPhase(); - - return rawTriggerTreeCache_.get(); - } else if (!performedSwitchOver_ and entryNumber_ < rawTriggerSwitchOverEntry_) { - // The raw trigger has fired and it contents are valid. - return rawTriggerTreeCache_.get(); - } else if (rawTriggerSwitchOverEntry_ > 0) { - // The raw trigger has fired, but we are out of the cache. Use the - // triggerCache instead. - if (!performedSwitchOver_) { - rawTriggerTreeCache_.reset(); - performedSwitchOver_ = true; - - // Train the triggerCache - triggerTreeCache_ = createCacheWithSize(5 * 1024 * 1024); - triggerTreeCache_->SetEnablePrefetching(false); - triggerTreeCache_->SetLearnEntries(0); - triggerTreeCache_->SetEntryRange(entryNumber, tree_->GetEntries()); - for (std::unordered_set::const_iterator it = triggerSet_.begin(), itEnd = triggerSet_.end(); - it != itEnd; - it++) { - triggerTreeCache_->AddBranch(*it, kTRUE); - } - triggerTreeCache_->StopLearningPhase(); - } - return triggerTreeCache_.get(); - } - - // By construction, this case should be impossible. - assert(false); - return nullptr; - } - - inline TTreeCache* RootRNTuple::selectCache(TBranch* branch, EntryNumber entryNumber) const { - TTreeCache* triggerCache = nullptr; - if (promptRead_) { - return rawTreeCache_.get(); - } - if (!treeCache_) { - return nullptr; - } else if (treeCache_->IsLearning() && rawTreeCache_) { - treeCache_->AddBranch(branch, kTRUE); - trainedSet_.insert(branch); - return rawTreeCache_.get(); - } else if ((triggerCache = checkTriggerCache(branch, entryNumber))) { - // A NULL return value from checkTriggerCache indicates the trigger cache case - // does not apply, and we should continue below. - return triggerCache; - } else { - // The "normal" TTreeCache case. - return treeCache_.get(); - } - } - TTreeCache* RootRNTuple::getAuxCache(TBranch* auxBranch) const { - if (not auxCache_ and cacheSize_ > 0) { - auxCache_ = createCacheWithSize(1 * 1024 * 1024); - if (auxCache_) { - auxCache_->SetEnablePrefetching(enablePrefetching_); - auxCache_->SetLearnEntries(0); - auxCache_->StartLearningPhase(); - auxCache_->SetEntryRange(0, tree_->GetEntries()); - auxCache_->AddBranch(auxBranch->GetName(), kTRUE); - auxCache_->StopLearningPhase(); - } - } - return auxCache_.get(); } void RootRNTuple::getEntryForAllBranches() const { - oneapi::tbb::this_task_arena::isolate([&]() { - auto guard = filePtr_->setCacheReadTemporarily(treeCache_.get(), tree_); - tree_->GetEntry(entryNumber_); - }); + oneapi::tbb::this_task_arena::isolate([&]() { tree_->GetEntry(entryNumber_); }); } void RootRNTuple::getEntry(TBranch* branch, EntryNumber entryNumber) const { - getEntryUsingCache(branch, entryNumber, selectCache(branch, entryNumber)); - } - - inline void RootRNTuple::getEntryUsingCache(TBranch* branch, EntryNumber entryNumber, TTreeCache* cache) const { LogTrace("IOTrace").format( "RootRNTuple::getEntryUsingCache() begin for branch {} entry {}", branch->GetName(), entryNumber); try { - auto guard = filePtr_->setCacheReadTemporarily(cache, tree_); branch->GetEntry(entryNumber); } catch (cms::Exception const& e) { // We make sure the treeCache_ is detached from the file, @@ -481,107 +266,15 @@ namespace edm::rntuple_temp { return retval; } - void RootRNTuple::startTraining() { - if (cacheSize_ == 0) { - return; - } - assert(treeCache_); - assert(branchType_ == InEvent); - assert(!rawTreeCache_); - treeCache_->SetLearnEntries(learningEntries_); - rawTreeCache_ = createCacheWithSize(cacheSize_); - rawTreeCache_->SetEnablePrefetching(false); - rawTreeCache_->SetLearnEntries(0); - if (promptRead_) { - switchOverEntry_ = entries_; - } else { - switchOverEntry_ = entryNumber_ + learningEntries_; - } - auto rawStart = entryNumber_; - auto rawEnd = switchOverEntry_; - auto treeStart = switchOverEntry_; - if (switchOverEntry_ >= tree_->GetEntries()) { - treeStart = switchOverEntry_ - tree_->GetEntries(); - rawEnd = tree_->GetEntries(); - } - rawTreeCache_->StartLearningPhase(); - rawTreeCache_->SetEntryRange(rawStart, rawEnd); - rawTreeCache_->AddBranch("*", kTRUE); - rawTreeCache_->StopLearningPhase(); - - treeCache_->StartLearningPhase(); - treeCache_->SetEntryRange(treeStart, tree_->GetEntries()); - // Make sure that 'branchListIndexes' branch exist in input file - if (filePtr_->Get(poolNames::branchListIndexesBranchName().c_str()) != nullptr) { - treeCache_->AddBranch(poolNames::branchListIndexesBranchName().c_str(), kTRUE); - } - treeCache_->AddBranch(BranchTypeToAuxiliaryBranchName(branchType_).c_str(), kTRUE); - trainedSet_.clear(); - triggerSet_.clear(); - assert(treeCache_->GetTree() == tree_); - } - - void RootRNTuple::stopTraining() { - auto guard = filePtr_->setCacheReadTemporarily(treeCache_.get(), tree_); - treeCache_->StopLearningPhase(); - rawTreeCache_.reset(); - } - void RootRNTuple::close() { // The TFile is about to be closed, and destructed. // Just to play it safe, zero all pointers to quantities that are owned by the TFile. - auxBranch_ = branchEntryInfoBranch_ = nullptr; - tree_ = metaTree_ = infoTree_ = nullptr; - // We own the treeCache_. - // We make sure the treeCache_ is detached from the file, - // so that ROOT does not also delete it. - filePtr_->clearCacheRead(tree_); - // We *must* delete the TTreeCache here because the TFilePrefetch object - // references the TFile. If TFile is closed, before the TTreeCache is - // deleted, the TFilePrefetch may continue to do TFile operations, causing - // deadlocks or exceptions. - treeCache_.reset(); - rawTreeCache_.reset(); - triggerTreeCache_.reset(); - rawTriggerTreeCache_.reset(); - auxCache_.reset(); + auxBranch_ = nullptr; + tree_ = nullptr; // We give up our shared ownership of the TFile itself. filePtr_.reset(); } - void RootRNTuple::trainCache(char const* branchNames) { - if (cacheSize_ == 0) { - return; - } - tree_->LoadTree(0); - assert(treeCache_); - { - auto guard = filePtr_->setCacheReadTemporarily(treeCache_.get(), tree_); - treeCache_->StartLearningPhase(); - treeCache_->SetEntryRange(0, tree_->GetEntries()); - treeCache_->AddBranch(branchNames, kTRUE); - treeCache_->StopLearningPhase(); - assert(treeCache_->GetTree() == tree_); - // We own the treeCache_. - // We make sure the treeCache_ is detached from the file, - // so that ROOT does not also delete it. - - //want guard to end here - } - - if (branchType_ == InEvent) { - // Must also manually add things to the trained set. - TObjArray* branches = tree_->GetListOfBranches(); - int branchCount = branches->GetEntriesFast(); - for (int i = 0; i < branchCount; i++) { - TBranch* branch = (TBranch*)branches->UncheckedAt(i); - if ((branchNames[0] == '*') || (strcmp(branchNames, branch->GetName()) == 0)) { - trainedSet_.insert(branch); - } - } - } - } - void RootRNTuple::setSignals( signalslot::Signal const* preEventReadSource, signalslot::Signal const* postEventReadSource) { @@ -608,20 +301,5 @@ namespace edm::rntuple_temp { } return n; } - - std::unique_ptr trainCache(TTree* tree, - InputFile& file, - unsigned int cacheSize, - char const* branchNames) { - tree->LoadTree(0); - std::unique_ptr treeCache = file.createCacheWithSize(*tree, cacheSize); - if (nullptr != treeCache.get()) { - treeCache->StartLearningPhase(); - treeCache->SetEntryRange(0, tree->GetEntries()); - treeCache->AddBranch(branchNames, kTRUE); - treeCache->StopLearningPhase(); - } - return treeCache; - } } // namespace rootrntuple } // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempInput/src/RootRNTuple.h b/FWIO/RNTupleTempInput/src/RootRNTuple.h index 8dc6e8b85ea76..b795fc210d695 100644 --- a/FWIO/RNTupleTempInput/src/RootRNTuple.h +++ b/FWIO/RNTupleTempInput/src/RootRNTuple.h @@ -28,7 +28,6 @@ RootRNTuple.h // used by ROOT input sources class TClass; class TTree; -class TTreeCache; namespace edm::rntuple_temp { class InputFile; @@ -40,8 +39,8 @@ namespace edm::rntuple_temp { unsigned int const defaultLearningEntries = 20U; unsigned int const defaultNonEventLearningEntries = 1U; using EntryNumber = IndexIntoFile::EntryNumber_t; - struct BranchInfo { - BranchInfo(ProductDescription const& prod) + struct ProductInfo { + ProductInfo(ProductDescription const& prod) : productDescription_(prod), productBranch_(nullptr), classCache_(nullptr), offsetToWrapperBase_(0) {} ProductDescription const productDescription_; void setBranch(TBranch* branch, TClass const* wrapperBaseTClass); @@ -54,14 +53,14 @@ namespace edm::rntuple_temp { Int_t offsetToWrapperBase_; }; - class BranchMap { + class ProductMap { public: - using Map = std::unordered_map; + using Map = std::unordered_map; void reserve(Map::size_type iSize) { map_.reserve(iSize); } - void insert(edm::BranchID const& iKey, BranchInfo const& iInfo) { map_.emplace(iKey.id(), iInfo); } - BranchInfo const* find(BranchID const& iKey) const { return find(iKey.id()); } - BranchInfo const* find(unsigned int iKey) const { + void insert(edm::BranchID const& iKey, ProductInfo const& iInfo) { map_.emplace(iKey.id(), iInfo); } + ProductInfo const* find(BranchID const& iKey) const { return find(iKey.id()); } + ProductInfo const* find(unsigned int iKey) const { auto itFound = map_.find(iKey); if (itFound == map_.end()) { return nullptr; @@ -80,15 +79,11 @@ namespace edm::rntuple_temp { Int_t getEntry(TBranch* branch, EntryNumber entryNumber); Int_t getEntry(TTree* tree, EntryNumber entryNumber); - std::unique_ptr trainCache(TTree* tree, - InputFile& file, - unsigned int cacheSize, - char const* branchNames); } // namespace rootrntuple class RootRNTuple { public: - using BranchMap = rootrntuple::BranchMap; + using ProductMap = rootrntuple::ProductMap; using EntryNumber = rootrntuple::EntryNumber; struct Options { unsigned int treeCacheSize = 0U; @@ -124,7 +119,7 @@ namespace edm::rntuple_temp { RootRNTuple& operator=(RootRNTuple const&) = delete; // Disallow copying and moving bool isValid() const; - void numberOfBranchesToAdd(BranchMap::Map::size_type iSize) { branches_.reserve(iSize); } + void numberOfBranchesToAdd(ProductMap::Map::size_type iSize) { branches_.reserve(iSize); } void addBranch(ProductDescription const& prod, std::string const& oldBranchName); void dropBranch(std::string const& oldBranchName); void getEntry(TBranch* branch, EntryNumber entry) const; @@ -150,8 +145,7 @@ namespace edm::rntuple_temp { template void fillAux(T*& pAux) { auxBranch_->SetAddress(&pAux); - auto cache = getAuxCache(auxBranch_); - getEntryUsingCache(auxBranch_, entryNumber_, cache); + getEntry(auxBranch_, entryNumber_); auxBranch_->SetAddress(nullptr); } @@ -164,13 +158,7 @@ namespace edm::rntuple_temp { template void fillBranchEntryMeta(TBranch* branch, EntryNumber entryNumber, T*& pbuf) { - if (metaTree_ != nullptr) { - // Metadata was in separate tree. Not cached. - branch->SetAddress(&pbuf); - rootrntuple::getEntry(branch, entryNumber); - } else { - fillBranchEntry(branch, entryNumber, pbuf); - } + fillBranchEntry(branch, entryNumber, pbuf); } template @@ -181,18 +169,7 @@ namespace edm::rntuple_temp { TTree const* tree() const { return tree_; } TTree* tree() { return tree_; } - TTree const* metaTree() const { return metaTree_; } - TTree* metaTree() { return metaTree_; } - BranchMap const& branches() const; - - //For backwards compatibility - TBranch* branchEntryInfoBranch() const { return branchEntryInfoBranch_; } - - inline TTreeCache* checkTriggerCache(TBranch* branch, EntryNumber entryNumber) const; - TTreeCache* checkTriggerCacheImpl(TBranch* branch, EntryNumber entryNumber) const; - inline TTreeCache* selectCache(TBranch* branch, EntryNumber entryNumber) const; - void trainCache(char const* branchNames); - void resetTraining() { trainNow_ = true; } + ProductMap const& branches() const; BranchType branchType() const { return branchType_; } std::string const& processName() const { return processName_; } @@ -210,56 +187,27 @@ namespace edm::rntuple_temp { bool promptRead, InputType inputType); - std::shared_ptr createCacheWithSize(unsigned int cacheSize) const; void setCacheSize(unsigned int cacheSize); void setTreeMaxVirtualSize(int treeMaxVirtualSize); - void startTraining(); - void stopTraining(); - void getEntryUsingCache(TBranch* branch, EntryNumber entry, TTreeCache*) const; - TTreeCache* getAuxCache(TBranch* auxBranch) const; std::shared_ptr filePtr_; // We use bare pointers for pointers to some ROOT entities. // Root owns them and uses bare pointers internally. // Therefore,using smart pointers here will do no good. TTree* tree_ = nullptr; - TTree* metaTree_ = nullptr; BranchType branchType_; std::string processName_; TBranch* auxBranch_ = nullptr; - // We use a smart pointer to own the TTreeCache. - // Unfortunately, ROOT owns it when attached to a TFile, but not after it is detached. - // So, we make sure to it is detached before closing the TFile so there is no double delete. - std::shared_ptr treeCache_; - std::shared_ptr rawTreeCache_; - CMS_SA_ALLOW mutable std::shared_ptr auxCache_; - //All access to a ROOT file is serialized - CMS_SA_ALLOW mutable std::shared_ptr triggerTreeCache_; - CMS_SA_ALLOW mutable std::shared_ptr rawTriggerTreeCache_; - CMS_SA_ALLOW mutable std::unordered_set trainedSet_; - CMS_SA_ALLOW mutable std::unordered_set triggerSet_; EntryNumber entries_ = 0; EntryNumber entryNumber_ = IndexIntoFile::invalidEntry; std::unique_ptr > entryNumberForIndex_; std::vector branchNames_; - BranchMap branches_; - bool trainNow_ = false; - EntryNumber switchOverEntry_ = -1; - CMS_SA_ALLOW mutable EntryNumber rawTriggerSwitchOverEntry_ = -1; - CMS_SA_ALLOW mutable bool performedSwitchOver_ = false; - unsigned int learningEntries_; + ProductMap branches_; unsigned int cacheSize_ = 0; unsigned long treeAutoFlush_ = 0; - // Enable asynchronous I/O in ROOT (done in a separate thread). Only takes - // effect on the primary treeCache_; all other caches have this explicitly disabled. bool enablePrefetching_; - bool enableTriggerCache_; bool promptRead_; std::unique_ptr rootDelayedReader_; - - TBranch* branchEntryInfoBranch_ = nullptr; //backwards compatibility - // below for backward compatibility - TTree* infoTree_ = nullptr; // backward compatibility }; } // namespace edm::rntuple_temp #endif diff --git a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc index 232f13f5286e7..4839ae009ecf4 100644 --- a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc +++ b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc @@ -117,7 +117,7 @@ namespace edm { bool produced) { auto field = ROOT::RFieldBase::Create(branchName, className).Unwrap(); model_->AddField(std::move(field)); - producedBranches_.push_back(model_->GetToken(branchName.c_str())); + producedBranches_.push_back(model_->GetToken(branchName)); producedBranchPointers_.push_back(const_cast(pProd)); } diff --git a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h index 7f9cb7da77f29..84537bc867bce 100644 --- a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h +++ b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h @@ -39,7 +39,7 @@ namespace edm { assert(model_); auto field = std::make_unique>(branchName); model_->AddField(std::move(field)); - auxBranches_.push_back(model_->GetToken(branchName.c_str())); + auxBranches_.push_back(model_->GetToken(branchName)); auxBranchPointers_.push_back(reinterpret_cast(const_cast(pAux))); } From 893970c7d6d86d218964bf6ac8d857aa96597ac3 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Wed, 1 Oct 2025 16:03:29 -0500 Subject: [PATCH 08/19] edmRNTupleTempFileUtil now complete --- FWIO/RNTupleTempInput/bin/CollUtil.cc | 205 ++++-------------- .../bin/EdmRNTupleTempFileUtil.cpp | 16 +- 2 files changed, 55 insertions(+), 166 deletions(-) diff --git a/FWIO/RNTupleTempInput/bin/CollUtil.cc b/FWIO/RNTupleTempInput/bin/CollUtil.cc index fde8b4078ff7e..80eb2798d0605 100644 --- a/FWIO/RNTupleTempInput/bin/CollUtil.cc +++ b/FWIO/RNTupleTempInput/bin/CollUtil.cc @@ -17,6 +17,7 @@ #include "TTree.h" #include "ROOT/RNTupleReader.hxx" #include "ROOT/RNTupleDescriptor.hxx" +#include "ROOT/RNTuple.hxx" #include #include @@ -52,13 +53,19 @@ namespace edm::rntuple_temp { // number of entries in a tree Long64_t numEntries(TFile *hdl, std::string const &trname) { - TTree *tree = (TTree *)hdl->Get(trname.c_str()); - if (tree) { - return tree->GetEntries(); + auto tuple = hdl->Get(trname.c_str()); + if (tuple) { + auto reader = ROOT::RNTupleReader::Open(*tuple); + return reader->GetNEntries(); } else { - std::cout << "ERR cannot find a TTree named \"" << trname << "\"" << std::endl; - return -1; + // Try as a TTree + TTree *tree = (TTree *)hdl->Get(trname.c_str()); + if (tree) { + return tree->GetEntries(); + } } + std::cout << "ERR cannot find a RNTuple named \"" << trname << "\"" << std::endl; + return -1; } namespace { @@ -344,75 +351,31 @@ namespace edm::rntuple_temp { void printUuids(ROOT::RNTupleReader *uuidTree) { std::cout << "UUID: " << getUuid(uuidTree) << std::endl; } - static void preIndexIntoFilePrintEventLists(TFile *, - FileFormatVersion const &fileFormatVersion, - TTree *metaDataTree) { - FileIndex fileIndex; - FileIndex *findexPtr = &fileIndex; - if (metaDataTree->FindBranch(poolNames::fileIndexBranchName().c_str()) != nullptr) { - TBranch *fndx = metaDataTree->GetBranch(poolNames::fileIndexBranchName().c_str()); - fndx->SetAddress(&findexPtr); - fndx->GetEntry(0); - } else { - std::cout << "FileIndex not found. If this input file was created with release 1_8_0 or later\n" - "this indicates a problem with the file. This condition should be expected with\n" - "files created with earlier releases and printout of the event list will fail.\n"; + static void postIndexIntoFilePrintEventLists(TFile *tfl, ROOT::RNTupleReader &metaDataReader) { + if (metaDataReader.GetModel().GetFieldNames().find(poolNames::indexIntoFileBranchName()) == + metaDataReader.GetModel().GetFieldNames().end()) { + std::cout << "IndexIntoFile not found this indicates a problem with the file."; return; } - - std::cout << "\n" << fileIndex; - - std::cout << "\nFileFormatVersion = " << fileFormatVersion << ". "; - if (fileFormatVersion.fastCopyPossible()) - std::cout << "This version supports fast copy\n"; - else - std::cout << "This version does not support fast copy\n"; - - if (fileIndex.allEventsInEntryOrder()) { - std::cout << "Events are sorted such that fast copy is possible in the \"noEventSort = False\" mode\n"; - } else { - std::cout << "Events are sorted such that fast copy is NOT possible in the \"noEventSort = False\" mode\n"; - } - - fileIndex.sortBy_Run_Lumi_EventEntry(); - if (fileIndex.allEventsInEntryOrder()) { - std::cout << "Events are sorted such that fast copy is possible in the \"noEventSort\" mode\n"; - } else { - std::cout << "Events are sorted such that fast copy is NOT possible in the \"noEventSort\" mode\n"; - } - std::cout << "(Note that other factors can prevent fast copy from occurring)\n\n"; - } - - static void postIndexIntoFilePrintEventLists(TFile *tfl, - FileFormatVersion const &fileFormatVersion, - TTree *metaDataTree) { IndexIntoFile indexIntoFile; - IndexIntoFile *findexPtr = &indexIntoFile; - if (metaDataTree->FindBranch(poolNames::indexIntoFileBranchName().c_str()) != nullptr) { - TBranch *fndx = metaDataTree->GetBranch(poolNames::indexIntoFileBranchName().c_str()); - fndx->SetAddress(&findexPtr); - fndx->GetEntry(0); - } else { - std::cout << "IndexIntoFile not found. If this input file was created with release 1_8_0 or later\n" - "this indicates a problem with the file. This condition should be expected with\n" - "files created with earlier releases and printout of the event list will fail.\n"; - return; - } + auto metaDataEntry = metaDataReader.GetModel().CreateEntry(); + metaDataEntry->BindRawPtr(poolNames::indexIntoFileBranchName(), &indexIntoFile); + metaDataReader.LoadEntry(0, *metaDataEntry); + //need to read event # from the EventAuxiliary branch - TTree *eventsTree = dynamic_cast(tfl->Get(poolNames::eventTreeName().c_str())); - TBranch *eventAuxBranch = nullptr; - assert(nullptr != eventsTree); - char const *const kEventAuxiliaryBranchName = "EventAuxiliary"; - if (eventsTree->FindBranch(kEventAuxiliaryBranchName) != nullptr) { - eventAuxBranch = eventsTree->GetBranch(kEventAuxiliaryBranchName); - } else { - std::cout << "Failed to find " << kEventAuxiliaryBranchName - << " branch in Events TTree. Something is wrong with this file." << std::endl; + auto *eventsTuple = tfl->Get(poolNames::eventTreeName().c_str()); + assert(nullptr != eventsTuple); + auto eventsReader = ROOT::RNTupleReader::Open(*eventsTuple); + if (eventsReader->GetModel().GetFieldNames().find("EventAuxiliary") == + eventsReader->GetModel().GetFieldNames().end()) { + std::cout << "Failed to find EventAuxiliary Field in Events RNTuple. Something is wrong with this file." + << std::endl; return; } + EventAuxiliary eventAuxiliary; - EventAuxiliary *eAPtr = &eventAuxiliary; - eventAuxBranch->SetAddress(&eAPtr); + auto eventAuxEntry = eventsReader->GetModel().CreateEntry(); + eventAuxEntry->BindRawPtr("EventAuxiliary", &eventAuxiliary); std::cout << "\nPrinting IndexIntoFile contents. This includes a list of all Runs, LuminosityBlocks\n" << "and Events stored in the root file.\n\n"; std::cout << std::setw(15) << "Run" << std::setw(15) << "Lumi" << std::setw(15) << "Event" << std::setw(15) @@ -435,7 +398,7 @@ namespace edm::rntuple_temp { type = "(Lumi)"; break; case IndexIntoFile::kEvent: - eventAuxBranch->GetEntry(it.entry()); + eventsReader->LoadEntry(it.entry(), *eventAuxEntry); eventNum = eventAuxiliary.id().event(); break; default: @@ -444,12 +407,6 @@ namespace edm::rntuple_temp { std::cout << std::setw(15) << eventNum << std::setw(15) << it.entry() << " " << type << std::endl; } - std::cout << "\nFileFormatVersion = " << fileFormatVersion << ". "; - if (fileFormatVersion.fastCopyPossible()) - std::cout << "This version supports fast copy\n"; - else - std::cout << "This version does not support fast copy\n"; - if (indexIntoFile.iterationWillBeInEntryOrder(IndexIntoFile::firstAppearanceOrder)) { std::cout << "Events are sorted such that fast copy is possible in the \"noEventSort = false\" mode\n"; } else { @@ -467,79 +424,21 @@ namespace edm::rntuple_temp { } void printEventLists(TFile *tfl) { - TTree *metaDataTree = dynamic_cast(tfl->Get(poolNames::metaDataTreeName().c_str())); - assert(nullptr != metaDataTree); - - FileFormatVersion fileFormatVersion; - FileFormatVersion *fftPtr = &fileFormatVersion; - if (metaDataTree->FindBranch(poolNames::fileFormatVersionBranchName().c_str()) != nullptr) { - TBranch *fft = metaDataTree->GetBranch(poolNames::fileFormatVersionBranchName().c_str()); - fft->SetAddress(&fftPtr); - fft->GetEntry(0); - } - if (fileFormatVersion.hasIndexIntoFile()) { - postIndexIntoFilePrintEventLists(tfl, fileFormatVersion, metaDataTree); - } else { - preIndexIntoFilePrintEventLists(tfl, fileFormatVersion, metaDataTree); - } - } + auto metaDataTuple = tfl->Get(edm::poolNames::metaDataTreeName().c_str()); + assert(nullptr != metaDataTuple); + auto reader = ROOT::RNTupleReader::Open(*metaDataTuple); - static void preIndexIntoFilePrintEventsInLumis(TFile *, - FileFormatVersion const &fileFormatVersion, - TTree *metaDataTree) { - FileIndex fileIndex; - FileIndex *findexPtr = &fileIndex; - if (metaDataTree->FindBranch(poolNames::fileIndexBranchName().c_str()) != nullptr) { - TBranch *fndx = metaDataTree->GetBranch(poolNames::fileIndexBranchName().c_str()); - fndx->SetAddress(&findexPtr); - fndx->GetEntry(0); - } else { - std::cout << "FileIndex not found. If this input file was created with release 1_8_0 or later\n" - "this indicates a problem with the file. This condition should be expected with\n" - "files created with earlier releases and printout of the event list will fail.\n"; - return; - } - - std::cout << "\n" - << std::setw(15) << "Run" << std::setw(15) << "Lumi" << std::setw(15) << "# Events" - << "\n"; - unsigned long nEvents = 0; - unsigned long runID = 0; - unsigned long lumiID = 0; - for (std::vector::const_iterator it = fileIndex.begin(), itEnd = fileIndex.end(); it != itEnd; - ++it) { - if (it->getEntryType() == FileIndex::kEvent) { - ++nEvents; - } else if (it->getEntryType() == FileIndex::kLumi) { - if (runID != it->run_ || lumiID != it->lumi_) { - //print the previous one - if (lumiID != 0) { - std::cout << std::setw(15) << runID << std::setw(15) << lumiID << std::setw(15) << nEvents << "\n"; - } - nEvents = 0; - runID = it->run_; - lumiID = it->lumi_; - } - } - } - //print the last one - if (lumiID != 0) { - std::cout << std::setw(15) << runID << std::setw(15) << lumiID << std::setw(15) << nEvents << "\n"; - } - - std::cout << "\n"; + postIndexIntoFilePrintEventLists(tfl, *reader); } - static void postIndexIntoFilePrintEventsInLumis(TFile *tfl, - FileFormatVersion const &fileFormatVersion, - TTree *metaDataTree) { + static void postIndexIntoFilePrintEventsInLumis(ROOT::RNTupleReader &metaDataReader) { IndexIntoFile indexIntoFile; - IndexIntoFile *findexPtr = &indexIntoFile; - if (metaDataTree->FindBranch(poolNames::indexIntoFileBranchName().c_str()) != nullptr) { - TBranch *fndx = metaDataTree->GetBranch(poolNames::indexIntoFileBranchName().c_str()); - fndx->SetAddress(&findexPtr); - fndx->GetEntry(0); - } else { + auto metaDataEntry = metaDataReader.GetModel().CreateEntry(); + metaDataEntry->BindRawPtr(poolNames::indexIntoFileBranchName(), &indexIntoFile); + metaDataReader.LoadEntry(0, *metaDataEntry); + + if (metaDataReader.GetModel().GetFieldNames().find(poolNames::indexIntoFileBranchName()) == + metaDataReader.GetModel().GetFieldNames().end()) { std::cout << "IndexIntoFile not found. If this input file was created with release 1_8_0 or later\n" "this indicates a problem with the file. This condition should be expected with\n" "files created with earlier releases and printout of the event list will fail.\n"; @@ -587,20 +486,10 @@ namespace edm::rntuple_temp { } void printEventsInLumis(TFile *tfl) { - TTree *metaDataTree = dynamic_cast(tfl->Get(poolNames::metaDataTreeName().c_str())); - assert(nullptr != metaDataTree); - - FileFormatVersion fileFormatVersion; - FileFormatVersion *fftPtr = &fileFormatVersion; - if (metaDataTree->FindBranch(poolNames::fileFormatVersionBranchName().c_str()) != nullptr) { - TBranch *fft = metaDataTree->GetBranch(poolNames::fileFormatVersionBranchName().c_str()); - fft->SetAddress(&fftPtr); - fft->GetEntry(0); - } - if (fileFormatVersion.hasIndexIntoFile()) { - postIndexIntoFilePrintEventsInLumis(tfl, fileFormatVersion, metaDataTree); - } else { - preIndexIntoFilePrintEventsInLumis(tfl, fileFormatVersion, metaDataTree); - } + auto metaDataTuple = tfl->Get(edm::poolNames::metaDataTreeName().c_str()); + assert(nullptr != metaDataTuple); + auto reader = ROOT::RNTupleReader::Open(*metaDataTuple); + + postIndexIntoFilePrintEventsInLumis(*reader); } } // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempInput/bin/EdmRNTupleTempFileUtil.cpp b/FWIO/RNTupleTempInput/bin/EdmRNTupleTempFileUtil.cpp index f5c7820306553..9fdb8c4ad2f50 100644 --- a/FWIO/RNTupleTempInput/bin/EdmRNTupleTempFileUtil.cpp +++ b/FWIO/RNTupleTempInput/bin/EdmRNTupleTempFileUtil.cpp @@ -58,9 +58,9 @@ int main(int argc, char* argv[]) { "eventsInLumis", "Print how many Events are in each LuminosityBlock."); // What trees do we require for this to be a valid collection? - std::vector expectedTrees; - expectedTrees.push_back(edm::poolNames::metaDataTreeName()); - expectedTrees.push_back(edm::poolNames::eventTreeName()); + std::vector expectedRNTuples; + expectedRNTuples.push_back(edm::poolNames::metaDataTreeName()); + expectedRNTuples.push_back(edm::poolNames::eventTreeName()); boost::program_options::positional_options_description p; p.add("file", -1); @@ -187,20 +187,20 @@ int main(int argc, char* argv[]) { } // Ok. Do we have the expected trees? - for (unsigned int i = 0; i < expectedTrees.size(); ++i) { - TTree* t = (TTree*)tfile->Get(expectedTrees[i].c_str()); + for (unsigned int i = 0; i < expectedRNTuples.size(); ++i) { + auto* t = tfile->Get(expectedRNTuples[i].c_str()); if (t == nullptr) { - std::cout << "Tree " << expectedTrees[i] << " appears to be missing. Not a valid collection\n"; + std::cout << "RNTuple " << expectedRNTuples[i] << " appears to be missing. Not a valid collection\n"; std::cout << "Exiting\n"; return 1; } else { if (verbose) - std::cout << "ECU:: Found Tree " << expectedTrees[i] << std::endl; + std::cout << "ECU:: Found RNTuple " << expectedRNTuples[i] << std::endl; } } if (verbose) - std::cout << "ECU:: Found all expected trees\n"; + std::cout << "ECU:: Found all expected RNTuples\n"; std::ostringstream auout; if (adler32) { From fc0d434cf1c13f490e21b0d2d89a29d52807d0cf Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Thu, 2 Oct 2025 08:55:57 -0500 Subject: [PATCH 09/19] Updated edmRNTupleTempProvDump to only use RNTuple Removed all references to TTree used during the transition. --- .../bin/EdmRNTupleTempProvDump.cc | 231 ++++++++++-------- 1 file changed, 127 insertions(+), 104 deletions(-) diff --git a/FWIO/RNTupleTempInput/bin/EdmRNTupleTempProvDump.cc b/FWIO/RNTupleTempInput/bin/EdmRNTupleTempProvDump.cc index 0b5a187954431..2fc79213d5519 100644 --- a/FWIO/RNTupleTempInput/bin/EdmRNTupleTempProvDump.cc +++ b/FWIO/RNTupleTempInput/bin/EdmRNTupleTempProvDump.cc @@ -23,7 +23,6 @@ #include "TError.h" #include "TFile.h" -#include "TTree.h" #include "ROOT/RNTuple.hxx" #include "ROOT/RNTupleReader.hxx" @@ -440,31 +439,31 @@ void ProvenanceDumper::dumpEventFilteringParameterSets(edm::EventSelectionIDVect } void ProvenanceDumper::dumpEventFilteringParameterSets_(TFile* file, ProcessSimpleIDsType const& processSimpleIDs) { - TTree* history = dynamic_cast(file->Get(edm::poolNames::eventHistoryTreeName().c_str())); - if (history != nullptr) { - edm::History h; - edm::History* ph = &h; - - history->SetBranchAddress(edm::poolNames::eventHistoryBranchName().c_str(), &ph); - if (history->GetEntry(0) <= 0) { - std::cout << "No event filtering information is available; the event history tree has no entries\n"; - } else { - dumpEventFilteringParameterSets(h.eventSelectionIDs(), processSimpleIDs); - } - } else { - TTree* events = dynamic_cast(file->Get(edm::poolNames::eventTreeName().c_str())); - assert(events != nullptr); - TBranch* eventSelectionsBranch = events->GetBranch(edm::poolNames::eventSelectionsBranchName().c_str()); - if (eventSelectionsBranch == nullptr) - return; + auto eventsRNTuple = file->Get(edm::poolNames::eventTreeName().c_str()); + assert(eventsRNTuple != nullptr); + auto eventsReader = ROOT::RNTupleReader::Open(*eventsRNTuple); + + if (eventsReader->GetNEntries() == 0) { + std::cout << "No event filtering information is available; the event selections branch has no entries\n"; + return; + } + + if (eventsReader->GetModel().GetFieldNames().find(edm::poolNames::eventSelectionsBranchName()) == + eventsReader->GetModel().GetFieldNames().end()) { + std::cout << "No event filtering information is available; the event selections field is not present\n"; + return; + } + + try { + auto entry = eventsReader->GetModel().CreateEntry(); edm::EventSelectionIDVector ids; - edm::EventSelectionIDVector* pids = &ids; - eventSelectionsBranch->SetAddress(&pids); - if (eventSelectionsBranch->GetEntry(0) <= 0) { - std::cout << "No event filtering information is available; the event selections branch has no entries\n"; - } else { - dumpEventFilteringParameterSets(ids, processSimpleIDs); - } + entry->BindRawPtr(edm::poolNames::eventSelectionsBranchName(), &ids); + eventsReader->LoadEntry(0, *entry); + dumpEventFilteringParameterSets(ids, processSimpleIDs); + } catch (ROOT::RException const& iExcept) { + std::cout << "The following exception occured while reading the event filtering information:\n" + << iExcept.what() << std::endl; + return; } } @@ -681,49 +680,76 @@ ProvenanceDumper::makeBranchIDListHelper() { return {}; } - TTree* metaTree = dynamic_cast(inputFile_->Get(edm::poolNames::metaDataTreeName().c_str())); - if (nullptr == metaTree) { - //std::cerr << "Did not find " << edm::poolNames::metaDataTreeName() << " tree" << std::endl; + // Read BranchIDLists from metadata RNTuple + auto metaRNTuple = inputFile_->Get(edm::poolNames::metaDataTreeName().c_str()); + if (nullptr == metaRNTuple) { + std::cerr << "Did not find " << edm::poolNames::metaDataTreeName() << " RNTuple" << std::endl; return {}; } - TBranch* branchIDListsBranch = metaTree->GetBranch(edm::poolNames::branchIDListBranchName().c_str()); - if (nullptr == branchIDListsBranch) { - /* + auto metaReader = ROOT::RNTupleReader::Open(*metaRNTuple); + if (metaReader->GetNEntries() == 0) { + std::cerr << "No entries in " << edm::poolNames::metaDataTreeName() << " RNTuple" << std::endl; + return {}; + } + + // Check if the required field exists in the metadata RNTuple + auto metaFieldNames = metaReader->GetModel().GetFieldNames(); + if (metaFieldNames.find(edm::poolNames::branchIDListBranchName()) == metaFieldNames.end()) { std::cerr << "Did not find " << edm::poolNames::branchIDListBranchName() << " from " - << edm::poolNames::metaDataTreeName() << " tree" << std::endl; - */ + << edm::poolNames::metaDataTreeName() << " RNTuple" << std::endl; return {}; } edm::BranchIDLists branchIDLists; - edm::BranchIDLists* branchIDListsPtr = &branchIDLists; - branchIDListsBranch->SetAddress(&branchIDListsPtr); - if (branchIDListsBranch->GetEntry(0) <= 0) { - //std::cerr << "Failed to read an entry from " << edm::poolNames::branchIDListBranchName() << std::endl; + try { + auto metaEntry = metaReader->GetModel().CreateEntry(); + metaEntry->BindRawPtr(edm::poolNames::branchIDListBranchName(), &branchIDLists); + metaReader->LoadEntry(0, *metaEntry); + } catch (ROOT::RException const& iExcept) { + std::cerr << "The following exception occurred while reading " << edm::poolNames::branchIDListBranchName() + << " from " << edm::poolNames::metaDataTreeName() << " RNTuple:\n" + << iExcept.what() << std::endl; return {}; } edm::BranchIDListHelper branchIDListHelper; branchIDListHelper.updateFromInput(branchIDLists); - TTree* events = dynamic_cast(inputFile_->Get(edm::poolNames::eventTreeName().c_str())); - assert(events != nullptr); - TBranch* branchListIndexesBranch = events->GetBranch(edm::poolNames::branchListIndexesBranchName().c_str()); - if (nullptr == branchListIndexesBranch) { - /* + // Read BranchListIndexes from events RNTuple + auto eventsRNTuple = inputFile_->Get(edm::poolNames::eventTreeName().c_str()); + assert(eventsRNTuple != nullptr); + auto eventsReader = ROOT::RNTupleReader::Open(*eventsRNTuple); + + if (eventsReader->GetNEntries() <= static_cast(productIDEntry_)) { + std::cerr << "Requested entry " << productIDEntry_ << " but only " << eventsReader->GetNEntries() << " entries in " + << edm::poolNames::eventTreeName() << " RNTuple" << std::endl; + return {}; + } + + // Check if the required field exists in the events RNTuple + auto eventsFieldNames = eventsReader->GetModel().GetFieldNames(); + if (eventsFieldNames.find(edm::poolNames::branchListIndexesBranchName()) == eventsFieldNames.end()) { std::cerr << "Did not find " << edm::poolNames::branchListIndexesBranchName() << " from " - << edm::poolNames::eventTreeName() << " tree" << std::endl; - */ + << edm::poolNames::eventTreeName() << " RNTuple" << std::endl; return {}; } + edm::BranchListIndexes branchListIndexes; - edm::BranchListIndexes* pbranchListIndexes = &branchListIndexes; - branchListIndexesBranch->SetAddress(&pbranchListIndexes); - if (branchListIndexesBranch->GetEntry(productIDEntry_) <= 0 or branchListIndexes.empty()) { + try { + auto eventsEntry = eventsReader->GetModel().CreateEntry(); + eventsEntry->BindRawPtr(edm::poolNames::branchListIndexesBranchName(), &branchListIndexes); + eventsReader->LoadEntry(productIDEntry_, *eventsEntry); + } catch (ROOT::RException const& iExcept) { + std::cerr << "The following exception occurred while reading " << edm::poolNames::branchListIndexesBranchName() + << " from " << edm::poolNames::eventTreeName() << " RNTuple:\n" + << iExcept.what() << std::endl; + return {}; + } + + if (branchListIndexes.empty()) { /* - std::cerr << "Failed to read entry from " << edm::poolNames::branchListIndexesBranchName() << ", or it is empty" - << std::endl; + std::cerr << "BranchListIndexes is empty" << std::endl; */ return {}; } @@ -741,7 +767,6 @@ ProvenanceDumper::makeBranchIDListHelper() { void ProvenanceDumper::work_() { auto metaRNTuple = inputFile_->Get(edm::poolNames::metaDataTreeName().c_str()); - //TTree* meta = dynamic_cast(inputFile_->Get(edm::poolNames::metaDataTreeName().c_str())); assert(metaRNTuple); auto reader = ROOT::RNTupleReader::Open(*metaRNTuple); @@ -755,7 +780,7 @@ void ProvenanceDumper::work_() { assert(rntuple); auto psets = ROOT::RNTupleReader::Open(*rntuple); assert(psets.get()); - auto entry = psets->GetModel().CreateBareEntry(); + auto entry = psets->GetModel().CreateEntry(); std::pair idToBlob; entry->BindRawPtr("IdToParameterSetsBlobs", &idToBlob); @@ -817,74 +842,72 @@ void ProvenanceDumper::work_() { std::map> perProductParentage; if (showDependencies_ || extendedAncestors_ || extendedDescendants_) { - TTree* parentageTree = dynamic_cast(inputFile_->Get(edm::poolNames::parentageTreeName().c_str())); - if (nullptr == parentageTree) { - std::cerr << "ERROR, no Parentage tree available so cannot show dependencies, ancestors, or descendants.\n"; + auto parentageRNTuple = inputFile_->Get(edm::poolNames::parentageTreeName().c_str()); + if (nullptr == parentageRNTuple) { + std::cerr << "ERROR, no Parentage RNTuple available so cannot show dependencies, ancestors, or descendants.\n"; std::cerr << "Possibly this is not a standard EDM format file. For example, dependency, ancestor, and\n"; std::cerr << "descendant options to edmProvDump will not work with nanoAOD format files.\n\n"; showDependencies_ = false; extendedAncestors_ = false; extendedDescendants_ = false; } else { - edm::ParentageRegistry& registry = *edm::ParentageRegistry::instance(); + auto parentageReader = ROOT::RNTupleReader::Open(*parentageRNTuple); - std::vector orderedParentageIDs; - orderedParentageIDs.reserve(parentageTree->GetEntries()); - for (Long64_t i = 0, numEntries = parentageTree->GetEntries(); i < numEntries; ++i) { - edm::Parentage parentageBuffer; - edm::Parentage* pParentageBuffer = &parentageBuffer; - parentageTree->SetBranchAddress(edm::poolNames::parentageBranchName().c_str(), &pParentageBuffer); - parentageTree->GetEntry(i); - registry.insertMapped(parentageBuffer); - orderedParentageIDs.push_back(parentageBuffer.id()); - } - parentageTree->SetBranchAddress(edm::poolNames::parentageBranchName().c_str(), nullptr); - - TTree* eventMetaTree = - dynamic_cast(inputFile_->Get(edm::BranchTypeToMetaDataTreeName(edm::InEvent).c_str())); - if (nullptr == eventMetaTree) { - eventMetaTree = dynamic_cast(inputFile_->Get(edm::BranchTypeToProductTreeName(edm::InEvent).c_str())); - } - if (nullptr == eventMetaTree) { - std::cerr << "ERROR, no '" << edm::BranchTypeToProductTreeName(edm::InEvent) - << "' Tree in file so can not show dependencies\n"; + // Check if the required field exists in the parentage RNTuple + auto parentageFieldNames = parentageReader->GetModel().GetFieldNames(); + if (parentageFieldNames.find(edm::poolNames::parentageBranchName()) == parentageFieldNames.end()) { + std::cerr << "ERROR, no '" << edm::poolNames::parentageBranchName() + << "' field in Parentage RNTuple so cannot show dependencies\n"; showDependencies_ = false; extendedAncestors_ = false; extendedDescendants_ = false; } else { - TBranch* storedProvBranch = - eventMetaTree->GetBranch(edm::BranchTypeToProductProvenanceBranchName(edm::InEvent).c_str()); - - if (nullptr != storedProvBranch) { - std::vector info; - std::vector* pInfo = &info; - storedProvBranch->SetAddress(&pInfo); - for (Long64_t i = 0, numEntries = eventMetaTree->GetEntries(); i < numEntries; ++i) { - storedProvBranch->GetEntry(i); - for (auto const& item : info) { - edm::BranchID bid(item.branchID_); - perProductParentage[bid].insert(orderedParentageIDs.at(item.parentageIDIndex_)); - } - } + edm::ParentageRegistry& registry = *edm::ParentageRegistry::instance(); + + std::vector orderedParentageIDs; + orderedParentageIDs.reserve(parentageReader->GetNEntries()); + + auto parentageEntry = parentageReader->GetModel().CreateEntry(); + edm::Parentage parentageBuffer; + parentageEntry->BindRawPtr(edm::poolNames::parentageBranchName(), &parentageBuffer); + + for (ROOT::NTupleSize_t i = 0, numEntries = parentageReader->GetNEntries(); i < numEntries; ++i) { + parentageReader->LoadEntry(i, *parentageEntry); + registry.insertMapped(parentageBuffer); + orderedParentageIDs.push_back(parentageBuffer.id()); + } + + auto eventsRNTuple = inputFile_->Get(edm::BranchTypeToProductTreeName(edm::InEvent).c_str()); + if (nullptr == eventsRNTuple) { + std::cerr << "ERROR, no '" << edm::BranchTypeToProductTreeName(edm::InEvent) + << "' RNTuple in file so can not show dependencies\n"; + showDependencies_ = false; + extendedAncestors_ = false; + extendedDescendants_ = false; } else { - //backwards compatible check - TBranch* productProvBranch = - eventMetaTree->GetBranch(edm::BranchTypeToBranchEntryInfoBranchName(edm::InEvent).c_str()); - if (nullptr != productProvBranch) { - std::vector info; - std::vector* pInfo = &info; - productProvBranch->SetAddress(&pInfo); - for (Long64_t i = 0, numEntries = eventMetaTree->GetEntries(); i < numEntries; ++i) { - productProvBranch->GetEntry(i); - for (auto const& item : info) { - perProductParentage[item.branchID()].insert(item.parentageID()); - } - } - } else { - std::cerr << " ERROR, could not find provenance information so can not show dependencies\n"; + auto eventsReader = ROOT::RNTupleReader::Open(*eventsRNTuple); + + // Check if the required field exists in the events RNTuple + auto eventsFieldNames = eventsReader->GetModel().GetFieldNames(); + if (eventsFieldNames.find(edm::BranchTypeToProductProvenanceBranchName(edm::InEvent)) == + eventsFieldNames.end()) { + std::cerr << "ERROR, no '" << edm::BranchTypeToProductProvenanceBranchName(edm::InEvent) + << "' field in events RNTuple so cannot show dependencies\n"; showDependencies_ = false; extendedAncestors_ = false; extendedDescendants_ = false; + } else { + auto eventsEntry = eventsReader->GetModel().CreateEntry(); + std::vector info; + eventsEntry->BindRawPtr(edm::BranchTypeToProductProvenanceBranchName(edm::InEvent), &info); + + for (ROOT::NTupleSize_t i = 0, numEntries = eventsReader->GetNEntries(); i < numEntries; ++i) { + eventsReader->LoadEntry(i, *eventsEntry); + for (auto const& item : info) { + edm::BranchID bid(item.branchID_); + perProductParentage[bid].insert(orderedParentageIDs.at(item.parentageIDIndex_)); + } + } } } } From 556a6fbfa106fa37958b090e2c5f447cfdb22618 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Sat, 4 Oct 2025 08:08:12 -0500 Subject: [PATCH 10/19] Removed unneeded header file --- FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc b/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc index a48b171ed8884..2c93cb09a8fb8 100644 --- a/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc +++ b/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc @@ -20,7 +20,6 @@ #include "FWCore/Utilities/interface/TimeOfDay.h" #include "FWCore/Utilities/interface/WrappedClassName.h" -#include "TBranchElement.h" #include "TObjArray.h" #include "RVersion.h" #include "TDictAttributeMap.h" From 4ca6c132777e3d239359835fbf4c4438664819e6 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Sat, 4 Oct 2025 08:08:44 -0500 Subject: [PATCH 11/19] Move input to only use RNTuple --- .../RNTupleTempInput/src/RootDelayedReader.cc | 20 +-- FWIO/RNTupleTempInput/src/RootDelayedReader.h | 4 +- FWIO/RNTupleTempInput/src/RootFile.cc | 135 +++++--------- FWIO/RNTupleTempInput/src/RootFile.h | 6 +- .../src/RootPromptReadDelayedReader.cc | 27 +-- .../src/RootPromptReadDelayedReader.h | 7 +- FWIO/RNTupleTempInput/src/RootRNTuple.cc | 168 +++++++----------- FWIO/RNTupleTempInput/src/RootRNTuple.h | 80 +++++---- FWIO/RNTupleTempInput/test/BuildFile.xml | 1 - .../test/testNoParentDictionary.sh | 78 -------- FWIO/RNTupleTempOutput/test/BuildFile.xml | 1 - .../test/run_TestProcessBlock.sh | 20 +-- .../test/testLooperEventNavigation2_cfg.py | 2 +- .../test/testLooperEventNavigation3_cfg.py | 2 +- .../testProcessBlockMergeOfMergedFiles_cfg.py | 2 +- ...estProcessBlockNOMergeOfMergedFiles_cfg.py | 2 +- 16 files changed, 179 insertions(+), 376 deletions(-) delete mode 100755 FWIO/RNTupleTempInput/test/testNoParentDictionary.sh diff --git a/FWIO/RNTupleTempInput/src/RootDelayedReader.cc b/FWIO/RNTupleTempInput/src/RootDelayedReader.cc index beb3f38c1f755..a1080b6d85402 100644 --- a/FWIO/RNTupleTempInput/src/RootDelayedReader.cc +++ b/FWIO/RNTupleTempInput/src/RootDelayedReader.cc @@ -13,15 +13,16 @@ #include "FWCore/Utilities/interface/EDMException.h" -#include "TBranch.h" #include "TClass.h" #include namespace edm::rntuple_temp { - RootDelayedReader::RootDelayedReader(RootRNTuple const& tree, std::shared_ptr filePtr, InputType inputType) - : tree_(tree), filePtr_(filePtr), nextReader_(), inputType_(inputType) { + RootDelayedReader::RootDelayedReader(RootRNTuple const& tuple, + std::shared_ptr filePtr, + InputType inputType) + : rntuple_(tuple), filePtr_(filePtr), nextReader_(), inputType_(inputType) { if (inputType == InputType::Primary) { auto resources = SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader(); resourceAcquirer_ = std::make_unique(std::move(resources.first)); @@ -60,8 +61,7 @@ namespace edm::rntuple_temp { return std::shared_ptr(); } } - TBranch* br = branchInfo->productBranch_; - if (br == nullptr) { + if (not branchInfo->valid()) { if (nextReader_) { return nextReader_->getProduct(k, ep); } else { @@ -71,19 +71,19 @@ namespace edm::rntuple_temp { RefCoreStreamerGuard guard(ep); std::unique_ptr edp = branchInfo->newWrapper(); - void* edpPtr = edp.get(); - branchInfo->productBranch_->SetAddress(&edpPtr); + branchInfo->view().BindRawPtr(edp.get()); try { //Run, Lumi, and ProcessBlock only have 1 entry number, which is index 0 - tree_.getEntry(br, tree_.entryNumberForIndex(tree_.branchType() == InEvent ? ep->transitionIndex() : 0)); + rntuple_.getEntry(branchInfo->view(), + rntuple_.entryNumberForIndex(rntuple_.branchType() == InEvent ? ep->transitionIndex() : 0)); } catch (...) { lastException_ = std::current_exception(); std::rethrow_exception(lastException_); } - if (tree_.branchType() == InEvent) { + if (rntuple_.branchType() == InEvent) { // CMS-THREADING For the primary input source calls to this function need to be serialized - InputFile::reportReadBranch(inputType_, std::string(br->GetName())); + InputFile::reportReadBranch(inputType_, std::string(branchInfo->productDescription().branchName())); } return edp; } diff --git a/FWIO/RNTupleTempInput/src/RootDelayedReader.h b/FWIO/RNTupleTempInput/src/RootDelayedReader.h index 67e016135d81a..377a6ae44d78d 100644 --- a/FWIO/RNTupleTempInput/src/RootDelayedReader.h +++ b/FWIO/RNTupleTempInput/src/RootDelayedReader.h @@ -52,11 +52,11 @@ namespace edm::rntuple_temp { void reset_() override { nextReader_ = nullptr; } std::pair sharedResources_() const override; - ProductMap const& branches() const { return tree_.branches(); } + ProductMap const& branches() const { return rntuple_.branches(); } ProductInfo const* getProductInfo(BranchID const& k) const { return branches().find(k); } // NOTE: filePtr_ appears to be unused, but is needed to prevent // the file containing the branch from being reclaimed. - RootRNTuple const& tree_; + RootRNTuple const& rntuple_; edm::propagate_const> filePtr_; edm::propagate_const nextReader_; std::unique_ptr diff --git a/FWIO/RNTupleTempInput/src/RootFile.cc b/FWIO/RNTupleTempInput/src/RootFile.cc index edb1f043ee2fa..165ce64356af9 100644 --- a/FWIO/RNTupleTempInput/src/RootFile.cc +++ b/FWIO/RNTupleTempInput/src/RootFile.cc @@ -76,6 +76,14 @@ #include namespace edm::rntuple_temp { + namespace { + std::string fixName(std::string_view iName) { + if (not iName.empty() and iName.back() == '.') { + iName.remove_suffix(1); + } + return std::string(iName); + } + } // namespace // Algorithm classes for making ProvenanceReader: class MakeReducedProvenanceReader : public MakeProvenanceReader { @@ -186,10 +194,9 @@ namespace edm::rntuple_temp { processingMode_(processingOptions.processingMode), runHelper_(crossFileInfo.runHelper), newBranchToOldBranch_(), - eventToProcessBlockIndexesBranch_( - inputType == InputType::Primary - ? eventTree_.tree()->GetBranch(poolNames::eventToProcessBlockIndexesBranchName().c_str()) - : nullptr), + eventToProcessBlockIndexesView_(inputType == InputType::Primary + ? eventTree_.view(poolNames::eventToProcessBlockIndexesBranchName()) + : std::optional>()), productDependencies_(new ProductDependencies), duplicateChecker_(crossFileInfo.duplicateChecker), provenanceReaderMaker_(), @@ -491,11 +498,11 @@ namespace edm::rntuple_temp { auto it = std::find(processes.begin(), processes.end(), prod.processName()); if (it != processes.end()) { auto index = std::distance(processes.begin(), it); - treePointers_[numberOfRunLumiEventProductTrees + index]->addBranch(prod, - newBranchToOldBranch(prod.branchName())); + treePointers_[numberOfRunLumiEventProductTrees + index]->addBranch( + prod, fixName(newBranchToOldBranch(prod.branchName()))); } } else { - treePointers_[prod.branchType()]->addBranch(prod, newBranchToOldBranch(prod.branchName())); + treePointers_[prod.branchType()]->addBranch(prod, fixName(newBranchToOldBranch(prod.branchName()))); } } @@ -506,51 +513,6 @@ namespace edm::rntuple_temp { RootFile::~RootFile() {} bool RootFile::empty() const { return runTree_.entries() == 0; } - void RootFile::readEntryDescriptionTree(EntryDescriptionMap& entryDescriptionMap, InputType inputType) { - // Called only for old format files. - // We use a smart pointer so the tree will be deleted after use, and not kept for the life of the file. - std::unique_ptr entryDescriptionTree( - dynamic_cast(filePtr_->Get(poolNames::entryDescriptionTreeName().c_str()))); - if (nullptr == entryDescriptionTree.get()) { - throw Exception(errors::FileReadError) - << "Could not find tree " << poolNames::entryDescriptionTreeName() << " in the input file.\n"; - } - - EntryDescriptionID idBuffer; - EntryDescriptionID* pidBuffer = &idBuffer; - entryDescriptionTree->SetBranchAddress(poolNames::entryDescriptionIDBranchName().c_str(), &pidBuffer); - - EventEntryDescription entryDescriptionBuffer; - EventEntryDescription* pEntryDescriptionBuffer = &entryDescriptionBuffer; - entryDescriptionTree->SetBranchAddress(poolNames::entryDescriptionBranchName().c_str(), &pEntryDescriptionBuffer); - - // Fill in the parentage registry. - ParentageRegistry& registry = *ParentageRegistry::instance(); - - for (Long64_t i = 0, numEntries = entryDescriptionTree->GetEntries(); i < numEntries; ++i) { - rootrntuple::getEntry(entryDescriptionTree.get(), i); - if (idBuffer != entryDescriptionBuffer.id()) { - throw Exception(errors::EventCorruption) << "Corruption of EntryDescription tree detected.\n"; - } - entryDescriptionMap.insert(std::make_pair(entryDescriptionBuffer.id(), entryDescriptionBuffer)); - Parentage parents; - parents.setParents(entryDescriptionBuffer.parents()); - if (daqProvenanceHelper_) { - ParentageID const oldID = parents.id(); - daqProvenanceHelper_->fixMetaData(parents.parentsForUpdate()); - ParentageID newID = parents.id(); - if (newID != oldID) { - daqProvenanceHelper_->setOldParentageIDToNew(oldID, newID); - } - } - // For thread safety, don't update global registries when a secondary source opens a file. - if (inputType != InputType::SecondarySource) { - registry.insertMapped(parents); - } - } - entryDescriptionTree->SetBranchAddress(poolNames::entryDescriptionIDBranchName().c_str(), nullptr); - entryDescriptionTree->SetBranchAddress(poolNames::entryDescriptionBranchName().c_str(), nullptr); - } void RootFile::readParentageTree(InputType inputType) { // New format file @@ -593,16 +555,12 @@ namespace edm::rntuple_temp { std::vector processesWithProcessBlockTrees; processBlockTrees.reserve(processBlockTrees_.size()); processesWithProcessBlockTrees.reserve(processBlockTrees_.size()); - for (auto& processBlockTree : processBlockTrees_) { - processBlockTrees.push_back(processBlockTree->tree()); - processesWithProcessBlockTrees.push_back(processBlockTree->processName()); - } return std::make_shared(fileFormatVersion(), - eventTree_.tree(), nullptr, - lumiTree_.tree(), nullptr, - runTree_.tree(), + nullptr, + nullptr, + nullptr, nullptr, std::move(processBlockTrees), std::move(processesWithProcessBlockTrees), @@ -617,17 +575,11 @@ namespace edm::rntuple_temp { void RootFile::updateFileBlock(FileBlock& fileBlock) { std::vector processBlockTrees; std::vector processesWithProcessBlockTrees; - processBlockTrees.reserve(processBlockTrees_.size()); - processesWithProcessBlockTrees.reserve(processBlockTrees_.size()); - for (auto& processBlockTree : processBlockTrees_) { - processBlockTrees.push_back(processBlockTree->tree()); - processesWithProcessBlockTrees.push_back(processBlockTree->processName()); - } - fileBlock.updateTTreePointers(eventTree_.tree(), + fileBlock.updateTTreePointers(nullptr, + nullptr, + nullptr, nullptr, - lumiTree_.tree(), nullptr, - runTree_.tree(), nullptr, std::move(processBlockTrees), std::move(processesWithProcessBlockTrees)); @@ -1143,8 +1095,7 @@ namespace edm::rntuple_temp { } void RootFile::fillEventToProcessBlockIndexes() { - TBranch* eventToProcessBlockIndexesBranch = get_underlying_safe(eventToProcessBlockIndexesBranch_); - if (eventToProcessBlockIndexesBranch == nullptr) { + if (not eventToProcessBlockIndexesView_.has_value()) { if (processBlockHelper_.get() == nullptr) { eventToProcessBlockIndexes_.setIndex(0); } else { @@ -1154,8 +1105,8 @@ namespace edm::rntuple_temp { if (processBlockHelper_->cacheIndexVectorsPerFile().back() == 1u) { eventToProcessBlockIndexes_.setIndex(processBlockHelper_->outerOffset()); } else { - EventToProcessBlockIndexes* pEventToProcessBlockIndexes = &eventToProcessBlockIndexes_; - eventTree_.fillBranchEntry(eventToProcessBlockIndexesBranch, pEventToProcessBlockIndexes); + eventToProcessBlockIndexesView_->BindRawPtr(&eventToProcessBlockIndexes_); + eventTree_.fillEntry(*eventToProcessBlockIndexesView_); unsigned int updatedIndex = eventToProcessBlockIndexes_.index() + processBlockHelper_->outerOffset(); eventToProcessBlockIndexes_.setIndex(updatedIndex); } @@ -1176,14 +1127,14 @@ namespace edm::rntuple_temp { // for backward compatibility. } else if (fileFormatVersion().noMetaDataTrees()) { // Current format - EventSelectionIDVector* pESV = &eventSelectionIDs; - TBranch* eventSelectionIDBranch = eventTree_.tree()->GetBranch(poolNames::eventSelectionsBranchName().c_str()); - assert(eventSelectionIDBranch != nullptr); - eventTree_.fillBranchEntry(eventSelectionIDBranch, pESV); - BranchListIndexes* pBLI = &branchListIndexes; - TBranch* branchListIndexesBranch = eventTree_.tree()->GetBranch(poolNames::branchListIndexesBranchName().c_str()); - assert(branchListIndexesBranch != nullptr); - eventTree_.fillBranchEntry(branchListIndexesBranch, pBLI); + auto eventSelectionIDsView = eventTree_.view(poolNames::eventSelectionsBranchName()); + assert(eventSelectionIDsView.has_value()); + eventSelectionIDsView->BindRawPtr(&eventSelectionIDs); + eventTree_.fillEntry(*eventSelectionIDsView); + auto branchListIndexesView = eventTree_.view(poolNames::branchListIndexesBranchName()); + assert(branchListIndexesView.has_value()); + branchListIndexesView->BindRawPtr(&branchListIndexes); + eventTree_.fillEntry(*branchListIndexesView); } if (daqProvenanceHelper_) { evtAux.setProcessHistoryID(daqProvenanceHelper_->mapProcessHistoryID(evtAux.processHistoryID())); @@ -1701,14 +1652,14 @@ namespace edm::rntuple_temp { auto it = std::find(processes.begin(), processes.end(), prod.processName()); if (it != processes.end()) { auto index = std::distance(processes.begin(), it); - processBlockTrees_[index]->setPresence(prod, newBranchToOldBranch(prod.branchName())); + processBlockTrees_[index]->setPresence(prod, fixName(newBranchToOldBranch(prod.branchName()))); } else { // Given current rules for saving ProductDescriptions, this case should only occur // in non-Primary sequences. prod.setDropped(true); } } else { - treePointers_[prod.branchType()]->setPresence(prod, newBranchToOldBranch(prod.branchName())); + treePointers_[prod.branchType()]->setPresence(prod, fixName(newBranchToOldBranch(prod.branchName()))); } if (prod.present()) { prod.initFromDictionary(); @@ -1829,9 +1780,9 @@ namespace edm::rntuple_temp { auto it = std::find(processes.begin(), processes.end(), prod.processName()); assert(it != processes.end()); auto index = std::distance(processes.begin(), it); - processBlockTrees_[index]->dropBranch(newBranchToOldBranch(prod.branchName())); + processBlockTrees_[index]->dropBranch(fixName(newBranchToOldBranch(prod.branchName()))); } else { - treePointers_[prod.branchType()]->dropBranch(newBranchToOldBranch(prod.branchName())); + treePointers_[prod.branchType()]->dropBranch(fixName(newBranchToOldBranch(prod.branchName()))); } hasNewlyDroppedBranch_[prod.branchType()] = true; } @@ -1859,7 +1810,7 @@ namespace edm::rntuple_temp { int offset = cp->GetBaseClassOffset(edProductClass_); std::unique_ptr edp = getWrapperBasePtr(p, offset); if (edp->isMergeable()) { - treePointers_[prod.branchType()]->dropBranch(newBranchToOldBranch(prod.branchName())); + treePointers_[prod.branchType()]->dropBranch(fixName(newBranchToOldBranch(prod.branchName()))); ProductRegistry::ProductList::iterator icopy = it; ++it; prodList.erase(icopy); @@ -1990,7 +1941,7 @@ namespace edm::rntuple_temp { std::atomic*>& writeTo) const noexcept override; edm::propagate_const rootTree_; - edm::propagate_const provBranch_; + ROOT::DescriptorId_t provID_; StoredProductProvenanceVector provVector_; StoredProductProvenanceVector const* pProvVector_; std::vector const& parentageIDLookup_; @@ -2004,14 +1955,12 @@ namespace edm::rntuple_temp { DaqProvenanceHelper const* daqProvenanceHelper) : ProvenanceReaderBase(), rootTree_(iRootRNTuple), + provID_(rootTree_->descriptorFor(BranchTypeToProductProvenanceBranchName(rootTree_->branchType()))), pProvVector_(&provVector_), parentageIDLookup_(iParentageIDLookup), daqProvenanceHelper_(daqProvenanceHelper), mutex_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().second), - acquirer_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().first) { - provBranch_ = - rootTree_->tree()->GetBranch(BranchTypeToProductProvenanceBranchName(rootTree_->branchType()).c_str()); - } + acquirer_(SharedResourcesRegistry::instance()->createAcquirerForSourceDelayedReader().first) {} namespace { using SignalType = signalslot::Signal; @@ -2091,16 +2040,14 @@ namespace edm::rntuple_temp { // void ReducedProvenanceReader::unsafe_fillProvenance(unsigned int transitionIndex) const { ReducedProvenanceReader* me = const_cast(this); - me->rootTree_->fillBranchEntry( - me->provBranch_, me->rootTree_->entryNumberForIndex(transitionIndex), me->pProvVector_); + me->rootTree_->fillEntry(me->provID_, me->rootTree_->entryNumberForIndex(transitionIndex), &me->provVector_); } std::set ReducedProvenanceReader::readProvenance(unsigned int transitionIndex) const { if (provVector_.empty()) { std::lock_guard guard(*mutex_); ReducedProvenanceReader* me = const_cast(this); - me->rootTree_->fillBranchEntry( - me->provBranch_, me->rootTree_->entryNumberForIndex(transitionIndex), me->pProvVector_); + me->rootTree_->fillEntry(me->provID_, me->rootTree_->entryNumberForIndex(transitionIndex), &me->provVector_); } std::set retValue; if (daqProvenanceHelper_) { diff --git a/FWIO/RNTupleTempInput/src/RootFile.h b/FWIO/RNTupleTempInput/src/RootFile.h index 394d44d073894..c97aa8ccbd619 100644 --- a/FWIO/RNTupleTempInput/src/RootFile.h +++ b/FWIO/RNTupleTempInput/src/RootFile.h @@ -30,8 +30,6 @@ RootFile.h // used by ROOT input sources #include "FWCore/Utilities/interface/get_underlying_safe.h" #include "FWCore/Utilities/interface/propagate_const.h" -#include "TBranch.h" - #include #include #include @@ -243,8 +241,6 @@ namespace edm { ProcessBlockHelper const*); void readParentageTree(InputType inputType); - void readEntryDescriptionTree(EntryDescriptionMap& entryDescriptionMap, - InputType inputType); // backward compatibility bool isDuplicateEvent(); void initializeDuplicateChecker(std::vector> const& indexesIntoFiles, @@ -308,7 +304,7 @@ namespace edm { edm::propagate_const runHelper_; std::map newBranchToOldBranch_; EventToProcessBlockIndexes eventToProcessBlockIndexes_; - edm::propagate_const eventToProcessBlockIndexesBranch_; + std::optional> eventToProcessBlockIndexesView_; edm::propagate_const> productDependencies_; edm::propagate_const> duplicateChecker_; edm::propagate_const> provenanceReaderMaker_; diff --git a/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.cc b/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.cc index b0dd7c7cfb7e7..869ca9b4f0606 100644 --- a/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.cc +++ b/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.cc @@ -13,9 +13,7 @@ #include "FWCore/Utilities/interface/EDMException.h" -#include "TBranch.h" #include "TClass.h" -#include "TTree.h" #include @@ -62,14 +60,14 @@ namespace edm::rntuple_temp { auto& cacheMap = cacheMaps_[indexFor(tree_, ep)]; auto itFound = cacheMap.find(k.id()); if (itFound != cacheMap.end()) { - auto& cache = itFound->second; - if (cache.wrapperBase_) { + auto cache = std::move(itFound->second); + if (cache) { if (tree_.branchType() == InEvent) { // CMS-THREADING For the primary input source calls to this function need to be serialized - InputFile::reportReadBranch(inputType_, - std::string(tree_.branches().find(itFound->first)->productBranch_->GetName())); + InputFile::reportReadBranch( + inputType_, std::string(tree_.branches().find(itFound->first)->productDescription().branchName())); } - return std::shared_ptr(std::move(cache.wrapperBase_)); + return std::shared_ptr(std::move(cache)); } } if (nextReader_) { @@ -85,25 +83,14 @@ namespace edm::rntuple_temp { for (auto& cacheMap : cacheMaps_) { cacheMap.reserve(tree_.branches().size()); for (auto const& branch : tree_.branches()) { - cacheMap.emplace(branch.first, Cache{}); + cacheMap[branch.first]; } } } - for (auto& it : cacheMap) { - auto branchInfo = getProductInfo(it.first); - if (branchInfo == nullptr || branchInfo->productBranch_ == nullptr) { - continue; // Skip if branch info or product branch is not available - } - auto& cache = it.second; - cache.wrapperBase_ = branchInfo->newWrapper(); - cache.wrapperBasePtr_ = cache.wrapperBase_.get(); - branchInfo->productBranch_->SetAddress(&cache.wrapperBasePtr_); - } - { // ROOT might use multiple threads while reading the entries MultiThreadRefCoreStreamerGuard epGuard(ep); - tree_.getEntryForAllBranches(); + tree_.getEntryForAllBranches(cacheMap); } } } // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.h b/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.h index 57df25bbdecf1..c9793f061d916 100644 --- a/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.h +++ b/FWIO/RNTupleTempInput/src/RootPromptReadDelayedReader.h @@ -48,11 +48,6 @@ namespace edm::rntuple_temp { RootPromptReadDelayedReader(RootPromptReadDelayedReader&&) = delete; RootPromptReadDelayedReader& operator=(RootPromptReadDelayedReader&&) = delete; - struct Cache { - std::unique_ptr wrapperBase_; - edm::WrapperBase* wrapperBasePtr_ = nullptr; // TBranch::SetAddress() needs a long live pointer to reference. - }; - void readAllProductsNow(EDProductGetter const* ep) override; private: @@ -63,7 +58,7 @@ namespace edm::rntuple_temp { ProductMap const& branches() const { return tree_.branches(); } ProductInfo const* getProductInfo(unsigned int k) const { return branches().find(k); } - std::vector> cacheMaps_; + std::vector>> cacheMaps_; // NOTE: filePtr_ appears to be unused, but is needed to prevent // the file containing the branch from being reclaimed. RootRNTuple const& tree_; diff --git a/FWIO/RNTupleTempInput/src/RootRNTuple.cc b/FWIO/RNTupleTempInput/src/RootRNTuple.cc index cbecb00dab7ac..9fdc977276ba8 100644 --- a/FWIO/RNTupleTempInput/src/RootRNTuple.cc +++ b/FWIO/RNTupleTempInput/src/RootRNTuple.cc @@ -9,24 +9,13 @@ #include "RootDelayedReader.h" #include "RootPromptReadDelayedReader.h" -#include "TTree.h" -#include "TLeaf.h" - #include "oneapi/tbb/task_arena.h" #include namespace edm::rntuple_temp { namespace { - TBranch* getAuxiliaryBranch(TTree* tree, BranchType const& branchType) { - TBranch* branch = tree->GetBranch(BranchTypeToAuxiliaryBranchName(branchType).c_str()); - if (branch == nullptr) { - branch = tree->GetBranch(BranchTypeToAuxBranchName(branchType).c_str()); - } - return branch; - } - TBranch* getProductProvenanceBranch(TTree* tree, BranchType const& branchType) { - TBranch* branch = tree->GetBranch(BranchTypeToBranchEntryInfoBranchName(branchType).c_str()); - return branch; + ROOT::DescriptorId_t getAuxiliaryFieldId(ROOT::RNTupleReader& reader, BranchType const& branchType) { + return reader.GetDescriptor().FindFieldId(BranchTypeToAuxiliaryBranchName(branchType)); } std::unique_ptr makeRootDelayedReader(RootRNTuple const& tree, @@ -67,7 +56,7 @@ namespace edm::rntuple_temp { : RootRNTuple( filePtr, branchType, nIndexes, learningEntries, options.enablePrefetching, options.promptReading, inputType) { init(BranchTypeToProductTreeName(branchType), options.treeMaxVirtualSize, options.treeCacheSize); - auxBranch_ = getAuxiliaryBranch(tree_, branchType_); + auxDesc_ = getAuxiliaryFieldId(*reader_, branchType_); } // Used for ProcessBlock RootRNTuples @@ -86,31 +75,32 @@ namespace edm::rntuple_temp { void RootRNTuple::init(std::string const& productTreeName, unsigned int maxVirtualSize, unsigned int cacheSize) { if (filePtr_.get() != nullptr) { - tree_ = dynamic_cast(filePtr_->Get(productTreeName.c_str())); + auto tuple = filePtr_->Get(productTreeName.c_str()); + if (tuple != nullptr) { + reader_ = ROOT::RNTupleReader::Open(*tuple); + } } - if (not tree_) { + if (not reader_) { throw cms::Exception("WrongFileFormat") << "The ROOT file does not contain a TTree named " << productTreeName << "\n This is either not an edm ROOT file or is one that has been corrupted."; } - entries_ = tree_->GetEntries(); + entries_ = reader_->GetNEntries(); - // On merged files in older releases of ROOT, the autoFlush setting is always negative; we must guess. - // TODO: On newer merged files, we should be able to get this from the cluster iterator. - long treeAutoFlush = tree_->GetAutoFlush(); - if (treeAutoFlush < 0) { - // The "+1" is here to avoid divide-by-zero in degenerate cases. - Long64_t averageEventSizeBytes = tree_->GetZipBytes() / (tree_->GetEntries() + 1) + 1; - treeAutoFlush_ = cacheSize / averageEventSizeBytes + 1; - } else { - treeAutoFlush_ = treeAutoFlush; - } setTreeMaxVirtualSize(maxVirtualSize); setCacheSize(cacheSize); } RootRNTuple::~RootRNTuple() {} + std::optional> RootRNTuple::view(std::string_view iName) { + auto id = reader_->GetDescriptor().FindFieldId(iName); + if (id == ROOT::kInvalidDescriptorId) { + return std::nullopt; + } + return reader_->GetView(id, std::shared_ptr()); + } + RootRNTuple::EntryNumber const& RootRNTuple::entryNumberForIndex(unsigned int index) const { assert(index < entryNumberForIndex_->size()); return (*entryNumberForIndex_)[index]; @@ -124,10 +114,10 @@ namespace edm::rntuple_temp { bool RootRNTuple::isValid() const { // ProcessBlock if (branchType_ == InProcess) { - return tree_ != nullptr; + return bool(reader_); } // Run/Lumi/Event - return tree_ != nullptr && auxBranch_ != nullptr; + return bool(reader_) && auxDesc_ != ROOT::kInvalidDescriptorId; } DelayedReader* RootRNTuple::resetAndGetRootDelayedReader() const { @@ -139,17 +129,18 @@ namespace edm::rntuple_temp { void RootRNTuple::setPresence(ProductDescription& prod, std::string const& oldBranchName) { assert(isValid()); - if (tree_->GetBranch(oldBranchName.c_str()) == nullptr) { + if (reader_->GetDescriptor().FindFieldId(oldBranchName) == ROOT::kInvalidDescriptorId) { prod.setDropped(true); } } - void rootrntuple::ProductInfo::setBranch(TBranch* branch, TClass const* wrapperBaseTClass) { - productBranch_ = branch; - if (branch) { - classCache_ = TClass::GetClass(productDescription_.wrappedName().c_str()); - offsetToWrapperBase_ = classCache_->GetBaseClassOffset(wrapperBaseTClass); - } + void rootrntuple::ProductInfo::setField(ROOT::RFieldToken token, + ROOT::RNTupleView view, + TClass const* wrapperBaseTClass) { + token_ = token; + view_ = std::move(view); + classCache_ = TClass::GetClass(productDescription_.wrappedName().c_str()); + offsetToWrapperBase_ = classCache_->GetBaseClassOffset(wrapperBaseTClass); } std::unique_ptr rootrntuple::ProductInfo::newWrapper() const { assert(nullptr != classCache_); @@ -161,49 +152,25 @@ namespace edm::rntuple_temp { assert(isValid()); static TClass const* const wrapperBaseTClass = TClass::GetClass("edm::WrapperBase"); //use the translated branch name - TBranch* branch = tree_->GetBranch(oldBranchName.c_str()); - rootrntuple::ProductInfo info = rootrntuple::ProductInfo(prod); - info.productBranch_ = nullptr; - if (prod.present()) { - info.setBranch(branch, wrapperBaseTClass); + auto id = reader_->GetDescriptor().FindFieldId(oldBranchName); + rootrntuple::ProductInfo info(prod); + if (prod.present() and id != ROOT::kInvalidDescriptorId) { + info.setField(reader_->GetModel().GetToken(oldBranchName), + reader_->GetView(id, std::shared_ptr()), + wrapperBaseTClass); //we want the new branch name for the JobReport branchNames_.push_back(prod.branchName()); } - branches_.insert(prod.branchID(), info); + branches_.insert(prod.branchID(), std::move(info)); } - void RootRNTuple::dropBranch(std::string const& oldBranchName) { - //use the translated branch name - TBranch* branch = tree_->GetBranch(oldBranchName.c_str()); - if (branch != nullptr) { - TObjArray* leaves = tree_->GetListOfLeaves(); - int entries = leaves->GetEntries(); - for (int i = 0; i < entries; ++i) { - TLeaf* leaf = (TLeaf*)(*leaves)[i]; - if (leaf == nullptr) - continue; - TBranch* br = leaf->GetBranch(); - if (br == nullptr) - continue; - if (br->GetMother() == branch) { - leaves->Remove(leaf); - } - } - leaves->Compress(); - tree_->GetListOfBranches()->Remove(branch); - tree_->GetListOfBranches()->Compress(); - delete branch; - } - } + void RootRNTuple::dropBranch(std::string const& oldBranchName) {} rootrntuple::ProductMap const& RootRNTuple::branches() const { return branches_; } void RootRNTuple::setCacheSize(unsigned int cacheSize) {} - void RootRNTuple::setTreeMaxVirtualSize(int treeMaxVirtualSize) { - if (treeMaxVirtualSize >= 0) - tree_->SetMaxVirtualSize(static_cast(treeMaxVirtualSize)); - } + void RootRNTuple::setTreeMaxVirtualSize(int treeMaxVirtualSize) {} bool RootRNTuple::nextWithCache() { bool returnValue = ++entryNumber_ < entries_; @@ -213,42 +180,48 @@ namespace edm::rntuple_temp { return returnValue; } - void RootRNTuple::setEntryNumber(EntryNumber theEntryNumber) { - { - entryNumber_ = theEntryNumber; - tree_->LoadTree(entryNumber_); - //want guard to end here - } - } + void RootRNTuple::setEntryNumber(EntryNumber theEntryNumber) { entryNumber_ = theEntryNumber; } - void RootRNTuple::getEntryForAllBranches() const { - oneapi::tbb::this_task_arena::isolate([&]() { tree_->GetEntry(entryNumber_); }); + void RootRNTuple::getEntryForAllBranches( + std::unordered_map>& iFields) const { + oneapi::tbb::this_task_arena::isolate([&]() { + auto entry = reader_->GetModel().CreateEntry(); + for (auto& iField : iFields) { + auto const& prod = branches_.find(iField.first); + if (prod == nullptr or not prod->valid()) { + continue; + } + iField.second = prod->newWrapper(); + entry->BindRawPtr(prod->token(), reinterpret_cast(iField.second.get())); + } + reader_->LoadEntry(entryNumber_, *entry); + }); } - void RootRNTuple::getEntry(TBranch* branch, EntryNumber entryNumber) const { + void RootRNTuple::getEntry(ROOT::RNTupleView& view, EntryNumber entryNumber) const { LogTrace("IOTrace").format( - "RootRNTuple::getEntryUsingCache() begin for branch {} entry {}", branch->GetName(), entryNumber); + "RootRNTuple::getEntryUsingCache() begin for branch {} entry {}", view.GetField().GetFieldName(), entryNumber); try { - branch->GetEntry(entryNumber); + view(entryNumber); } catch (cms::Exception const& e) { // We make sure the treeCache_ is detached from the file, // so that ROOT does not also delete it. Exception t(errors::FileReadError, "", e); - t.addContext(std::string("Reading branch ") + branch->GetName()); + t.addContext(std::string("Reading branch ") + view.GetField().GetFieldName()); throw t; } catch (std::exception const& e) { Exception t(errors::FileReadError); t << e.what(); - t.addContext(std::string("Reading branch ") + branch->GetName()); + t.addContext(std::string("Reading branch ") + view.GetField().GetFieldName()); throw t; } catch (...) { Exception t(errors::FileReadError); t << "An exception of unknown type was thrown."; - t.addContext(std::string("Reading branch ") + branch->GetName()); + t.addContext(std::string("Reading branch ") + view.GetField().GetFieldName()); throw t; } LogTrace("IOTrace").format( - "RootRNTuple::getEntryUsingCache() end for branch {} entry {}", branch->GetName(), entryNumber); + "RootRNTuple::getEntryUsingCache() end for branch {} entry {}", view.GetField().GetFieldName(), entryNumber); } bool RootRNTuple::skipEntries(unsigned int& offset) { @@ -269,8 +242,8 @@ namespace edm::rntuple_temp { void RootRNTuple::close() { // The TFile is about to be closed, and destructed. // Just to play it safe, zero all pointers to quantities that are owned by the TFile. - auxBranch_ = nullptr; - tree_ = nullptr; + auxDesc_ = ROOT::kInvalidDescriptorId; + //reader_.reset(); //if there are any outstanding views, they will be invalidated // We give up our shared ownership of the TFile itself. filePtr_.reset(); } @@ -281,25 +254,4 @@ namespace edm::rntuple_temp { rootDelayedReader_->setSignals(preEventReadSource, postEventReadSource); } - namespace rootrntuple { - Int_t getEntry(TBranch* branch, EntryNumber entryNumber) { - Int_t n = 0; - try { - n = branch->GetEntry(entryNumber); - } catch (cms::Exception const& e) { - throw Exception(errors::FileReadError, "", e); - } - return n; - } - - Int_t getEntry(TTree* tree, EntryNumber entryNumber) { - Int_t n = 0; - try { - n = tree->GetEntry(entryNumber); - } catch (cms::Exception const& e) { - throw Exception(errors::FileReadError, "", e); - } - return n; - } - } // namespace rootrntuple } // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempInput/src/RootRNTuple.h b/FWIO/RNTupleTempInput/src/RootRNTuple.h index b795fc210d695..907fa23dc9dc7 100644 --- a/FWIO/RNTupleTempInput/src/RootRNTuple.h +++ b/FWIO/RNTupleTempInput/src/RootRNTuple.h @@ -16,9 +16,9 @@ RootRNTuple.h // used by ROOT input sources #include "FWCore/Utilities/interface/InputType.h" #include "FWCore/Utilities/interface/Signal.h" #include "FWCore/Utilities/interface/thread_safety_macros.h" +#include "FWCore/Utilities/interface/EDMException.h" #include "Rtypes.h" -#include "TBranch.h" #include #include @@ -26,8 +26,10 @@ RootRNTuple.h // used by ROOT input sources #include #include +#include "ROOT/RNTuple.hxx" +#include "ROOT/RNTupleReader.hxx" +#include "ROOT/RNTupleView.hxx" class TClass; -class TTree; namespace edm::rntuple_temp { class InputFile; @@ -40,17 +42,26 @@ namespace edm::rntuple_temp { unsigned int const defaultNonEventLearningEntries = 1U; using EntryNumber = IndexIntoFile::EntryNumber_t; struct ProductInfo { - ProductInfo(ProductDescription const& prod) - : productDescription_(prod), productBranch_(nullptr), classCache_(nullptr), offsetToWrapperBase_(0) {} - ProductDescription const productDescription_; - void setBranch(TBranch* branch, TClass const* wrapperBaseTClass); + ProductInfo(ProductDescription const& prod) : productDescription_(prod) {} + ProductInfo(ProductInfo const&) = default; + ProductInfo& operator=(ProductInfo const&) = default; + ProductInfo(ProductInfo&&) = default; + ProductInfo& operator=(ProductInfo&&) = default; + void setField(ROOT::RFieldToken token, ROOT::RNTupleView view, TClass const* wrapperBaseTClass); std::unique_ptr newWrapper() const; - TBranch* productBranch_; + bool valid() const { return view_.has_value(); } + ROOT::RNTupleView& view() const { return view_.value(); } + ProductDescription const& productDescription() const { return productDescription_; } + + ROOT::RFieldToken token() const { return token_; } private: + ProductDescription const productDescription_; + ROOT::RFieldToken token_; + mutable std::optional> view_; //All access to a ROOT file is serialized - TClass* classCache_; - Int_t offsetToWrapperBase_; + TClass* classCache_ = nullptr; + Int_t offsetToWrapperBase_ = 0; }; class ProductMap { @@ -58,7 +69,7 @@ namespace edm::rntuple_temp { using Map = std::unordered_map; void reserve(Map::size_type iSize) { map_.reserve(iSize); } - void insert(edm::BranchID const& iKey, ProductInfo const& iInfo) { map_.emplace(iKey.id(), iInfo); } + void insert(edm::BranchID const& iKey, ProductInfo iInfo) { map_.emplace(iKey.id(), std::move(iInfo)); } ProductInfo const* find(BranchID const& iKey) const { return find(iKey.id()); } ProductInfo const* find(unsigned int iKey) const { auto itFound = map_.find(iKey); @@ -76,9 +87,6 @@ namespace edm::rntuple_temp { private: Map map_; }; - - Int_t getEntry(TBranch* branch, EntryNumber entryNumber); - Int_t getEntry(TTree* tree, EntryNumber entryNumber); } // namespace rootrntuple class RootRNTuple { @@ -122,8 +130,8 @@ namespace edm::rntuple_temp { void numberOfBranchesToAdd(ProductMap::Map::size_type iSize) { branches_.reserve(iSize); } void addBranch(ProductDescription const& prod, std::string const& oldBranchName); void dropBranch(std::string const& oldBranchName); - void getEntry(TBranch* branch, EntryNumber entry) const; - void getEntryForAllBranches() const; + void getEntry(ROOT::RNTupleView& view, EntryNumber entry) const; + void getEntryForAllBranches(std::unordered_map>&) const; void setPresence(ProductDescription& prod, std::string const& oldBranchName); bool next() { return ++entryNumber_ < entries_; } @@ -144,31 +152,29 @@ namespace edm::rntuple_temp { DelayedReader* resetAndGetRootDelayedReader() const; template void fillAux(T*& pAux) { - auxBranch_->SetAddress(&pAux); - getEntry(auxBranch_, entryNumber_); - auxBranch_->SetAddress(nullptr); + try { + auto view = reader_->GetView(auxDesc_, pAux); + view(entryNumber_); + } catch (cms::Exception const& e) { + throw Exception(errors::FileReadError, "", e); + } catch (std::exception const& e) { + Exception t(errors::FileReadError); + t << e.what(); + throw t; + } } - template - void fillBranchEntry(TBranch* branch, T*& pbuf) { - branch->SetAddress(&pbuf); - getEntry(branch, entryNumber_); - branch->SetAddress(nullptr); - } + std::optional> view(std::string_view iName); + ROOT::DescriptorId_t descriptorFor(std::string_view iName) { return reader_->GetDescriptor().FindFieldId(iName); } - template - void fillBranchEntryMeta(TBranch* branch, EntryNumber entryNumber, T*& pbuf) { - fillBranchEntry(branch, entryNumber, pbuf); - } + void fillEntry(ROOT::RNTupleView& view) { getEntry(view, entryNumber_); } + void fillEntry(ROOT::RNTupleView& view, EntryNumber entryNumber) { getEntry(view, entryNumber); } - template - void fillBranchEntry(TBranch* branch, EntryNumber entryNumber, T*& pbuf) { - branch->SetAddress(&pbuf); - getEntry(branch, entryNumber); + void fillEntry(ROOT::DescriptorId_t id, EntryNumber entryNumber, void* iData) { + auto view = reader_->GetView(id, iData); + getEntry(view, entryNumber); } - TTree const* tree() const { return tree_; } - TTree* tree() { return tree_; } ProductMap const& branches() const; BranchType branchType() const { return branchType_; } @@ -194,13 +200,13 @@ namespace edm::rntuple_temp { // We use bare pointers for pointers to some ROOT entities. // Root owns them and uses bare pointers internally. // Therefore,using smart pointers here will do no good. - TTree* tree_ = nullptr; + std::unique_ptr reader_; BranchType branchType_; std::string processName_; - TBranch* auxBranch_ = nullptr; + ROOT::DescriptorId_t auxDesc_ = ROOT::kInvalidDescriptorId; EntryNumber entries_ = 0; EntryNumber entryNumber_ = IndexIntoFile::invalidEntry; - std::unique_ptr > entryNumberForIndex_; + std::unique_ptr> entryNumberForIndex_; std::vector branchNames_; ProductMap branches_; unsigned int cacheSize_ = 0; diff --git a/FWIO/RNTupleTempInput/test/BuildFile.xml b/FWIO/RNTupleTempInput/test/BuildFile.xml index c60f57eb138c7..dfd77bc88cb1e 100644 --- a/FWIO/RNTupleTempInput/test/BuildFile.xml +++ b/FWIO/RNTupleTempInput/test/BuildFile.xml @@ -7,7 +7,6 @@ - diff --git a/FWIO/RNTupleTempInput/test/testNoParentDictionary.sh b/FWIO/RNTupleTempInput/test/testNoParentDictionary.sh deleted file mode 100755 index 1650603ef7715..0000000000000 --- a/FWIO/RNTupleTempInput/test/testNoParentDictionary.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash -e - -function die { echo $1: status $2 ; exit $2; } - -SCRAM_TEST_NAME=TestIOPoolInputNoParentDictionary -rm -rf $SCRAM_TEST_NAME -mkdir $SCRAM_TEST_NAME -cd $SCRAM_TEST_NAME - -# Create a new CMSSW dev area and build modified DataFormats/TestObjects in it -NEW_CMSSW_BASE=$(/bin/pwd -P)/$CMSSW_VERSION -scram -a $SCRAM_ARCH project $CMSSW_VERSION -pushd $CMSSW_VERSION/src -mkdir DataFormats - -# Copy DataFormats/Common if it was modified locally -if [ -d ${CMSSW_BASE}/src/DataFormats/Common ]; then - cp -Lr ${CMSSW_BASE}/src/DataFormats/Common DataFormats/ -fi -if [ -d ${CMSSW_BASE}/src/DataFormats/Provenance ]; then - cp -Lr ${CMSSW_BASE}/src/DataFormats/Provenance DataFormats/ -fi -#DataFormats/Common and DataFormat/Provenance depend on Utilities -if [ -d ${CMSSW_BASE}/src/FWCore/Utilities ]; then - mkdir -p FWCore - cp -Lr ${CMSSW_BASE}/src/FWCore/Utilities FWCore/ -fi -#DataFormats/Common depends on MessageLogger -if [ -d ${CMSSW_BASE}/src/FWCore/MessageLogger ]; then - mkdir -p FWCore - cp -Lr ${CMSSW_BASE}/src/FWCore/MessageLogger FWCore/ -fi -#DataFormats/Provenance depends on Reflection -if [ -d ${CMSSW_BASE}/src/FWCore/Reflection ]; then - mkdir -p FWCore - cp -Lr ${CMSSW_BASE}/src/FWCore/Reflection FWCore/ -fi - -#DataFormats/TestObjects depends on FWCore/SOA -if [ -d ${CMSSW_BASE}/src/FWCore/SOA ]; then - mkdir -p FWCore - cp -Lr ${CMSSW_BASE}/src/FWCore/SOA FWCore/ -fi - -# Copy DataFormats/TestObjects code to be able to edit it to make ROOT header parsing to fail -for DIR in ${CMSSW_BASE} ${CMSSW_RELEASE_BASE} ${CMSSW_FULL_RELEASE_BASE} ; do - if [ -d ${DIR}/src/DataFormats/TestObjects ]; then - cp -Lr ${DIR}/src/DataFormats/TestObjects DataFormats/ - break - fi -done -if [ ! -e DataFormats/TestObjects ]; then - echo "Failed to copy DataFormats/TestObjects from local or release area" - exit 1; -fi - -# Enable TransientIntParentT dictionaries -cat DataFormats/TestObjects/test/BuildFile_extra.xml >> DataFormats/TestObjects/test/BuildFile.xml -#Set env and build in sub-shel -(eval $(scram run -sh) ; scram build -j $(nproc)) - -popd - -# Prepend NEW_CMSSW_BASE's lib/src paths in to LD_LIBRARY_PATH and ROOT_INCLUDE_PATH -export LD_LIBRARY_PATH=${NEW_CMSSW_BASE}/lib/${SCRAM_ARCH}:${LD_LIBRARY_PATH} -export ROOT_INCLUDE_PATH=${NEW_CMSSW_BASE}/src:${ROOT_INCLUDE_PATH} - -echo "Produce a file with TransientIntParentT<1> product" -cmsRun ${LOCALTOP}/src/IOPool/Input/test/PoolNoParentDictionaryTestStep1_cfg.py || die 'Failed cmsRun PoolNoParentDictionaryTestStep1_cfg.py' $? - - -# Then make attempt to load TransientIntParentT<1> to fail -echo "PREVENT_HEADER_PARSING" >> ${NEW_CMSSW_BASE}/src/DataFormats/TestObjects/interface/ToyProducts.h -rm ${NEW_CMSSW_BASE}/lib/${SCRAM_ARCH}/DataFormatsTestObjectsParent1_xr_rdict.pcm ${NEW_CMSSW_BASE}/lib/${SCRAM_ARCH}/libDataFormatsTestObjectsParent1.so -sed -i -e 's/libDataFormatsTestObjectsParent1.so/libDataFormatsTestObjectsParent2.so/' ${NEW_CMSSW_BASE}/lib/${SCRAM_ARCH}/DataFormatsTestObjectsParent1_xr.rootmap - -echo "Read the file without TransientIntParentT<1> dictionary" -cmsRun ${LOCALTOP}/src/IOPool/Input/test/PoolNoParentDictionaryTestStep2_cfg.py || die 'Failed cmsRun PoolNoParentDictionaryTestStep2_cfg.py' $? diff --git a/FWIO/RNTupleTempOutput/test/BuildFile.xml b/FWIO/RNTupleTempOutput/test/BuildFile.xml index 5b584654ba366..14c823861d11a 100644 --- a/FWIO/RNTupleTempOutput/test/BuildFile.xml +++ b/FWIO/RNTupleTempOutput/test/BuildFile.xml @@ -1,2 +1 @@ - diff --git a/FWIO/RNTupleTempTests/test/run_TestProcessBlock.sh b/FWIO/RNTupleTempTests/test/run_TestProcessBlock.sh index b7ef24cefa445..c636365d1d69d 100755 --- a/FWIO/RNTupleTempTests/test/run_TestProcessBlock.sh +++ b/FWIO/RNTupleTempTests/test/run_TestProcessBlock.sh @@ -23,14 +23,14 @@ then echo "testProcessBlock1" cmsRun ${LOCAL_TEST_DIR}/testProcessBlock1_cfg.py &> testProcessBlock1.log || die "cmsRun testProcessBlock1_cfg.py" $? - # The MetaData ProcessBlock branch and the TTree should exist to hold the ProcessBlock + # The MetaData ProcessBlock branch and the RNTuple should exist to hold the ProcessBlock # data. The Events branch should not exist because there were not any ProcessBlock branches # saved from an input file. Test that here: edmRNTupleTempFileUtil -l -t MetaData -P file:testProcessBlock1.root > testProcessBlock1ContentsM.txt grep "Field.* ProcessBlockHelper " testProcessBlock1ContentsM.txt || die "Check for existence of ProcessBlockHelper branch" $? - grep "TTree.*ProcessBlocksPROD1" testProcessBlock1ContentsM.txt || die "Check for existence of ProcessBlocksPROD1 TTree" $? + grep "RNTuple.*ProcessBlocksPROD1" testProcessBlock1ContentsM.txt || die "Check for existence of ProcessBlocksPROD1 RNTuple" $? edmRNTupleTempFileUtil -t Events -P file:testProcessBlock1.root > testProcessBlock1ContentsE.txt - grep "Branch.* EventToProcessBlockIndexes " testProcessBlock1ContentsE.txt && die "Check for non-existence of eventToProcessBlockIndexes branch" 1 + grep "Field.* EventToProcessBlockIndexes " testProcessBlock1ContentsE.txt && die "Check for non-existence of eventToProcessBlockIndexes branch" 1 fi if [ $1 -eq 2 ] @@ -56,13 +56,13 @@ then echo "testProcessBlockMerge" cmsRun ${LOCAL_TEST_DIR}/testProcessBlockMerge_cfg.py &> testProcessBlockMerge.log || die "cmsRun testProcessBlockMerge_cfg.py" $? - # The ProcessBlock Branches and TTrees should exist in this case. Test that here: + # The ProcessBlock Branches and RNTuples should exist in this case. Test that here: edmRNTupleTempFileUtil -l -t MetaData -P file:testProcessBlockMerge.root > testProcessBlockMContentsM.txt grep "Field.* ProcessBlockHelper " testProcessBlockMContentsM.txt || die "Check for existence of ProcessBlockHelper branch" $? - grep "TTree.*ProcessBlocksPROD1" testProcessBlockMContentsM.txt || die "Check for existence of ProcessBlocksPROD1 TTree" $? - grep "TTree.*ProcessBlocksMERGE" testProcessBlockMContentsM.txt || die "Check for existence of ProcessBlocksMERGE TTree" $? + grep "RNTuple.*ProcessBlocksPROD1" testProcessBlockMContentsM.txt || die "Check for existence of ProcessBlocksPROD1 RNTuple" $? + grep "RNTuple.*ProcessBlocksMERGE" testProcessBlockMContentsM.txt || die "Check for existence of ProcessBlocksMERGE RNTuple" $? edmRNTupleTempFileUtil -t Events -P file:testProcessBlockMerge.root > testProcessBlockMContentsE.txt - grep "Branch.* EventToProcessBlockIndexes " testProcessBlockMContentsE.txt || die "Check for existence of eventToProcessBlockIndexes branch" $? + grep "Field.* EventToProcessBlockIndexes " testProcessBlockMContentsE.txt || die "Check for existence of eventToProcessBlockIndexes branch" $? fi if [ $1 -eq 6 ] @@ -82,13 +82,13 @@ then echo "testProcessBlock2Dropped" cmsRun ${LOCAL_TEST_DIR}/testProcessBlock2Dropped_cfg.py &> testProcessBlock2Dropped.log || die "cmsRun testProcessBlock2Dropped_cfg.py" $? - # The ProcessBlock Branches and TTrees should not exist in this case because + # The ProcessBlock Branches and RNTuples should not exist in this case because # all the ProcessBlock products are dropped. Test that here: edmRNTupleTempFileUtil -l -t MetaData -P file:testProcessBlock2Dropped.root > testProcessBlock2DroppedContentsM.txt grep "Field.* ProcessBlockHelper " testProcessBlock2DroppedContentsM.txt && die "Check for non-existence of ProcessBlockHelper branch" 1 - grep "TTree.*ProcessBlocksPROD1" testProcessBlock2DroppedContentsM.txt && die "Check for non-existence of ProcessBlocksPROD1 TTree" 1 + grep "RNTuple.*ProcessBlocksPROD1" testProcessBlock2DroppedContentsM.txt && die "Check for non-existence of ProcessBlocksPROD1 RNTuple" 1 edmRNTupleTempFileUtil -t Events -P file:testProcessBlock2Dropped.root > testProcessBlock2DroppedContentsE.txt - grep "Branch.* EventToProcessBlockIndexes " testProcessBlock2DroppedContentsE.txt && die "Check for non-existence of eventToProcessBlockIndexes branch" 1 + grep "Field.* EventToProcessBlockIndexes " testProcessBlock2DroppedContentsE.txt && die "Check for non-existence of eventToProcessBlockIndexes branch" 1 fi if [ $1 -eq 8 ] diff --git a/FWIO/RNTupleTempTests/test/testLooperEventNavigation2_cfg.py b/FWIO/RNTupleTempTests/test/testLooperEventNavigation2_cfg.py index a73d28176e46c..4413e22823778 100644 --- a/FWIO/RNTupleTempTests/test/testLooperEventNavigation2_cfg.py +++ b/FWIO/RNTupleTempTests/test/testLooperEventNavigation2_cfg.py @@ -26,7 +26,7 @@ expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), expectedProcessNamesAtWrite = cms.untracked.vstring('PROD1', 'PROD1', 'MERGE', 'PROD1', 'PROD1', 'MERGE', 'PROD1', 'PROD1', 'MERGE', 'PROD1', 'PROD1', 'MERGE', 'PROD1', 'PROD1', 'MERGE', 'TEST'), expectedWriteProcessBlockTransitions = cms.untracked.int32(16), - testTTreesInFileBlock = cms.untracked.bool(True), + testTTreesInFileBlock = cms.untracked.bool(False), expectedCacheIndexSize = cms.untracked.vuint32(2, 2, 2, 4, 4, 4, 6, 6, 6, 8, 8, 8, 10, 10, 10) ) diff --git a/FWIO/RNTupleTempTests/test/testLooperEventNavigation3_cfg.py b/FWIO/RNTupleTempTests/test/testLooperEventNavigation3_cfg.py index 2e4166abad1a5..facae5cc094de 100644 --- a/FWIO/RNTupleTempTests/test/testLooperEventNavigation3_cfg.py +++ b/FWIO/RNTupleTempTests/test/testLooperEventNavigation3_cfg.py @@ -27,7 +27,7 @@ expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE'), expectedProcessNamesAtWrite = cms.untracked.vstring('PROD1', 'PROD1', 'MERGE', 'PROD1', 'PROD1', 'MERGE', 'TEST'), expectedWriteProcessBlockTransitions = cms.untracked.int32(7), - testTTreesInFileBlock = cms.untracked.bool(True), + testTTreesInFileBlock = cms.untracked.bool(False), expectedCacheIndexSize = cms.untracked.vuint32(2, 2, 2, 4, 4, 4, 4) ) diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockMergeOfMergedFiles_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockMergeOfMergedFiles_cfg.py index c4fde0d1b53d6..d8a436b938c22 100644 --- a/FWIO/RNTupleTempTests/test/testProcessBlockMergeOfMergedFiles_cfg.py +++ b/FWIO/RNTupleTempTests/test/testProcessBlockMergeOfMergedFiles_cfg.py @@ -66,7 +66,7 @@ expectedTopAddedProcesses = cms.untracked.vstring('MERGEOFMERGED'), expectedProcessNamesAtWrite = cms.untracked.vstring('PROD1', 'PROD1', 'MERGE', 'PROD1', 'PROD1', 'MERGE', 'MERGEOFMERGED'), expectedWriteProcessBlockTransitions = cms.untracked.int32(7), - testTTreesInFileBlock = cms.untracked.bool(True), + testTTreesInFileBlock = cms.untracked.bool(False), expectedCacheIndexSize = cms.untracked.vuint32(2, 2, 2, 4, 4, 4, 4), expectedNAddedProcesses = cms.untracked.uint32(1), expectedTopCacheIndices0 = cms.untracked.vuint32(0, 2, 1, 2), diff --git a/FWIO/RNTupleTempTests/test/testProcessBlockNOMergeOfMergedFiles_cfg.py b/FWIO/RNTupleTempTests/test/testProcessBlockNOMergeOfMergedFiles_cfg.py index 9275ed557964e..6dd81f8adcd1c 100644 --- a/FWIO/RNTupleTempTests/test/testProcessBlockNOMergeOfMergedFiles_cfg.py +++ b/FWIO/RNTupleTempTests/test/testProcessBlockNOMergeOfMergedFiles_cfg.py @@ -67,7 +67,7 @@ verbose = cms.untracked.bool(False), expectedProcessesWithProcessBlockProducts = cms.untracked.vstring('PROD1', 'MERGE', 'MERGEOFMERGED'), expectedWriteProcessBlockTransitions = cms.untracked.int32(8), - testTTreesInFileBlock = cms.untracked.bool(True), + testTTreesInFileBlock = cms.untracked.bool(False), expectedCacheIndexSize = cms.untracked.vuint32(2, 2, 2, 2, 2, 2, 2, 2) ) From 2553a20deed582fd15c2d9f704b665c38c31fe99 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Mon, 6 Oct 2025 14:27:50 -0500 Subject: [PATCH 12/19] Better configuration of RNTupleTemp* - made TTree only parameters optional - added RNTuple specific options --- .../src/EmbeddedRNTupleTempSource.cc | 20 +++- .../src/EmbeddedRNTupleTempSource.h | 6 + .../RNTupleTempInput/src/RNTupleTempSource.cc | 19 ++- FWIO/RNTupleTempInput/src/RNTupleTempSource.h | 5 + .../src/RootEmbeddedFileSequence.cc | 7 +- FWIO/RNTupleTempInput/src/RootFile.cc | 26 +--- .../src/RootPrimaryFileSequence.cc | 9 +- FWIO/RNTupleTempInput/src/RootRNTuple.cc | 25 ++-- FWIO/RNTupleTempInput/src/RootRNTuple.h | 21 +--- .../src/RootSecondaryFileSequence.cc | 3 +- .../interface/RNTupleTempOutputModule.h | 26 ++-- .../src/RNTupleTempOutputModule.cc | 112 ++++++++++-------- FWIO/RNTupleTempOutput/src/RootOutputFile.cc | 33 ++++-- .../src/RootOutputRNTuple.cc | 29 ++++- .../RNTupleTempOutput/src/RootOutputRNTuple.h | 20 +++- 15 files changed, 204 insertions(+), 157 deletions(-) diff --git a/FWIO/RNTupleTempInput/src/EmbeddedRNTupleTempSource.cc b/FWIO/RNTupleTempInput/src/EmbeddedRNTupleTempSource.cc index 8de0ad820369a..d1db4b14dd3d2 100644 --- a/FWIO/RNTupleTempInput/src/EmbeddedRNTupleTempSource.cc +++ b/FWIO/RNTupleTempInput/src/EmbeddedRNTupleTempSource.cc @@ -14,6 +14,14 @@ namespace edm { class EventPrincipal; } // namespace edm namespace edm::rntuple_temp { + namespace { + EmbeddedRNTupleTempSource::Optimizations fromConfig(edm::ParameterSet const& iConfig) { + EmbeddedRNTupleTempSource::Optimizations opts; + opts.useClusterCache = iConfig.getUntrackedParameter("useClusterCache"); + return opts; + } + + } // namespace EmbeddedRNTupleTempSource::EmbeddedRNTupleTempSource(ParameterSet const& pset, VectorInputSourceDescription const& desc) @@ -30,7 +38,8 @@ namespace edm::rntuple_temp { // skipBadFiles_(pset.getUntrackedParameter("skipBadFiles", false)), bypassVersionCheck_(pset.getUntrackedParameter("bypassVersionCheck", false)), - treeMaxVirtualSize_(pset.getUntrackedParameter("treeMaxVirtualSize", -1)), + treeMaxVirtualSize_(0), + optimizations_(fromConfig(pset.getUntrackedParameterSet("rntupleReadOptions"))), productSelectorRules_(pset, "inputCommands", "InputSource"), runHelper_(new DefaultInputSourceRunHelper()), catalog_(pset.getUntrackedParameter >("fileNames"), @@ -91,8 +100,13 @@ namespace edm::rntuple_temp { ->setComment( "True: Bypass release version check.\n" "False: Throw exception if reading file in a release prior to the release in which the file was written."); - desc.addUntracked("treeMaxVirtualSize", -1) - ->setComment("Size of ROOT TTree TBasket cache. Affects performance."); + desc.addUntracked("treeMaxVirtualSize", -1)->setComment("Not used by RNTuple."); + { + ParameterSetDescription rntupleReadOptions; + rntupleReadOptions.addUntracked("useClusterCache", true) + ->setComment("True: use ROOT cluster cache. False: do not use cluster cache."); + desc.addUntracked("rntupleReadOptions", rntupleReadOptions); + } ProductSelectorRules::fillDescription(desc, "inputCommands"); RootEmbeddedFileSequence::fillDescription(desc); diff --git a/FWIO/RNTupleTempInput/src/EmbeddedRNTupleTempSource.h b/FWIO/RNTupleTempInput/src/EmbeddedRNTupleTempSource.h index 7e4bc176cdf7e..9bac7e17d506b 100644 --- a/FWIO/RNTupleTempInput/src/EmbeddedRNTupleTempSource.h +++ b/FWIO/RNTupleTempInput/src/EmbeddedRNTupleTempSource.h @@ -35,6 +35,10 @@ namespace edm::rntuple_temp { class EmbeddedRNTupleTempSource : public VectorInputSource { public: + struct Optimizations { + bool useClusterCache = true; + }; + explicit EmbeddedRNTupleTempSource(ParameterSet const& pset, VectorInputSourceDescription const& desc); ~EmbeddedRNTupleTempSource() override; using VectorInputSource::processHistoryRegistryForUpdate; @@ -45,6 +49,7 @@ namespace edm::rntuple_temp { bool bypassVersionCheck() const { return bypassVersionCheck_; } unsigned int nStreams() const { return nStreams_; } int treeMaxVirtualSize() const { return treeMaxVirtualSize_; } + Optimizations const& optimizations() const { return optimizations_; } ProductSelectorRules const& productSelectorRules() const { return productSelectorRules_; } InputSourceRunHelperBase* runHelper() { return runHelper_.get(); } @@ -68,6 +73,7 @@ namespace edm::rntuple_temp { bool skipBadFiles_; bool bypassVersionCheck_; int const treeMaxVirtualSize_; + Optimizations optimizations_; ProductSelectorRules productSelectorRules_; std::unique_ptr runHelper_; diff --git a/FWIO/RNTupleTempInput/src/RNTupleTempSource.cc b/FWIO/RNTupleTempInput/src/RNTupleTempSource.cc index 26d2f638ce04f..2d53c0f90ef09 100644 --- a/FWIO/RNTupleTempInput/src/RNTupleTempSource.cc +++ b/FWIO/RNTupleTempInput/src/RNTupleTempSource.cc @@ -62,6 +62,11 @@ namespace edm::rntuple_temp { << primary.id() << " has inconsistent RunAuxiliary data in the primary and secondary file\n"; } } + RNTupleTempSource::Optimizations fromConfig(edm::ParameterSet const& iConfig) { + RNTupleTempSource::Optimizations opts; + opts.useClusterCache = iConfig.getUntrackedParameter("useClusterCache"); + return opts; + } } // namespace RNTupleTempSource::RNTupleTempSource(ParameterSet const& pset, InputSourceDescription const& desc) @@ -79,7 +84,8 @@ namespace edm::rntuple_temp { nStreams_(desc.allocations_->numberOfStreams()), skipBadFiles_(pset.getUntrackedParameter("skipBadFiles")), bypassVersionCheck_(pset.getUntrackedParameter("bypassVersionCheck")), - treeMaxVirtualSize_(pset.getUntrackedParameter("treeMaxVirtualSize")), + treeMaxVirtualSize_(0), + optimizations_(fromConfig(pset.getUntrackedParameterSet("rntupleReadOptions"))), productSelectorRules_(pset, "inputCommands", "InputSource"), dropDescendants_(pset.getUntrackedParameter("dropDescendantsOfDroppedBranches")), labelRawDataLikeMC_(pset.getUntrackedParameter("labelRawDataLikeMC")), @@ -301,7 +307,7 @@ namespace edm::rntuple_temp { ParameterSetDescription desc; std::vector defaultStrings; - desc.setComment("Reads EDM/Root files."); + desc.setComment("Reads EDM/RNTuple Root files."); desc.addUntracked >("fileNames")->setComment("Names of files to be processed."); desc.addUntracked >("secondaryFileNames", defaultStrings) ->setComment("Names of secondary files to be processed."); @@ -316,8 +322,13 @@ namespace edm::rntuple_temp { ->setComment( "True: Bypass release version check.\n" "False: Throw exception if reading file in a release prior to the release in which the file was written."); - desc.addUntracked("treeMaxVirtualSize", -1) - ->setComment("Size of ROOT TTree TBasket cache. Affects performance."); + desc.addOptionalUntracked("treeMaxVirtualSize", -1)->setComment("Not used by RNTuple."); + { + ParameterSetDescription rntupleReadOptions; + rntupleReadOptions.addUntracked("useClusterCache", true) + ->setComment("True: use ROOT cluster cache. False: do not use cluster cache."); + desc.addUntracked("rntupleReadOptions", rntupleReadOptions); + } desc.addUntracked("dropDescendantsOfDroppedBranches", true) ->setComment("If True, also drop on input any descendent of any branch dropped on input."); desc.addUntracked("labelRawDataLikeMC", true) diff --git a/FWIO/RNTupleTempInput/src/RNTupleTempSource.h b/FWIO/RNTupleTempInput/src/RNTupleTempSource.h index e57cf5433a665..c746fe2e4832a 100644 --- a/FWIO/RNTupleTempInput/src/RNTupleTempSource.h +++ b/FWIO/RNTupleTempInput/src/RNTupleTempSource.h @@ -32,6 +32,9 @@ namespace edm::rntuple_temp { class RNTupleTempSource : public InputSource { public: + struct Optimizations { + bool useClusterCache = true; + }; explicit RNTupleTempSource(ParameterSet const& pset, InputSourceDescription const& desc); ~RNTupleTempSource() override; using InputSource::processHistoryRegistryForUpdate; @@ -45,6 +48,7 @@ namespace edm::rntuple_temp { bool delayReadingEventProducts() const { return delayReadingEventProducts_; } unsigned int nStreams() const { return nStreams_; } int treeMaxVirtualSize() const { return treeMaxVirtualSize_; } + Optimizations const& optimizations() const { return optimizations_; } ProductSelectorRules const& productSelectorRules() const { return productSelectorRules_; } InputSourceRunHelperBase* runHelper() { return runHelper_.get(); } @@ -87,6 +91,7 @@ namespace edm::rntuple_temp { bool skipBadFiles_; bool bypassVersionCheck_; int const treeMaxVirtualSize_; + Optimizations optimizations_; ProductSelectorRules productSelectorRules_; bool dropDescendants_; bool labelRawDataLikeMC_; diff --git a/FWIO/RNTupleTempInput/src/RootEmbeddedFileSequence.cc b/FWIO/RNTupleTempInput/src/RootEmbeddedFileSequence.cc index 36651340bbaad..d04dd865347c4 100644 --- a/FWIO/RNTupleTempInput/src/RootEmbeddedFileSequence.cc +++ b/FWIO/RNTupleTempInput/src/RootEmbeddedFileSequence.cc @@ -144,9 +144,7 @@ namespace edm::rntuple_temp { .enforceGUIDInFileName = enforceGUIDInFileName_}, InputType::SecondarySource, RootFile::ProcessingOptions{}, - RootFile::TTreeOptions{.treeCacheSize = treeCacheSize_, - .treeMaxVirtualSize = input_.treeMaxVirtualSize(), - .enablePrefetching = enablePrefetching_}, + RootFile::TTreeOptions{.useClusterCache = input_.optimizations().useClusterCache}, RootFile::ProductChoices{.productSelectorRules = input_.productSelectorRules()}, RootFile::CrossFileInfo{.runHelper = input_.runHelper(), .indexesIntoFiles = indexesIntoFiles(), @@ -379,8 +377,7 @@ namespace edm::rntuple_temp { ->setComment( "How many files to try if 'sequential' is False and 'skipBadFiles' is True.\n" "Defaults to 3 (or # of files if smaller)."); - desc.addUntracked("cacheSize", rootrntuple::defaultCacheSize) - ->setComment("Size of ROOT TTree prefetch cache. Affects performance."); + desc.addOptionalUntracked("cacheSize", 0)->setComment("Not used by RNTuple."); desc.addUntracked("enforceGUIDInFileName", false) ->setComment( "True: file name part is required to be equal to the GUID of the file\n" diff --git a/FWIO/RNTupleTempInput/src/RootFile.cc b/FWIO/RNTupleTempInput/src/RootFile.cc index 165ce64356af9..9899ddcc75203 100644 --- a/FWIO/RNTupleTempInput/src/RootFile.cc +++ b/FWIO/RNTupleTempInput/src/RootFile.cc @@ -169,20 +169,9 @@ namespace edm::rntuple_temp { hasNewlyDroppedBranch_(), branchListIndexesUnchanged_(false), eventAuxCache_(), - eventTree_( - fileOptions.filePtr, InEvent, nStreams, ttreeOptions, rootrntuple::defaultLearningEntries, inputType), - lumiTree_(fileOptions.filePtr, - InLumi, - 1, - ttreeOptions.usingDefaultNonEventOptions(), - rootrntuple::defaultNonEventLearningEntries, - inputType), - runTree_(fileOptions.filePtr, - InRun, - 1, - ttreeOptions.usingDefaultNonEventOptions(), - rootrntuple::defaultNonEventLearningEntries, - inputType), + eventTree_(fileOptions.filePtr, InEvent, nStreams, ttreeOptions, inputType), + lumiTree_(fileOptions.filePtr, InLumi, 1, ttreeOptions.usingDefaultNonEventOptions(), inputType), + runTree_(fileOptions.filePtr, InRun, 1, ttreeOptions.usingDefaultNonEventOptions(), inputType), treePointers_(), lastEventEntryNumberRead_(IndexIntoFile::invalidEntry), productRegistry_(), @@ -1895,13 +1884,8 @@ namespace edm::rntuple_temp { // the ProcessBlock TTree's in the file. (later in the RootFile constructor, dropOnInput might // remove some and also reordering may occur). for (auto const& process : storedProcessBlockHelper.processesWithProcessBlockProducts()) { - processBlockTrees_.emplace_back(std::make_unique(filePtr, - InProcess, - process, - 1, - options.usingDefaultNonEventOptions(), - rootrntuple::defaultNonEventLearningEntries, - inputType)); + processBlockTrees_.emplace_back(std::make_unique( + filePtr, InProcess, process, 1, options.usingDefaultNonEventOptions(), inputType)); } } diff --git a/FWIO/RNTupleTempInput/src/RootPrimaryFileSequence.cc b/FWIO/RNTupleTempInput/src/RootPrimaryFileSequence.cc index 0db444c268d08..7d7af10b0b1f3 100644 --- a/FWIO/RNTupleTempInput/src/RootPrimaryFileSequence.cc +++ b/FWIO/RNTupleTempInput/src/RootPrimaryFileSequence.cc @@ -32,7 +32,7 @@ namespace edm::rntuple_temp { initialNumberOfEventsToSkip_(pset.getUntrackedParameter("skipEvents")), noRunLumiSort_(pset.getUntrackedParameter("noRunLumiSort")), noEventSort_(noRunLumiSort_ ? true : pset.getUntrackedParameter("noEventSort")), - treeCacheSize_(noEventSort_ ? pset.getUntrackedParameter("cacheSize") : 0U), + treeCacheSize_(0U), duplicateChecker_(new DuplicateChecker(pset)), usingGoToEvent_(false), enablePrefetching_(false), @@ -161,9 +161,7 @@ namespace edm::rntuple_temp { .noRunLumiSort = noRunLumiSort_, .noEventSort = noEventSort_, .usingGoToEvent = usingGoToEvent_}, - RootFile::TTreeOptions{.treeCacheSize = treeCacheSize_, - .treeMaxVirtualSize = input_.treeMaxVirtualSize(), - .enablePrefetching = enablePrefetching_, + RootFile::TTreeOptions{.useClusterCache = input_.optimizations().useClusterCache, .promptReading = not input_.delayReadingEventProducts()}, RootFile::ProductChoices{.productSelectorRules = input_.productSelectorRules(), .associationsFromSecondary = nullptr, // associationsFromSecondary @@ -427,8 +425,7 @@ namespace edm::rntuple_temp { ->setComment( "True: Process runs, lumis and events in the order they appear in the file.\n" "False: Follow settings based on 'noEventSort' setting."); - desc.addUntracked("cacheSize", rootrntuple::defaultCacheSize) - ->setComment("Size of ROOT TTree prefetch cache. Affects performance."); + desc.addOptionalUntracked("cacheSize", 0)->setComment("Not used by RNTuple"); std::string defaultString("permissive"); desc.addUntracked("branchesMustMatch", defaultString) ->setComment( diff --git a/FWIO/RNTupleTempInput/src/RootRNTuple.cc b/FWIO/RNTupleTempInput/src/RootRNTuple.cc index 9fdc977276ba8..be2e496bbb9b7 100644 --- a/FWIO/RNTupleTempInput/src/RootRNTuple.cc +++ b/FWIO/RNTupleTempInput/src/RootRNTuple.cc @@ -35,14 +35,11 @@ namespace edm::rntuple_temp { RootRNTuple::RootRNTuple(std::shared_ptr filePtr, BranchType const& branchType, unsigned int nIndexes, - unsigned int learningEntries, - bool enablePrefetching, bool promptRead, InputType inputType) : filePtr_(filePtr), branchType_(branchType), entryNumberForIndex_(std::make_unique>(nIndexes, IndexIntoFile::invalidEntry)), - enablePrefetching_(enablePrefetching), promptRead_(promptRead), rootDelayedReader_(makeRootDelayedReader(*this, filePtr, inputType, nIndexes, promptRead)) {} @@ -51,11 +48,9 @@ namespace edm::rntuple_temp { BranchType const& branchType, unsigned int nIndexes, Options const& options, - unsigned int learningEntries, InputType inputType) - : RootRNTuple( - filePtr, branchType, nIndexes, learningEntries, options.enablePrefetching, options.promptReading, inputType) { - init(BranchTypeToProductTreeName(branchType), options.treeMaxVirtualSize, options.treeCacheSize); + : RootRNTuple(filePtr, branchType, nIndexes, options.promptReading, inputType) { + init(BranchTypeToProductTreeName(branchType), options); auxDesc_ = getAuxiliaryFieldId(*reader_, branchType_); } @@ -65,19 +60,20 @@ namespace edm::rntuple_temp { std::string const& processName, unsigned int nIndexes, Options const& options, - unsigned int learningEntries, InputType inputType) - : RootRNTuple( - filePtr, branchType, nIndexes, learningEntries, options.enablePrefetching, options.promptReading, inputType) { + : RootRNTuple(filePtr, branchType, nIndexes, options.promptReading, inputType) { processName_ = processName; - init(BranchTypeToProductTreeName(branchType, processName), options.treeMaxVirtualSize, options.treeCacheSize); + init(BranchTypeToProductTreeName(branchType, processName), options); } - void RootRNTuple::init(std::string const& productTreeName, unsigned int maxVirtualSize, unsigned int cacheSize) { + void RootRNTuple::init(std::string const& productTreeName, Options const& options) { if (filePtr_.get() != nullptr) { auto tuple = filePtr_->Get(productTreeName.c_str()); if (tuple != nullptr) { - reader_ = ROOT::RNTupleReader::Open(*tuple); + ROOT::RNTupleReadOptions rntupleOptions; + rntupleOptions.SetClusterCache(options.useClusterCache ? ROOT::RNTupleReadOptions::EClusterCache::kOn + : ROOT::RNTupleReadOptions::EClusterCache::kOff); + reader_ = ROOT::RNTupleReader::Open(*tuple, rntupleOptions); } } if (not reader_) { @@ -86,9 +82,6 @@ namespace edm::rntuple_temp { << "\n This is either not an edm ROOT file or is one that has been corrupted."; } entries_ = reader_->GetNEntries(); - - setTreeMaxVirtualSize(maxVirtualSize); - setCacheSize(cacheSize); } RootRNTuple::~RootRNTuple() {} diff --git a/FWIO/RNTupleTempInput/src/RootRNTuple.h b/FWIO/RNTupleTempInput/src/RootRNTuple.h index 907fa23dc9dc7..dbf73c023704b 100644 --- a/FWIO/RNTupleTempInput/src/RootRNTuple.h +++ b/FWIO/RNTupleTempInput/src/RootRNTuple.h @@ -36,10 +36,6 @@ namespace edm::rntuple_temp { class RootDelayedReaderBase; namespace rootrntuple { - unsigned int const defaultCacheSize = 20U * 1024 * 1024; - unsigned int const defaultNonEventCacheSize = 1U * 1024 * 1024; - unsigned int const defaultLearningEntries = 20U; - unsigned int const defaultNonEventLearningEntries = 1U; using EntryNumber = IndexIntoFile::EntryNumber_t; struct ProductInfo { ProductInfo(ProductDescription const& prod) : productDescription_(prod) {} @@ -94,21 +90,16 @@ namespace edm::rntuple_temp { using ProductMap = rootrntuple::ProductMap; using EntryNumber = rootrntuple::EntryNumber; struct Options { - unsigned int treeCacheSize = 0U; - int treeMaxVirtualSize; - bool enablePrefetching; + bool useClusterCache = true; bool promptReading = false; - Options usingDefaultNonEventOptions() const { - return {rootrntuple::defaultNonEventCacheSize, treeMaxVirtualSize, enablePrefetching, false}; - } + Options usingDefaultNonEventOptions() const { return {}; } }; RootRNTuple(std::shared_ptr filePtr, BranchType const& branchType, unsigned int nIndexes, Options const& options, - unsigned int learningEntries, InputType inputType); RootRNTuple(std::shared_ptr filePtr, @@ -116,11 +107,8 @@ namespace edm::rntuple_temp { std::string const& processName, unsigned int nIndexes, Options const& options, - unsigned int learningEntries, InputType inputType); - void init(std::string const& productTreeName, unsigned int maxVirtualSize, unsigned int cacheSize); - ~RootRNTuple(); RootRNTuple(RootRNTuple const&) = delete; // Disallow copying and moving @@ -185,11 +173,11 @@ namespace edm::rntuple_temp { signalslot::Signal const* postEventReadSource); private: + void init(std::string const& productTreeName, Options const& options); + RootRNTuple(std::shared_ptr filePtr, BranchType const& branchType, unsigned int nIndexes, - unsigned int learningEntries, - bool enablePrefetching, bool promptRead, InputType inputType); @@ -211,7 +199,6 @@ namespace edm::rntuple_temp { ProductMap branches_; unsigned int cacheSize_ = 0; unsigned long treeAutoFlush_ = 0; - bool enablePrefetching_; bool promptRead_; std::unique_ptr rootDelayedReader_; }; diff --git a/FWIO/RNTupleTempInput/src/RootSecondaryFileSequence.cc b/FWIO/RNTupleTempInput/src/RootSecondaryFileSequence.cc index 998226770f384..6534eba5aa992 100644 --- a/FWIO/RNTupleTempInput/src/RootSecondaryFileSequence.cc +++ b/FWIO/RNTupleTempInput/src/RootSecondaryFileSequence.cc @@ -82,8 +82,7 @@ namespace edm::rntuple_temp { RootFile::ProcessingOptions{ .processingMode = input_.processingMode(), }, - RootFile::TTreeOptions{.treeMaxVirtualSize = input_.treeMaxVirtualSize(), - .enablePrefetching = enablePrefetching_, + RootFile::TTreeOptions{.useClusterCache = input_.optimizations().useClusterCache, .promptReading = not input_.delayReadingEventProducts()}, RootFile::ProductChoices{.productSelectorRules = input_.productSelectorRules(), .associationsFromSecondary = &associationsFromSecondary_, diff --git a/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h b/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h index fc7a7d36aed6b..7672cc0ef865c 100644 --- a/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h +++ b/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h @@ -43,6 +43,15 @@ namespace edm::rntuple_temp { class RNTupleTempOutputModule : public one::OutputModule { public: enum DropMetaData { DropNone, DropDroppedPrior, DropPrior, DropAll }; + struct Optimizations { + unsigned long long approxZippedClusterSize; + unsigned long long maxUnzippedClusterSize; + unsigned long long initialUnzippedPageSize; + unsigned long long maxUnzippedPageSize; + unsigned long long pageBufferBudget; + bool useBufferedWrite; + bool useDirectIO; + }; explicit RNTupleTempOutputModule(ParameterSet const& ps); ~RNTupleTempOutputModule() override; RNTupleTempOutputModule(RNTupleTempOutputModule const&) = delete; // Disallow copying and moving @@ -51,14 +60,7 @@ namespace edm::rntuple_temp { std::string const& logicalFileName() const { return logicalFileName_; } int compressionLevel() const { return compressionLevel_; } std::string const& compressionAlgorithm() const { return compressionAlgorithm_; } - int basketSize() const { return basketSize_; } - int eventAuxiliaryBasketSize() const { return eventAuxBasketSize_; } - int eventAutoFlushSize() const { return eventAutoFlushSize_; } - int splitLevel() const { return splitLevel_; } - std::string const& basketOrder() const { return basketOrder_; } - int treeMaxVirtualSize() const { return treeMaxVirtualSize_; } - bool overrideInputFileSplitLevels() const { return overrideInputFileSplitLevels_; } - bool compactEventAuxiliary() const { return compactEventAuxiliary_; } + Optimizations const& optimizations() const { return optimizations_; } bool mergeJob() const { return mergeJob_; } DropMetaData const& dropMetaData() const { return dropMetaData_; } std::string const& catalog() const { return catalog_; } @@ -193,12 +195,7 @@ namespace edm::rntuple_temp { unsigned int const maxFileSize_; int const compressionLevel_; std::string const compressionAlgorithm_; - int const basketSize_; - int const eventAuxBasketSize_; - int const eventAutoFlushSize_; - int const splitLevel_; - std::string basketOrder_; - int const treeMaxVirtualSize_; + Optimizations const optimizations_; DropMetaData dropMetaData_; std::string const moduleLabel_; bool initializedFromInput_; @@ -208,7 +205,6 @@ namespace edm::rntuple_temp { ProductDependencies productDependencies_; std::vector producedBranches_; bool overrideInputFileSplitLevels_; - bool compactEventAuxiliary_; bool mergeJob_; edm::propagate_const> rootOutputFile_; std::string statusFileName_; diff --git a/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc b/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc index 2c93cb09a8fb8..35ad517dfa7d2 100644 --- a/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc +++ b/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc @@ -29,6 +29,19 @@ #include #include "boost/algorithm/string.hpp" +namespace { + edm::rntuple_temp::RNTupleTempOutputModule::Optimizations fromConfig(edm::ParameterSet const& iConfig) { + edm::rntuple_temp::RNTupleTempOutputModule::Optimizations opts; + opts.approxZippedClusterSize = iConfig.getUntrackedParameter("approxZippedClusterSize"); + opts.maxUnzippedClusterSize = iConfig.getUntrackedParameter("maxUnzippedClusterSize"); + opts.initialUnzippedPageSize = iConfig.getUntrackedParameter("initialUnzippedPageSize"); + opts.maxUnzippedPageSize = iConfig.getUntrackedParameter("maxUnzippedPageSize"); + opts.pageBufferBudget = iConfig.getUntrackedParameter("pageBufferBudget"); + opts.useBufferedWrite = iConfig.getUntrackedParameter("useBufferedWrite"); + opts.useDirectIO = iConfig.getUntrackedParameter("useDirectIO"); + return opts; + } +} // namespace namespace edm::rntuple_temp { RNTupleTempOutputModule::RNTupleTempOutputModule(ParameterSet const& pset) : edm::one::OutputModuleBase::OutputModuleBase(pset), @@ -42,12 +55,7 @@ namespace edm::rntuple_temp { maxFileSize_(pset.getUntrackedParameter("maxSize")), compressionLevel_(pset.getUntrackedParameter("compressionLevel")), compressionAlgorithm_(pset.getUntrackedParameter("compressionAlgorithm")), - basketSize_(pset.getUntrackedParameter("basketSize")), - eventAuxBasketSize_(pset.getUntrackedParameter("eventAuxiliaryBasketSize")), - eventAutoFlushSize_(pset.getUntrackedParameter("eventAutoFlushCompressedSize")), - splitLevel_(std::min(pset.getUntrackedParameter("splitLevel") + 1, 99)), - basketOrder_(pset.getUntrackedParameter("sortBaskets")), - treeMaxVirtualSize_(pset.getUntrackedParameter("treeMaxVirtualSize")), + optimizations_(fromConfig(pset.getUntrackedParameterSet("rntupleWriteOptions"))), dropMetaData_(DropNone), moduleLabel_(pset.getParameter("@module_label")), initializedFromInput_(false), @@ -55,9 +63,6 @@ namespace edm::rntuple_temp { inputFileCount_(0), branchParents_(), productDependencies_(), - overrideInputFileSplitLevels_(pset.getUntrackedParameter("overrideInputFileSplitLevels")), - compactEventAuxiliary_(pset.getUntrackedParameter("compactEventAuxiliary")), - mergeJob_(pset.getUntrackedParameter("mergeJob")), rootOutputFile_(), statusFileName_(), overrideGUID_(pset.getUntrackedParameter("overrideGUID")) { @@ -84,20 +89,6 @@ namespace edm::rntuple_temp { << "Legal values are 'NONE', 'DROPPED', 'PRIOR', and 'ALL'.\n"; } - auto const& specialSplit{pset.getUntrackedParameterSetVector("overrideBranchesSplitLevel")}; - - specialSplitLevelForBranches_.reserve(specialSplit.size()); - for (auto const& s : specialSplit) { - specialSplitLevelForBranches_.emplace_back(s.getUntrackedParameter("branch"), - s.getUntrackedParameter("splitLevel")); - } - - auto const& branchAliases{pset.getUntrackedParameterSetVector("branchAliases")}; - aliasForBranches_.reserve(branchAliases.size()); - for (auto const& a : branchAliases) { - aliasForBranches_.emplace_back(a.getUntrackedParameter("branch"), - a.getUntrackedParameter("alias")); - } // We don't use this next parameter, but we read it anyway because it is part // of the configuration of this module. An external parser creates the // configuration by reading this source code. @@ -391,30 +382,20 @@ namespace edm::rntuple_temp { desc.addUntracked("compressionAlgorithm", "ZSTD") ->setComment( "Algorithm used to compress data in the ROOT output file, allowed values are ZLIB, LZMA, LZ4, and ZSTD"); - desc.addUntracked("basketSize", 16384)->setComment("Default ROOT basket size in output file."); - desc.addUntracked("eventAuxiliaryBasketSize", 16384) + desc.addOptionalUntracked("basketSize", 16384)->setComment("Default ROOT basket size in output file."); + desc.addOptionalUntracked("eventAuxiliaryBasketSize", 16384) ->setComment("Default ROOT basket size in output file for EventAuxiliary branch."); - desc.addUntracked("eventAutoFlushCompressedSize", 20 * 1024 * 1024)->setComment("Not used by RNTuple"); - desc.addUntracked("splitLevel", 99)->setComment("Default ROOT branch split level in output file."); - desc.addUntracked("sortBaskets", std::string("sortbasketsbyoffset")) + desc.addOptionalUntracked("eventAutoFlushCompressedSize", 20 * 1024 * 1024)->setComment("Not used by RNTuple"); + desc.addOptionalUntracked("splitLevel", 99)->setComment("Default ROOT branch split level in output file."); + desc.addOptionalUntracked("sortBaskets", std::string("sortbasketsbyoffset")) ->setComment( "Legal values: 'sortbasketsbyoffset', 'sortbasketsbybranch', 'sortbasketsbyentry'.\n" "Used by ROOT when fast copying. Affects performance."); - desc.addUntracked("treeMaxVirtualSize", -1)->setComment("Not used by RNTuple."); - desc.addUntracked("fastCloning", false)->setComment("Not used by RNTuple"); - desc.addUntracked("mergeJob", false) - ->setComment( - "If set to true and fast copying is disabled, copy input file compression and basket sizes to the output " - "file."); - desc.addUntracked("compactEventAuxiliary", false) - ->setComment( - "False: Write EventAuxiliary as we go like any other event metadata branch.\n" - "True: Optimize the file layout by deferring writing the EventAuxiliary branch until the output file is " - "closed."); - desc.addUntracked("overrideInputFileSplitLevels", false) - ->setComment( - "False: Use branch split levels and basket sizes from input file, if possible.\n" - "True: Always use specified or default split levels and basket sizes."); + desc.addOptionalUntracked("treeMaxVirtualSize", -1)->setComment("Not used by RNTuple."); + desc.addOptionalUntracked("fastCloning", false)->setComment("Not used by RNTuple"); + desc.addOptionalUntracked("mergeJob", false)->setComment("Not used by RNTuple."); + desc.addOptionalUntracked("compactEventAuxiliary", false)->setComment("Not used by RNTuple."); + desc.addOptionalUntracked("overrideInputFileSplitLevels", false)->setComment("Not used by RNTuple."); desc.addUntracked("writeStatusFile", false) ->setComment("Write a status file. Intended for use by workflow management."); desc.addUntracked("dropMetaData", defaultString) @@ -439,17 +420,48 @@ namespace edm::rntuple_temp { } { ParameterSetDescription specialSplit; - specialSplit.addUntracked("branch")->setComment( - "Name of branch needing a special split level. The name can contain wildcards '*' and '?'"); - specialSplit.addUntracked("splitLevel")->setComment("The special split level for the branch"); + specialSplit.addOptionalUntracked("branch")->setComment("Not used by RNTuple."); + specialSplit.addOptionalUntracked("splitLevel")->setComment("Not used by RNTuple."); desc.addVPSetUntracked("overrideBranchesSplitLevel", specialSplit, std::vector()); } { ParameterSetDescription alias; - alias.addUntracked("branch")->setComment( - "Name of branch which will get alias. The name can contain wildcards '*' and '?'"); - alias.addUntracked("alias")->setComment("The alias to give to the TBranch"); - desc.addVPSetUntracked("branchAliases", alias, std::vector()); + alias.addOptionalUntracked("branch")->setComment("Not used by RNTuple."); + alias.addOptionalUntracked("alias")->setComment("The alias to give to the TBranch"); + desc.addVPSetOptionalUntracked("branchAliases", alias, std::vector()); + } + { + ParameterSetDescription optimizations; + + ROOT::RNTupleWriteOptions ops; + optimizations.addUntracked("approxZippedClusterSize", ops.GetApproxZippedClusterSize()) + ->setComment("Approximation of the target compressed cluster size"); + optimizations.addUntracked("maxUnzippedClusterSize", ops.GetMaxUnzippedClusterSize()) + ->setComment("Memory limit for committing a cluster. High compression leads to high IO buffer size."); + + optimizations.addUntracked("initialUnzippedPageSize", ops.GetInitialUnzippedPageSize()) + ->setComment("Initially, columns start with a page of this size (bytes)."); + optimizations.addUntracked("maxUnzippedPageSize", ops.GetMaxUnzippedPageSize()) + ->setComment("Pages can grow only to the given limit (bytes)."); + optimizations.addUntracked("pageBufferBudget", 0) + ->setComment( + "The maximum size that the sum of all page buffers used for writing into a persistent sink are allowed " + "to " + "use." + " If set to zero, RNTuple will auto-adjust the budget based on the value of 'approxZippedClusterSize'." + " If set manually, the size needs to be large enough to hold all initial page buffers."); + + optimizations.addUntracked("useBufferedWrite", ops.GetUseBufferedWrite()) + ->setComment( + "Turn on use of buffered writing. This buffers compressed pages in memory, reorders them to keep pages " + "of " + "the same column adjacent, and coalesces the writes when committing a cluster."); + optimizations.addUntracked("useDirectIO", ops.GetUseDirectIO()) + ->setComment( + "Set use of direct IO. this introduces alignment requirements that may vary between filesystems and " + "platforms"); + desc.addUntracked("rntupleWriteOptions", optimizations) + ->setComment("Options to control RNTuple specific output features."); } OutputModule::fillDescription(desc); } diff --git a/FWIO/RNTupleTempOutput/src/RootOutputFile.cc b/FWIO/RNTupleTempOutput/src/RootOutputFile.cc index 41da46ae1782a..426d4324ec335 100644 --- a/FWIO/RNTupleTempOutput/src/RootOutputFile.cc +++ b/FWIO/RNTupleTempOutput/src/RootOutputFile.cc @@ -114,9 +114,9 @@ namespace edm::rntuple_temp { pEventEntryInfoVector_(&eventEntryInfoVector_), pBranchListIndexes_(nullptr), pEventSelectionIDs_(nullptr), - eventRNTuple_(filePtr(), InEvent, om_->splitLevel(), om_->treeMaxVirtualSize()), - lumiRNTuple_(filePtr(), InLumi, om_->splitLevel(), om_->treeMaxVirtualSize()), - runRNTuple_(filePtr(), InRun, om_->splitLevel(), om_->treeMaxVirtualSize()), + eventRNTuple_(filePtr(), InEvent), + lumiRNTuple_(filePtr(), InLumi), + runRNTuple_(filePtr(), InRun), dataTypeReported_(false), processHistoryRegistry_(), parentageIDs_(), @@ -125,8 +125,7 @@ namespace edm::rntuple_temp { std::vector const& processesWithProcessBlockProducts = om_->outputProcessBlockHelper().processesWithProcessBlockProducts(); for (auto const& processName : processesWithProcessBlockProducts) { - processBlockRNTuples_.emplace_back(std::make_unique( - filePtr(), InProcess, om_->splitLevel(), om_->treeMaxVirtualSize(), processName)); + processBlockRNTuples_.emplace_back(std::make_unique(filePtr(), InProcess, processName)); } if (om_->compressionAlgorithm() == std::string("ZLIB")) { @@ -143,9 +142,6 @@ namespace edm::rntuple_temp { << "'\n" << "Allowed compression algorithms are ZLIB, LZMA, LZ4, and ZSTD\n"; } - if (-1 != om->eventAutoFlushSize()) { - eventRNTuple_.setAutoFlush(-1 * om->eventAutoFlushSize()); - } eventRNTuple_.addAuxiliary( BranchTypeToAuxiliaryBranchName(InEvent), &pEventAux_, om_->auxItems()[InEvent].basketSize_); @@ -191,8 +187,23 @@ namespace edm::rntuple_temp { branchesWithStoredHistory_.insert(item.branchID()); } } + RootOutputRNTuple::Config config; + config.compressionAlgo = + om_->compressionAlgorithm() == std::string("LZMA") ? RootOutputRNTuple::Config::CompressionAlgos::kLZMA + : om_->compressionAlgorithm() == std::string("ZSTD") ? RootOutputRNTuple::Config::CompressionAlgos::kZSTD + : om_->compressionAlgorithm() == std::string("LZ4") ? RootOutputRNTuple::Config::CompressionAlgos::kLZ4 + : RootOutputRNTuple::Config::CompressionAlgos::kZLIB; + config.compressionLevel = om_->compressionLevel(); + auto& optimizations = om_->optimizations(); + config.approxZippedClusterSize = optimizations.approxZippedClusterSize; + config.maxUnzippedClusterSize = optimizations.maxUnzippedClusterSize; + config.initialUnzippedPageSize = optimizations.initialUnzippedPageSize; + config.maxUnzippedPageSize = optimizations.maxUnzippedPageSize; + config.pageBufferBudget = optimizations.pageBufferBudget; + config.useBufferedWrite = optimizations.useBufferedWrite; + config.useDirectIO = optimizations.useDirectIO; for (auto& tree : treePointers_) { - tree->finishInitialization(); + tree->finishInitialization(config); } if (overrideGUID.empty()) { @@ -250,9 +261,7 @@ namespace edm::rntuple_temp { void RootOutputFile::respondToCloseInputFile(FileBlock const&) { // We can't do setEntries() on the event tree if the EventAuxiliary branch is empty & disabled - if (not om_->compactEventAuxiliary()) { - eventRNTuple_.setEntries(); - } + eventRNTuple_.setEntries(); lumiRNTuple_.setEntries(); runRNTuple_.setEntries(); } diff --git a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc index 4839ae009ecf4..1b8a0028259b0 100644 --- a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc +++ b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc @@ -27,8 +27,6 @@ namespace edm { RootOutputRNTuple::RootOutputRNTuple(std::shared_ptr filePtr, BranchType const& branchType, - int splitLevel, - int treeMaxVirtualSize, std::string const& processName) : filePtr_(filePtr), name_(processName.empty() ? BranchTypeToProductTreeName(branchType) @@ -105,7 +103,32 @@ namespace edm { } } - void RootOutputRNTuple::finishInitialization() { + void RootOutputRNTuple::finishInitialization(Config const& config) { + ROOT::RNTupleWriteOptions options; + switch (config.compressionAlgo) { + case Config::CompressionAlgos::kLZMA: + options.SetCompression(ROOT::RCompressionSetting::EAlgorithm::kLZMA, config.compressionLevel); + break; + case Config::CompressionAlgos::kZSTD: + options.SetCompression(ROOT::RCompressionSetting::EAlgorithm::kZSTD, config.compressionLevel); + break; + case Config::CompressionAlgos::kZLIB: + options.SetCompression(ROOT::RCompressionSetting::EAlgorithm::kZLIB, config.compressionLevel); + break; + case Config::CompressionAlgos::kLZ4: + options.SetCompression(ROOT::RCompressionSetting::EAlgorithm::kLZ4, config.compressionLevel); + break; + default: + throw edm::Exception(edm::errors::Configuration) + << "Unknown compression algorithm enum value: " << static_cast(config.compressionAlgo) << "\n"; + } + options.SetApproxZippedClusterSize(config.approxZippedClusterSize); + options.SetMaxUnzippedClusterSize(config.maxUnzippedClusterSize); + options.SetInitialUnzippedPageSize(config.initialUnzippedPageSize); + options.SetMaxUnzippedPageSize(config.maxUnzippedPageSize); + options.SetPageBufferBudget(config.pageBufferBudget); + options.SetUseBufferedWrite(config.useBufferedWrite); + options.SetUseDirectIO(config.useDirectIO); writer_ = ROOT::RNTupleWriter::Append(std::move(model_), name_, *filePtr_, ROOT::RNTupleWriteOptions()); } diff --git a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h index 84537bc867bce..e22fd5b03a7d0 100644 --- a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h +++ b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h @@ -21,12 +21,26 @@ RootOutputRNTuple.h // used by ROOT output modules class TFile; namespace edm { + class RootOutputRNTuple { public: + struct Config { + enum class CompressionAlgos { kLZMA, kZSTD, kZLIB, kLZ4 }; + + std::vector doNotSplitSubFields; + CompressionAlgos compressionAlgo = CompressionAlgos::kZSTD; + int compressionLevel = 4; + unsigned long long approxZippedClusterSize; + unsigned long long maxUnzippedClusterSize; + unsigned long long initialUnzippedPageSize; + unsigned long long maxUnzippedPageSize; + unsigned long long pageBufferBudget; + bool useBufferedWrite; + bool useDirectIO; + }; + RootOutputRNTuple(std::shared_ptr filePtr, BranchType const& branchType, - int splitLevel, - int treeMaxVirtualSize, std::string const& processName = std::string()); ~RootOutputRNTuple() {} @@ -54,7 +68,7 @@ namespace edm { void fill(); - void finishInitialization(); + void finishInitialization(Config const& config); std::string const& name() const { return name_; } From 11cfc9f3d48d2582afd648eae2fb566088fdbc7d Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Tue, 21 Oct 2025 08:17:30 -0500 Subject: [PATCH 13/19] Update to use std::format --- FWIO/RNTupleTempInput/src/RootInputFileSequence.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/FWIO/RNTupleTempInput/src/RootInputFileSequence.cc b/FWIO/RNTupleTempInput/src/RootInputFileSequence.cc index c3b57d161bc0a..dcfdd2604825e 100644 --- a/FWIO/RNTupleTempInput/src/RootInputFileSequence.cc +++ b/FWIO/RNTupleTempInput/src/RootInputFileSequence.cc @@ -281,11 +281,11 @@ namespace edm::rntuple_temp { throw ex; } else { exInfo.push_back("Calling RootInputFileSequence::initTheFile(): fail to open the file with name " + (*it)); - additionalMessage.push_back(fmt::format( + additionalMessage.push_back(std::format( "Input file {} could not be opened, and fallback was attempted.\nAdditional information:", *it)); char c = 'a'; for (auto const& ai : e.additionalInfo()) { - additionalMessage.push_back(fmt::format(" [{}] {}", c, ai)); + additionalMessage.push_back(std::format(" [{}] {}", c, ai)); ++c; } } From ed1bba5f821af65d6995444d2f8a7ab2d928eea8 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Tue, 21 Oct 2025 08:17:57 -0500 Subject: [PATCH 14/19] Handle RNTuple optimizations - properly pass optimizations to RNTupleWriter - can specify per Field optimizations --- .../interface/RNTupleTempOutputModule.h | 39 ++++--- .../src/RNTupleTempOutputModule.cc | 110 +++++++++++++----- FWIO/RNTupleTempOutput/src/RootOutputFile.cc | 5 +- .../src/RootOutputRNTuple.cc | 75 +++++++++++- .../RNTupleTempOutput/src/RootOutputRNTuple.h | 5 +- 5 files changed, 175 insertions(+), 59 deletions(-) diff --git a/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h b/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h index 7672cc0ef865c..4a4a096328f75 100644 --- a/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h +++ b/FWIO/RNTupleTempOutput/interface/RNTupleTempOutputModule.h @@ -84,7 +84,7 @@ namespace edm::rntuple_temp { AuxItemArray const& auxItems() const { return auxItems_; } struct OutputItem { - explicit OutputItem(ProductDescription const* bd, EDGetToken const& token, int splitLevel, int basketSize); + explicit OutputItem(ProductDescription const* bd, EDGetToken const& token, bool streamerProduct); BranchID branchID() const { return productDescription_->branchID(); } std::string const& branchName() const { return productDescription_->branchName(); } @@ -97,31 +97,18 @@ namespace edm::rntuple_temp { void const*& product() { return product_; } void const** productPtr() { return &product_; } void setProduct(void const* iProduct) { product_ = iProduct; } - int splitLevel() const { return splitLevel_; } - int basketSize() const { return basketSize_; } + + bool streamerProduct() const { return streamerProduct_; } private: ProductDescription const* productDescription_; EDGetToken token_; void const* product_; - int splitLevel_; - int basketSize_; + bool streamerProduct_; }; using OutputItemList = std::vector; - struct SpecialSplitLevelForBranch { - SpecialSplitLevelForBranch(std::string const& iBranchName, int iSplitLevel) - : branch_(convert(iBranchName)), - splitLevel_(iSplitLevel < 1 ? 1 : iSplitLevel) //minimum is 1 - {} - bool match(std::string const& iBranchName) const; - std::regex convert(std::string const& iGlobBranchExpression) const; - - std::regex branch_; - int splitLevel_; - }; - struct AliasForBranch { AliasForBranch(std::string const& iBranchName, std::string const& iAlias) : branch_{convert(iBranchName)}, alias_{iAlias} {} @@ -133,6 +120,16 @@ namespace edm::rntuple_temp { std::string alias_; }; + struct SetStreamerForDataProduct { + SetStreamerForDataProduct(std::string const& iName, bool iUseStreamer) + : branch_(convert(iName)), useStreamer_(iUseStreamer) {} + bool match(std::string const& iName) const; + std::regex convert(std::string const& iGlobBranchExpression) const; + + std::regex branch_; + bool useStreamer_; + }; + std::vector const& selectedOutputItemList() const { return selectedOutputItemList_; } std::vector& selectedOutputItemList() { return selectedOutputItemList_; } @@ -141,6 +138,9 @@ namespace edm::rntuple_temp { std::vector const& aliasForBranches() const { return aliasForBranches_; } + std::vector const& noSplitSubFields() const { return noSplitSubFields_; } + bool allProductsUseStreamer() const { return allProductsUseStreamer_; } + protected: ///allow inheriting classes to override but still be able to call this method in the overridden version bool shouldWeCloseFile() const override; @@ -186,7 +186,6 @@ namespace edm::rntuple_temp { RootServiceChecker rootServiceChecker_; AuxItemArray auxItems_; std::vector selectedOutputItemList_; - std::vector specialSplitLevelForBranches_; std::vector aliasForBranches_; std::unique_ptr reg_; std::string const fileName_; @@ -210,6 +209,10 @@ namespace edm::rntuple_temp { std::string statusFileName_; std::string overrideGUID_; std::vector processesWithSelectedMergeableRunProducts_; + + std::vector noSplitSubFields_; + std::vector overrideStreamer_; + bool allProductsUseStreamer_; }; } // namespace edm::rntuple_temp diff --git a/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc b/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc index 35ad517dfa7d2..9e649e3d2663c 100644 --- a/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc +++ b/FWIO/RNTupleTempOutput/src/RNTupleTempOutputModule.cc @@ -29,19 +29,56 @@ #include #include "boost/algorithm/string.hpp" -namespace { - edm::rntuple_temp::RNTupleTempOutputModule::Optimizations fromConfig(edm::ParameterSet const& iConfig) { - edm::rntuple_temp::RNTupleTempOutputModule::Optimizations opts; - opts.approxZippedClusterSize = iConfig.getUntrackedParameter("approxZippedClusterSize"); - opts.maxUnzippedClusterSize = iConfig.getUntrackedParameter("maxUnzippedClusterSize"); - opts.initialUnzippedPageSize = iConfig.getUntrackedParameter("initialUnzippedPageSize"); - opts.maxUnzippedPageSize = iConfig.getUntrackedParameter("maxUnzippedPageSize"); - opts.pageBufferBudget = iConfig.getUntrackedParameter("pageBufferBudget"); - opts.useBufferedWrite = iConfig.getUntrackedParameter("useBufferedWrite"); - opts.useDirectIO = iConfig.getUntrackedParameter("useDirectIO"); - return opts; +namespace edm::rntuple_temp { + inline bool RNTupleTempOutputModule::SetStreamerForDataProduct::match(std::string const& iBranchName) const { + return std::regex_match(iBranchName, branch_); + } + + std::regex RNTupleTempOutputModule::SetStreamerForDataProduct::convert( + std::string const& iGlobBranchExpression) const { + std::string tmp(iGlobBranchExpression); + boost::replace_all(tmp, "*", ".*"); + boost::replace_all(tmp, "?", "."); + return std::regex(tmp); } -} // namespace + + namespace { + std::vector fromConfig( + std::vector const& iConfig) { + std::vector returnValue; + returnValue.reserve(iConfig.size()); + + for (auto const& prod : iConfig) { + returnValue.emplace_back(prod.getUntrackedParameter("product"), + prod.getUntrackedParameter("useStreamer")); + } + return returnValue; + } + + std::optional useStreamer(std::string const& iName, + std::vector const& iSpecial) { + auto nameNoDot = iName.substr(0, iName.size() - 1); + for (auto const& prod : iSpecial) { + if (prod.match(nameNoDot)) { + return prod.useStreamer_; + } + } + return {}; + } + + edm::rntuple_temp::RNTupleTempOutputModule::Optimizations fromConfig(edm::ParameterSet const& iConfig) { + edm::rntuple_temp::RNTupleTempOutputModule::Optimizations opts; + opts.approxZippedClusterSize = iConfig.getUntrackedParameter("approxZippedClusterSize"); + opts.maxUnzippedClusterSize = iConfig.getUntrackedParameter("maxUnzippedClusterSize"); + opts.initialUnzippedPageSize = iConfig.getUntrackedParameter("initialUnzippedPageSize"); + opts.maxUnzippedPageSize = iConfig.getUntrackedParameter("maxUnzippedPageSize"); + opts.pageBufferBudget = iConfig.getUntrackedParameter("pageBufferBudget"); + opts.useBufferedWrite = iConfig.getUntrackedParameter("useBufferedWrite"); + opts.useDirectIO = iConfig.getUntrackedParameter("useDirectIO"); + return opts; + } + } // namespace +} // namespace edm::rntuple_temp namespace edm::rntuple_temp { RNTupleTempOutputModule::RNTupleTempOutputModule(ParameterSet const& pset) : edm::one::OutputModuleBase::OutputModuleBase(pset), @@ -65,7 +102,14 @@ namespace edm::rntuple_temp { productDependencies_(), rootOutputFile_(), statusFileName_(), - overrideGUID_(pset.getUntrackedParameter("overrideGUID")) { + overrideGUID_(pset.getUntrackedParameter("overrideGUID")), + noSplitSubFields_(pset.getUntrackedParameterSet("fieldLevelOptimizations") + .getUntrackedParameter>("noSplitSubFields")), + overrideStreamer_( + fromConfig(pset.getUntrackedParameterSet("fieldLevelOptimizations") + .getUntrackedParameter>("overrideDataProductStreamer"))), + allProductsUseStreamer_( + pset.getUntrackedParameterSet("fieldLevelOptimizations").getUntrackedParameter("useStreamer")) { if (pset.getUntrackedParameter("writeStatusFile")) { std::ostringstream statusfilename; statusfilename << moduleLabel_ << '_' << getpid(); @@ -107,9 +151,8 @@ namespace edm::rntuple_temp { RNTupleTempOutputModule::OutputItem::OutputItem(ProductDescription const* bd, EDGetToken const& token, - int splitLevel, - int basketSize) - : productDescription_(bd), token_(token), product_(nullptr), splitLevel_(splitLevel), basketSize_(basketSize) {} + bool streamerProduct) + : productDescription_(bd), token_(token), product_(nullptr), streamerProduct_(streamerProduct) {} namespace { std::regex convertBranchExpression(std::string const& iGlobBranchExpression) { @@ -120,15 +163,6 @@ namespace edm::rntuple_temp { } } // namespace - inline bool RNTupleTempOutputModule::SpecialSplitLevelForBranch::match(std::string const& iBranchName) const { - return std::regex_match(iBranchName, branch_); - } - - std::regex RNTupleTempOutputModule::SpecialSplitLevelForBranch::convert( - std::string const& iGlobBranchExpression) const { - return convertBranchExpression(iGlobBranchExpression); - } - bool RNTupleTempOutputModule::AliasForBranch::match(std::string const& iBranchName) const { return std::regex_match(iBranchName, branch_); } @@ -144,14 +178,12 @@ namespace edm::rntuple_temp { // Fill outputItemList with an entry for each branch. for (auto const& kept : keptVector) { - int splitLevel = ProductDescription::invalidSplitLevel; - int basketSize = ProductDescription::invalidBasketSize; - ProductDescription const& prod = *kept.first; if (branchType == InProcess && processName != prod.processName()) { continue; } - outputItemList.emplace_back(&prod, kept.second, splitLevel, basketSize); + bool streamerProduct = allProductsUseStreamer_ or useStreamer(prod.branchName(), overrideStreamer_); + outputItemList.emplace_back(&prod, kept.second, streamerProduct); } } @@ -463,6 +495,26 @@ namespace edm::rntuple_temp { desc.addUntracked("rntupleWriteOptions", optimizations) ->setComment("Options to control RNTuple specific output features."); } + { + ParameterSetDescription fieldLevel; + fieldLevel.addUntracked>("noSplitSubFields", {}) + ->setComment( + "fully qualified subfield names for fields which should not be split. A single value of 'all' means all " + "possible subfields will be unsplit"); + fieldLevel.addUntracked("useStreamer", false) + ->setComment("Use streamer storage for top level fields when storing data products"); + + { + ParameterSetDescription specialStreamer; + specialStreamer.addUntracked("product")->setComment( + "Name of data product needing a special split setting. The name can contain wildcards '*' and '?'"); + specialStreamer.addUntracked("useStreamer", true) + ->setComment("Explicitly set if should or should not use streamer (default is to use streamer)"); + fieldLevel.addVPSetUntracked("overrideDataProductStreamer", specialStreamer, {}); + } + desc.addUntracked("fieldLevelOptimizations", fieldLevel) + ->setComment("Options to control specializing how Fields are stored."); + } OutputModule::fillDescription(desc); } diff --git a/FWIO/RNTupleTempOutput/src/RootOutputFile.cc b/FWIO/RNTupleTempOutput/src/RootOutputFile.cc index 426d4324ec335..d8b5328b437c5 100644 --- a/FWIO/RNTupleTempOutput/src/RootOutputFile.cc +++ b/FWIO/RNTupleTempOutput/src/RootOutputFile.cc @@ -180,9 +180,8 @@ namespace edm::rntuple_temp { theRNTuple->addField(fixName(desc.branchName()), desc.wrappedName(), item.productPtr(), - item.splitLevel(), - item.basketSize(), - item.productDescription()->produced()); + item.streamerProduct() or om_->allProductsUseStreamer(), + om_->noSplitSubFields()); //make sure we always store product registry info for all branches we create branchesWithStoredHistory_.insert(item.branchID()); } diff --git a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc index 1b8a0028259b0..6ec6288fe83ec 100644 --- a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc +++ b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.cc @@ -129,17 +129,80 @@ namespace edm { options.SetPageBufferBudget(config.pageBufferBudget); options.SetUseBufferedWrite(config.useBufferedWrite); options.SetUseDirectIO(config.useDirectIO); - writer_ = ROOT::RNTupleWriter::Append(std::move(model_), name_, *filePtr_, ROOT::RNTupleWriteOptions()); + writer_ = ROOT::RNTupleWriter::Append(std::move(model_), name_, *filePtr_, options); } + namespace { + /* By default RNTuple will take a multi-byte intrinsic data type and break +it into multiple output fields to separate the high-bytes from the low-bytes (or mantessa from exponent). +This typically allows for better compression. Empirically we have found that some important +member data of some classes actually take more space on disk when this is done. +This function allows one to override the default RNTuple behavior and instead store +all bytes of a data type in one field. To do that one must find the storage type (typeName) and +explicitly pass the correct variable to `SetColumnRepresentatives`). + */ + void noSplitField(ROOT::RFieldBase& iField) { + auto const& typeName = iField.GetTypeName(); + if (typeName == "std::uint16_t") { + iField.SetColumnRepresentatives({{ROOT::ENTupleColumnType::kUInt16}}); + } else if (typeName == "std::uint32_t") { + iField.SetColumnRepresentatives({{ROOT::ENTupleColumnType::kUInt32}}); + } else if (typeName == "std::uint64_t") { + iField.SetColumnRepresentatives({{ROOT::ENTupleColumnType::kUInt64}}); + } else if (typeName == "std::int16_t") { + iField.SetColumnRepresentatives({{ROOT::ENTupleColumnType::kInt16}}); + } else if (typeName == "std::int32_t") { + iField.SetColumnRepresentatives({{ROOT::ENTupleColumnType::kInt32}}); + } else if (typeName == "std::int64_t") { + iField.SetColumnRepresentatives({{ROOT::ENTupleColumnType::kInt64}}); + } else if (typeName == "float") { + iField.SetColumnRepresentatives({{ROOT::ENTupleColumnType::kReal32}}); + } else if (typeName == "double") { + iField.SetColumnRepresentatives({{ROOT::ENTupleColumnType::kReal64}}); + } + } + + void findSubFieldsForNoSplitThenApply(ROOT::RFieldBase& iField, std::vector const& iNoSplitFields) { + for (auto const& name : iNoSplitFields) { + if (name.starts_with(iField.GetFieldName())) { + bool found = false; + for (auto& subfield : iField) { + if (subfield.GetQualifiedFieldName() == name) { + found = true; + noSplitField(subfield); + break; + } + } + if (not found) { + throw edm::Exception(edm::errors::Configuration) + << "The data product was found but the requested subfield '" << name << "' is not part of the class"; + } + } + } + } + } // namespace + void RootOutputRNTuple::addField(std::string const& branchName, std::string const& className, void const** pProd, - int splitLevel, - int basketSize, - bool produced) { - auto field = ROOT::RFieldBase::Create(branchName, className).Unwrap(); - model_->AddField(std::move(field)); + bool useStreamer, + std::vector const& iNoSplitFields) { + const bool noSplitSubFields = (iNoSplitFields.size() == 1 and iNoSplitFields[0] == "all") ? true : false; + if (useStreamer) { + auto field = std::make_unique(branchName, className); + model_->AddField(std::move(field)); + } else { + auto field = ROOT::RFieldBase::Create(branchName, className).Unwrap(); + if (noSplitSubFields) { + //use the 'conventional' way to store fields + for (auto& subfield : *field) { + noSplitField(subfield); + } + } else if (not iNoSplitFields.empty()) { + findSubFieldsForNoSplitThenApply(*field, iNoSplitFields); + } + model_->AddField(std::move(field)); + } producedBranches_.push_back(model_->GetToken(branchName)); producedBranchPointers_.push_back(const_cast(pProd)); } diff --git a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h index e22fd5b03a7d0..8325281f12b95 100644 --- a/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h +++ b/FWIO/RNTupleTempOutput/src/RootOutputRNTuple.h @@ -62,9 +62,8 @@ namespace edm { void addField(std::string const& branchName, std::string const& className, void const** pProd, - int splitLevel, - int basketSize, - bool produced); + bool useStreamer, + std::vector const& iNoSplitFields); void fill(); From 16a8720a8c834df23c8faceb0108eb3dbe0a2677 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Fri, 24 Oct 2025 03:36:40 -0500 Subject: [PATCH 15/19] Use config to decide meta data compression in RNTupleTemp --- FWIO/RNTupleTempOutput/src/RootOutputFile.cc | 25 +++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/FWIO/RNTupleTempOutput/src/RootOutputFile.cc b/FWIO/RNTupleTempOutput/src/RootOutputFile.cc index d8b5328b437c5..aa769930520c3 100644 --- a/FWIO/RNTupleTempOutput/src/RootOutputFile.cc +++ b/FWIO/RNTupleTempOutput/src/RootOutputFile.cc @@ -379,6 +379,19 @@ namespace edm::rntuple_temp { treePointers_[ttreeIndex]->optimizeBaskets(10ULL * 1024 * 1024); } + namespace { + ROOT::RNTupleWriteOptions optionsFrom(RNTupleTempOutputModule const& iOM) { + auto writeOptions = ROOT::RNTupleWriteOptions(); + auto compressionAlgo = + iOM.compressionAlgorithm() == std::string("LZMA") ? ROOT::RCompressionSetting::EAlgorithm::kLZMA + : iOM.compressionAlgorithm() == std::string("ZSTD") ? ROOT::RCompressionSetting::EAlgorithm::kZSTD + : iOM.compressionAlgorithm() == std::string("LZ4") ? ROOT::RCompressionSetting::EAlgorithm::kLZ4 + : ROOT::RCompressionSetting::EAlgorithm::kZLIB; + writeOptions.SetCompression(compressionAlgo, iOM.compressionLevel()); + return writeOptions; + } + } // namespace + void RootOutputFile::writeMetaData(ProductRegistry const& iReg) { auto model = ROOT::RNTupleModel::CreateBare(); { @@ -396,10 +409,8 @@ namespace edm::rntuple_temp { } } - auto writeOptions = ROOT::RNTupleWriteOptions(); - //writeOptions.SetCompression(convert(iConfig.compressionAlgo), iConfig.compressionLevel); auto metaData = - ROOT::RNTupleWriter::Append(std::move(model), poolNames::metaDataTreeName(), *filePtr_, writeOptions); + ROOT::RNTupleWriter::Append(std::move(model), poolNames::metaDataTreeName(), *filePtr_, optionsFrom(*om_)); auto rentry = metaData->CreateEntry(); @@ -420,10 +431,8 @@ namespace edm::rntuple_temp { auto model = ROOT::RNTupleModel::CreateBare(); model->AddField(ROOT::RFieldBase::Create(poolNames::parentageBranchName(), "edm::Parentage").Unwrap()); - auto writeOptions = ROOT::RNTupleWriteOptions(); - //writeOptions.SetCompression(convert(iConfig.compressionAlgo), iConfig.compressionLevel); auto parentageWriter = - ROOT::RNTupleWriter::Append(std::move(model), poolNames::parentageTreeName(), *filePtr_, writeOptions); + ROOT::RNTupleWriter::Append(std::move(model), poolNames::parentageTreeName(), *filePtr_, optionsFrom(*om_)); ParentageRegistry& ptReg = *ParentageRegistry::instance(); @@ -569,10 +578,8 @@ namespace edm::rntuple_temp { ROOT::RFieldBase::Create("IdToParameterSetsBlobs", "std::pair,edm::ParameterSetBlob>").Unwrap(); model->AddField(std::move(field)); } - auto writeOptions = ROOT::RNTupleWriteOptions(); - //writeOptions.SetCompression(convert(iConfig.compressionAlgo), iConfig.compressionLevel); auto parameterSets = - ROOT::RNTupleWriter::Append(std::move(model), poolNames::parameterSetsTreeName(), *filePtr_, writeOptions); + ROOT::RNTupleWriter::Append(std::move(model), poolNames::parameterSetsTreeName(), *filePtr_, optionsFrom(*om_)); std::pair idToBlob; From 8fa87a86844842beb4e9f27abebdb387ea5df091 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Fri, 24 Oct 2025 03:37:35 -0500 Subject: [PATCH 16/19] Remove redundant FixMissingStreamerInfos --- .../src/FixMissingStreamerInfos.cc | 74 ------------------- 1 file changed, 74 deletions(-) delete mode 100644 FWIO/RNTupleTempInput/src/FixMissingStreamerInfos.cc diff --git a/FWIO/RNTupleTempInput/src/FixMissingStreamerInfos.cc b/FWIO/RNTupleTempInput/src/FixMissingStreamerInfos.cc deleted file mode 100644 index d6e41df2e56ae..0000000000000 --- a/FWIO/RNTupleTempInput/src/FixMissingStreamerInfos.cc +++ /dev/null @@ -1,74 +0,0 @@ -// -*- C++ -*- -// -// Package: Services -// Class : FixMissingStreamerInfos -// -// Implementation: - -/** \class edm::service::FixMissingStreamerInfos - -This service is used to open and close a ROOT file that contains -StreamerInfo objects causing them to be saved in memory. It is -used when reading a file written with a version of ROOT with a -bug that caused it to fail to write out StreamerInfo objects. -(see Issue 41246). - -CMSSW_13_0_0 had such a problem and files were written with -this problem. When using this service to read files written -with this release set the "fileInPath" parameter to the string -"IOPool/Input/data/fileContainingStreamerInfos_13_0_0.root". -This file is saved in the cms-data repository for IOPool/Input. -Note that it was difficult to identify all the problem classes -and we might have missed some. If there are additional problem -classes a new version of this file can be generated with script -IOPool/Input/scripts/makeFileContainingStreamerInfos.C. If the -problem ever recurs in ROOT with a different release, one could -use that script to generate a file containing StreamerInfos for -other releases. - - \author W. David Dagenhart, created 30 October, 2023 - -*/ - -#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" -#include "FWCore/ParameterSet/interface/ParameterSet.h" -#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" -#include "FWCore/ServiceRegistry/interface/ActivityRegistry.h" -#include "FWCore/ServiceRegistry/interface/ServiceMaker.h" -#include "FWCore/Utilities/interface/EDMException.h" -#include "FWCore/Utilities/interface/FileInPath.h" - -#include "TFile.h" - -namespace edm::rntuple_temp { - namespace service { - - class FixMissingStreamerInfos { - public: - FixMissingStreamerInfos(ParameterSet const&, ActivityRegistry&); - static void fillDescriptions(ConfigurationDescriptions&); - - private: - FileInPath fileInPath_; - }; - - FixMissingStreamerInfos::FixMissingStreamerInfos(ParameterSet const& pset, edm::ActivityRegistry&) - : fileInPath_(pset.getUntrackedParameter("fileInPath")) { - auto tFile = TFile::Open(fileInPath_.fullPath().c_str()); - if (!tFile || tFile->IsZombie()) { - throw cms::Exception("FixMissingStreamerInfo") - << "Failed opening file containing missing StreamerInfos: " << fileInPath_.fullPath(); - } - tFile->Close(); - } - - void FixMissingStreamerInfos::fillDescriptions(ConfigurationDescriptions& descriptions) { - ParameterSetDescription desc; - desc.addUntracked("fileInPath"); - descriptions.add("FixMissingStreamerInfos", desc); - } - } // namespace service -} // namespace edm::rntuple_temp - -using namespace edm::rntuple_temp::service; -DEFINE_FWK_SERVICE(FixMissingStreamerInfos); From 532b458922ca50e34c5795b6a2857ef741a77bae Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Tue, 4 Nov 2025 09:46:53 -0600 Subject: [PATCH 17/19] Update TestRNTupleTempGetBy to new Tracer output The Tracer service added new state transition output so the results of the test need to be updated. --- .../test/unit_test_outputs/testGetBy1.log | 13 +++++++++++++ .../test/unit_test_outputs/testGetBy2.log | 13 +++++++++++++ 2 files changed, 26 insertions(+) diff --git a/FWIO/RNTupleTempTests/test/unit_test_outputs/testGetBy1.log b/FWIO/RNTupleTempTests/test/unit_test_outputs/testGetBy1.log index 4b6d957087269..4da1cc2342343 100644 --- a/FWIO/RNTupleTempTests/test/unit_test_outputs/testGetBy1.log +++ b/FWIO/RNTupleTempTests/test/unit_test_outputs/testGetBy1.log @@ -1,3 +1,6 @@ +++ finished: construction of Services +++ starting: construction of EventSetup modules +++ finished: construction of EventSetup modules ++ starting: constructing source: IntSource Module type=IntSource, Module label=source, Parameter Set ID=6f0b3d3a362a6270c801489bb066414a ++ finished: constructing source: IntSource @@ -32,7 +35,17 @@ Module type=IntProducer, Module label=intProducerA, Parameter Set ID=56ea7c8bbb0 ++++ finished: constructing module with label 'intProducerU' id = 12 ++++ starting: constructing module with label 'intVectorProducer' id = 13 ++++ finished: constructing module with label 'intVectorProducer' id = 13 +++ starting: finalize Schedule +++ finished: finalize Schedule +++ starting: creation of Run, LuminosityBlock, and Event Principals +++ finished: creation of Run, LuminosityBlock, and Event Principals ++ preallocate: 1 concurrent runs, 1 concurrent luminosity sections, 1 streams +++ starting: schedule consistency check +++ finished: schedule consistency check +++ starting: finalize EventSetup configuration +++ finished: finalize EventSetup configuration +++ starting: finalize EDModules' initialization +++ finished: finalize EDModules' initialization ++ starting: begin job ++++ starting: begin job for module with label 'TriggerResults' id = 1 ++++ finished: begin job for module with label 'TriggerResults' id = 1 diff --git a/FWIO/RNTupleTempTests/test/unit_test_outputs/testGetBy2.log b/FWIO/RNTupleTempTests/test/unit_test_outputs/testGetBy2.log index 25056eef44cdb..8cdf16d85e316 100644 --- a/FWIO/RNTupleTempTests/test/unit_test_outputs/testGetBy2.log +++ b/FWIO/RNTupleTempTests/test/unit_test_outputs/testGetBy2.log @@ -1,3 +1,6 @@ +++ finished: construction of Services +++ starting: construction of EventSetup modules +++ finished: construction of EventSetup modules ++ starting: constructing source: RNTupleTempSource Module type=RNTupleTempSource, Module label=source, Parameter Set ID=f45c0cea718e3436a3db8abdc232e8c6 ++++ starting: open input file: lfn = file:testGetBy1.root @@ -18,7 +21,17 @@ Module type=IntProducer, Module label=intProducer, Parameter Set ID=b4b90439a301 ++++ finished: constructing module with label 'intProducerU' id = 5 ++++ starting: constructing module with label 'intVectorProducer' id = 6 ++++ finished: constructing module with label 'intVectorProducer' id = 6 +++ starting: finalize Schedule +++ finished: finalize Schedule +++ starting: creation of Run, LuminosityBlock, and Event Principals +++ finished: creation of Run, LuminosityBlock, and Event Principals ++ preallocate: 1 concurrent runs, 1 concurrent luminosity sections, 1 streams +++ starting: schedule consistency check +++ finished: schedule consistency check +++ starting: finalize EventSetup configuration +++ finished: finalize EventSetup configuration +++ starting: finalize EDModules' initialization +++ finished: finalize EDModules' initialization ++ starting: begin job ++++ starting: begin job for module with label 'TriggerResults' id = 1 ++++ finished: begin job for module with label 'TriggerResults' id = 1 From f673be0f48effe68070ba3fbb1a4b6fd728282c8 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Tue, 2 Dec 2025 14:35:28 -0600 Subject: [PATCH 18/19] Added missing header --- IOPool/Input/src/RootFile.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/IOPool/Input/src/RootFile.cc b/IOPool/Input/src/RootFile.cc index 3ae777a547901..8320874344d49 100644 --- a/IOPool/Input/src/RootFile.cc +++ b/IOPool/Input/src/RootFile.cc @@ -5,6 +5,7 @@ #include "DuplicateChecker.h" #include "InputFile.h" #include "ProvenanceAdaptor.h" +#include "RootDelayedReader.h" #include "FWCore/Sources/interface/InputSourceRunHelper.h" #include "DataFormats/Common/interface/setIsMergeable.h" From b3bf9f0f60b38c9e705b27a97d4c3ea95f8a1927 Mon Sep 17 00:00:00 2001 From: Christopher Jones Date: Tue, 2 Dec 2025 15:30:13 -0600 Subject: [PATCH 19/19] Fix clang compilation issue The default assignment/copy methods do not work because of restrictions from the member data. --- FWIO/RNTupleTempInput/src/RootRNTuple.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/FWIO/RNTupleTempInput/src/RootRNTuple.h b/FWIO/RNTupleTempInput/src/RootRNTuple.h index dbf73c023704b..0ae3f07d8d255 100644 --- a/FWIO/RNTupleTempInput/src/RootRNTuple.h +++ b/FWIO/RNTupleTempInput/src/RootRNTuple.h @@ -39,10 +39,10 @@ namespace edm::rntuple_temp { using EntryNumber = IndexIntoFile::EntryNumber_t; struct ProductInfo { ProductInfo(ProductDescription const& prod) : productDescription_(prod) {} - ProductInfo(ProductInfo const&) = default; - ProductInfo& operator=(ProductInfo const&) = default; + ProductInfo(ProductInfo const&) = delete; + ProductInfo& operator=(ProductInfo const&) = delete; ProductInfo(ProductInfo&&) = default; - ProductInfo& operator=(ProductInfo&&) = default; + ProductInfo& operator=(ProductInfo&&) = delete; void setField(ROOT::RFieldToken token, ROOT::RNTupleView view, TClass const* wrapperBaseTClass); std::unique_ptr newWrapper() const; bool valid() const { return view_.has_value(); }