Skip to content

Add RNTupleTempSource and RNTupleTempOutputModule #49532

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cmsbuild merged 19 commits into from
Dec 4, 2025
Merged

Add RNTupleTempSource and RNTupleTempOutputModule #49532

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
678f32b
Moved RunHelper to FWCore/Sources and renamed
Dr15Jones May 14, 2025
fd9ee5f
All RNTupleTemp related files
Dr15Jones Sep 16, 2025
36aeb10
Replaced meta data TTrees with RNTuple
Dr15Jones Sep 19, 2025
f6b725e
Added FWIO/RNTupleTempTests items
Dr15Jones Sep 23, 2025
2cac53f
Removed cloning code from RNTuple implementation
Dr15Jones Sep 29, 2025
149ace9
Moved output to use only RNTuple
Dr15Jones Sep 30, 2025
b9c4d2c
Simplifying input code for RNTuple migration
Dr15Jones Oct 1, 2025
893970c
edmRNTupleTempFileUtil now complete
Dr15Jones Oct 1, 2025
fc0d434
Updated edmRNTupleTempProvDump to only use RNTuple
Dr15Jones Oct 2, 2025
556a6fb
Removed unneeded header file
Dr15Jones Oct 4, 2025
4ca6c13
Move input to only use RNTuple
Dr15Jones Oct 4, 2025
2553a20
Better configuration of RNTupleTemp*
Dr15Jones Oct 6, 2025
11cfc9f
Update to use std::format
Dr15Jones Oct 21, 2025
ed1bba5
Handle RNTuple optimizations
Dr15Jones Oct 21, 2025
16a8720
Use config to decide meta data compression in RNTupleTemp
Dr15Jones Oct 24, 2025
8fa87a8
Remove redundant FixMissingStreamerInfos
Dr15Jones Oct 24, 2025
532b458
Update TestRNTupleTempGetBy to new Tracer output
Dr15Jones Nov 4, 2025
f673be0
Added missing header
Dr15Jones Dec 2, 2025
b3bf9f0
Fix clang compilation issue
Dr15Jones Dec 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
30 changes: 15 additions & 15 deletions IOPool/Input/src/RunHelper.h → .../Sources/interface/InputSourceRunHelper.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#ifndef IOPool_Input_RunHelper_h
#define IOPool_Input_RunHelper_h
#ifndef IOPool_Input_InputSourceRunHelper_h
#define IOPool_Input_InputSourceRunHelper_h

#include "DataFormats/Provenance/interface/RunLumiEventNumber.h"
#include "FWCore/Framework/interface/Frameworkfwd.h"
Expand All @@ -12,10 +12,10 @@ namespace edm {
class IndexIntoFile;
class ParameterSetDescription;

class RunHelperBase {
class InputSourceRunHelperBase {
public:
explicit RunHelperBase() = default;
virtual ~RunHelperBase();
explicit InputSourceRunHelperBase() = default;
virtual ~InputSourceRunHelperBase();

virtual InputSource::ItemType nextItemType(InputSource::ItemType const& previousItemType,
InputSource::ItemType const& newItemType,
Expand All @@ -38,16 +38,16 @@ namespace edm {
static void fillDescription(ParameterSetDescription& desc);
};

class DefaultRunHelper : public RunHelperBase {
class DefaultInputSourceRunHelper : public InputSourceRunHelperBase {
public:
explicit DefaultRunHelper() = default;
~DefaultRunHelper() override;
explicit DefaultInputSourceRunHelper() = default;
~DefaultInputSourceRunHelper() override;
};

class SetRunHelper : public RunHelperBase {
class SetInputSourceRunHelper : public InputSourceRunHelperBase {
public:
explicit SetRunHelper(ParameterSet const& pset);
~SetRunHelper() override;
explicit SetInputSourceRunHelper(ParameterSet const& pset);
~SetInputSourceRunHelper() override;

void setForcedRunOffset(RunNumber_t firstRun) override;

Expand All @@ -62,7 +62,7 @@ namespace edm {
bool firstTime_;
};

class SetRunForEachLumiHelper : public RunHelperBase {
class SetRunForEachLumiHelper : public InputSourceRunHelperBase {
public:
explicit SetRunForEachLumiHelper(ParameterSet const& pset);
~SetRunForEachLumiHelper() override;
Expand All @@ -89,9 +89,9 @@ namespace edm {
bool firstTime_;
};

class FirstLuminosityBlockForEachRunHelper : public RunHelperBase {
class FirstLuminosityBlockForEachInputSourceRunHelper : public InputSourceRunHelperBase {
public:
explicit FirstLuminosityBlockForEachRunHelper(ParameterSet const& pset);
explicit FirstLuminosityBlockForEachInputSourceRunHelper(ParameterSet const& pset);

InputSource::ItemType nextItemType(InputSource::ItemType const& previousItemType,
InputSource::ItemType const& newIemType,
Expand All @@ -114,7 +114,7 @@ namespace edm {
RunNumber_t lastUsedRunNumber_;
bool fakeNewRun_;
};
std::unique_ptr<RunHelperBase> makeRunHelper(ParameterSet const& pset);
std::unique_ptr<InputSourceRunHelperBase> makeInputSourceRunHelper(ParameterSet const& pset);
} // namespace edm

#endif
78 changes: 43 additions & 35 deletions IOPool/Input/src/RunHelper.cc → FWCore/Sources/src/InputSourceRunHelper.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
/*----------------------------------------------------------------------
----------------------------------------------------------------------*/
#include "RunHelper.h"
#include "FWCore/Sources/interface/InputSourceRunHelper.h"

#include "FWCore/ParameterSet/interface/ParameterSetDescription.h"
#include "FWCore/Utilities/interface/EDMException.h"
Expand All @@ -9,11 +9,11 @@

namespace edm {

std::unique_ptr<RunHelperBase> makeRunHelper(ParameterSet const& pset) {
std::unique_ptr<InputSourceRunHelperBase> makeInputSourceRunHelper(ParameterSet const& pset) {
if (pset.exists("setRunNumber")) {
RunNumber_t run = pset.getUntrackedParameter<unsigned int>("setRunNumber");
if (run != 0U) {
return std::make_unique<SetRunHelper>(pset);
return std::make_unique<SetInputSourceRunHelper>(pset);
}
}
if (pset.exists("setRunNumberForEachLumi")) {
Expand All @@ -24,33 +24,34 @@ namespace edm {
}
if (pset.exists("firstLuminosityBlockForEachRun")) {
if (not pset.getUntrackedParameter<std::vector<LuminosityBlockID>>("firstLuminosityBlockForEachRun").empty()) {
return std::make_unique<FirstLuminosityBlockForEachRunHelper>(pset);
return std::make_unique<FirstLuminosityBlockForEachInputSourceRunHelper>(pset);
}
}
return std::make_unique<DefaultRunHelper>();
return std::make_unique<DefaultInputSourceRunHelper>();
}

RunHelperBase::~RunHelperBase() {}
InputSourceRunHelperBase::~InputSourceRunHelperBase() {}

void RunHelperBase::checkLumiConsistency(LuminosityBlockNumber_t lumi, LuminosityBlockNumber_t originalLumi) const {
void InputSourceRunHelperBase::checkLumiConsistency(LuminosityBlockNumber_t lumi,
LuminosityBlockNumber_t originalLumi) const {
assert(lumi == originalLumi);
}

void RunHelperBase::checkRunConsistency(RunNumber_t run, RunNumber_t originalRun) const {
void InputSourceRunHelperBase::checkRunConsistency(RunNumber_t run, RunNumber_t originalRun) const {
assert(run == originalRun);
}

DefaultRunHelper::~DefaultRunHelper() {}
DefaultInputSourceRunHelper::~DefaultInputSourceRunHelper() {}

SetRunHelper::SetRunHelper(ParameterSet const& pset)
: RunHelperBase(),
SetInputSourceRunHelper::SetInputSourceRunHelper(ParameterSet const& pset)
: InputSourceRunHelperBase(),
setRun_(pset.getUntrackedParameter<unsigned int>("setRunNumber")),
forcedRunOffset_(0),
firstTime_(true) {}

SetRunHelper::~SetRunHelper() {}
SetInputSourceRunHelper::~SetInputSourceRunHelper() {}

void SetRunHelper::setForcedRunOffset(RunNumber_t firstRun) {
void SetInputSourceRunHelper::setForcedRunOffset(RunNumber_t firstRun) {
if (firstTime_ && setRun_ != 0) {
forcedRunOffset_ = setRun_ - firstRun;
if (forcedRunOffset_ < 0) {
Expand All @@ -63,21 +64,21 @@ namespace edm {
firstTime_ = false;
}

void SetRunHelper::overrideRunNumber(RunID& id) {
void SetInputSourceRunHelper::overrideRunNumber(RunID& id) {
id = RunID(id.run() + forcedRunOffset_);
if (id < RunID::firstValidRun())
id = RunID::firstValidRun();
}

void SetRunHelper::overrideRunNumber(LuminosityBlockID& id) {
void SetInputSourceRunHelper::overrideRunNumber(LuminosityBlockID& id) {
id = LuminosityBlockID(id.run() + forcedRunOffset_, id.luminosityBlock());
if (RunID(id.run()) < RunID::firstValidRun())
id = LuminosityBlockID(RunID::firstValidRun().run(), id.luminosityBlock());
}

void SetRunHelper::overrideRunNumber(EventID& id, bool isRealData) {
void SetInputSourceRunHelper::overrideRunNumber(EventID& id, bool isRealData) {
if (isRealData) {
throw Exception(errors::Configuration, "SetRunHelper::overrideRunNumber()")
throw Exception(errors::Configuration, "SetInputSourceRunHelper::overrideRunNumber()")
<< "The 'setRunNumber' parameter of PoolSource cannot be used with real data.\n";
}
id = EventID(id.run() + forcedRunOffset_, id.luminosityBlock(), id.event());
Expand All @@ -87,12 +88,12 @@ namespace edm {
}
}

void SetRunHelper::checkRunConsistency(RunNumber_t run, RunNumber_t originalRun) const {
void SetInputSourceRunHelper::checkRunConsistency(RunNumber_t run, RunNumber_t originalRun) const {
assert(run == originalRun + forcedRunOffset_);
}

SetRunForEachLumiHelper::SetRunForEachLumiHelper(ParameterSet const& pset)
: RunHelperBase(),
: InputSourceRunHelperBase(),
setRunNumberForEachLumi_(pset.getUntrackedParameter<std::vector<unsigned int>>("setRunNumberForEachLumi")),
indexOfNextRunNumber_(0),
realRunNumber_(0),
Expand Down Expand Up @@ -163,17 +164,19 @@ namespace edm {
assert(run == runNumberToUseForThisLumi());
}

FirstLuminosityBlockForEachRunHelper::FirstLuminosityBlockForEachRunHelper(ParameterSet const& pset)
FirstLuminosityBlockForEachInputSourceRunHelper::FirstLuminosityBlockForEachInputSourceRunHelper(
ParameterSet const& pset)
: lumiToRun_(pset.getUntrackedParameter<std::vector<edm::LuminosityBlockID>>("firstLuminosityBlockForEachRun")),
realRunNumber_{0},
lastUsedRunNumber_{0},
fakeNewRun_{false} {}

InputSource::ItemType FirstLuminosityBlockForEachRunHelper::nextItemType(InputSource::ItemType const& previousItemType,
InputSource::ItemType const& newItemType,
RunNumber_t,
LuminosityBlockNumber_t iLumi,
EventNumber_t) {
InputSource::ItemType FirstLuminosityBlockForEachInputSourceRunHelper::nextItemType(
InputSource::ItemType const& previousItemType,
InputSource::ItemType const& newItemType,
RunNumber_t,
LuminosityBlockNumber_t iLumi,
EventNumber_t) {
if (newItemType == InputSource::ItemType::IsLumi && previousItemType != InputSource::ItemType::IsRun) {
auto run = findRunFromLumi(iLumi);
if (run == 0) {
Expand All @@ -189,9 +192,11 @@ namespace edm {
return newItemType;
}

RunNumber_t FirstLuminosityBlockForEachRunHelper::runNumberToUseForThisLumi() const { return lastUsedRunNumber_; }
RunNumber_t FirstLuminosityBlockForEachInputSourceRunHelper::runNumberToUseForThisLumi() const {
return lastUsedRunNumber_;
}

void FirstLuminosityBlockForEachRunHelper::checkForNewRun(RunNumber_t run, LuminosityBlockNumber_t iLumi) {
void FirstLuminosityBlockForEachInputSourceRunHelper::checkForNewRun(RunNumber_t run, LuminosityBlockNumber_t iLumi) {
if (realRunNumber_ != 0 && run != realRunNumber_) {
throw Exception(errors::MismatchedInputFiles, "PoolSource::checkForNewRun")
<< " Parameter 'firstLuminosityBlockForEachRun' can only process a single run.\n"
Expand All @@ -202,25 +207,28 @@ namespace edm {
fakeNewRun_ = false;
}

void FirstLuminosityBlockForEachRunHelper::overrideRunNumber(RunID& id) { id = RunID(runNumberToUseForThisLumi()); }
void FirstLuminosityBlockForEachInputSourceRunHelper::overrideRunNumber(RunID& id) {
id = RunID(runNumberToUseForThisLumi());
}

void FirstLuminosityBlockForEachRunHelper::overrideRunNumber(LuminosityBlockID& id) {
void FirstLuminosityBlockForEachInputSourceRunHelper::overrideRunNumber(LuminosityBlockID& id) {
id = LuminosityBlockID(runNumberToUseForThisLumi(), id.luminosityBlock());
}

void FirstLuminosityBlockForEachRunHelper::overrideRunNumber(EventID& id, bool isRealData) {
void FirstLuminosityBlockForEachInputSourceRunHelper::overrideRunNumber(EventID& id, bool isRealData) {
if (isRealData) {
throw Exception(errors::Configuration, "FirstLuminosityBlockForEachRunHelper::overrideRunNumber()")
throw Exception(errors::Configuration, "FirstLuminosityBlockForEachInputSourceRunHelper::overrideRunNumber()")
<< "The 'firstLuminosityBlockForEachRun' parameter of PoolSource cannot be used with real data.\n";
}
id = EventID(runNumberToUseForThisLumi(), id.luminosityBlock(), id.event());
}

void FirstLuminosityBlockForEachRunHelper::checkRunConsistency(RunNumber_t run, RunNumber_t originalRun) const {
void FirstLuminosityBlockForEachInputSourceRunHelper::checkRunConsistency(RunNumber_t run,
RunNumber_t originalRun) const {
assert(run == runNumberToUseForThisLumi());
}

RunNumber_t FirstLuminosityBlockForEachRunHelper::findRunFromLumi(LuminosityBlockNumber_t iLumi) const {
RunNumber_t FirstLuminosityBlockForEachInputSourceRunHelper::findRunFromLumi(LuminosityBlockNumber_t iLumi) const {
RunNumber_t run = 0;
for (auto const& lumiID : lumiToRun_) {
if (lumiID.luminosityBlock() > iLumi) {
Expand All @@ -229,15 +237,15 @@ namespace edm {
run = lumiID.run();
}
if (0 == run) {
throw Exception(errors::Configuration, "FirstLuminosityBlockForEachRunHelper::findRunFromLumi()")
throw Exception(errors::Configuration, "FirstLuminosityBlockForEachInputSourceRunHelper::findRunFromLumi()")
<< "The 'firstLuminosityBlockForEachRun' parameter does not have a matching Run number for LuminosityBlock "
"number: "
<< iLumi << ".\n";
}
return run;
}

void RunHelperBase::fillDescription(ParameterSetDescription& desc) {
void InputSourceRunHelperBase::fillDescription(ParameterSetDescription& desc) {
desc.addOptionalNode(ParameterDescription<unsigned int>("setRunNumber", 0U, false) xor
ParameterDescription<std::vector<unsigned int>>(
"setRunNumberForEachLumi", std::vector<unsigned int>(), false) xor
Expand Down
15 changes: 15 additions & 0 deletions FWIO/RNTupleTempInput/BuildFile.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
<use name="DataFormats/Common"/>
<use name="DataFormats/Provenance"/>
<use name="FWCore/Catalog"/>
<use name="FWCore/Framework"/>
<use name="FWCore/MessageLogger"/>
<use name="FWCore/ParameterSet"/>
<use name="FWCore/ServiceRegistry"/>
<use name="FWCore/Sources"/>
<use name="FWCore/Utilities"/>
<use name="IOPool/Common"/>
<use name="Utilities/StorageFactory"/>
<use name="clhep"/>
<use name="rootcore"/>
<use name="rootntuple"/>
<flags EDM_PLUGIN="1"/>
27 changes: 27 additions & 0 deletions FWIO/RNTupleTempInput/bin/BuildFile.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
<bin name="edmRNTupleTempProvDump" file="EdmRNTupleTempProvDump.cc,">
<use name="boost_program_options"/>
<use name="fmt"/>
<use name="rootcore"/>
<use name="rootntuple"/>
<use name="DataFormats/Common"/>
<use name="DataFormats/Provenance"/>
<use name="FWCore/Catalog"/>
<use name="FWCore/ParameterSet"/>
<use name="FWCore/ServiceRegistry"/>
<use name="FWCore/Services"/>
<use name="FWCore/Utilities"/>
</bin>

<bin name="edmRNTupleTempFileUtil" file="EdmRNTupleTempFileUtil.cpp,CollUtil.cc">
<use name="boost"/>
<use name="boost_program_options"/>
<use name="rootcore"/>
<use name="rootntuple"/>
<use name="roothistmatrix"/>
<use name="DataFormats/Provenance"/>
<use name="FWCore/Catalog"/>
<use name="FWCore/ParameterSet"/>
<use name="FWCore/PluginManager"/>
<use name="FWCore/ServiceRegistry"/>
<use name="FWCore/Services"/>
</bin>
Loading