diff --git a/DataFormats/Provenance/interface/BranchDescription.h b/DataFormats/Provenance/interface/BranchDescription.h index 6d913bca365df..0bc43da5fe26a 100644 --- a/DataFormats/Provenance/interface/BranchDescription.h +++ b/DataFormats/Provenance/interface/BranchDescription.h @@ -81,6 +81,8 @@ namespace edm { std::string const& productInstanceName() const { return productInstanceName_; } bool produced() const { return transient_.produced_; } void setProduced(bool isProduced) { transient_.produced_ = isProduced; } + bool isTransform() const { return transient_.isTransform_; } + void setIsTransform(bool isTransform) { transient_.isTransform_ = isTransform; } bool present() const { return !transient_.dropped_; } bool dropped() const { return transient_.dropped_; } void setDropped(bool isDropped) { transient_.dropped_ = isDropped; } @@ -178,6 +180,9 @@ namespace edm { // This item is set only in the framework, not by FWLite. bool onDemand_; + // Was this branch produced in this current process via the transform ability + bool isTransform_; + // Has the branch been dropped from the product tree in this file // (or if this is a merged product registry, in the first file). // This item is set only in the framework, not by FWLite. diff --git a/DataFormats/Provenance/src/BranchDescription.cc b/DataFormats/Provenance/src/BranchDescription.cc index dd59b7387ab3c..3abe10225beb8 100644 --- a/DataFormats/Provenance/src/BranchDescription.cc +++ b/DataFormats/Provenance/src/BranchDescription.cc @@ -27,6 +27,7 @@ namespace edm { basketSize_(), produced_(false), onDemand_(false), + isTransform_(false), dropped_(false), transient_(false), availableOnlyAtEndTransition_(false), diff --git a/FWCore/Framework/interface/EDConsumerBase.h b/FWCore/Framework/interface/EDConsumerBase.h index 43c7f89a2b38a..16ac44352097b 100644 --- a/FWCore/Framework/interface/EDConsumerBase.h +++ b/FWCore/Framework/interface/EDConsumerBase.h @@ -243,6 +243,7 @@ namespace edm { void resetItemsToGetFrom(BranchType iType) { itemsToGetFromBranch_[iType].clear(); } private: + virtual void extendUpdateLookup(BranchType iBranchType, ProductResolverIndexHelper const&); virtual void registerLateConsumes(eventsetup::ESRecordsToProxyIndices const&) {} unsigned int recordConsumes(BranchType iBranch, TypeToGet const& iType, edm::InputTag const& iTag, bool iAlwaysGets); ESTokenIndex recordESConsumes(Transition, diff --git a/FWCore/Framework/interface/EventForTransformer.h b/FWCore/Framework/interface/EventForTransformer.h new file mode 100644 index 0000000000000..6cddefe4263ed --- /dev/null +++ b/FWCore/Framework/interface/EventForTransformer.h @@ -0,0 +1,42 @@ +#ifndef FWCore_Framework_EventForTransformer_h +#define FWCore_Framework_EventForTransformer_h + +// -*- C++ -*- +// +// Package: Framework +// Class : EventForTransformer +// +/**\class edm::EventForTransformer + +*/ +/*---------------------------------------------------------------------- +----------------------------------------------------------------------*/ + +#include "DataFormats/Common/interface/BasicHandle.h" +#include "DataFormats/Common/interface/WrapperBase.h" + +#include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/Utilities/interface/TypeID.h" +#include "FWCore/Utilities/interface/ProductResolverIndex.h" + +#include + +namespace edm { + + class EventPrincipal; + class ModuleCallingContext; + + class EventForTransformer { + public: + EventForTransformer(EventPrincipal const&, ModuleCallingContext const*); + + BasicHandle get(edm::TypeID const& iTypeID, ProductResolverIndex iIndex) const; + + void put(ProductResolverIndex index, std::unique_ptr edp, BasicHandle const& iGetHandle); + + private: + EventPrincipal const& eventPrincipal_; + ModuleCallingContext const* mcc_; + }; +} // namespace edm +#endif diff --git a/FWCore/Framework/interface/Principal.h b/FWCore/Framework/interface/Principal.h index 9b1b6cfc38c9a..3b1043fa52fad 100644 --- a/FWCore/Framework/interface/Principal.h +++ b/FWCore/Framework/interface/Principal.h @@ -229,6 +229,7 @@ namespace edm { void addDelayedReaderInputProduct(std::shared_ptr bd); void addPutOnReadInputProduct(std::shared_ptr bd); void addUnscheduledProduct(std::shared_ptr bd); + void addTransformProduct(std::shared_ptr bd); void addAliasedProduct(std::shared_ptr bd); void addSwitchProducerProduct(std::shared_ptr bd); void addSwitchAliasProduct(std::shared_ptr bd); diff --git a/FWCore/Framework/interface/ProducerBase.h b/FWCore/Framework/interface/ProducerBase.h index 4a971b3bdf85c..7e11dd28ccfb9 100644 --- a/FWCore/Framework/interface/ProducerBase.h +++ b/FWCore/Framework/interface/ProducerBase.h @@ -82,6 +82,9 @@ namespace edm { using ProductRegistryHelper::recordProvenanceList; using ProductRegistryHelper::typeLabelList; + template + using BranchAliasSetterT = ProductRegistryHelper::BranchAliasSetterT; + void callWhenNewProductsRegistered(std::function const& func) { callWhenNewProductsRegistered_ = func; } @@ -114,6 +117,7 @@ namespace edm { friend class limited::EDProducerBase; friend class limited::EDFilterBase; friend class PuttableSourceBase; + friend class TransformerBase; template friend class stream::ProducingModuleAdaptorBase; @@ -127,6 +131,8 @@ namespace edm { iPrincipal.commit_(putIndicies_[producerbasehelper::PrincipalTraits

::kBranchType], iID); } + using ProductRegistryHelper::transforms; + std::function callWhenNewProductsRegistered_; std::array, edm::NumBranchTypes> putIndicies_; std::vector putTokenToResolverIndex_; diff --git a/FWCore/Framework/interface/ProductRegistryHelper.h b/FWCore/Framework/interface/ProductRegistryHelper.h index c13a95a11fe16..b55e76894137a 100644 --- a/FWCore/Framework/interface/ProductRegistryHelper.h +++ b/FWCore/Framework/interface/ProductRegistryHelper.h @@ -36,19 +36,21 @@ namespace edm { }; struct TypeLabelItem { - enum class AliasType { kBranchAlias, kSwitchAlias }; + enum class AliasType : char { kBranchAlias, kSwitchAlias }; TypeLabelItem(Transition const& transition, TypeID const& tid, std::string pin) : transition_(transition), typeID_(tid), productInstanceName_(std::move(pin)), branchAlias_(), - aliasType_(AliasType::kBranchAlias) {} + aliasType_(AliasType::kBranchAlias), + isTransform_(false) {} Transition transition_; TypeID typeID_; std::string productInstanceName_; std::string branchAlias_; AliasType aliasType_; + bool isTransform_; }; struct BranchAliasSetter { @@ -181,6 +183,14 @@ namespace edm { return BranchAliasSetter{typeLabelList_.back(), EDPutToken{index}}; } + EDPutToken transforms(const TypeID& id, std::string instanceName) { + unsigned int index = typeLabelList_.size(); + typeLabelList_.emplace_back(Transition::Event, id, std::move(instanceName)); + typeLabelList_.back().isTransform_ = true; + recordProvenanceList_.push_back(true); + return EDPutToken{index}; + } + virtual bool hasAbilityToProduceInBeginProcessBlocks() const { return false; } virtual bool hasAbilityToProduceInEndProcessBlocks() const { return false; } diff --git a/FWCore/Framework/interface/TransformerBase.h b/FWCore/Framework/interface/TransformerBase.h new file mode 100644 index 0000000000000..7e0b15cb27e73 --- /dev/null +++ b/FWCore/Framework/interface/TransformerBase.h @@ -0,0 +1,54 @@ +// +// TransformerBase.h +// CMSSW +// +// Created by Chris Jones on 6/02/22. +// + +#ifndef FWCore_Framework_TransformerBase_h +#define FWCore_Framework_TransformerBase_h + +#include "FWCore/Utilities/interface/EDPutToken.h" +#include "FWCore/Utilities/interface/SoATuple.h" +#include "FWCore/Utilities/interface/TypeID.h" +#include "FWCore/Utilities/interface/ProductResolverIndex.h" + +#include +#include +#include + +namespace edm { + class ProducerBase; + class TypeID; + class WrapperBase; + class EventForTransformer; + class BranchDescription; + class ProductResolverIndexHelper; + class ModuleDescription; + + class TransformerBase { + public: + TransformerBase() = default; + virtual ~TransformerBase() noexcept(false) = default; + + protected: + //The function takes the WrapperBase corresponding to the data product from the EDPutToken + // and returns the WrapperBase associated to the id and instanceName + using TransformFunction = std::function(edm::WrapperBase const&)>; + + void registerTransformImp(ProducerBase&, EDPutToken, const TypeID& id, std::string instanceName, TransformFunction); + + std::size_t findMatchingIndex(ProducerBase const& iBase, edm::BranchDescription const&) const; + ProductResolverIndex prefetchImp(std::size_t iIndex) const { return transformInfo_.get<0>(iIndex); } + void transformImp(std::size_t iIndex, ProducerBase const& iBase, edm::EventForTransformer&) const; + + void extendUpdateLookup(ProducerBase const&, + ModuleDescription const& iModuleDesc, + ProductResolverIndexHelper const& iHelper); + + private: + SoATuple transformInfo_; + }; +} // namespace edm + +#endif /* TransformerBase_h */ diff --git a/FWCore/Framework/interface/global/EDFilterBase.h b/FWCore/Framework/interface/global/EDFilterBase.h index 5b68eb730a739..9f29b3570d84f 100644 --- a/FWCore/Framework/interface/global/EDFilterBase.h +++ b/FWCore/Framework/interface/global/EDFilterBase.h @@ -38,6 +38,7 @@ namespace edm { class ActivityRegistry; class ThinnedAssociationsHelper; class WaitingTaskWithArenaHolder; + class EventForTransformer; namespace maker { template @@ -77,6 +78,10 @@ namespace edm { ActivityRegistry*, ModuleCallingContext const*, WaitingTaskWithArenaHolder&); + void doTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ActivityRegistry*, + ModuleCallingContext const*); //For now this is a placeholder /*virtual*/ void preActionBeforeRunEventAsync(WaitingTaskHolder iTask, ModuleCallingContext const& iModuleCallingContext, @@ -147,6 +152,10 @@ namespace edm { virtual void doBeginLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); virtual void doEndLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); + virtual size_t transformIndex_(edm::BranchDescription const& iBranch) const; + virtual ProductResolverIndex transformPrefetch_(std::size_t iIndex) const; + virtual void transform_(std::size_t iIndex, edm::EventForTransformer& iEvent) const; + virtual void clearInputProcessBlockCaches(); virtual bool hasAcquire() const { return false; } bool hasAccumulator() const { return false; } diff --git a/FWCore/Framework/interface/global/EDProducerBase.h b/FWCore/Framework/interface/global/EDProducerBase.h index f1eeb792e9e7e..2efddaed5bfed 100644 --- a/FWCore/Framework/interface/global/EDProducerBase.h +++ b/FWCore/Framework/interface/global/EDProducerBase.h @@ -39,6 +39,7 @@ namespace edm { class ActivityRegistry; class ThinnedAssociationsHelper; class WaitingTaskWithArenaHolder; + class EventForTransformer; namespace maker { template @@ -80,6 +81,10 @@ namespace edm { ActivityRegistry*, ModuleCallingContext const*, WaitingTaskWithArenaHolder&); + void doTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ActivityRegistry*, + ModuleCallingContext const*); void doPreallocate(PreallocationConfiguration const&); void doBeginJob(); void doEndJob(); @@ -150,6 +155,10 @@ namespace edm { virtual void doBeginLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); virtual void doEndLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); + virtual size_t transformIndex_(edm::BranchDescription const& iBranch) const; + virtual ProductResolverIndex transformPrefetch_(std::size_t iIndex) const; + virtual void transform_(std::size_t iIndex, edm::EventForTransformer& iEvent) const; + virtual void clearInputProcessBlockCaches(); virtual bool hasAccumulator() const { return false; } diff --git a/FWCore/Framework/interface/global/filterAbilityToImplementor.h b/FWCore/Framework/interface/global/filterAbilityToImplementor.h index 8867b849489dc..e6e9eadd3b203 100644 --- a/FWCore/Framework/interface/global/filterAbilityToImplementor.h +++ b/FWCore/Framework/interface/global/filterAbilityToImplementor.h @@ -101,6 +101,11 @@ namespace edm { using Type = edm::global::impl::ExternalWork; }; + template <> + struct AbilityToImplementor { + using Type = edm::global::impl::Transformer; + }; + template struct SpecializeAbilityToImplementor { using Type = typename AbilityToImplementor::Type; diff --git a/FWCore/Framework/interface/global/implementors.h b/FWCore/Framework/interface/global/implementors.h index 220b31e55b935..0b048c4806893 100644 --- a/FWCore/Framework/interface/global/implementors.h +++ b/FWCore/Framework/interface/global/implementors.h @@ -33,12 +33,15 @@ #include "FWCore/Framework/interface/Frameworkfwd.h" #include "FWCore/Framework/interface/InputProcessBlockCacheImpl.h" #include "FWCore/Framework/interface/LuminosityBlock.h" +#include "FWCore/Framework/interface/TransformerBase.h" +#include "FWCore/Framework/interface/ProductRegistryHelper.h" #include "FWCore/Utilities/interface/EDGetToken.h" #include "FWCore/Utilities/interface/StreamID.h" #include "FWCore/Utilities/interface/ProcessBlockIndex.h" #include "FWCore/Utilities/interface/RunIndex.h" #include "FWCore/Utilities/interface/LuminosityBlockIndex.h" #include "FWCore/Utilities/interface/propagate_const.h" +#include "DataFormats/Common/interface/Wrapper.h" // forward declarations namespace edm { @@ -439,6 +442,53 @@ namespace edm { virtual void accumulate(StreamID streamID, Event const& ev, EventSetup const& es) const = 0; }; + + template + class Transformer : public virtual T, private TransformerBase { + public: + Transformer() = default; + Transformer(Transformer const&) = delete; + Transformer& operator=(Transformer const&) = delete; + ~Transformer() noexcept(false) override{}; + + template + void registerTransform(ProductRegistryHelper::BranchAliasSetterT iSetter, + F&& iF, + std::string productInstance = std::string()) { + registerTransform(edm::EDPutTokenT(iSetter), std::forward(iF), std::move(productInstance)); + } + + template + void registerTransform(edm::EDPutTokenT iToken, F iF, std::string productInstance = std::string()) { + using ReturnTypeT = decltype(iF(std::declval())); + TypeID returnType(typeid(ReturnTypeT)); + TransformerBase::registerTransformImp(*this, + EDPutToken(iToken), + returnType, + std::move(productInstance), + [f = std::move(iF)](edm::WrapperBase const& iGotProduct) { + return std::make_unique>( + WrapperBase::Emplace{}, + f(*static_cast const&>(iGotProduct).product())); + }); + } + + private: + size_t transformIndex_(edm::BranchDescription const& iBranch) const final { + return TransformerBase::findMatchingIndex(*this, iBranch); + } + ProductResolverIndex transformPrefetch_(std::size_t iIndex) const final { + return TransformerBase::prefetchImp(iIndex); + } + void transform_(std::size_t iIndex, edm::EventForTransformer& iEvent) const final { + return TransformerBase::transformImp(iIndex, *this, iEvent); + } + void extendUpdateLookup(BranchType iBranchType, ProductResolverIndexHelper const& iHelper) override { + if (iBranchType == InEvent) { + TransformerBase::extendUpdateLookup(*this, this->moduleDescription(), iHelper); + } + } + }; } // namespace impl } // namespace global } // namespace edm diff --git a/FWCore/Framework/interface/global/producerAbilityToImplementor.h b/FWCore/Framework/interface/global/producerAbilityToImplementor.h index 54085f462b1d0..a168902238f27 100644 --- a/FWCore/Framework/interface/global/producerAbilityToImplementor.h +++ b/FWCore/Framework/interface/global/producerAbilityToImplementor.h @@ -102,6 +102,11 @@ namespace edm { using Type = edm::global::impl::ExternalWork; }; + template <> + struct AbilityToImplementor { + using Type = edm::global::impl::Transformer; + }; + template <> struct AbilityToImplementor { using Type = edm::global::impl::Accumulator; diff --git a/FWCore/Framework/interface/limited/EDFilterBase.h b/FWCore/Framework/interface/limited/EDFilterBase.h index f0d8c54c710f5..8382b359ad47a 100644 --- a/FWCore/Framework/interface/limited/EDFilterBase.h +++ b/FWCore/Framework/interface/limited/EDFilterBase.h @@ -38,6 +38,7 @@ namespace edm { class StreamID; class ActivityRegistry; class ThinnedAssociationsHelper; + class EventForTransformer; namespace maker { template @@ -77,6 +78,11 @@ namespace edm { private: bool doEvent(EventTransitionInfo const&, ActivityRegistry*, ModuleCallingContext const*); + void doTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ActivityRegistry*, + ModuleCallingContext const*); + //For now this is a placeholder /*virtual*/ void preActionBeforeRunEventAsync(WaitingTaskHolder, ModuleCallingContext const&, @@ -147,6 +153,10 @@ namespace edm { virtual void doBeginLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); virtual void doEndLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); + virtual size_t transformIndex_(edm::BranchDescription const& iBranch) const; + virtual ProductResolverIndex transformPrefetch_(std::size_t iIndex) const; + virtual void transform_(std::size_t iIndex, edm::EventForTransformer& iEvent) const; + virtual void clearInputProcessBlockCaches(); bool hasAcquire() const { return false; } diff --git a/FWCore/Framework/interface/limited/EDProducerBase.h b/FWCore/Framework/interface/limited/EDProducerBase.h index 7c63a4f63da7a..6085e827b4361 100644 --- a/FWCore/Framework/interface/limited/EDProducerBase.h +++ b/FWCore/Framework/interface/limited/EDProducerBase.h @@ -39,6 +39,7 @@ namespace edm { class GlobalSchedule; class ActivityRegistry; class ThinnedAssociationsHelper; + class EventForTransformer; namespace maker { template @@ -80,6 +81,10 @@ namespace edm { private: bool doEvent(EventTransitionInfo const&, ActivityRegistry*, ModuleCallingContext const*); + void doTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ActivityRegistry*, + ModuleCallingContext const*); void doPreallocate(PreallocationConfiguration const&); void doBeginJob(); void doEndJob(); @@ -151,6 +156,10 @@ namespace edm { virtual void doBeginLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); virtual void doEndLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); + virtual size_t transformIndex_(edm::BranchDescription const& iBranch) const; + virtual ProductResolverIndex transformPrefetch_(std::size_t iIndex) const; + virtual void transform_(std::size_t iIndex, edm::EventForTransformer& iEvent) const; + virtual void clearInputProcessBlockCaches(); virtual bool hasAccumulator() const { return false; } diff --git a/FWCore/Framework/interface/limited/filterAbilityToImplementor.h b/FWCore/Framework/interface/limited/filterAbilityToImplementor.h index 8d3ef2fef6f3c..7829c69cfb99a 100644 --- a/FWCore/Framework/interface/limited/filterAbilityToImplementor.h +++ b/FWCore/Framework/interface/limited/filterAbilityToImplementor.h @@ -97,6 +97,11 @@ namespace edm { using Type = edm::limited::impl::EndLuminosityBlockProducer; }; + template <> + struct AbilityToImplementor { + using Type = edm::limited::impl::Transformer; + }; + template struct SpecializeAbilityToImplementor { using Type = typename AbilityToImplementor::Type; diff --git a/FWCore/Framework/interface/limited/implementors.h b/FWCore/Framework/interface/limited/implementors.h index e69659e4cd452..2452ec0e61dca 100644 --- a/FWCore/Framework/interface/limited/implementors.h +++ b/FWCore/Framework/interface/limited/implementors.h @@ -32,12 +32,15 @@ #include "FWCore/Framework/interface/Frameworkfwd.h" #include "FWCore/Framework/interface/InputProcessBlockCacheImpl.h" #include "FWCore/Framework/interface/LuminosityBlock.h" +#include "FWCore/Framework/interface/TransformerBase.h" +#include "FWCore/Framework/interface/ProductRegistryHelper.h" #include "FWCore/Utilities/interface/EDGetToken.h" #include "FWCore/Utilities/interface/StreamID.h" #include "FWCore/Utilities/interface/ProcessBlockIndex.h" #include "FWCore/Utilities/interface/RunIndex.h" #include "FWCore/Utilities/interface/LuminosityBlockIndex.h" #include "FWCore/Utilities/interface/propagate_const.h" +#include "DataFormats/Common/interface/Wrapper.h" // forward declarations namespace edm { @@ -425,6 +428,54 @@ namespace edm { virtual void accumulate(StreamID streamID, Event const& ev, EventSetup const& es) const = 0; }; + + template + class Transformer : public virtual T, private TransformerBase { + public: + Transformer(edm::ParameterSet const& iPSet) : T(iPSet) {} + Transformer() = default; + Transformer(Transformer const&) = delete; + Transformer& operator=(Transformer const&) = delete; + ~Transformer() noexcept(false) override{}; + + template + void registerTransform(ProductRegistryHelper::BranchAliasSetterT iSetter, + F&& iF, + std::string productInstance = std::string()) { + registerTransform(edm::EDPutTokenT(iSetter), std::forward(iF), std::move(productInstance)); + } + + template + void registerTransform(edm::EDPutTokenT iToken, F iF, std::string productInstance = std::string()) { + using ReturnTypeT = decltype(iF(std::declval())); + TypeID returnType(typeid(ReturnTypeT)); + TransformerBase::registerTransformImp(*this, + EDPutToken(iToken), + returnType, + std::move(productInstance), + [f = std::move(iF)](edm::WrapperBase const& iGotProduct) { + return std::make_unique>( + WrapperBase::Emplace{}, + f(*static_cast const&>(iGotProduct).product())); + }); + } + + private: + size_t transformIndex_(edm::BranchDescription const& iBranch) const final { + return TransformerBase::findMatchingIndex(*this, iBranch); + } + ProductResolverIndex transformPrefetch_(std::size_t iIndex) const final { + return TransformerBase::prefetchImp(iIndex); + } + void transform_(std::size_t iIndex, edm::EventForTransformer& iEvent) const final { + return TransformerBase::transformImp(iIndex, *this, iEvent); + } + void extendUpdateLookup(BranchType iBranchType, ProductResolverIndexHelper const& iHelper) override { + if (iBranchType == InEvent) { + TransformerBase::extendUpdateLookup(*this, this->moduleDescription(), iHelper); + } + } + }; } // namespace impl } // namespace limited } // namespace edm diff --git a/FWCore/Framework/interface/limited/producerAbilityToImplementor.h b/FWCore/Framework/interface/limited/producerAbilityToImplementor.h index 6f179e2055713..657f91f534f49 100644 --- a/FWCore/Framework/interface/limited/producerAbilityToImplementor.h +++ b/FWCore/Framework/interface/limited/producerAbilityToImplementor.h @@ -97,6 +97,11 @@ namespace edm { using Type = edm::limited::impl::EndLuminosityBlockProducer; }; + template <> + struct AbilityToImplementor { + using Type = edm::limited::impl::Transformer; + }; + template <> struct AbilityToImplementor { using Type = edm::limited::impl::Accumulator; diff --git a/FWCore/Framework/interface/maker/Worker.h b/FWCore/Framework/interface/maker/Worker.h index aab872680fc3c..c36fe2409b207 100644 --- a/FWCore/Framework/interface/maker/Worker.h +++ b/FWCore/Framework/interface/maker/Worker.h @@ -165,6 +165,15 @@ namespace edm { ParentContext const&, typename T::Context const*); + virtual size_t transformIndex(edm::BranchDescription const&) const = 0; + void doTransformAsync(WaitingTaskHolder, + size_t iTransformIndex, + EventPrincipal const&, + ServiceToken const&, + StreamID, + ModuleCallingContext const&, + StreamContext const*); + void callWhenDoneAsync(WaitingTaskHolder task) { waitingTasks_.add(std::move(task)); } void skipOnPath(EventPrincipal const& iEvent); void beginJob(); @@ -257,6 +266,9 @@ namespace edm { ModuleCallingContext const*, WaitingTaskWithArenaHolder&) = 0; + virtual void implDoTransform(size_t iTransformIndex, EventPrincipal const&, ParentContext const&) = 0; + virtual ProductResolverIndex itemToGetForTransform(size_t iTransformIndex) const = 0; + virtual bool implDoPrePrefetchSelection(StreamID, EventPrincipal const&, ModuleCallingContext const*) = 0; virtual bool implDoBeginProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*) = 0; virtual bool implDoAccessInputProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*) = 0; @@ -289,7 +301,6 @@ namespace edm { virtual std::vector const& esItemsToGetFrom(Transition) const = 0; virtual std::vector const& esRecordsToGetFrom(Transition) const = 0; - virtual std::vector const& itemsShouldPutInEvent() const = 0; virtual void preActionBeforeRunEventAsync(WaitingTaskHolder iTask, ModuleCallingContext const& moduleCallingContext, diff --git a/FWCore/Framework/interface/maker/WorkerT.h b/FWCore/Framework/interface/maker/WorkerT.h index 88154fcd949c2..29d0973aadac0 100644 --- a/FWCore/Framework/interface/maker/WorkerT.h +++ b/FWCore/Framework/interface/maker/WorkerT.h @@ -92,6 +92,10 @@ namespace edm { void implDoAcquire(EventTransitionInfo const&, ModuleCallingContext const*, WaitingTaskWithArenaHolder&) final; + size_t transformIndex(edm::BranchDescription const&) const final; + void implDoTransform(size_t iTransformIndex, EventPrincipal const&, ParentContext const&) final; + ProductResolverIndex itemToGetForTransform(size_t iTransformIndex) const final; + bool implDoPrePrefetchSelection(StreamID, EventPrincipal const&, ModuleCallingContext const*) override; bool implDoBeginProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*) override; bool implDoAccessInputProcessBlock(ProcessBlockPrincipal const&, ModuleCallingContext const*) override; @@ -149,8 +153,6 @@ namespace edm { return module_->esGetTokenRecordIndicesVector(iTransition); } - std::vector const& itemsShouldPutInEvent() const override; - void preActionBeforeRunEventAsync(WaitingTaskHolder iTask, ModuleCallingContext const& iModuleCallingContext, Principal const& iPrincipal) const override { diff --git a/FWCore/Framework/interface/moduleAbilities.h b/FWCore/Framework/interface/moduleAbilities.h index ddbaf7a6c9f87..861c62bd3aa4e 100644 --- a/FWCore/Framework/interface/moduleAbilities.h +++ b/FWCore/Framework/interface/moduleAbilities.h @@ -123,6 +123,11 @@ namespace edm { typedef module::Empty Type; }; + struct Transformer { + static constexpr module::Abilities kAbilities = module::Abilities::kTransformer; + using Type = module::Empty; + }; + //Recursively checks VArgs template arguments looking for the ABILITY template struct CheckAbility; diff --git a/FWCore/Framework/interface/moduleAbilityEnums.h b/FWCore/Framework/interface/moduleAbilityEnums.h index bf89bc70f57d5..9b2c2a4488c9b 100644 --- a/FWCore/Framework/interface/moduleAbilityEnums.h +++ b/FWCore/Framework/interface/moduleAbilityEnums.h @@ -47,7 +47,8 @@ namespace edm { kOneWatchLuminosityBlocks, kWatchInputFiles, kExternalWork, - kAccumulator + kAccumulator, + kTransformer }; namespace AbilityBits { diff --git a/FWCore/Framework/interface/one/EDFilterBase.h b/FWCore/Framework/interface/one/EDFilterBase.h index be07237025116..36455a857661d 100644 --- a/FWCore/Framework/interface/one/EDFilterBase.h +++ b/FWCore/Framework/interface/one/EDFilterBase.h @@ -36,6 +36,7 @@ namespace edm { class PreallocationConfiguration; class ActivityRegistry; class ThinnedAssociationsHelper; + class EventForTransformer; namespace maker { template @@ -74,6 +75,10 @@ namespace edm { private: bool doEvent(EventTransitionInfo const&, ActivityRegistry*, ModuleCallingContext const*); + void doTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ActivityRegistry*, + ModuleCallingContext const*); //For now this is a placeholder /*virtual*/ void preActionBeforeRunEventAsync(WaitingTaskHolder, ModuleCallingContext const&, @@ -125,6 +130,10 @@ namespace edm { virtual void doBeginLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); virtual void doEndLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); + virtual size_t transformIndex_(edm::BranchDescription const& iBranch) const; + virtual ProductResolverIndex transformPrefetch_(std::size_t iIndex) const; + virtual void transform_(std::size_t iIndex, edm::EventForTransformer& iEvent) const; + virtual void clearInputProcessBlockCaches(); bool hasAcquire() const { return false; } diff --git a/FWCore/Framework/interface/one/EDProducerBase.h b/FWCore/Framework/interface/one/EDProducerBase.h index f906832e2168b..8526cfee60246 100644 --- a/FWCore/Framework/interface/one/EDProducerBase.h +++ b/FWCore/Framework/interface/one/EDProducerBase.h @@ -36,6 +36,7 @@ namespace edm { class PreallocationConfiguration; class ActivityRegistry; class ThinnedAssociationsHelper; + class EventForTransformer; namespace maker { template @@ -79,6 +80,10 @@ namespace edm { ModuleCallingContext const&, Principal const&) const {} + void doTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ActivityRegistry*, + ModuleCallingContext const*); void doPreallocate(PreallocationConfiguration const&); virtual void preallocLumis(unsigned int); void doBeginJob(); @@ -125,6 +130,10 @@ namespace edm { virtual void doBeginLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); virtual void doEndLuminosityBlockProduce_(LuminosityBlock& lbp, EventSetup const& c); + virtual size_t transformIndex_(edm::BranchDescription const& iBranch) const; + virtual ProductResolverIndex transformPrefetch_(std::size_t iIndex) const; + virtual void transform_(std::size_t iIndex, edm::EventForTransformer& iEvent) const; + virtual void clearInputProcessBlockCaches(); virtual bool hasAccumulator() const { return false; } diff --git a/FWCore/Framework/interface/one/filterAbilityToImplementor.h b/FWCore/Framework/interface/one/filterAbilityToImplementor.h index e112879884d41..c02a44f8a4670 100644 --- a/FWCore/Framework/interface/one/filterAbilityToImplementor.h +++ b/FWCore/Framework/interface/one/filterAbilityToImplementor.h @@ -84,6 +84,11 @@ namespace edm { using Type = edm::one::impl::EndLuminosityBlockProducer; }; + template <> + struct AbilityToImplementor { + using Type = edm::one::impl::Transformer; + }; + template struct AbilityToImplementor> { using Type = edm::one::impl::InputProcessBlockCacheHolder; diff --git a/FWCore/Framework/interface/one/implementors.h b/FWCore/Framework/interface/one/implementors.h index 7ffbe162fc8fb..6ab659a1caa23 100644 --- a/FWCore/Framework/interface/one/implementors.h +++ b/FWCore/Framework/interface/one/implementors.h @@ -33,11 +33,14 @@ #include "FWCore/Framework/interface/Frameworkfwd.h" #include "FWCore/Framework/interface/InputProcessBlockCacheImpl.h" #include "FWCore/Framework/interface/LuminosityBlock.h" +#include "FWCore/Framework/interface/TransformerBase.h" +#include "FWCore/Framework/interface/ProductRegistryHelper.h" #include "FWCore/Utilities/interface/EDGetToken.h" #include "FWCore/Utilities/interface/ProcessBlockIndex.h" #include "FWCore/Utilities/interface/RunIndex.h" #include "FWCore/Utilities/interface/LuminosityBlockIndex.h" #include "FWCore/Utilities/interface/propagate_const.h" +#include "DataFormats/Common/interface/Wrapper.h" // forward declarations @@ -327,6 +330,53 @@ namespace edm { virtual void accumulate(Event const& ev, EventSetup const& es) = 0; }; + + template + class Transformer : public virtual T, private TransformerBase { + public: + Transformer() = default; + Transformer(Transformer const&) = delete; + Transformer& operator=(Transformer const&) = delete; + ~Transformer() noexcept(false) override{}; + + template + void registerTransform(ProductRegistryHelper::BranchAliasSetterT iSetter, + F&& iF, + std::string productInstance = std::string()) { + registerTransform(edm::EDPutTokenT(iSetter), std::forward(iF), std::move(productInstance)); + } + + template + void registerTransform(edm::EDPutTokenT iToken, F iF, std::string productInstance = std::string()) { + using ReturnTypeT = decltype(iF(std::declval())); + TypeID returnType(typeid(ReturnTypeT)); + TransformerBase::registerTransformImp(*this, + EDPutToken(iToken), + returnType, + std::move(productInstance), + [f = std::move(iF)](edm::WrapperBase const& iGotProduct) { + return std::make_unique>( + WrapperBase::Emplace{}, + f(*static_cast const&>(iGotProduct).product())); + }); + } + + private: + size_t transformIndex_(edm::BranchDescription const& iBranch) const final { + return TransformerBase::findMatchingIndex(*this, iBranch); + } + ProductResolverIndex transformPrefetch_(std::size_t iIndex) const final { + return TransformerBase::prefetchImp(iIndex); + } + void transform_(std::size_t iIndex, edm::EventForTransformer& iEvent) const final { + return TransformerBase::transformImp(iIndex, *this, iEvent); + } + void extendUpdateLookup(BranchType iBranchType, ProductResolverIndexHelper const& iHelper) override { + if (iBranchType == InEvent) { + TransformerBase::extendUpdateLookup(*this, this->moduleDescription(), iHelper); + } + } + }; } // namespace impl } // namespace one } // namespace edm diff --git a/FWCore/Framework/interface/one/producerAbilityToImplementor.h b/FWCore/Framework/interface/one/producerAbilityToImplementor.h index 85b0a64f21859..79a1a3a492175 100644 --- a/FWCore/Framework/interface/one/producerAbilityToImplementor.h +++ b/FWCore/Framework/interface/one/producerAbilityToImplementor.h @@ -84,6 +84,11 @@ namespace edm { using Type = edm::one::impl::EndLuminosityBlockProducer; }; + template <> + struct AbilityToImplementor { + using Type = edm::one::impl::Transformer; + }; + template struct AbilityToImplementor> { using Type = edm::one::impl::InputProcessBlockCacheHolder; diff --git a/FWCore/Framework/interface/stream/AbilityChecker.h b/FWCore/Framework/interface/stream/AbilityChecker.h index c73010589f7ab..7b74ba896fe02 100644 --- a/FWCore/Framework/interface/stream/AbilityChecker.h +++ b/FWCore/Framework/interface/stream/AbilityChecker.h @@ -102,6 +102,11 @@ namespace edm { static constexpr bool kExternalWork = true; }; + template + struct HasAbility : public HasAbility { + static constexpr bool kTransformer = true; + }; + template struct HasAbility : public HasAbility { static constexpr bool kAccumulator = true; @@ -124,6 +129,7 @@ namespace edm { static constexpr bool kEndLuminosityBlockProducer = false; static constexpr bool kExternalWork = false; static constexpr bool kAccumulator = false; + static constexpr bool kTransformer = false; }; } // namespace impl template diff --git a/FWCore/Framework/interface/stream/AbilityToImplementor.h b/FWCore/Framework/interface/stream/AbilityToImplementor.h index 8fc9117dd8fba..bf63478c0fa5c 100644 --- a/FWCore/Framework/interface/stream/AbilityToImplementor.h +++ b/FWCore/Framework/interface/stream/AbilityToImplementor.h @@ -100,6 +100,13 @@ namespace edm { using Type = edm::stream::impl::ExternalWork; }; + // As currently implemented this ability only works + // with EDProducer, not with EDAnalyzers or EDFilters! + template <> + struct AbilityToImplementor { + using Type = edm::stream::impl::Transformer; + }; + // As currently implemented this ability only works // with EDProducer, not with EDAnalyzers or EDFilters! template <> diff --git a/FWCore/Framework/interface/stream/EDProducer.h b/FWCore/Framework/interface/stream/EDProducer.h index eae7abd0cb267..71a7dbd2e3a88 100644 --- a/FWCore/Framework/interface/stream/EDProducer.h +++ b/FWCore/Framework/interface/stream/EDProducer.h @@ -34,7 +34,8 @@ namespace edm { template class EDProducer : public AbilityToImplementor::Type..., - public std::conditional::kHasIt, + public std::conditional::kHasIt or + CheckAbility::kHasIt, impl::EmptyType, EDProducerBase>::type { public: diff --git a/FWCore/Framework/interface/stream/EDProducerBase.h b/FWCore/Framework/interface/stream/EDProducerBase.h index cfea5f2b79f6d..f1a561b6ac95b 100644 --- a/FWCore/Framework/interface/stream/EDProducerBase.h +++ b/FWCore/Framework/interface/stream/EDProducerBase.h @@ -27,6 +27,7 @@ #include "FWCore/Framework/interface/stream/EDProducerAdaptor.h" #include "DataFormats/Provenance/interface/ModuleDescription.h" #include "FWCore/ParameterSet/interface/ParameterSetfwd.h" +#include "FWCore/Utilities/interface/ProductResolverIndex.h" // forward declarations namespace edm { @@ -35,6 +36,7 @@ namespace edm { class ProductRegistry; class ThinnedAssociationsHelper; class WaitingTaskWithArenaHolder; + class EventForTransformer; namespace stream { class EDProducerAdaptorBase; @@ -73,6 +75,9 @@ namespace edm { virtual void registerThinnedAssociations(ProductRegistry const&, ThinnedAssociationsHelper&) {} virtual void doAcquire_(Event const&, EventSetup const&, WaitingTaskWithArenaHolder&) = 0; + virtual size_t transformIndex_(edm::BranchDescription const& iBranch) const; + virtual ProductResolverIndex transformPrefetch_(std::size_t iIndex) const; + virtual void transform_(std::size_t iIndex, edm::EventForTransformer& iEvent) const; void setModuleDescriptionPtr(ModuleDescription const* iDesc) { moduleDescriptionPtr_ = iDesc; } // ---------- member data -------------------------------- diff --git a/FWCore/Framework/interface/stream/ProducingModuleAdaptorBase.h b/FWCore/Framework/interface/stream/ProducingModuleAdaptorBase.h index 9456392188bde..212710286f388 100644 --- a/FWCore/Framework/interface/stream/ProducingModuleAdaptorBase.h +++ b/FWCore/Framework/interface/stream/ProducingModuleAdaptorBase.h @@ -53,6 +53,7 @@ namespace edm { class PreallocationConfiguration; class ProductResolverIndexAndSkipBit; class ThinnedAssociationsHelper; + class ActivityRegistry; namespace maker { template @@ -125,6 +126,13 @@ namespace edm { std::vector const& indiciesForPutProducts(BranchType iBranchType) const; + ProductResolverIndex transformPrefetch_(size_t iTransformIndex) const; + size_t transformIndex_(edm::BranchDescription const& iBranch) const; + void doTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ActivityRegistry*, + ModuleCallingContext const*); + protected: template void createStreamModules(F iFunc) { diff --git a/FWCore/Framework/interface/stream/implementors.h b/FWCore/Framework/interface/stream/implementors.h index 021e561e2671f..6467473edd4ca 100644 --- a/FWCore/Framework/interface/stream/implementors.h +++ b/FWCore/Framework/interface/stream/implementors.h @@ -31,6 +31,7 @@ #include "FWCore/Framework/interface/stream/EDProducerBase.h" #include "FWCore/Framework/interface/Frameworkfwd.h" #include "FWCore/Framework/interface/InputProcessBlockCacheImpl.h" +#include "FWCore/Framework/interface/TransformerBase.h" #include "FWCore/Utilities/interface/EDGetToken.h" #include "FWCore/Utilities/interface/EDMException.h" #include "FWCore/Utilities/interface/StreamID.h" @@ -289,6 +290,52 @@ namespace edm { virtual void acquire(Event const&, edm::EventSetup const&, WaitingTaskWithArenaHolder) = 0; }; + class Transformer : private TransformerBase, public EDProducerBase { + public: + Transformer() = default; + Transformer(Transformer const&) = delete; + Transformer& operator=(Transformer const&) = delete; + ~Transformer() noexcept(false) override{}; + + template + void registerTransform(ProducerBase::BranchAliasSetterT iSetter, + F&& iF, + std::string productInstance = std::string()) { + registerTransform(edm::EDPutTokenT(iSetter), std::forward(iF), std::move(productInstance)); + } + + template + void registerTransform(edm::EDPutTokenT iToken, F iF, std::string productInstance = std::string()) { + using ReturnTypeT = decltype(iF(std::declval())); + TypeID returnType(typeid(ReturnTypeT)); + TransformerBase::registerTransformImp(*this, + EDPutToken(iToken), + returnType, + std::move(productInstance), + [f = std::move(iF)](edm::WrapperBase const& iGotProduct) { + return std::make_unique>( + WrapperBase::Emplace{}, + f(*static_cast const&>(iGotProduct).product())); + }); + } + + private: + size_t transformIndex_(edm::BranchDescription const& iBranch) const final { + return TransformerBase::findMatchingIndex(*this, iBranch); + } + ProductResolverIndex transformPrefetch_(std::size_t iIndex) const final { + return TransformerBase::prefetchImp(iIndex); + } + void transform_(std::size_t iIndex, edm::EventForTransformer& iEvent) const final { + return TransformerBase::transformImp(iIndex, *this, iEvent); + } + void extendUpdateLookup(BranchType iBranchType, ProductResolverIndexHelper const& iHelper) override { + if (iBranchType == InEvent) { + TransformerBase::extendUpdateLookup(*this, this->moduleDescription(), iHelper); + } + } + }; + class Accumulator : public EDProducerBase { public: Accumulator() = default; diff --git a/FWCore/Framework/src/EDConsumerBase.cc b/FWCore/Framework/src/EDConsumerBase.cc index 3f17ef96aa81f..f8ca6b545291e 100644 --- a/FWCore/Framework/src/EDConsumerBase.cc +++ b/FWCore/Framework/src/EDConsumerBase.cc @@ -110,11 +110,14 @@ unsigned int EDConsumerBase::recordConsumes(BranchType iBranch, return index; } +void EDConsumerBase::extendUpdateLookup(BranchType, ProductResolverIndexHelper const&) {} + void EDConsumerBase::updateLookup(BranchType iBranchType, ProductResolverIndexHelper const& iHelper, bool iPrefetchMayGet) { frozen_ = true; assert(!containsCurrentProcessAlias_); + extendUpdateLookup(iBranchType, iHelper); { auto itKind = m_tokenInfo.begin(); auto itLabels = m_tokenInfo.begin(); diff --git a/FWCore/Framework/src/EventForTransformer.cc b/FWCore/Framework/src/EventForTransformer.cc new file mode 100644 index 0000000000000..3f13f55ed8110 --- /dev/null +++ b/FWCore/Framework/src/EventForTransformer.cc @@ -0,0 +1,28 @@ +#include "FWCore/Framework/interface/EventForTransformer.h" + +#include "DataFormats/Common/interface/TriggerResults.h" +#include "FWCore/Common/interface/TriggerResultsByName.h" +#include "FWCore/Framework/interface/EventPrincipal.h" +#include "FWCore/Framework/interface/LuminosityBlockForOutput.h" +#include "FWCore/Framework/interface/TransitionInfoTypes.h" +#include "FWCore/ParameterSet/interface/Registry.h" +#include "FWCore/Utilities/interface/Algorithms.h" +#include "FWCore/Utilities/interface/InputTag.h" + +namespace edm { + + EventForTransformer::EventForTransformer(EventPrincipal const& ep, ModuleCallingContext const* moduleCallingContext) + : eventPrincipal_{ep}, mcc_{moduleCallingContext} {} + + BasicHandle EventForTransformer::get(edm::TypeID const& iTypeID, ProductResolverIndex iIndex) const { + bool amb = false; + return eventPrincipal_.getByToken(PRODUCT_TYPE, iTypeID, iIndex, false, amb, nullptr, mcc_); + } + + void EventForTransformer::put(ProductResolverIndex index, + std::unique_ptr edp, + BasicHandle const& iGetHandle) { + eventPrincipal_.put(index, std::move(edp), iGetHandle.provenance()->productProvenance()->parentageID()); + } + +} // namespace edm diff --git a/FWCore/Framework/src/Principal.cc b/FWCore/Framework/src/Principal.cc index beefa28fe9424..d3bd9a5eef1e3 100644 --- a/FWCore/Framework/src/Principal.cc +++ b/FWCore/Framework/src/Principal.cc @@ -162,7 +162,11 @@ namespace edm { addSourceProduct(cbd); } else if (bd.onDemand()) { assert(branchType_ == InEvent); - addUnscheduledProduct(cbd); + if (bd.isTransform()) { + addTransformProduct(cbd); + } else { + addUnscheduledProduct(cbd); + } } else { addScheduledProduct(cbd); } @@ -350,6 +354,10 @@ namespace edm { addProductOrThrow(std::make_unique(std::move(bd))); } + void Principal::addTransformProduct(std::shared_ptr bd) { + addProductOrThrow(std::make_unique(std::move(bd))); + } + void Principal::addAliasedProduct(std::shared_ptr bd) { ProductResolverIndex index = preg_->indexFrom(bd->originalBranchID()); assert(index != ProductResolverIndexInvalid); diff --git a/FWCore/Framework/src/ProductRegistryHelper.cc b/FWCore/Framework/src/ProductRegistryHelper.cc index 6f62f3baa5bd1..c2e890259299b 100644 --- a/FWCore/Framework/src/ProductRegistryHelper.cc +++ b/FWCore/Framework/src/ProductRegistryHelper.cc @@ -94,6 +94,10 @@ namespace edm { } pdesc.setSwitchAliasModuleLabel(p->branchAlias_); } + if (p->isTransform_) { + pdesc.setOnDemand(true); + pdesc.setIsTransform(true); + } setIsMergeable(pdesc); if (pdesc.transient()) { diff --git a/FWCore/Framework/src/ProductResolvers.cc b/FWCore/Framework/src/ProductResolvers.cc index d6acf88e3e55d..426c97a0ceaa6 100644 --- a/FWCore/Framework/src/ProductResolvers.cc +++ b/FWCore/Framework/src/ProductResolvers.cc @@ -522,6 +522,82 @@ namespace edm { DataManagingProductResolver::resetProductData_(deleteEarly); } + void TransformingProductResolver::setupUnscheduled(UnscheduledConfigurator const& iConfigure) { + aux_ = iConfigure.auxiliary(); + worker_ = iConfigure.findWorker(branchDescription().moduleLabel()); + index_ = worker_->transformIndex(branchDescription()); + } + + ProductResolverBase::Resolution TransformingProductResolver::resolveProduct_(Principal const&, + bool skipCurrentProcess, + SharedResourcesAcquirer*, + ModuleCallingContext const*) const { + if (!skipCurrentProcess and worker_) { + return resolveProductImpl([] {}); + } + return Resolution(nullptr); + } + + void TransformingProductResolver::prefetchAsync_(WaitingTaskHolder waitTask, + Principal const& principal, + bool skipCurrentProcess, + ServiceToken const& token, + SharedResourcesAcquirer* sra, + ModuleCallingContext const* mcc) const { + if (skipCurrentProcess) { + return; + } + if (worker_ == nullptr) { + throw cms::Exception("LogicError") << "TransformingProductResolver::prefetchAsync_() called with null worker_. " + "This should not happen, please contact framework developers."; + } + //need to try changing prefetchRequested_ before adding to waitingTasks_ + bool expected = false; + bool prefetchRequested = prefetchRequested_.compare_exchange_strong(expected, true); + waitingTasks_.add(waitTask); + if (prefetchRequested) { + //Have to create a new task which will make sure the state for TransformingProductResolver + // is properly set after the module has run + auto t = make_waiting_task([this](std::exception_ptr const* iPtr) { + //The exception is being rethrown because resolveProductImpl sets the ProductResolver to a failed + // state for the case where an exception occurs during the call to the function. + // Caught exception is propagated via WaitingTaskList + CMS_SA_ALLOW try { + resolveProductImpl([iPtr]() { + if (iPtr) { + std::rethrow_exception(*iPtr); + } + }); + } catch (...) { + waitingTasks_.doneWaiting(std::current_exception()); + return; + } + waitingTasks_.doneWaiting(nullptr); + }); + + //This gives a lifetime greater than this call + ParentContext parent(mcc); + mcc_ = ModuleCallingContext(worker_->description(), ModuleCallingContext::State::kPrefetching, parent, nullptr); + + EventTransitionInfo const& info = aux_->eventTransitionInfo(); + worker_->doTransformAsync(WaitingTaskHolder(*waitTask.group(), t), + index_, + info.principal(), + token, + info.principal().streamID(), + mcc_, + mcc->getStreamContext()); + } + } + + void TransformingProductResolver::resetProductData_(bool deleteEarly) { + if (not deleteEarly) { + prefetchRequested_ = false; + waitingTasks_.reset(); + } + DataManagingProductResolver::resetProductData_(deleteEarly); + } + void ProducedProductResolver::putProduct(std::unique_ptr edp) const { if (status() != defaultStatus()) { throw Exception(errors::InsertFailure) diff --git a/FWCore/Framework/src/ProductResolvers.h b/FWCore/Framework/src/ProductResolvers.h index 530cffea7b805..8ca0faccff8d7 100644 --- a/FWCore/Framework/src/ProductResolvers.h +++ b/FWCore/Framework/src/ProductResolvers.h @@ -254,6 +254,36 @@ namespace edm { mutable std::atomic prefetchRequested_ = false; }; + class TransformingProductResolver : public ProducedProductResolver { + public: + explicit TransformingProductResolver(std::shared_ptr bd) + : ProducedProductResolver(bd, ProductStatus::ResolveFailed), mcc_(nullptr) {} + + void setupUnscheduled(UnscheduledConfigurator const&) final; + + private: + Resolution resolveProduct_(Principal const& principal, + bool skipCurrentProcess, + SharedResourcesAcquirer* sra, + ModuleCallingContext const* mcc) const override; + void prefetchAsync_(WaitingTaskHolder waitTask, + Principal const& principal, + bool skipCurrentProcess, + ServiceToken const& token, + SharedResourcesAcquirer* sra, + ModuleCallingContext const* mcc) const override; + bool unscheduledWasNotRun_() const override { return status() == ProductStatus::ResolveNotRun; } + + void resetProductData_(bool deleteEarly) override; + + CMS_THREAD_SAFE mutable WaitingTaskList waitingTasks_; + UnscheduledAuxiliary const* aux_ = nullptr; + Worker* worker_ = nullptr; + CMS_THREAD_GUARD(prefetchRequested_) mutable ModuleCallingContext mcc_; + size_t index_; + mutable std::atomic prefetchRequested_ = false; + }; + class AliasProductResolver : public DataManagingOrAliasProductResolver { public: typedef ProducedProductResolver::ProductStatus ProductStatus; diff --git a/FWCore/Framework/src/TransformerBase.cc b/FWCore/Framework/src/TransformerBase.cc new file mode 100644 index 0000000000000..c407d18dd25ea --- /dev/null +++ b/FWCore/Framework/src/TransformerBase.cc @@ -0,0 +1,63 @@ +#include "FWCore/Framework/interface/TransformerBase.h" +#include "FWCore/Framework/interface/ProducerBase.h" +#include "FWCore/Framework/interface/EventForTransformer.h" +#include "DataFormats/Provenance/interface/ProductResolverIndexHelper.h" +#include "DataFormats/Provenance/interface/BranchDescription.h" +#include "DataFormats/Provenance/interface/ModuleDescription.h" + +namespace edm { + void TransformerBase::registerTransformImp( + ProducerBase& iBase, EDPutToken iToken, const TypeID& id, std::string instanceName, TransformFunction iFunc) { + auto transformPut = iBase.transforms(id, std::move(instanceName)); + transformInfo_.emplace_back(iToken.index(), id, transformPut, std::move(iFunc)); + } + + std::size_t TransformerBase::findMatchingIndex(ProducerBase const& iBase, + edm::BranchDescription const& iBranch) const { + auto const& list = iBase.typeLabelList(); + + std::size_t index = 0; + bool found = false; + for (auto const& element : list) { + if (not element.isTransform_) { + continue; + } + if (element.typeID_ == iBranch.unwrappedTypeID() && + element.productInstanceName_ == iBranch.productInstanceName()) { + found = true; + break; + } + ++index; + } + assert(found); + return index; + } + + void TransformerBase::extendUpdateLookup(ProducerBase const& iBase, + ModuleDescription const& iModuleDesc, + ProductResolverIndexHelper const& iHelper) { + auto const& list = iBase.typeLabelList(); + + for (auto it = transformInfo_.begin<0>(); it != transformInfo_.end<0>(); ++it) { + auto const& putInfo = list[*it]; + *it = iHelper.index(PRODUCT_TYPE, + putInfo.typeID_, + iModuleDesc.moduleLabel().c_str(), + putInfo.productInstanceName_.c_str(), + iModuleDesc.processName().c_str()); + } + } + + void TransformerBase::transformImp(std::size_t iIndex, + ProducerBase const& iBase, + edm::EventForTransformer& iEvent) const { + auto handle = iEvent.get(transformInfo_.get<1>(iIndex), transformInfo_.get<0>(iIndex)); + + if (handle.wrapper()) { + iEvent.put(iBase.putTokenIndexToProductResolverIndex()[transformInfo_.get<2>(iIndex).index()], + transformInfo_.get<3>(iIndex)(*handle.wrapper()), + handle); + } + } + +} // namespace edm diff --git a/FWCore/Framework/src/Worker.cc b/FWCore/Framework/src/Worker.cc index ef360f8759e56..ce736b0b14d4e 100644 --- a/FWCore/Framework/src/Worker.cc +++ b/FWCore/Framework/src/Worker.cc @@ -236,6 +236,38 @@ namespace edm { void Worker::setEarlyDeleteHelper(EarlyDeleteHelper* iHelper) { earlyDeleteHelper_ = iHelper; } + size_t Worker::transformIndex(edm::BranchDescription const&) const { return -1; } + void Worker::doTransformAsync(WaitingTaskHolder iTask, + size_t iTransformIndex, + EventPrincipal const& iPrincipal, + ServiceToken const& iToken, + StreamID, + ModuleCallingContext const& mcc, + StreamContext const*) { + ServiceWeakToken weakToken = iToken; + + //Need to make the services available early so other services can see them + auto task = make_waiting_task([this, iTask, weakToken, &iPrincipal, iTransformIndex, parent = mcc.parent()]( + std::exception_ptr const* iExcept) mutable { + if (iExcept) { + iTask.doneWaiting(*iExcept); + } + try { + ServiceRegistry::Operate guard(weakToken.lock()); + implDoTransform(iTransformIndex, iPrincipal, parent); + } catch (...) { + iTask.doneWaiting(std::current_exception()); + return; + } + iTask.doneWaiting(std::exception_ptr{}); + }); + + //NOTE: need different ModuleCallingContext. The ProductResolver will copy the context in order to get + // a longer lifetime than this function call. + iPrincipal.prefetchAsync( + WaitingTaskHolder(*iTask.group(), task), itemToGetForTransform(iTransformIndex), false, iToken, &mcc); + } + void Worker::resetModuleDescription(ModuleDescription const* iDesc) { ModuleCallingContext temp(iDesc, moduleCallingContext_.state(), diff --git a/FWCore/Framework/src/WorkerT.cc b/FWCore/Framework/src/WorkerT.cc index 77ae243fecaaf..d56033b76be56 100644 --- a/FWCore/Framework/src/WorkerT.cc +++ b/FWCore/Framework/src/WorkerT.cc @@ -268,6 +268,132 @@ namespace edm { module_->doAcquire(info, activityRegistry(), mcc, holder); } + template + inline void WorkerT::implDoTransform(size_t iTransformIndex, EventPrincipal const&, ParentContext const&) {} + template <> + inline void WorkerT::implDoTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ParentContext const& iParent) { + ModuleCallingContext mcc( + &module_->moduleDescription(), ModuleCallingContext::State::kPrefetching, iParent, nullptr); + module_->doTransform(iTransformIndex, iEvent, activityRegistry(), &mcc); + } + template <> + inline void WorkerT::implDoTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ParentContext const& iParent) { + ModuleCallingContext mcc( + &module_->moduleDescription(), ModuleCallingContext::State::kPrefetching, iParent, nullptr); + module_->doTransform(iTransformIndex, iEvent, activityRegistry(), &mcc); + } + template <> + inline void WorkerT::implDoTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ParentContext const& iParent) { + ModuleCallingContext mcc( + &module_->moduleDescription(), ModuleCallingContext::State::kPrefetching, iParent, nullptr); + module_->doTransform(iTransformIndex, iEvent, activityRegistry(), &mcc); + } + template <> + inline void WorkerT::implDoTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ParentContext const& iParent) { + ModuleCallingContext mcc( + &module_->moduleDescription(), ModuleCallingContext::State::kPrefetching, iParent, nullptr); + module_->doTransform(iTransformIndex, iEvent, activityRegistry(), &mcc); + } + template <> + inline void WorkerT::implDoTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ParentContext const& iParent) { + ModuleCallingContext mcc( + &module_->moduleDescription(), ModuleCallingContext::State::kPrefetching, iParent, nullptr); + module_->doTransform(iTransformIndex, iEvent, activityRegistry(), &mcc); + } + template <> + inline void WorkerT::implDoTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ParentContext const& iParent) { + ModuleCallingContext mcc( + &module_->moduleDescription(), ModuleCallingContext::State::kPrefetching, iParent, nullptr); + module_->doTransform(iTransformIndex, iEvent, activityRegistry(), &mcc); + } + template <> + inline void WorkerT::implDoTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ParentContext const& iParent) { + ModuleCallingContext mcc( + &module_->moduleDescription(), ModuleCallingContext::State::kPrefetching, iParent, nullptr); + module_->doTransform(iTransformIndex, iEvent, activityRegistry(), &mcc); + } + + template + inline size_t WorkerT::transformIndex(edm::BranchDescription const&) const { + return -1; + } + template <> + inline size_t WorkerT::transformIndex(edm::BranchDescription const& iBranch) const { + return module_->transformIndex_(iBranch); + } + template <> + inline size_t WorkerT::transformIndex(edm::BranchDescription const& iBranch) const { + return module_->transformIndex_(iBranch); + } + template <> + inline size_t WorkerT::transformIndex(edm::BranchDescription const& iBranch) const { + return module_->transformIndex_(iBranch); + } + template <> + inline size_t WorkerT::transformIndex(edm::BranchDescription const& iBranch) const { + return module_->transformIndex_(iBranch); + } + template <> + inline size_t WorkerT::transformIndex(edm::BranchDescription const& iBranch) const { + return module_->transformIndex_(iBranch); + } + template <> + inline size_t WorkerT::transformIndex(edm::BranchDescription const& iBranch) const { + return module_->transformIndex_(iBranch); + } + template <> + inline size_t WorkerT::transformIndex(edm::BranchDescription const& iBranch) const { + return module_->transformIndex_(iBranch); + } + + template + inline ProductResolverIndex WorkerT::itemToGetForTransform(size_t iTransformIndex) const { + return -1; + } + template <> + inline ProductResolverIndex WorkerT::itemToGetForTransform(size_t iTransformIndex) const { + return module_->transformPrefetch_(iTransformIndex); + } + template <> + inline ProductResolverIndex WorkerT::itemToGetForTransform(size_t iTransformIndex) const { + return module_->transformPrefetch_(iTransformIndex); + } + template <> + inline ProductResolverIndex WorkerT::itemToGetForTransform( + size_t iTransformIndex) const { + return module_->transformPrefetch_(iTransformIndex); + } + template <> + inline ProductResolverIndex WorkerT::itemToGetForTransform(size_t iTransformIndex) const { + return module_->transformPrefetch_(iTransformIndex); + } + template <> + inline ProductResolverIndex WorkerT::itemToGetForTransform(size_t iTransformIndex) const { + return module_->transformPrefetch_(iTransformIndex); + } + template <> + inline ProductResolverIndex WorkerT::itemToGetForTransform(size_t iTransformIndex) const { + return module_->transformPrefetch_(iTransformIndex); + } + template <> + inline ProductResolverIndex WorkerT::itemToGetForTransform(size_t iTransformIndex) const { + return module_->transformPrefetch_(iTransformIndex); + } + template inline bool WorkerT::implNeedToRunSelection() const { return false; @@ -696,22 +822,6 @@ namespace edm { iProd->resolvePutIndicies(iBranchType, iIndicies, iModuleLabel); } - std::vector s_emptyIndexList; - - std::vector const& itemsShouldPutInEventImpl(void const*) { return s_emptyIndexList; } - - std::vector const& itemsShouldPutInEventImpl(ProducerBase const* iProd) { - return iProd->indiciesForPutProducts(edm::InEvent); - } - - std::vector const& itemsShouldPutInEventImpl(edm::stream::EDProducerAdaptorBase const* iProd) { - return iProd->indiciesForPutProducts(edm::InEvent); - } - - std::vector const& itemsShouldPutInEventImpl(edm::stream::EDFilterAdaptorBase const* iProd) { - return iProd->indiciesForPutProducts(edm::InEvent); - } - } // namespace template @@ -722,11 +832,6 @@ namespace edm { resolvePutIndiciesImpl(&module(), iBranchType, iIndicies, description()->moduleLabel()); } - template - std::vector const& WorkerT::itemsShouldPutInEvent() const { - return itemsShouldPutInEventImpl(&module()); - } - template <> Worker::Types WorkerT::moduleType() const { return Worker::kAnalyzer; diff --git a/FWCore/Framework/src/global/EDFilterBase.cc b/FWCore/Framework/src/global/EDFilterBase.cc index 152ce661235ad..ed83b6090ee60 100644 --- a/FWCore/Framework/src/global/EDFilterBase.cc +++ b/FWCore/Framework/src/global/EDFilterBase.cc @@ -26,6 +26,7 @@ #include "FWCore/Framework/interface/PreallocationConfiguration.h" #include "FWCore/Framework/src/edmodule_mightGet_config.h" #include "FWCore/Framework/interface/TransitionInfoTypes.h" +#include "FWCore/Framework/interface/EventForTransformer.h" #include "FWCore/ServiceRegistry/interface/ESParentContext.h" #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" @@ -83,6 +84,18 @@ namespace edm { this->doAcquire_(e.streamID(), e, c, holder); } + void EDFilterBase::doTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ActivityRegistry*, + ModuleCallingContext const* iMCC) { + EventForTransformer ev(iEvent, iMCC); + transform_(iTransformIndex, ev); + } + + size_t EDFilterBase::transformIndex_(edm::BranchDescription const& iBranch) const { return -1; } + ProductResolverIndex EDFilterBase::transformPrefetch_(std::size_t iIndex) const { return 0; } + void EDFilterBase::transform_(std::size_t iIndex, edm::EventForTransformer& iEvent) const {} + void EDFilterBase::doPreallocate(PreallocationConfiguration const& iPrealloc) { const auto nStreams = iPrealloc.numberOfStreams(); previousParentages_ = std::make_unique[]>(nStreams); diff --git a/FWCore/Framework/src/global/EDProducerBase.cc b/FWCore/Framework/src/global/EDProducerBase.cc index 22524b7b7c54b..dd191240270a4 100644 --- a/FWCore/Framework/src/global/EDProducerBase.cc +++ b/FWCore/Framework/src/global/EDProducerBase.cc @@ -26,6 +26,7 @@ #include "FWCore/Framework/interface/PreallocationConfiguration.h" #include "FWCore/Framework/src/edmodule_mightGet_config.h" #include "FWCore/Framework/interface/TransitionInfoTypes.h" +#include "FWCore/Framework/interface/EventForTransformer.h" #include "FWCore/ServiceRegistry/interface/ESParentContext.h" #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" @@ -89,6 +90,18 @@ namespace edm { this->doAcquire_(e.streamID(), e, c, holder); } + void EDProducerBase::doTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ActivityRegistry*, + ModuleCallingContext const* iMCC) { + EventForTransformer ev(iEvent, iMCC); + transform_(iTransformIndex, ev); + } + + size_t EDProducerBase::transformIndex_(edm::BranchDescription const& iBranch) const { return -1; } + ProductResolverIndex EDProducerBase::transformPrefetch_(std::size_t iIndex) const { return 0; } + void EDProducerBase::transform_(std::size_t iIndex, edm::EventForTransformer& iEvent) const {} + void EDProducerBase::doPreallocate(PreallocationConfiguration const& iPrealloc) { auto const nStreams = iPrealloc.numberOfStreams(); previousParentages_ = std::make_unique[]>(nStreams); diff --git a/FWCore/Framework/src/limited/EDFilterBase.cc b/FWCore/Framework/src/limited/EDFilterBase.cc index 10ddcef23e9b3..c709cf640763f 100644 --- a/FWCore/Framework/src/limited/EDFilterBase.cc +++ b/FWCore/Framework/src/limited/EDFilterBase.cc @@ -25,6 +25,7 @@ #include "FWCore/Framework/interface/PreallocationConfiguration.h" #include "FWCore/Framework/src/edmodule_mightGet_config.h" #include "FWCore/Framework/interface/TransitionInfoTypes.h" +#include "FWCore/Framework/interface/EventForTransformer.h" #include "FWCore/ServiceRegistry/interface/ESParentContext.h" #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" @@ -67,6 +68,18 @@ namespace edm { return returnValue; } + void EDFilterBase::doTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ActivityRegistry*, + ModuleCallingContext const* iMCC) { + EventForTransformer ev(iEvent, iMCC); + transform_(iTransformIndex, ev); + } + + size_t EDFilterBase::transformIndex_(edm::BranchDescription const& iBranch) const { return -1; } + ProductResolverIndex EDFilterBase::transformPrefetch_(std::size_t iIndex) const { return 0; } + void EDFilterBase::transform_(std::size_t iIndex, edm::EventForTransformer& iEvent) const {} + void EDFilterBase::doPreallocate(PreallocationConfiguration const& iPrealloc) { const auto nStreams = iPrealloc.numberOfStreams(); previousParentages_ = std::make_unique[]>(nStreams); diff --git a/FWCore/Framework/src/limited/EDProducerBase.cc b/FWCore/Framework/src/limited/EDProducerBase.cc index 4cbd373ddbdab..74d5da78ada98 100644 --- a/FWCore/Framework/src/limited/EDProducerBase.cc +++ b/FWCore/Framework/src/limited/EDProducerBase.cc @@ -25,6 +25,7 @@ #include "FWCore/Framework/interface/PreallocationConfiguration.h" #include "FWCore/Framework/src/edmodule_mightGet_config.h" #include "FWCore/Framework/interface/TransitionInfoTypes.h" +#include "FWCore/Framework/interface/EventForTransformer.h" #include "FWCore/ServiceRegistry/interface/ESParentContext.h" #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" @@ -67,6 +68,18 @@ namespace edm { return true; } + void EDProducerBase::doTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ActivityRegistry*, + ModuleCallingContext const* iMCC) { + EventForTransformer ev(iEvent, iMCC); + transform_(iTransformIndex, ev); + } + + size_t EDProducerBase::transformIndex_(edm::BranchDescription const& iBranch) const { return -1; } + ProductResolverIndex EDProducerBase::transformPrefetch_(std::size_t iIndex) const { return 0; } + void EDProducerBase::transform_(std::size_t iIndex, edm::EventForTransformer& iEvent) const {} + void EDProducerBase::doPreallocate(PreallocationConfiguration const& iPrealloc) { auto const nStreams = iPrealloc.numberOfStreams(); previousParentages_ = std::make_unique[]>(nStreams); diff --git a/FWCore/Framework/src/one/EDFilterBase.cc b/FWCore/Framework/src/one/EDFilterBase.cc index a302105b296ad..fc80c30562933 100644 --- a/FWCore/Framework/src/one/EDFilterBase.cc +++ b/FWCore/Framework/src/one/EDFilterBase.cc @@ -23,6 +23,7 @@ #include "FWCore/Framework/interface/PreallocationConfiguration.h" #include "FWCore/Framework/src/EventSignalsSentry.h" #include "FWCore/Framework/interface/TransitionInfoTypes.h" +#include "FWCore/Framework/interface/EventForTransformer.h" #include "FWCore/ServiceRegistry/interface/ESParentContext.h" #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" @@ -79,6 +80,18 @@ namespace edm { void EDFilterBase::doEndJob() { this->endJob(); } + void EDFilterBase::doTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ActivityRegistry*, + ModuleCallingContext const* iMCC) { + EventForTransformer ev(iEvent, iMCC); + transform_(iTransformIndex, ev); + } + + size_t EDFilterBase::transformIndex_(edm::BranchDescription const& iBranch) const { return -1; } + ProductResolverIndex EDFilterBase::transformPrefetch_(std::size_t iIndex) const { return 0; } + void EDFilterBase::transform_(std::size_t iIndex, edm::EventForTransformer& iEvent) const {} + void EDFilterBase::doPreallocate(PreallocationConfiguration const& iPrealloc) { auto const nThreads = iPrealloc.numberOfThreads(); preallocThreads(nThreads); diff --git a/FWCore/Framework/src/one/EDProducerBase.cc b/FWCore/Framework/src/one/EDProducerBase.cc index 4f96084edbac8..090a8a4682dc9 100644 --- a/FWCore/Framework/src/one/EDProducerBase.cc +++ b/FWCore/Framework/src/one/EDProducerBase.cc @@ -23,6 +23,7 @@ #include "FWCore/Framework/interface/PreallocationConfiguration.h" #include "FWCore/Framework/src/EventSignalsSentry.h" #include "FWCore/Framework/interface/TransitionInfoTypes.h" +#include "FWCore/Framework/interface/EventForTransformer.h" #include "FWCore/ServiceRegistry/interface/ESParentContext.h" #include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" @@ -70,6 +71,18 @@ namespace edm { SerialTaskQueue* EDProducerBase::globalRunsQueue() { return nullptr; } SerialTaskQueue* EDProducerBase::globalLuminosityBlocksQueue() { return nullptr; }; + void EDProducerBase::doTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ActivityRegistry*, + ModuleCallingContext const* iMCC) { + EventForTransformer ev(iEvent, iMCC); + transform_(iTransformIndex, ev); + } + + size_t EDProducerBase::transformIndex_(edm::BranchDescription const& iBranch) const { return -1; } + ProductResolverIndex EDProducerBase::transformPrefetch_(std::size_t iIndex) const { return 0; } + void EDProducerBase::transform_(std::size_t iIndex, edm::EventForTransformer& iEvent) const {} + void EDProducerBase::doBeginJob() { resourcesAcquirer_ = createAcquirer(); diff --git a/FWCore/Framework/src/stream/EDProducerAdaptorBase.cc b/FWCore/Framework/src/stream/EDProducerAdaptorBase.cc index 7d2bcc13772b1..c72f0baef4578 100644 --- a/FWCore/Framework/src/stream/EDProducerAdaptorBase.cc +++ b/FWCore/Framework/src/stream/EDProducerAdaptorBase.cc @@ -32,6 +32,25 @@ using namespace edm::stream; namespace edm { namespace stream { + template <> + ProductResolverIndex ProducingModuleAdaptorBase::transformPrefetch_( + size_t iTransformIndex) const { + return m_streamModules[0]->transformPrefetch_(iTransformIndex); + } + template <> + size_t ProducingModuleAdaptorBase::transformIndex_( + edm::BranchDescription const& iBranch) const { + return m_streamModules[0]->transformIndex_(iBranch); + } + template <> + void ProducingModuleAdaptorBase::doTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ActivityRegistry*, + ModuleCallingContext const* iMCC) { + EventForTransformer ev(iEvent, iMCC); + m_streamModules[iEvent.streamID()]->transform_(iTransformIndex, ev); + } + // // constants, enums and typedefs // diff --git a/FWCore/Framework/src/stream/EDProducerBase.cc b/FWCore/Framework/src/stream/EDProducerBase.cc index c3bc96be3d9dd..cadf2114e9d99 100644 --- a/FWCore/Framework/src/stream/EDProducerBase.cc +++ b/FWCore/Framework/src/stream/EDProducerBase.cc @@ -69,6 +69,10 @@ void EDProducerBase::fillDescriptions(ConfigurationDescriptions& descriptions) { descriptions.addDefault(desc); } +size_t EDProducerBase::transformIndex_(edm::BranchDescription const& iBranch) const { return -1; } +edm::ProductResolverIndex EDProducerBase::transformPrefetch_(std::size_t iIndex) const { return 0; } +void EDProducerBase::transform_(std::size_t iIndex, edm::EventForTransformer& iEvent) const {} + void EDProducerBase::prevalidate(ConfigurationDescriptions& iConfig) { edmodule_mightGet_config(iConfig); } static const std::string kBaseType("EDProducer"); diff --git a/FWCore/Framework/src/stream/ProducingModuleAdaptorBase.cc b/FWCore/Framework/src/stream/ProducingModuleAdaptorBase.cc index b9e0e33c0ef41..7046b38d726ec 100644 --- a/FWCore/Framework/src/stream/ProducingModuleAdaptorBase.cc +++ b/FWCore/Framework/src/stream/ProducingModuleAdaptorBase.cc @@ -23,6 +23,7 @@ #include "FWCore/Framework/interface/RunPrincipal.h" #include "FWCore/Framework/interface/PreallocationConfiguration.h" #include "FWCore/Framework/interface/TransitionInfoTypes.h" +#include "FWCore/Framework/interface/EventForTransformer.h" #include "FWCore/ServiceRegistry/interface/ESParentContext.h" // @@ -181,6 +182,20 @@ namespace edm { return m_streamModules[0]->indiciesForPutProducts(iBranchType); } + template + ProductResolverIndex ProducingModuleAdaptorBase::transformPrefetch_(size_t iTransformIndex) const { + return 0; + } + template + size_t ProducingModuleAdaptorBase::transformIndex_(edm::BranchDescription const& iBranch) const { + return 0; + } + template + void ProducingModuleAdaptorBase::doTransform(size_t iTransformIndex, + EventPrincipal const& iEvent, + ActivityRegistry*, + ModuleCallingContext const* iMCC) {} + template void ProducingModuleAdaptorBase::doBeginStream(StreamID id) { m_streamModules[id]->beginStream(id); diff --git a/FWCore/Framework/test/global_filter_t.cppunit.cc b/FWCore/Framework/test/global_filter_t.cppunit.cc index 8c04d1189b8bb..4a70920dc2832 100644 --- a/FWCore/Framework/test/global_filter_t.cppunit.cc +++ b/FWCore/Framework/test/global_filter_t.cppunit.cc @@ -156,18 +156,14 @@ class testGlobalFilter : public CppUnit::TestFixture { return std::unique_ptr{}; } - virtual void streamBeginRun(edm::StreamID, edm::Run const&, edm::EventSetup const&) const override { ++m_count; } - virtual void streamBeginLuminosityBlock(edm::StreamID, - edm::LuminosityBlock const&, - edm::EventSetup const&) const override { + void streamBeginRun(edm::StreamID, edm::Run const&, edm::EventSetup const&) const override { ++m_count; } + void streamBeginLuminosityBlock(edm::StreamID, edm::LuminosityBlock const&, edm::EventSetup const&) const override { ++m_count; } - virtual void streamEndLuminosityBlock(edm::StreamID, - edm::LuminosityBlock const&, - edm::EventSetup const&) const override { + void streamEndLuminosityBlock(edm::StreamID, edm::LuminosityBlock const&, edm::EventSetup const&) const override { ++m_count; } - virtual void streamEndRun(edm::StreamID, edm::Run const&, edm::EventSetup const&) const override { ++m_count; } + void streamEndRun(edm::StreamID, edm::Run const&, edm::EventSetup const&) const override { ++m_count; } void endStream(edm::StreamID) const override { ++m_count; } }; @@ -356,6 +352,22 @@ class testGlobalFilter : public CppUnit::TestFixture { m_globalEndLuminosityBlockSummaryCalled = false; } }; + + class TransformProd : public edm::global::EDFilter { + public: + TransformProd(edm::ParameterSet const&) { + token_ = produces(); + registerTransform(token_, [](float iV) { return int(iV); }); + } + + bool filter(edm::StreamID, edm::Event& iEvent, edm::EventSetup const&) const { + //iEvent.emplace(token_, 3.625); + return true; + } + + private: + edm::EDPutTokenT token_; + }; }; ///registration of the test so that the runner can find it diff --git a/FWCore/Framework/test/global_producer_t.cppunit.cc b/FWCore/Framework/test/global_producer_t.cppunit.cc index dd4f44412cd41..d50eece8c80a3 100644 --- a/FWCore/Framework/test/global_producer_t.cppunit.cc +++ b/FWCore/Framework/test/global_producer_t.cppunit.cc @@ -321,6 +321,20 @@ class testGlobalProducer : public CppUnit::TestFixture { m_globalEndLuminosityBlockSummaryCalled = false; } }; + class TransformProd : public edm::global::EDProducer { + public: + TransformProd(edm::ParameterSet const&) { + token_ = produces(); + registerTransform(token_, [](float iV) { return int(iV); }); + } + + void produce(edm::StreamID, edm::Event& iEvent, edm::EventSetup const&) const { + //iEvent.emplace(token_, 3.625); + } + + private: + edm::EDPutTokenT token_; + }; }; ///registration of the test so that the runner can find it diff --git a/FWCore/Framework/test/limited_filter_t.cppunit.cc b/FWCore/Framework/test/limited_filter_t.cppunit.cc index f7388c644bf12..3e0dc765f3850 100644 --- a/FWCore/Framework/test/limited_filter_t.cppunit.cc +++ b/FWCore/Framework/test/limited_filter_t.cppunit.cc @@ -385,6 +385,23 @@ class testLimitedFilter : public CppUnit::TestFixture { m_globalEndLuminosityBlockSummaryCalled = false; } }; + + class TransformProd : public edm::limited::EDFilter { + public: + TransformProd(edm::ParameterSet const&) + : edm::limited::EDFilterBase(s_pset), edm::limited::EDFilter(s_pset) { + token_ = produces(); + registerTransform(token_, [](float iV) { return int(iV); }); + } + + bool filter(edm::StreamID, edm::Event& iEvent, edm::EventSetup const&) const { + //iEvent.emplace(token_, 3.625); + return true; + } + + private: + edm::EDPutTokenT token_; + }; }; ///registration of the test so that the runner can find it diff --git a/FWCore/Framework/test/limited_producer_t.cppunit.cc b/FWCore/Framework/test/limited_producer_t.cppunit.cc index 41bdbf38a52b2..11ca0df90ab77 100644 --- a/FWCore/Framework/test/limited_producer_t.cppunit.cc +++ b/FWCore/Framework/test/limited_producer_t.cppunit.cc @@ -352,6 +352,21 @@ class testLimitedProducer : public CppUnit::TestFixture { m_globalEndLuminosityBlockSummaryCalled = false; } }; + class TransformProd : public edm::limited::EDProducer { + public: + TransformProd(edm::ParameterSet const&) + : edm::limited::EDProducerBase(s_pset), edm::limited::EDProducer(s_pset) { + token_ = produces(); + registerTransform(token_, [](float iV) { return int(iV); }); + } + + void produce(edm::StreamID, edm::Event& iEvent, edm::EventSetup const&) const { + //iEvent.emplace(token_, 3.625); + } + + private: + edm::EDPutTokenT token_; + }; }; ///registration of the test so that the runner can find it diff --git a/FWCore/Framework/test/stubs/TestGlobalProducers.cc b/FWCore/Framework/test/stubs/TestGlobalProducers.cc index 09bbf09b7cbbc..c3126c33025a0 100644 --- a/FWCore/Framework/test/stubs/TestGlobalProducers.cc +++ b/FWCore/Framework/test/stubs/TestGlobalProducers.cc @@ -947,7 +947,6 @@ namespace edmtest { std::vector expectedByRun_; int expectedSum_{0}; }; - } // namespace global } // namespace edmtest diff --git a/FWCore/Integration/test/BuildFile.xml b/FWCore/Integration/test/BuildFile.xml index 2bcfc3d7548a5..feb241f353299 100644 --- a/FWCore/Integration/test/BuildFile.xml +++ b/FWCore/Integration/test/BuildFile.xml @@ -440,6 +440,20 @@ + + + + + + + + + + + + + + diff --git a/FWCore/Integration/test/TransformIntProducer.cc b/FWCore/Integration/test/TransformIntProducer.cc new file mode 100644 index 0000000000000..29afc7e81d9e5 --- /dev/null +++ b/FWCore/Integration/test/TransformIntProducer.cc @@ -0,0 +1,54 @@ +#include "FWCore/Framework/interface/global/EDProducer.h" +#include "FWCore/Framework/interface/Event.h" +#include "FWCore/Framework/interface/MakerMacros.h" +#include "DataFormats/TestObjects/interface/ToyProducts.h" +#include "FWCore/Utilities/interface/Exception.h" + +namespace edmtest { + class TransformIntProducer : public edm::global::EDProducer { + public: + TransformIntProducer(edm::ParameterSet const& iPSet) + : getToken_(consumes(iPSet.getParameter("get"))), + offset_(iPSet.getParameter("offset")), + transformOffset_(iPSet.getParameter("transformOffset")), + noPut_(iPSet.getParameter("noPut")) { + putToken_ = produces(); + bool check = iPSet.getUntrackedParameter("checkTransformNotCalled"); + registerTransform( + putToken_, + [offset = transformOffset_, check](auto const& iFrom) { + if (check) { + throw cms::Exception("TransformShouldNotBeCalled"); + } + return IntProduct(iFrom.value + offset); + }, + "transform"); + } + + void produce(edm::StreamID, edm::Event& iEvent, edm::EventSetup const&) const override { + if (not noPut_) { + iEvent.emplace(putToken_, iEvent.get(getToken_).value + offset_); + } + } + static void fillDescriptions(edm::ConfigurationDescriptions& desc) { + edm::ParameterSetDescription pset; + pset.add("get"); + pset.add("offset", 0); + pset.add("transformOffset", 1); + pset.addUntracked("checkTransformNotCalled", false); + pset.add("noPut", false); + + desc.addDefault(pset); + } + + private: + const edm::EDGetTokenT getToken_; + edm::EDPutTokenT putToken_; + const unsigned int offset_; + const unsigned int transformOffset_; + const bool noPut_; + }; +} // namespace edmtest + +using edmtest::TransformIntProducer; +DEFINE_FWK_MODULE(TransformIntProducer); diff --git a/FWCore/Integration/test/TransformIntStreamProducer.cc b/FWCore/Integration/test/TransformIntStreamProducer.cc new file mode 100644 index 0000000000000..212287a35bebe --- /dev/null +++ b/FWCore/Integration/test/TransformIntStreamProducer.cc @@ -0,0 +1,49 @@ +#include "FWCore/Framework/interface/stream/EDProducer.h" +#include "FWCore/Framework/interface/Event.h" +#include "FWCore/Framework/interface/MakerMacros.h" +#include "DataFormats/TestObjects/interface/ToyProducts.h" +#include "FWCore/Utilities/interface/Exception.h" + +namespace edmtest { + class TransformIntStreamProducer : public edm::stream::EDProducer { + public: + TransformIntStreamProducer(edm::ParameterSet const& iPSet) + : getToken_(consumes(iPSet.getParameter("get"))), + offset_(iPSet.getParameter("offset")), + transformOffset_(iPSet.getParameter("transformOffset")) { + putToken_ = produces(); + bool check = iPSet.getUntrackedParameter("checkTransformNotCalled"); + registerTransform( + putToken_, + [offset = transformOffset_, check](auto const& iFrom) { + if (check) { + throw cms::Exception("TransformShouldNotBeCalled"); + } + return IntProduct(iFrom.value + offset); + }, + "transform"); + } + + void produce(edm::Event& iEvent, edm::EventSetup const&) override { + iEvent.emplace(putToken_, iEvent.get(getToken_).value + offset_); + } + static void fillDescriptions(edm::ConfigurationDescriptions& desc) { + edm::ParameterSetDescription pset; + pset.add("get"); + pset.add("offset", 0); + pset.add("transformOffset", 1); + pset.addUntracked("checkTransformNotCalled", false); + + desc.addDefault(pset); + } + + private: + const edm::EDGetTokenT getToken_; + edm::EDPutTokenT putToken_; + const unsigned int offset_; + const unsigned int transformOffset_; + }; +} // namespace edmtest + +using edmtest::TransformIntStreamProducer; +DEFINE_FWK_MODULE(TransformIntStreamProducer); diff --git a/FWCore/Integration/test/transformTest_cfg.py b/FWCore/Integration/test/transformTest_cfg.py new file mode 100644 index 0000000000000..696620619bc04 --- /dev/null +++ b/FWCore/Integration/test/transformTest_cfg.py @@ -0,0 +1,49 @@ +import FWCore.ParameterSet.Config as cms +import argparse +import sys + +parser = argparse.ArgumentParser(prog=sys.argv[0], description='Test module transform.') + +#do not call transform +#on Path/off Path + +parser.add_argument("--onPath", help="Put transform module on the Path", action="store_true") +parser.add_argument("--noTransform", help="do not consume transform", action="store_true") +parser.add_argument("--stream", help="use stream module", action="store_true") +parser.add_argument("--noPut", help="do not put data used by transform", action="store_true") +parser.add_argument("--addTracer", help="add Tracer service", action="store_true") + +argv = sys.argv[:] +if '--' in argv: + argv.remove("--") +args, unknown = parser.parse_known_args(argv) + + +process = cms.Process("TEST") + +process.source = cms.Source("EmptySource") + +process.maxEvents.input = 4 + +process.start = cms.EDProducer("IntProducer", ivalue = cms.int32(1)) +if args.stream: + process.t = cms.EDProducer("TransformIntStreamProducer", get = cms.InputTag("start"), offset = cms.uint32(1), checkTransformNotCalled = cms.untracked.bool(False)) +else: + process.t = cms.EDProducer("TransformIntProducer", get = cms.InputTag("start"), offset = cms.uint32(1), checkTransformNotCalled = cms.untracked.bool(False), noPut = cms.bool(args.noPut)) + +process.tester = cms.EDAnalyzer("IntTestAnalyzer", + moduleLabel = cms.untracked.InputTag("t","transform"), + valueMustMatch = cms.untracked.int32(3)) + +if args.noTransform: + process.tester.moduleLabel = "t" + process.tester.valueMustMatch = 2 + process.t.checkTransformNotCalled = True + +if args.onPath: + process.p = cms.Path(process.t+process.tester, cms.Task(process.start)) +else: + process.p = cms.Path(process.tester, cms.Task(process.start, process.t)) + +if args.addTracer: + process.add_(cms.Service("Tracer"))