SonicTriton feature updates, improvements, bug fixes

HeterogeneousCore/SonicCore/interface/SonicAcquirer.h

@@ -17,8 +17,8 @@ class SonicAcquirer : public Module {
   //typedef to simplify usage
   typedef typename Client::Input Input;
   //constructor
-  SonicAcquirer(edm::ParameterSet const& cfg, const std::string& debugName = "")
-      : clientPset_(cfg.getParameterSet("Client")), debugName_(debugName) {}
+  SonicAcquirer(edm::ParameterSet const& cfg, const std::string& debugName = "", bool verbose = true)
+      : clientPset_(cfg.getParameterSet("Client")), debugName_(debugName), verbose_(verbose) {}
   //destructor
   ~SonicAcquirer() override = default;
 
@@ -31,7 +31,8 @@ class SonicAcquirer : public Module {
   void acquire(edm::Event const& iEvent, edm::EventSetup const& iSetup, edm::WaitingTaskWithArenaHolder holder) final {
     auto t0 = std::chrono::high_resolution_clock::now();
     acquire(iEvent, iSetup, client_->input());
-    sonic_utils::printDebugTime(debugName_, "acquire() time: ", t0);
+    if (verbose_)
+      sonic_utils::printDebugTime(debugName_, "acquire() time: ", t0);
     t_dispatch_ = std::chrono::high_resolution_clock::now();
     client_->dispatch(holder);
   }
@@ -45,6 +46,7 @@ class SonicAcquirer : public Module {
   edm::ParameterSet clientPset_;
   std::unique_ptr<Client> client_;
   std::string debugName_;
+  bool verbose_;
   std::chrono::time_point<std::chrono::high_resolution_clock> t_dispatch_;
 };
 

HeterogeneousCore/SonicCore/interface/SonicClientBase.h

@@ -9,7 +9,6 @@
 #include "HeterogeneousCore/SonicCore/interface/SonicDispatcherPseudoAsync.h"
 
 #include <string>
-#include <chrono>
 #include <exception>
 #include <memory>
 #include <optional>
@@ -59,7 +58,6 @@ class SonicClientBase {
 
   //for logging/debugging
   std::string debugName_, clientName_, fullDebugName_;
-  std::chrono::time_point<std::chrono::high_resolution_clock> t0_;
 
   friend class SonicDispatcher;
   friend class SonicDispatcherPseudoAsync;

HeterogeneousCore/SonicCore/interface/SonicEDFilter.h

@@ -16,19 +16,21 @@ class SonicEDFilter : public SonicAcquirer<Client, edm::stream::EDFilter<edm::Ex
   //typedef to simplify usage
   typedef typename Client::Output Output;
   //constructor
-  SonicEDFilter(edm::ParameterSet const& cfg, const std::string& debugName)
-      : SonicAcquirer<Client, edm::stream::EDFilter<edm::ExternalWork, Capabilities...>>(cfg, debugName) {}
+  SonicEDFilter(edm::ParameterSet const& cfg, const std::string& debugName, bool verbose = true)
+      : SonicAcquirer<Client, edm::stream::EDFilter<edm::ExternalWork, Capabilities...>>(cfg, debugName, verbose) {}
   //destructor
   ~SonicEDFilter() override = default;
 
   //derived classes use a dedicated produce() interface that incorporates client_->output()
   bool filter(edm::Event& iEvent, edm::EventSetup const& iSetup) final {
     //measure time between acquire and produce
-    sonic_utils::printDebugTime(this->debugName_, "dispatch() time: ", this->t_dispatch_);
+    if (this->verbose_)
+      sonic_utils::printDebugTime(this->debugName_, "dispatch() time: ", this->t_dispatch_);
 
     auto t0 = std::chrono::high_resolution_clock::now();
     bool result = filter(iEvent, iSetup, this->client_->output());
-    sonic_utils::printDebugTime(this->debugName_, "filter() time: ", t0);
+    if (this->verbose_)
+      sonic_utils::printDebugTime(this->debugName_, "filter() time: ", t0);
 
     //reset client data
     this->client_->reset();

HeterogeneousCore/SonicCore/interface/SonicEDProducer.h

@@ -16,19 +16,21 @@ class SonicEDProducer : public SonicAcquirer<Client, edm::stream::EDProducer<edm
   //typedef to simplify usage
   typedef typename Client::Output Output;
   //constructor
-  SonicEDProducer(edm::ParameterSet const& cfg, const std::string& debugName)
-      : SonicAcquirer<Client, edm::stream::EDProducer<edm::ExternalWork, Capabilities...>>(cfg, debugName) {}
+  SonicEDProducer(edm::ParameterSet const& cfg, const std::string& debugName, bool verbose = true)
+      : SonicAcquirer<Client, edm::stream::EDProducer<edm::ExternalWork, Capabilities...>>(cfg, debugName, verbose) {}
   //destructor
   ~SonicEDProducer() override = default;
 
   //derived classes use a dedicated produce() interface that incorporates client_->output()
   void produce(edm::Event& iEvent, edm::EventSetup const& iSetup) final {
     //measure time between acquire and produce
-    sonic_utils::printDebugTime(this->debugName_, "dispatch() time: ", this->t_dispatch_);
+    if (this->verbose_)
+      sonic_utils::printDebugTime(this->debugName_, "dispatch() time: ", this->t_dispatch_);
 
     auto t0 = std::chrono::high_resolution_clock::now();
     produce(iEvent, iSetup, this->client_->output());
-    sonic_utils::printDebugTime(this->debugName_, "produce() time: ", t0);
+    if (this->verbose_)
+      sonic_utils::printDebugTime(this->debugName_, "produce() time: ", t0);
 
     //reset client data
     this->client_->reset();

HeterogeneousCore/SonicCore/interface/SonicOneEDAnalyzer.h

@@ -21,8 +21,8 @@ class SonicOneEDAnalyzer : public edm::one::EDAnalyzer<Capabilities...> {
   typedef typename Client::Input Input;
   typedef typename Client::Output Output;
   //constructor
-  SonicOneEDAnalyzer(edm::ParameterSet const& cfg, const std::string& debugName)
-      : clientPset_(cfg.getParameterSet("Client")), debugName_(debugName) {
+  SonicOneEDAnalyzer(edm::ParameterSet const& cfg, const std::string& debugName, bool verbose = true)
+      : clientPset_(cfg.getParameterSet("Client")), debugName_(debugName), verbose_(verbose) {
     //ExternalWork is not compatible with one modules, so Sync mode is enforced
     if (clientPset_.getParameter<std::string>("mode") != "Sync") {
       edm::LogWarning("ResetClientMode") << "Resetting client mode to Sync for SonicOneEDAnalyzer";
@@ -42,18 +42,21 @@ class SonicOneEDAnalyzer : public edm::one::EDAnalyzer<Capabilities...> {
   void analyze(edm::Event const& iEvent, edm::EventSetup const& iSetup) final {
     auto t0 = std::chrono::high_resolution_clock::now();
     acquire(iEvent, iSetup, client_->input());
-    sonic_utils::printDebugTime(debugName_, "acquire() time: ", t0);
+    if (verbose_)
+      sonic_utils::printDebugTime(debugName_, "acquire() time: ", t0);
 
     //pattern similar to ExternalWork, but blocking
     auto t1 = std::chrono::high_resolution_clock::now();
     client_->dispatch();
 
     //measure time between acquire and produce
-    sonic_utils::printDebugTime(debugName_, "dispatch() time: ", t1);
+    if (verbose_)
+      sonic_utils::printDebugTime(debugName_, "dispatch() time: ", t1);
 
     auto t2 = std::chrono::high_resolution_clock::now();
     analyze(iEvent, iSetup, client_->output());
-    sonic_utils::printDebugTime(debugName_, "analyze() time: ", t2);
+    if (verbose_)
+      sonic_utils::printDebugTime(debugName_, "analyze() time: ", t2);
 
     //reset client data
     client_->reset();
@@ -68,6 +71,7 @@ class SonicOneEDAnalyzer : public edm::one::EDAnalyzer<Capabilities...> {
   edm::ParameterSet clientPset_;
   std::unique_ptr<Client> client_;
   std::string debugName_;
+  bool verbose_;
 };
 
 #endif

HeterogeneousCore/SonicCore/src/SonicClientBase.cc

@@ -40,11 +40,7 @@ void SonicClientBase::start(edm::WaitingTaskWithArenaHolder holder) {
   holder_ = std::move(holder);
 }
 
-void SonicClientBase::start() {
-  tries_ = 0;
-  if (!debugName_.empty())
-    t0_ = std::chrono::high_resolution_clock::now();
-}
+void SonicClientBase::start() { tries_ = 0; }
 
 void SonicClientBase::finish(bool success, std::exception_ptr eptr) {
   //retries are only allowed if no exception was raised
@@ -63,11 +59,6 @@ void SonicClientBase::finish(bool success, std::exception_ptr eptr) {
       eptr = make_exception_ptr(ex);
     }
   }
-  if (!debugName_.empty()) {
-    auto t1 = std::chrono::high_resolution_clock::now();
-    edm::LogInfo(fullDebugName_) << "Client time: "
-                                 << std::chrono::duration_cast<std::chrono::microseconds>(t1 - t0_).count();
-  }
   if (holder_) {
     holder_->doneWaiting(eptr);
     holder_.reset();

HeterogeneousCore/SonicTriton/BuildFile.xml

@@ -5,7 +5,7 @@
 <use name="FWCore/Utilities"/>
 <use name="HeterogeneousCore/SonicCore"/>
 <use name="HeterogeneousCore/CUDAUtilities"/>
-<use name="triton-inference-server"/>
+<use name="triton-inference-client"/>
 <use name="protobuf"/>
 <iftool name="cuda">
   <use name="cuda"/>

HeterogeneousCore/SonicTriton/README.md

@@ -32,6 +32,7 @@ The model information from the server can be printed by enabling `verbose` outpu
 * `outputs`: optional, specify which output(s) the server should send
 * `verbose`: enable verbose printouts (default: false)
 * `useSharedMemory`: enable use of shared memory (see [below](#shared-memory)) with local servers (default: true)
+* `compression`: enable compression of input and output data to reduce bandwidth (using gzip or deflate) (default: none)
 
 The batch size should be set using the client accessor, in order to ensure a consistent value across all inputs:
 * `setBatchSize()`: set a new batch size
@@ -143,6 +144,15 @@ If the process modifiers `enableSonicTriton` or `allSonicTriton` are activated,
 the fallback server will launch automatically if needed and will use a local GPU if one is available.
 If the fallback server uses CPU, clients that use the fallback server will automatically be set to `Sync` mode.
 
+Servers have several available parameters:
+* `name`: unique identifier for each server (clients use this when specifying preferred server; also used internally by `TritonService`)
+* `address`: web address for server
+* `port`: port number for server (Triton server provides gRPC service on port 8001 by default)
+* `useSsl`: connect to server via SSL (default: false)
+* `rootCertificates`: for SSL, name of file containing PEM encoding of server root certificates, if any
+* `privateKey`: for SSL, name of file containing PEM encoding of user's private key, if any
+* `certificateChain`: for SSL, name of file containing PEM encoding of user's certificate chain, if any
+
 ## Examples
 
-Several example producers (running ResNet50 or Graph Attention Network) can be found in the [test](./test) directory.
+Several example producers (running image classification networks or Graph Attention Network) can be found in the [test](./test) directory.

HeterogeneousCore/SonicTriton/interface/TritonClient.h

@@ -50,7 +50,7 @@ class TritonClient : public SonicClient<TritonInputMap, TritonOutputMap> {
 
 protected:
   //helpers
-  void getResults(std::shared_ptr<nvidia::inferenceserver::client::InferResult> results);
+  void getResults(std::shared_ptr<triton::client::InferResult> results);
   void evaluate() override;
   template <typename F>
   bool handle_exception(F&& call);
@@ -68,14 +68,16 @@ class TritonClient : public SonicClient<TritonInputMap, TritonOutputMap> {
   bool verbose_;
   bool useSharedMemory_;
   TritonServerType serverType_;
+  grpc_compression_algorithm compressionAlgo_;
+  triton::client::Headers headers_;
 
   //IO pointers for triton
-  std::vector<nvidia::inferenceserver::client::InferInput*> inputsTriton_;
-  std::vector<const nvidia::inferenceserver::client::InferRequestedOutput*> outputsTriton_;
+  std::vector<triton::client::InferInput*> inputsTriton_;
+  std::vector<const triton::client::InferRequestedOutput*> outputsTriton_;
 
-  std::unique_ptr<nvidia::inferenceserver::client::InferenceServerGrpcClient> client_;
+  std::unique_ptr<triton::client::InferenceServerGrpcClient> client_;
   //stores timeout, model name and version
-  nvidia::inferenceserver::client::InferOptions options_;
+  triton::client::InferOptions options_;
 
 private:
   friend TritonInputData;

HeterogeneousCore/SonicTriton/interface/TritonData.h

@@ -42,7 +42,7 @@ using TritonInputContainer = std::shared_ptr<TritonInput<DT>>;
 template <typename IO>
 class TritonData {
 public:
-  using Result = nvidia::inferenceserver::client::InferResult;
+  using Result = triton::client::InferResult;
   using TensorMetadata = inference::ModelMetadataResponse_TensorMetadata;
   using ShapeType = std::vector<int64_t>;
   using ShapeView = edm::Span<ShapeType::const_iterator>;
@@ -93,7 +93,7 @@ class TritonData {
   void updateMem(size_t size);
   void computeSizes();
   void resetSizes();
-  nvidia::inferenceserver::client::InferenceServerGrpcClient* client();
+  triton::client::InferenceServerGrpcClient* client();
 
   //helpers
   bool anyNeg(const ShapeView& vec) const {
@@ -132,11 +132,12 @@ class TritonData {
   std::shared_ptr<void> holder_;
   std::shared_ptr<TritonMemResource<IO>> memResource_;
   std::shared_ptr<Result> result_;
+  mutable bool done_{};
 };
 
-using TritonInputData = TritonData<nvidia::inferenceserver::client::InferInput>;
+using TritonInputData = TritonData<triton::client::InferInput>;
 using TritonInputMap = std::unordered_map<std::string, TritonInputData>;
-using TritonOutputData = TritonData<nvidia::inferenceserver::client::InferRequestedOutput>;
+using TritonOutputData = TritonData<triton::client::InferRequestedOutput>;
 using TritonOutputMap = std::unordered_map<std::string, TritonOutputData>;
 
 //avoid "explicit specialization after instantiation" error
@@ -160,12 +161,12 @@ void TritonInputData::reset();
 template <>
 void TritonOutputData::reset();
 template <>
-void TritonInputData::createObject(nvidia::inferenceserver::client::InferInput** ioptr);
+void TritonInputData::createObject(triton::client::InferInput** ioptr);
 template <>
-void TritonOutputData::createObject(nvidia::inferenceserver::client::InferRequestedOutput** ioptr);
+void TritonOutputData::createObject(triton::client::InferRequestedOutput** ioptr);
 
 //explicit template instantiation declarations
-extern template class TritonData<nvidia::inferenceserver::client::InferInput>;
-extern template class TritonData<nvidia::inferenceserver::client::InferRequestedOutput>;
+extern template class TritonData<triton::client::InferInput>;
+extern template class TritonData<triton::client::InferRequestedOutput>;
 
 #endif

HeterogeneousCore/SonicTriton/interface/TritonEDFilter.h

@@ -13,7 +13,8 @@ template <typename G, typename... Capabilities>
 class TritonEDFilterT : public SonicEDFilter<TritonClient, edm::GlobalCache<G>, Capabilities...> {
 public:
   TritonEDFilterT(edm::ParameterSet const& cfg, const std::string& debugName)
-      : SonicEDFilter<TritonClient, edm::GlobalCache<G>, Capabilities...>(cfg, debugName) {}
+      : SonicEDFilter<TritonClient, edm::GlobalCache<G>, Capabilities...>(
+            cfg, debugName, cfg.getParameterSet("Client").getUntrackedParameter<bool>("verbose")) {}
 
   //use this function to avoid calling TritonService functions Nstreams times
   static std::unique_ptr<G> initializeGlobalCache(edm::ParameterSet const& pset) {

HeterogeneousCore/SonicTriton/interface/TritonEDProducer.h

@@ -13,7 +13,8 @@ template <typename G, typename... Capabilities>
 class TritonEDProducerT : public SonicEDProducer<TritonClient, edm::GlobalCache<G>, Capabilities...> {
 public:
   TritonEDProducerT(edm::ParameterSet const& cfg, const std::string& debugName)
-      : SonicEDProducer<TritonClient, edm::GlobalCache<G>, Capabilities...>(cfg, debugName) {}
+      : SonicEDProducer<TritonClient, edm::GlobalCache<G>, Capabilities...>(
+            cfg, debugName, cfg.getParameterSet("Client").getUntrackedParameter<bool>("verbose")) {}
 
   //use this function to avoid calling TritonService functions Nstreams times
   static std::unique_ptr<G> initializeGlobalCache(edm::ParameterSet const& pset) {

HeterogeneousCore/SonicTriton/interface/TritonMemResource.h

@@ -54,10 +54,10 @@ class TritonCpuShmResource : public TritonMemResource<IO> {
   const uint8_t* copyOutput() override { return nullptr; }
 };
 
-using TritonInputHeapResource = TritonHeapResource<nvidia::inferenceserver::client::InferInput>;
-using TritonInputCpuShmResource = TritonCpuShmResource<nvidia::inferenceserver::client::InferInput>;
-using TritonOutputHeapResource = TritonHeapResource<nvidia::inferenceserver::client::InferRequestedOutput>;
-using TritonOutputCpuShmResource = TritonCpuShmResource<nvidia::inferenceserver::client::InferRequestedOutput>;
+using TritonInputHeapResource = TritonHeapResource<triton::client::InferInput>;
+using TritonInputCpuShmResource = TritonCpuShmResource<triton::client::InferInput>;
+using TritonOutputHeapResource = TritonHeapResource<triton::client::InferRequestedOutput>;
+using TritonOutputCpuShmResource = TritonCpuShmResource<triton::client::InferRequestedOutput>;
 
 //avoid "explicit specialization after instantiation" error
 template <>
@@ -86,8 +86,8 @@ class TritonGpuShmResource : public TritonMemResource<IO> {
   std::shared_ptr<cudaIpcMemHandle_t> handle_;
 };
 
-using TritonInputGpuShmResource = TritonGpuShmResource<nvidia::inferenceserver::client::InferInput>;
-using TritonOutputGpuShmResource = TritonGpuShmResource<nvidia::inferenceserver::client::InferRequestedOutput>;
+using TritonInputGpuShmResource = TritonGpuShmResource<triton::client::InferInput>;
+using TritonOutputGpuShmResource = TritonGpuShmResource<triton::client::InferRequestedOutput>;
 
 //avoid "explicit specialization after instantiation" error
 template <>

HeterogeneousCore/SonicTriton/interface/TritonOneEDAnalyzer.h

@@ -13,7 +13,8 @@ template <typename... Capabilities>
 class TritonOneEDAnalyzer : public SonicOneEDAnalyzer<TritonClient, Capabilities...> {
 public:
   TritonOneEDAnalyzer(edm::ParameterSet const& cfg, const std::string& debugName)
-      : SonicOneEDAnalyzer<TritonClient, Capabilities...>(cfg, debugName) {
+      : SonicOneEDAnalyzer<TritonClient, Capabilities...>(
+            cfg, debugName, cfg.getParameterSet("Client").getUntrackedParameter<bool>("verbose")) {
     edm::Service<TritonService> ts;
     const auto& clientPset = cfg.getParameterSet("Client");
     ts->addModel(clientPset.getParameter<std::string>("modelName"),

HeterogeneousCore/SonicTriton/interface/TritonService.h

@@ -10,6 +10,8 @@
 #include <functional>
 #include <utility>
 
+#include "grpc_client.h"
+
 //forward declarations
 namespace edm {
   class ActivityRegistry;
@@ -34,7 +36,9 @@ class TritonService {
           retries(pset.getUntrackedParameter<int>("retries")),
           wait(pset.getUntrackedParameter<int>("wait")),
           instanceName(pset.getUntrackedParameter<std::string>("instanceName")),
-          tempDir(pset.getUntrackedParameter<std::string>("tempDir")) {}
+          tempDir(pset.getUntrackedParameter<std::string>("tempDir")),
+          imageName(pset.getUntrackedParameter<std::string>("imageName")),
+          sandboxName(pset.getUntrackedParameter<std::string>("sandboxName")) {}
 
     bool enable;
     bool debug;
@@ -45,17 +49,31 @@ class TritonService {
     int wait;
     std::string instanceName;
     std::string tempDir;
+    std::string imageName;
+    std::string sandboxName;
   };
   struct Server {
     Server(const edm::ParameterSet& pset)
         : url(pset.getUntrackedParameter<std::string>("address") + ":" +
               std::to_string(pset.getUntrackedParameter<unsigned>("port"))),
-          isFallback(pset.getUntrackedParameter<std::string>("name") == fallbackName) {}
-    Server(const std::string& name_, const std::string& url_) : url(url_), isFallback(name_ == fallbackName) {}
+          isFallback(pset.getUntrackedParameter<std::string>("name") == fallbackName),
+          type(TritonServerType::Remote),
+          useSsl(pset.getUntrackedParameter<bool>("useSsl")) {
+      if (useSsl) {
+        sslOptions.root_certificates = pset.getUntrackedParameter<std::string>("rootCertificates");
+        sslOptions.private_key = pset.getUntrackedParameter<std::string>("privateKey");
+        sslOptions.certificate_chain = pset.getUntrackedParameter<std::string>("certificateChain");
+      }
+    }
+    Server(const std::string& name_, const std::string& url_, TritonServerType type_)
+        : url(url_), isFallback(name_ == fallbackName), type(type_), useSsl(false) {}
 
     //members
     std::string url;
     bool isFallback;
+    TritonServerType type;
+    bool useSsl;
+    triton::client::SslOptions sslOptions;
     std::unordered_set<std::string> models;
     static const std::string fallbackName;
     static const std::string fallbackAddress;
@@ -81,8 +99,7 @@ class TritonService {
 
   //accessors
   void addModel(const std::string& modelName, const std::string& path);
-  std::pair<std::string, TritonServerType> serverAddress(const std::string& model,
-                                                         const std::string& preferred = "") const;
+  Server serverInfo(const std::string& model, const std::string& preferred = "") const;
   const std::string& pid() const { return pid_; }
 
   static void fillDescriptions(edm::ConfigurationDescriptions& descriptions);