1717#include " torch_tensorrt/executorch/TensorRTBackend.h"
1818
1919#include < NvInfer.h>
20+ #include < cuda.h>
2021
2122#include < cstdint>
2223#include < memory>
@@ -44,8 +45,23 @@ struct EngineHandle {
4445 std::vector<size_t > cached_input_sizes;
4546 std::vector<void *> cached_output_ptrs;
4647 std::vector<size_t > cached_output_sizes;
48+ // What exec_ctx holds, so execute() skips the TensorRT call when a shape or an address is the one it
49+ // already gave. Setting an unchanged input shape still costs about half a microsecond, and an address
50+ // or inferShapes tens of nanoseconds, and most callers pass the same ones every time. An unknown shape
51+ // has nbDims -1 and an unknown address is null. Only this delegate may set shapes, addresses or the
52+ // profile on exec_ctx, or these go stale.
53+ std::vector<nvinfer1::Dims> bound_input_dims;
54+ std::vector<void *> bound_addresses;
55+ // Changes whenever exec_ctx's input shapes or optimization profile may have changed, so a recorded
56+ // CUDA graph can tell that the shapes it was recorded under are gone.
57+ uint64_t shape_epoch = 0 ;
58+ // Whether inferShapes has succeeded since the shapes or the profile last changed.
59+ bool shapes_inferred = false ;
4760 size_t num_inputs = 0 ;
4861 size_t num_outputs = 0 ;
62+ // The ExecuTorch type of each binding, inputs then outputs, read once at init so execute() checks each
63+ // tensor without a TensorRT call. Undefined where ExecuTorch has no such type, so no tensor matches it.
64+ std::vector<::executorch::aten::ScalarType> binding_types;
4965 // Per output binding [0..num_outputs): index into input_binding_names of the
5066 // input it aliases (in-place KV-cache / user alias), or -1 for a normal output.
5167 // Built at init from the blob's aliased_io. Either way execute() binds the
@@ -66,6 +82,16 @@ struct EngineHandle {
6682 std::vector<bool > input_is_alias_target;
6783 size_t num_aliased_outputs = 0 ;
6884 int device_id = 0 ;
85+ // The regular CUDA context current when exec_ctx was created. It owns the engine and the buffers execute()
86+ // keeps across calls; a call on a green context stream still runs in that green context. With a driver older
87+ // than CUDA 12.5, a green context current at load is recorded here instead.
88+ CUcontext context = nullptr ;
89+ // That context's id, zero when the driver cannot report it. The driver can give a destroyed context's
90+ // handle to a new one, but never its id.
91+ unsigned long long context_id = 0 ;
92+ // The device's primary context, retained when a green context was current at load, so that context is
93+ // the one current while the engine and exec_ctx are created. Released when this handle is destroyed.
94+ CUcontext retained_primary = nullptr ;
6995 // Whether this device can reach pageable host memory through shared host page tables, which is the
7096 // question the uses of this flag ask. A discrete card answers no: it can reach pageable memory, but
7197 // only by faulting pages in one at a time, so it takes the staged-copy path instead.
@@ -81,7 +107,8 @@ struct EngineHandle {
81107 // needing activation scratch under some shape; a handle loaded with the option
82108 // off keeps this false and says nothing about what its engine needs, because it
83109 // has its own kSTATIC scratch either way. It is a predicate and not a size
84- // because the size is never the right amount to install: enqueueV3 refuses a
110+ // because the size is not the amount to install while TensorRT can size the
111+ // bound shapes (pooled_scratch_bytes_for_shapes): enqueueV3 refuses a
85112 // kUSER_MANAGED context with no device memory once the engine needs any, however
86113 // little the shapes bound to a given call need, so execute() must hand such a
87114 // call some buffer -- but the engine's own figure covers every shape in the
@@ -97,6 +124,16 @@ struct EngineHandle {
97124 // And a program gives an input the same dim order on every call.
98125 bool reordered_input_reported = false ;
99126
127+ // setTensorAddress for binding `index`, inputs then outputs, unless exec_ctx already holds `address`.
128+ bool bind_address (size_t index, void * address);
129+ // setInputShape for input `index`, unless exec_ctx already holds `dims` and `again` is false.
130+ bool bind_input_shape (size_t index, const nvinfer1::Dims& dims, bool again = false );
131+ // setOptimizationProfileAsync, after forgetting every shape and address exec_ctx holds and the active profile.
132+ bool switch_profile (int32_t profile, cudaStream_t stream);
133+ // Clears the record of bound addresses. Call it when exec_ctx may hold other addresses, so the next
134+ // bind_address sets each.
135+ void forget_bound_addresses ();
136+
100137 ~EngineHandle ();
101138};
102139
0 commit comments