@@ -406,12 +406,15 @@ Error load_engine(
406406 }
407407 // A plan serialized without its weights (built with STRIP_PLAN, or saved with EXCLUDE_WEIGHTS)
408408 // deserializes and runs without an error, but computes with placeholder weights, and this backend
409- // never refits. TensorRT reports -1 stripped bytes for a plan that carries its weights, and a
410- // stripped plan can report 0. A plan with no weights has nothing to restore, so the total is read
411- // first: querying the stripped size of an ordinary engine logs a TensorRT warning, and
412- // isRefittable() cannot skip it, since a plan built with STRIP_PLAN alone is not refittable.
413- if (engine->getEngineStat (nvinfer1::EngineStat::kTOTAL_WEIGHTS_SIZE ) > 0 &&
414- engine->getEngineStat (nvinfer1::EngineStat::kSTRIPPED_WEIGHTS_SIZE ) >= 0 ) {
409+ // never refits. For such a plan TensorRT reports the weight bytes it still holds, which equal the
410+ // total when there was nothing to strip. A plain re-save keeps that. Any other plan reports -1,
411+ // including a stripped one saved again with EXCLUDE_WEIGHTS cleared and no refit, so that case is
412+ // not caught here. A plan with no weights cannot lack any, and skipping the query for it avoids
413+ // the warning TensorRT logs for a plan that is not stripped.
414+ const int64_t total_weights = engine->getEngineStat (nvinfer1::EngineStat::kTOTAL_WEIGHTS_SIZE );
415+ const int64_t kept_weights =
416+ total_weights > 0 ? engine->getEngineStat (nvinfer1::EngineStat::kSTRIPPED_WEIGHTS_SIZE ) : -1 ;
417+ if (kept_weights >= 0 && kept_weights < total_weights) {
415418 ET_LOG (
416419 Error,
417420 " TensorRTBackend::init: the TensorRT engine was serialized without its weights, and this backend cannot refit them. Compile it with strip_engine_weights=False, or serialize it with its weights, and export again." );
@@ -684,6 +687,15 @@ Error validate_input_dims(const EngineHandle& handle, int32_t profile, const std
684687 return Error::Ok;
685688}
686689
690+ // setTensorAddress documents this alignment and does not check it. A tensor viewed at an offset into a
691+ // larger buffer misses it, and the engine then returns a wrong answer or faults and leaves the CUDA
692+ // context unusable.
693+ constexpr uintptr_t kTensorAddressAlignment = 256 ;
694+
695+ bool is_tensor_address_aligned (const void * ptr) {
696+ return reinterpret_cast <uintptr_t >(ptr) % kTensorAddressAlignment == 0 ;
697+ }
698+
687699bool is_cuda_accessible_ptr (const void * ptr) {
688700 if (ptr == nullptr ) {
689701 return false ;
@@ -954,7 +966,7 @@ constexpr size_t kMinPooledScratchBytes = 1;
954966// execute() calls this ahead of every CUDA call it makes that a capture cannot
955967// take, not just the pool's own: the cudaFree(nullptr) that makes a context
956968// current for a default stream, the cudaEventSynchronize on a previous enqueue,
957- // the cudaMalloc that grows a host- input staging buffer, and the
969+ // the cudaMalloc that grows an input staging buffer, and the
958970// cudaStreamSynchronize that ends a call this backend does not let return early.
959971// Any of them leaves a refusal made after it nothing to save. Only the device
960972// query and the device switch run earlier, and a capture takes both.
@@ -1800,6 +1812,23 @@ Error TensorRTBackend::execute(BackendExecutionContext& context, DelegateHandle*
18001812 // ------------------------------------------------------------------
18011813 // 2. Bind input shapes and addresses
18021814 // ------------------------------------------------------------------
1815+ const auto report_unaligned = [&engine](const std::string& name, const void * ptr) {
1816+ if (engine->unaligned_io_reported ) {
1817+ return ;
1818+ }
1819+ engine->unaligned_io_reported = true ;
1820+ ET_LOG (
1821+ Info,
1822+ " TensorRTBackend::execute: '%s' is at an address %zu bytes past a %zu-byte boundary, and TensorRT "
1823+ " needs that boundary, so it is copied through an aligned buffer (reported once per engine)" ,
1824+ name.c_str (),
1825+ static_cast <size_t >(reinterpret_cast <uintptr_t >(ptr) % kTensorAddressAlignment ),
1826+ static_cast <size_t >(kTensorAddressAlignment ));
1827+ };
1828+ // Caller-owned KV: (dst, src, nbytes) copies issued after enqueue. One kind reflects an engine
1829+ // in-place update into the delegate output EValue, so ExecuTorch's write-back copy_ sees the updated
1830+ // cache. The other carries that update from an aligned staging buffer back to the caller's input.
1831+ std::vector<std::tuple<void *, void *, size_t >> aliased_reflects;
18031832 for (size_t i = 0 ; i < num_inputs; ++i) {
18041833 exec_aten::Tensor et_in = args[i]->toTensor ();
18051834 const std::string& name = engine->input_binding_names [i];
@@ -1810,23 +1839,19 @@ Error TensorRTBackend::execute(BackendExecutionContext& context, DelegateHandle*
18101839 return Error::InvalidState;
18111840 }
18121841
1842+ const bool device_reachable =
1843+ et_in.nbytes () > 0 && (engine->pageable_host_access || is_cuda_accessible_ptr (et_in.const_data_ptr ()));
18131844 // Caller-owned aliased input: an aliased output binds in-place to this
1814- // input's device pointer, so its update must land in the caller's storage.
1815- // If it isn't device-resident the branches below would stage it through
1816- // delegate scratch, and the in-place update (bound to that scratch) would be
1817- // silently lost on the next execute() when the staging copy re-reads the
1818- // caller's unchanged buffer. Fail loudly instead.
1819- if (engine->input_is_alias_target [i]) {
1820- const bool device_resident =
1821- et_in.nbytes () > 0 && (engine->pageable_host_access || is_cuda_accessible_ptr (et_in.const_data_ptr ()));
1822- if (!device_resident) {
1823- ET_LOG (
1824- Error,
1825- " TensorRTBackend::execute: aliased input '%s' must be device-reachable without staging, "
1826- " because its caller-owned in-place update cannot go through host scratch" ,
1827- name.c_str ());
1828- return Error::InvalidArgument;
1829- }
1845+ // input's pointer, so its update must land in the caller's storage. Host
1846+ // memory is refused. A device buffer the branches below stage for alignment
1847+ // gets its update copied back after the enqueue.
1848+ if (engine->input_is_alias_target [i] && !device_reachable) {
1849+ ET_LOG (
1850+ Error,
1851+ " TensorRTBackend::execute: aliased input '%s' must be device-reachable, "
1852+ " because its caller-owned in-place update cannot go through host scratch" ,
1853+ name.c_str ());
1854+ return Error::InvalidArgument;
18301855 }
18311856
18321857 void * bind_ptr = nullptr ;
@@ -1839,9 +1864,12 @@ Error TensorRTBackend::execute(BackendExecutionContext& context, DelegateHandle*
18391864 engine->cached_input_sizes [i] = 1 ;
18401865 }
18411866 bind_ptr = engine->cached_input_ptrs [i];
1842- } else if (engine-> pageable_host_access || is_cuda_accessible_ptr (et_in.const_data_ptr ())) {
1867+ } else if (device_reachable && is_tensor_address_aligned (et_in.const_data_ptr ())) {
18431868 bind_ptr = et_in.mutable_data_ptr ();
18441869 } else {
1870+ if (device_reachable) {
1871+ report_unaligned (name, et_in.const_data_ptr ());
1872+ }
18451873 const size_t needed = et_in.nbytes ();
18461874 if (needed > engine->cached_input_sizes [i]) {
18471875 if (engine->cached_input_ptrs [i] != nullptr ) {
@@ -1856,16 +1884,19 @@ Error TensorRTBackend::execute(BackendExecutionContext& context, DelegateHandle*
18561884 engine->cached_input_sizes [i] = needed;
18571885 }
18581886 bind_ptr = engine->cached_input_ptrs [i];
1859- cuda_err = cudaMemcpyAsync (bind_ptr, et_in.const_data_ptr (), needed, cudaMemcpyHostToDevice , stream);
1887+ cuda_err = cudaMemcpyAsync (bind_ptr, et_in.const_data_ptr (), needed, cudaMemcpyDefault , stream);
18601888 drain_on_early_return.arm ();
18611889 if (cuda_err != cudaSuccess) {
18621890 ET_LOG (
18631891 Error,
1864- " TensorRTBackend::execute: H2D copy failed for input '%s': %s" ,
1892+ " TensorRTBackend::execute: staging copy failed for input '%s': %s" ,
18651893 name.c_str (),
18661894 cudaGetErrorString (cuda_err));
18671895 return Error::InvalidProgram;
18681896 }
1897+ if (engine->input_is_alias_target [i]) {
1898+ aliased_reflects.emplace_back (et_in.mutable_data_ptr (), bind_ptr, needed);
1899+ }
18691900 }
18701901
18711902 input_bind_ptrs[i] = bind_ptr;
@@ -1894,15 +1925,10 @@ Error TensorRTBackend::execute(BackendExecutionContext& context, DelegateHandle*
18941925 // dynamic models. After inferShapes() TRT knows the actual output
18951926 // dims, so update the ExecuTorch TensorImpl's sizes before computing
18961927 // nbytes() and before the Python binding reads back the shape.
1897- // If the buffer is CPU, stage through a temporary CUDA allocation.
1928+ // If the buffer is CPU or not aligned for TensorRT , stage through a CUDA allocation.
18981929 // ------------------------------------------------------------------
18991930 // (arg index, device_src ptr) for outputs staged through a device buffer.
19001931 std::vector<std::pair<size_t , void *>> outputs_needing_copy;
1901- // Caller-owned KV: (dst = delegate output EValue ptr, src = aliased input ptr,
1902- // nbytes). The engine updates the aliased input in place; reflect that into the
1903- // delegate output EValue after enqueue so ExecuTorch's write-back copy_ sees the
1904- // updated cache.
1905- std::vector<std::tuple<void *, void *, size_t >> aliased_reflects;
19061932 for (size_t o = 0 ; o < num_outputs; ++o) {
19071933 const std::string& name = engine->output_binding_names [o];
19081934
@@ -1920,7 +1946,8 @@ Error TensorRTBackend::execute(BackendExecutionContext& context, DelegateHandle*
19201946 return Error::InvalidState;
19211947 }
19221948 // Elided: setTensorAddress above pointed this output binding at the caller's
1923- // buffer, so the engine writes the buffer itself; nothing to reflect into.
1949+ // buffer, or at the aligned copy the input loop already copies back, so the
1950+ // engine's write reaches the buffer; nothing to reflect into.
19241951 if (aliased_outputs_elided) {
19251952 continue ;
19261953 }
@@ -2042,6 +2069,8 @@ Error TensorRTBackend::execute(BackendExecutionContext& context, DelegateHandle*
20422069
20432070 binding_bytes[num_inputs + o] = et_out.nbytes ();
20442071 void * bind_ptr = nullptr ;
2072+ const bool device_reachable =
2073+ et_out.nbytes () > 0 && (engine->pageable_host_access || is_cuda_accessible_ptr (et_out.const_data_ptr ()));
20452074 if (et_out.nbytes () == 0 ) {
20462075 if (engine->cached_output_sizes [o] == 0 ) {
20472076 cuda_err = cudaMalloc (&engine->cached_output_ptrs [o], 1 );
@@ -2051,9 +2080,12 @@ Error TensorRTBackend::execute(BackendExecutionContext& context, DelegateHandle*
20512080 engine->cached_output_sizes [o] = 1 ;
20522081 }
20532082 bind_ptr = engine->cached_output_ptrs [o];
2054- } else if (engine-> pageable_host_access || is_cuda_accessible_ptr (et_out.const_data_ptr ())) {
2083+ } else if (device_reachable && is_tensor_address_aligned (et_out.const_data_ptr ())) {
20552084 bind_ptr = et_out.mutable_data_ptr ();
20562085 } else {
2086+ if (device_reachable) {
2087+ report_unaligned (name, et_out.const_data_ptr ());
2088+ }
20572089 const size_t needed = et_out.nbytes ();
20582090 if (needed > engine->cached_output_sizes [o]) {
20592091 if (engine->cached_output_ptrs [o] != nullptr ) {
@@ -2176,7 +2208,7 @@ Error TensorRTBackend::execute(BackendExecutionContext& context, DelegateHandle*
21762208 }
21772209
21782210 // Caller-owned KV: reflect each engine in-place update into its delegate output
2179- // EValue (D2D on the same stream, after the engine work).
2211+ // EValue or the input it was staged from ( on the same stream, after the engine work).
21802212 for (const auto & r : aliased_reflects) {
21812213 cuda_err = cudaMemcpyAsync (std::get<0 >(r), std::get<1 >(r), std::get<2 >(r), cudaMemcpyDefault, stream);
21822214 drain_on_early_return.arm ();
@@ -2195,8 +2227,7 @@ Error TensorRTBackend::execute(BackendExecutionContext& context, DelegateHandle*
21952227 Error copy_err = Error::Ok;
21962228 for (auto & output : outputs_needing_copy) {
21972229 exec_aten::Tensor et_out = args[output.first ]->toTensor ();
2198- cuda_err =
2199- cudaMemcpyAsync (et_out.mutable_data_ptr (), output.second , et_out.nbytes (), cudaMemcpyDeviceToHost, stream);
2230+ cuda_err = cudaMemcpyAsync (et_out.mutable_data_ptr (), output.second , et_out.nbytes (), cudaMemcpyDefault, stream);
22002231 if (cuda_err != cudaSuccess) {
22012232 // Name the output and number it the way the caller does. output.first indexes the whole
22022233 // argument list, so on a one-input engine the first output read as "output 1", and the
0 commit comments