@@ -6,6 +6,7 @@ use crate::ptr;
6
6
use crate :: sync:: atomic:: { AtomicPtr , Ordering } ;
7
7
use crate :: sys:: c;
8
8
use crate :: sys:: common:: alloc:: { realloc_fallback, MIN_ALIGN } ;
9
+ use core:: mem:: MaybeUninit ;
9
10
10
11
#[ cfg( test) ]
11
12
mod tests;
@@ -94,29 +95,30 @@ static HEAP: AtomicPtr<c_void> = AtomicPtr::new(ptr::null_mut());
94
95
// a non-null handle returned by `GetProcessHeap`.
95
96
#[ inline]
96
97
fn init_or_get_process_heap ( ) -> c:: HANDLE {
97
- let heap = HEAP . load ( Ordering :: Relaxed ) ;
98
- if core:: intrinsics:: unlikely ( heap. is_null ( ) ) {
99
- // `HEAP` has not yet been successfully initialized
100
- let heap = unsafe { GetProcessHeap ( ) } ;
101
- if !heap. is_null ( ) {
102
- // SAFETY: No locking is needed because within the same process,
103
- // successful calls to `GetProcessHeap` will always return the same value, even on different threads.
104
- HEAP . store ( heap, Ordering :: Release ) ;
105
-
106
- // SAFETY: `HEAP` contains a non-null handle returned by `GetProcessHeap`
107
- heap
108
- } else {
109
- // Could not get the current process heap.
110
- ptr:: null_mut ( )
111
- }
112
- } else {
98
+ // `HEAP` has not yet been successfully initialized
99
+ let heap = unsafe { GetProcessHeap ( ) } ;
100
+ if !heap. is_null ( ) {
101
+ // SAFETY: No locking is needed because within the same process,
102
+ // successful calls to `GetProcessHeap` will always return the same value, even on different threads.
103
+ HEAP . store ( heap, Ordering :: Release ) ;
104
+
113
105
// SAFETY: `HEAP` contains a non-null handle returned by `GetProcessHeap`
114
106
heap
107
+ } else {
108
+ // Could not get the current process heap.
109
+ ptr:: null_mut ( )
115
110
}
116
111
}
117
112
113
+ /// This is outlined from `process_heap_alloc` so that `process_heap_alloc`
114
+ /// does not need any stack allocations.
118
115
#[ inline( never) ]
119
- fn process_heap_alloc ( flags : c:: DWORD , dwBytes : c:: SIZE_T ) -> c:: LPVOID {
116
+ #[ cold]
117
+ extern "C" fn process_heap_init_and_alloc (
118
+ _heap : MaybeUninit < c:: HANDLE > , // We pass this argument to match the ABI of `HeapAlloc`
119
+ flags : c:: DWORD ,
120
+ dwBytes : c:: SIZE_T ,
121
+ ) -> c:: LPVOID {
120
122
let heap = init_or_get_process_heap ( ) ;
121
123
if core:: intrinsics:: unlikely ( heap. is_null ( ) ) {
122
124
return ptr:: null_mut ( ) ;
@@ -125,6 +127,21 @@ fn process_heap_alloc(flags: c::DWORD, dwBytes: c::SIZE_T) -> c::LPVOID {
125
127
unsafe { HeapAlloc ( heap, flags, dwBytes) }
126
128
}
127
129
130
+ #[ inline( never) ]
131
+ fn process_heap_alloc (
132
+ _heap : MaybeUninit < c:: HANDLE > , // We pass this argument to match the ABI of `HeapAlloc`,
133
+ flags : c:: DWORD ,
134
+ dwBytes : c:: SIZE_T ,
135
+ ) -> c:: LPVOID {
136
+ let heap = HEAP . load ( Ordering :: Relaxed ) ;
137
+ if core:: intrinsics:: likely ( !heap. is_null ( ) ) {
138
+ // SAFETY: `heap` is a non-null handle returned by `GetProcessHeap`.
139
+ unsafe { HeapAlloc ( heap, flags, dwBytes) }
140
+ } else {
141
+ process_heap_init_and_alloc ( MaybeUninit :: uninit ( ) , flags, dwBytes)
142
+ }
143
+ }
144
+
128
145
// Get a non-null handle to the default heap of the current process.
129
146
// SAFETY: `HEAP` must have been successfully initialized.
130
147
#[ inline]
@@ -148,12 +165,12 @@ unsafe fn allocate(layout: Layout, zeroed: bool) -> *mut u8 {
148
165
149
166
if layout. align ( ) <= MIN_ALIGN {
150
167
// The returned pointer points to the start of an allocated block.
151
- process_heap_alloc ( flags, layout. size ( ) ) as * mut u8
168
+ process_heap_alloc ( MaybeUninit :: uninit ( ) , flags, layout. size ( ) ) as * mut u8
152
169
} else {
153
170
// Allocate extra padding in order to be able to satisfy the alignment.
154
171
let total = layout. align ( ) + layout. size ( ) ;
155
172
156
- let ptr = process_heap_alloc ( flags, total) as * mut u8 ;
173
+ let ptr = process_heap_alloc ( MaybeUninit :: uninit ( ) , flags, total) as * mut u8 ;
157
174
if ptr. is_null ( ) {
158
175
// Allocation has failed.
159
176
return ptr:: null_mut ( ) ;
0 commit comments