@@ -33,12 +33,16 @@ const WORD_COPY_THRESHOLD: usize = if 2 * WORD_SIZE > 16 {
3333 16
3434} ;
3535
36+ /// `x` must be valid for a read of `usize`.
3637#[ cfg( feature = "mem-unaligned" ) ]
3738unsafe fn read_usize_unaligned ( x : * const usize ) -> usize {
39+ type Arr = [ u8 ; core:: mem:: size_of :: < usize > ( ) ] ;
3840 // Do not use `core::ptr::read_unaligned` here, since it calls `copy_nonoverlapping` which
3941 // is translated to memcpy in LLVM.
40- let x_read = ( x as * const [ u8 ; core:: mem:: size_of :: < usize > ( ) ] ) . read ( ) ;
41- core:: mem:: transmute ( x_read)
42+ // SAFETY: x is valid for reads by preconditions.
43+ let x_read = unsafe { ( x as * const Arr ) . read ( ) } ;
44+ // SAFETY: Same-sized POD cast
45+ unsafe { core:: mem:: transmute :: < Arr , usize > ( x_read) }
4246}
4347
4448#[ inline( always) ]
@@ -47,7 +51,7 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize)
4751 unsafe fn copy_forward_bytes ( mut dest : * mut u8 , mut src : * const u8 , n : usize ) {
4852 let dest_end = dest. wrapping_add ( n) ;
4953 while dest < dest_end {
50- * dest = * src;
54+ unsafe { * dest = * src } ;
5155 dest = dest. wrapping_add ( 1 ) ;
5256 src = src. wrapping_add ( 1 ) ;
5357 }
@@ -60,7 +64,7 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize)
6064 let dest_end = dest. wrapping_add ( n) as * mut usize ;
6165
6266 while dest_usize < dest_end {
63- * dest_usize = * src_usize;
67+ unsafe { * dest_usize = * src_usize } ;
6468 dest_usize = dest_usize. wrapping_add ( 1 ) ;
6569 src_usize = src_usize. wrapping_add ( 1 ) ;
6670 }
@@ -108,7 +112,7 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize)
108112 let dest_end = dest. wrapping_add ( n) as * mut usize ;
109113
110114 while dest_usize < dest_end {
111- * dest_usize = read_usize_unaligned ( src_usize) ;
115+ unsafe { * dest_usize = read_usize_unaligned ( src_usize) } ;
112116 dest_usize = dest_usize. wrapping_add ( 1 ) ;
113117 src_usize = src_usize. wrapping_add ( 1 ) ;
114118 }
@@ -117,24 +121,26 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize)
117121 if n >= WORD_COPY_THRESHOLD {
118122 // Align dest
119123 // Because of n >= 2 * WORD_SIZE, dst_misalignment < n
120- let dest_misalignment = ( dest as usize ) . wrapping_neg ( ) & WORD_MASK ;
121- copy_forward_bytes ( dest, src, dest_misalignment) ;
122- dest = dest. wrapping_add ( dest_misalignment) ;
123- src = src. wrapping_add ( dest_misalignment) ;
124- n -= dest_misalignment;
125-
126- let n_words = n & !WORD_MASK ;
127- let src_misalignment = src as usize & WORD_MASK ;
128- if likely ( src_misalignment == 0 ) {
129- copy_forward_aligned_words ( dest, src, n_words) ;
130- } else {
131- copy_forward_misaligned_words ( dest, src, n_words) ;
124+ unsafe {
125+ let dest_misalignment = ( dest as usize ) . wrapping_neg ( ) & WORD_MASK ;
126+ copy_forward_bytes ( dest, src, dest_misalignment) ;
127+ dest = dest. wrapping_add ( dest_misalignment) ;
128+ src = src. wrapping_add ( dest_misalignment) ;
129+ n -= dest_misalignment;
130+ let n_words = n & !WORD_MASK ;
131+ let src_misalignment = src as usize & WORD_MASK ;
132+ if likely ( src_misalignment == 0 ) {
133+ copy_forward_aligned_words ( dest, src, n_words) ;
134+ } else {
135+ copy_forward_misaligned_words ( dest, src, n_words) ;
136+ }
137+ dest = dest. wrapping_add ( n_words) ;
138+ src = src. wrapping_add ( n_words) ;
139+ n -= n_words;
132140 }
133- dest = dest. wrapping_add ( n_words) ;
134- src = src. wrapping_add ( n_words) ;
135- n -= n_words;
136141 }
137- copy_forward_bytes ( dest, src, n) ;
142+
143+ unsafe { copy_forward_bytes ( dest, src, n) } ;
138144}
139145
140146#[ inline( always) ]
@@ -147,7 +153,7 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
147153 while dest_start < dest {
148154 dest = dest. wrapping_sub ( 1 ) ;
149155 src = src. wrapping_sub ( 1 ) ;
150- * dest = * src;
156+ unsafe { * dest = * src } ;
151157 }
152158 }
153159
@@ -160,7 +166,7 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
160166 while dest_start < dest_usize {
161167 dest_usize = dest_usize. wrapping_sub ( 1 ) ;
162168 src_usize = src_usize. wrapping_sub ( 1 ) ;
163- * dest_usize = * src_usize;
169+ unsafe { * dest_usize = * src_usize } ;
164170 }
165171 }
166172
@@ -180,13 +186,13 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
180186 // cfg needed because not all targets will have atomic loads that can be lowered
181187 // (e.g. BPF, MSP430), or provided by an external library (e.g. RV32I)
182188 #[ cfg( target_has_atomic_load_store = "ptr" ) ]
183- let mut prev_word = core:: intrinsics:: atomic_load_unordered ( src_aligned) ;
189+ let mut prev_word = unsafe { core:: intrinsics:: atomic_load_unordered ( src_aligned) } ;
184190 #[ cfg( not( target_has_atomic_load_store = "ptr" ) ) ]
185- let mut prev_word = core:: ptr:: read_volatile ( src_aligned) ;
191+ let mut prev_word = unsafe { core:: ptr:: read_volatile ( src_aligned) } ;
186192
187193 while dest_start < dest_usize {
188194 src_aligned = src_aligned. wrapping_sub ( 1 ) ;
189- let cur_word = * src_aligned;
195+ let cur_word = unsafe { * src_aligned } ;
190196 #[ cfg( target_endian = "little" ) ]
191197 let resembled = prev_word << ( WORD_SIZE * 8 - shift) | cur_word >> shift;
192198 #[ cfg( target_endian = "big" ) ]
@@ -208,7 +214,7 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
208214 while dest_start < dest_usize {
209215 dest_usize = dest_usize. wrapping_sub ( 1 ) ;
210216 src_usize = src_usize. wrapping_sub ( 1 ) ;
211- * dest_usize = read_usize_unaligned ( src_usize) ;
217+ unsafe { * dest_usize = read_usize_unaligned ( src_usize) } ;
212218 }
213219 }
214220
@@ -218,24 +224,26 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
218224 if n >= WORD_COPY_THRESHOLD {
219225 // Align dest
220226 // Because of n >= 2 * WORD_SIZE, dst_misalignment < n
221- let dest_misalignment = dest as usize & WORD_MASK ;
222- copy_backward_bytes ( dest, src, dest_misalignment) ;
223- dest = dest. wrapping_sub ( dest_misalignment) ;
224- src = src. wrapping_sub ( dest_misalignment) ;
225- n -= dest_misalignment;
226-
227- let n_words = n & !WORD_MASK ;
228- let src_misalignment = src as usize & WORD_MASK ;
229- if likely ( src_misalignment == 0 ) {
230- copy_backward_aligned_words ( dest, src, n_words) ;
231- } else {
232- copy_backward_misaligned_words ( dest, src, n_words) ;
227+ unsafe {
228+ let dest_misalignment = dest as usize & WORD_MASK ;
229+ copy_backward_bytes ( dest, src, dest_misalignment) ;
230+ dest = dest. wrapping_sub ( dest_misalignment) ;
231+ src = src. wrapping_sub ( dest_misalignment) ;
232+ n -= dest_misalignment;
233+
234+ let n_words = n & !WORD_MASK ;
235+ let src_misalignment = src as usize & WORD_MASK ;
236+ if likely ( src_misalignment == 0 ) {
237+ copy_backward_aligned_words ( dest, src, n_words) ;
238+ } else {
239+ copy_backward_misaligned_words ( dest, src, n_words) ;
240+ }
241+ dest = dest. wrapping_sub ( n_words) ;
242+ src = src. wrapping_sub ( n_words) ;
243+ n -= n_words;
233244 }
234- dest = dest. wrapping_sub ( n_words) ;
235- src = src. wrapping_sub ( n_words) ;
236- n -= n_words;
237245 }
238- copy_backward_bytes ( dest, src, n) ;
246+ unsafe { copy_backward_bytes ( dest, src, n) } ;
239247}
240248
241249#[ inline( always) ]
@@ -244,7 +252,7 @@ pub unsafe fn set_bytes(mut s: *mut u8, c: u8, mut n: usize) {
244252 pub unsafe fn set_bytes_bytes ( mut s : * mut u8 , c : u8 , n : usize ) {
245253 let end = s. wrapping_add ( n) ;
246254 while s < end {
247- * s = c;
255+ unsafe { * s = c } ;
248256 s = s. wrapping_add ( 1 ) ;
249257 }
250258 }
@@ -262,7 +270,7 @@ pub unsafe fn set_bytes(mut s: *mut u8, c: u8, mut n: usize) {
262270 let end = s. wrapping_add ( n) as * mut usize ;
263271
264272 while s_usize < end {
265- * s_usize = broadcast;
273+ unsafe { * s_usize = broadcast } ;
266274 s_usize = s_usize. wrapping_add ( 1 ) ;
267275 }
268276 }
@@ -271,24 +279,25 @@ pub unsafe fn set_bytes(mut s: *mut u8, c: u8, mut n: usize) {
271279 // Align s
272280 // Because of n >= 2 * WORD_SIZE, dst_misalignment < n
273281 let misalignment = ( s as usize ) . wrapping_neg ( ) & WORD_MASK ;
274- set_bytes_bytes ( s, c, misalignment) ;
282+ unsafe { set_bytes_bytes ( s, c, misalignment) } ;
275283 s = s. wrapping_add ( misalignment) ;
276284 n -= misalignment;
277285
278286 let n_words = n & !WORD_MASK ;
279- set_bytes_words ( s, c, n_words) ;
287+ unsafe { set_bytes_words ( s, c, n_words) } ;
280288 s = s. wrapping_add ( n_words) ;
281289 n -= n_words;
282290 }
283- set_bytes_bytes ( s, c, n) ;
291+ unsafe { set_bytes_bytes ( s, c, n) } ;
284292}
285293
294+ /// `memcmp` implementation. `s1` and `s2` must be valid for `n`.
286295#[ inline( always) ]
287296pub unsafe fn compare_bytes ( s1 : * const u8 , s2 : * const u8 , n : usize ) -> i32 {
288297 let mut i = 0 ;
289298 while i < n {
290- let a = * s1 . wrapping_add ( i ) ;
291- let b = * s2. wrapping_add ( i) ;
299+ // SAFETY: `i < n`, thus the reads are valid by preconditions.
300+ let ( a , b ) = unsafe { ( * s1 . wrapping_add ( i ) , * s2. wrapping_add ( i) ) } ;
292301 if a != b {
293302 return a as i32 - b as i32 ;
294303 }
@@ -297,10 +306,12 @@ pub unsafe fn compare_bytes(s1: *const u8, s2: *const u8, n: usize) -> i32 {
297306 0
298307}
299308
309+ /// `strlen` implementation. `s` must be valid for reads up to and including a null character.
300310#[ inline( always) ]
301311pub unsafe fn c_string_length ( mut s : * const core:: ffi:: c_char ) -> usize {
302312 let mut n = 0 ;
303- while * s != 0 {
313+ // SAFETY: safe to read until after the first `*s == 0`
314+ while unsafe { * s } != 0 {
304315 n += 1 ;
305316 s = s. wrapping_add ( 1 ) ;
306317 }
0 commit comments