Skip to content

Commit 739653e

Browse files
nbdd0121nbdd0121
committed
Address some feedbacks
1 parent 361c144 commit 739653e

File tree

1 file changed

+125
-127
lines changed

1 file changed

+125
-127
lines changed

‎src/mem/impls.rs

+125-127
Original file line numberDiff line numberDiff line change
@@ -3,66 +3,73 @@ use core::intrinsics::likely;
33
const WORD_SIZE: usize = core::mem::size_of::<usize>();
44
const WORD_MASK: usize = WORD_SIZE - 1;
55

6+
// If the number of bytes involved exceed this threshold we will opt in word-wise copy.
7+
// The value here selected is max(2 * WORD_SIZE, 16):
8+
// * We need at least 2 * WORD_SIZE bytes to guarantee that at least 1 word will be copied through
9+
// word-wise copy.
10+
// * The word-wise copy logic needs to perform some checks so it has some small overhead.
11+
// ensures that even on 32-bit platforms we have copied at least 8 bytes through
12+
// word-wise copy so the saving of word-wise copy outweights the fixed overhead.
613
const WORD_COPY_THRESHOLD: usize = if 2 * WORD_SIZE > 16 {
714
2 * WORD_SIZE
815
} else {
916
16
1017
};
1118

1219
#[inline(always)]
13-
unsafe fn copy_forward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) {
14-
let dest_end = dest.add(n);
15-
while dest < dest_end {
16-
*dest = *src;
17-
dest = dest.add(1);
18-
src = src.add(1);
20+
pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize) {
21+
#[inline(always)]
22+
unsafe fn copy_forward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) {
23+
let dest_end = dest.add(n);
24+
while dest < dest_end {
25+
*dest = *src;
26+
dest = dest.add(1);
27+
src = src.add(1);
28+
}
1929
}
20-
}
2130

22-
#[inline(always)]
23-
unsafe fn copy_forward_aligned_words(dest: *mut u8, src: *const u8, n: usize) {
24-
let mut dest_usize = dest as *mut usize;
25-
let mut src_usize = src as *mut usize;
26-
let dest_end = dest.add(n) as *mut usize;
27-
28-
while dest_usize < dest_end {
29-
*dest_usize = *src_usize;
30-
dest_usize = dest_usize.add(1);
31-
src_usize = src_usize.add(1);
31+
#[inline(always)]
32+
unsafe fn copy_forward_aligned_words(dest: *mut u8, src: *const u8, n: usize) {
33+
let mut dest_usize = dest as *mut usize;
34+
let mut src_usize = src as *mut usize;
35+
let dest_end = dest.add(n) as *mut usize;
36+
37+
while dest_usize < dest_end {
38+
*dest_usize = *src_usize;
39+
dest_usize = dest_usize.add(1);
40+
src_usize = src_usize.add(1);
41+
}
3242
}
33-
}
3443

35-
#[inline(always)]
36-
unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) {
37-
let mut dest_usize = dest as *mut usize;
38-
let dest_end = dest.add(n) as *mut usize;
39-
40-
// Calculate the misalignment offset and shift needed to reassemble value.
41-
let offset = src as usize & WORD_MASK;
42-
let shift = offset * 8;
43-
44-
// Realign src
45-
let mut src_aligned = (src as usize & !WORD_MASK) as *mut usize;
46-
// XXX: Could this possibly be UB?
47-
let mut prev_word = *src_aligned;
48-
49-
while dest_usize < dest_end {
50-
src_aligned = src_aligned.add(1);
51-
let cur_word = *src_aligned;
52-
#[cfg(target_endian = "little")]
53-
let resembled = prev_word >> shift | cur_word << (WORD_SIZE * 8 - shift);
54-
#[cfg(target_endian = "big")]
55-
let resembled = prev_word << shift | cur_word >> (WORD_SIZE * 8 - shift);
56-
prev_word = cur_word;
57-
58-
*dest_usize = resembled;
59-
dest_usize = dest_usize.add(1);
44+
#[inline(always)]
45+
unsafe fn copy_forward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) {
46+
let mut dest_usize = dest as *mut usize;
47+
let dest_end = dest.add(n) as *mut usize;
48+
49+
// Calculate the misalignment offset and shift needed to reassemble value.
50+
let offset = src as usize & WORD_MASK;
51+
let shift = offset * 8;
52+
53+
// Realign src
54+
let mut src_aligned = (src as usize & !WORD_MASK) as *mut usize;
55+
// XXX: Could this possibly be UB?
56+
let mut prev_word = *src_aligned;
57+
58+
while dest_usize < dest_end {
59+
src_aligned = src_aligned.add(1);
60+
let cur_word = *src_aligned;
61+
#[cfg(target_endian = "little")]
62+
let resembled = prev_word >> shift | cur_word << (WORD_SIZE * 8 - shift);
63+
#[cfg(target_endian = "big")]
64+
let resembled = prev_word << shift | cur_word >> (WORD_SIZE * 8 - shift);
65+
prev_word = cur_word;
66+
67+
*dest_usize = resembled;
68+
dest_usize = dest_usize.add(1);
69+
}
6070
}
61-
}
6271

63-
#[inline(always)]
64-
pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize) {
65-
if likely(n >= WORD_COPY_THRESHOLD) {
72+
if n >= WORD_COPY_THRESHOLD {
6673
// Align dest
6774
// Because of n >= 2 * WORD_SIZE, dst_misalignment < n
6875
let dest_misalignment = (dest as usize).wrapping_neg() & WORD_MASK;
@@ -85,66 +92,65 @@ pub unsafe fn copy_forward(mut dest: *mut u8, mut src: *const u8, mut n: usize)
8592
copy_forward_bytes(dest, src, n);
8693
}
8794

88-
// The following backward copy helper functions except the public-facing copy_backward
89-
// uses the pointers past the end as their inputs instead of pointers to the start!
90-
9195
#[inline(always)]
92-
unsafe fn copy_backward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) {
93-
let dest_start = dest.sub(n);
94-
while dest_start < dest {
95-
dest = dest.sub(1);
96-
src = src.sub(1);
97-
*dest = *src;
96+
pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
97+
// The following backward copy helper functions uses the pointers past the end
98+
// as their inputs instead of pointers to the start!
99+
#[inline(always)]
100+
unsafe fn copy_backward_bytes(mut dest: *mut u8, mut src: *const u8, n: usize) {
101+
let dest_start = dest.sub(n);
102+
while dest_start < dest {
103+
dest = dest.sub(1);
104+
src = src.sub(1);
105+
*dest = *src;
106+
}
98107
}
99-
}
100108

101-
#[inline(always)]
102-
unsafe fn copy_backward_aligned_words(dest: *mut u8, src: *const u8, n: usize) {
103-
let mut dest_usize = dest as *mut usize;
104-
let mut src_usize = src as *mut usize;
105-
let dest_start = dest.sub(n) as *mut usize;
106-
107-
while dest_start < dest_usize {
108-
dest_usize = dest_usize.sub(1);
109-
src_usize = src_usize.sub(1);
110-
*dest_usize = *src_usize;
109+
#[inline(always)]
110+
unsafe fn copy_backward_aligned_words(dest: *mut u8, src: *const u8, n: usize) {
111+
let mut dest_usize = dest as *mut usize;
112+
let mut src_usize = src as *mut usize;
113+
let dest_start = dest.sub(n) as *mut usize;
114+
115+
while dest_start < dest_usize {
116+
dest_usize = dest_usize.sub(1);
117+
src_usize = src_usize.sub(1);
118+
*dest_usize = *src_usize;
119+
}
111120
}
112-
}
113121

114-
#[inline(always)]
115-
unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) {
116-
let mut dest_usize = dest as *mut usize;
117-
let dest_start = dest.sub(n) as *mut usize;
118-
119-
// Calculate the misalignment offset and shift needed to reassemble value.
120-
let offset = src as usize & WORD_MASK;
121-
let shift = offset * 8;
122-
123-
// Realign src_aligned
124-
let mut src_aligned = (src as usize & !WORD_MASK) as *mut usize;
125-
// XXX: Could this possibly be UB?
126-
let mut prev_word = *src_aligned;
127-
128-
while dest_start < dest_usize {
129-
src_aligned = src_aligned.sub(1);
130-
let cur_word = *src_aligned;
131-
#[cfg(target_endian = "little")]
132-
let resembled = prev_word << (WORD_SIZE * 8 - shift) | cur_word >> shift;
133-
#[cfg(target_endian = "big")]
134-
let resembled = prev_word >> (WORD_SIZE * 8 - shift) | cur_word << shift;
135-
prev_word = cur_word;
136-
137-
dest_usize = dest_usize.sub(1);
138-
*dest_usize = resembled;
122+
#[inline(always)]
123+
unsafe fn copy_backward_misaligned_words(dest: *mut u8, src: *const u8, n: usize) {
124+
let mut dest_usize = dest as *mut usize;
125+
let dest_start = dest.sub(n) as *mut usize;
126+
127+
// Calculate the misalignment offset and shift needed to reassemble value.
128+
let offset = src as usize & WORD_MASK;
129+
let shift = offset * 8;
130+
131+
// Realign src_aligned
132+
let mut src_aligned = (src as usize & !WORD_MASK) as *mut usize;
133+
// XXX: Could this possibly be UB?
134+
let mut prev_word = *src_aligned;
135+
136+
while dest_start < dest_usize {
137+
src_aligned = src_aligned.sub(1);
138+
let cur_word = *src_aligned;
139+
#[cfg(target_endian = "little")]
140+
let resembled = prev_word << (WORD_SIZE * 8 - shift) | cur_word >> shift;
141+
#[cfg(target_endian = "big")]
142+
let resembled = prev_word >> (WORD_SIZE * 8 - shift) | cur_word << shift;
143+
prev_word = cur_word;
144+
145+
dest_usize = dest_usize.sub(1);
146+
*dest_usize = resembled;
147+
}
139148
}
140-
}
141149

142-
#[inline(always)]
143-
pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
144150
let mut dest = dest.add(n);
145151
let mut src = src.add(n);
146152

147-
if likely(n >= WORD_COPY_THRESHOLD) {
153+
if n >= WORD_COPY_THRESHOLD {
148154
// Align dest
149155
// Because of n >= 2 * WORD_SIZE, dst_misalignment < n
150156
let dest_misalignment = dest as usize & WORD_MASK;
@@ -168,44 +174,36 @@ pub unsafe fn copy_backward(dest: *mut u8, src: *const u8, mut n: usize) {
168174
}
169175

170176
#[inline(always)]
171-
pub unsafe fn set_bytes_bytes(mut s: *mut u8, c: u8, n: usize) {
172-
let end = s.add(n);
173-
while s < end {
174-
*s = c;
175-
s = s.add(1);
177+
pub unsafe fn set_bytes(mut s: *mut u8, c: u8, mut n: usize) {
178+
#[inline(always)]
179+
pub unsafe fn set_bytes_bytes(mut s: *mut u8, c: u8, n: usize) {
180+
let end = s.add(n);
181+
while s < end {
182+
*s = c;
183+
s = s.add(1);
184+
}
176185
}
177-
}
178186

179-
#[inline(always)]
180-
pub unsafe fn set_bytes_words(s: *mut u8, c: u8, n: usize) {
181-
let mut broadcast = c as usize;
182-
broadcast |= broadcast << 8;
183-
#[cfg(not(target_pointer_width = "16"))]
184-
{
185-
broadcast |= broadcast << 16;
186-
#[cfg(not(target_pointer_width = "32"))]
187-
{
188-
broadcast |= broadcast << 32;
189-
#[cfg(not(target_pointer_width = "64"))]
190-
{
191-
broadcast |= broadcast << 64;
192-
}
187+
#[inline(always)]
188+
pub unsafe fn set_bytes_words(s: *mut u8, c: u8, n: usize) {
189+
let mut broadcast = c as usize;
190+
let mut bits = 8;
191+
while bits < WORD_SIZE * 8 {
192+
broadcast |= broadcast << bits;
193+
bits *= 2;
193194
}
194-
}
195195

196-
let mut s_usize = s as *mut usize;
197-
let end = s.add(n) as *mut usize;
196+
let mut s_usize = s as *mut usize;
197+
let end = s.add(n) as *mut usize;
198198

199-
while s_usize < end {
200-
*s_usize = broadcast;
201-
s_usize = s_usize.add(1);
199+
while s_usize < end {
200+
*s_usize = broadcast;
201+
s_usize = s_usize.add(1);
202+
}
202203
}
203-
}
204204

205-
#[inline(always)]
206-
pub unsafe fn set_bytes(mut s: *mut u8, c: u8, mut n: usize) {
207205
if likely(n >= WORD_COPY_THRESHOLD) {
208-
// Bec// Align dest
206+
// Align s
209207
// Because of n >= 2 * WORD_SIZE, dst_misalignment < n
210208
let misalignment = (s as usize).wrapping_neg() & WORD_MASK;
211209
set_bytes_bytes(s, c, misalignment);

0 commit comments

Comments
 (0)