forked from rurban/smhasher
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathsse2neon.h
7576 lines (6948 loc) · 287 KB
/
sse2neon.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#ifndef SSE2NEON_H
#define SSE2NEON_H
// This header file provides a simple API translation layer
// between SSE intrinsics to their corresponding Arm/Aarch64 NEON versions
//
// This header file does not yet translate all of the SSE intrinsics.
//
// Contributors to this work are:
// John W. Ratcliff <[email protected]>
// Brandon Rowlett <[email protected]>
// Ken Fast <[email protected]>
// Eric van Beurden <[email protected]>
// Alexander Potylitsin <[email protected]>
// Hasindu Gamaarachchi <[email protected]>
// Jim Huang <[email protected]>
// Mark Cheng <[email protected]>
// Malcolm James MacLeod <[email protected]>
// Devin Hussey (easyaspi314) <[email protected]>
// Sebastian Pop <[email protected]>
// Developer Ecosystem Engineering <[email protected]>
// Danila Kutenin <[email protected]>
// François Turban (JishinMaster) <[email protected]>
// Pei-Hsuan Hung <[email protected]>
// Yang-Hao Yuan <[email protected]>
// Syoyo Fujita <[email protected]>
// Brecht Van Lommel <[email protected]>
/*
* sse2neon is freely redistributable under the MIT License.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
/* Tunable configurations */
/* Enable precise implementation of math operations
* This would slow down the computation a bit, but gives consistent result with
* x86 SSE2. (e.g. would solve a hole or NaN pixel in the rendering result)
*/
/* _mm_min_ps and _mm_max_ps */
#ifndef SSE2NEON_PRECISE_MINMAX
#define SSE2NEON_PRECISE_MINMAX (0)
#endif
/* _mm_rcp_ps and _mm_div_ps */
#ifndef SSE2NEON_PRECISE_DIV
#define SSE2NEON_PRECISE_DIV (0)
#endif
/* _mm_sqrt_ps and _mm_rsqrt_ps */
#ifndef SSE2NEON_PRECISE_SQRT
#define SSE2NEON_PRECISE_SQRT (0)
#endif
#if defined(__GNUC__) || defined(__clang__)
#pragma push_macro("FORCE_INLINE")
#pragma push_macro("ALIGN_STRUCT")
#ifndef FORCE_INLINE
#define FORCE_INLINE static inline __attribute__((always_inline))
#endif
#define ALIGN_STRUCT(x) __attribute__((aligned(x)))
#ifndef likely
#define likely(x) __builtin_expect(!!(x), 1)
#endif
#ifndef unlikely
#define unlikely(x) __builtin_expect(!!(x), 0)
#endif
#else
#error "Macro name collisions may happen with unsupported compiler."
#ifdef FORCE_INLINE
#undef FORCE_INLINE
#endif
#define FORCE_INLINE static inline
#ifndef ALIGN_STRUCT
#define ALIGN_STRUCT(x) __declspec(align(x))
#endif
#endif
#ifndef likely
#define likely(x) (x)
#endif
#ifndef unlikely
#define unlikely(x) (x)
#endif
#include <stdint.h>
#include <stdlib.h>
/* Architecture-specific build options */
/* FIXME: #pragma GCC push_options is only available on GCC */
#if defined(__GNUC__)
#if defined(__arm__) && __ARM_ARCH == 7
/* According to ARM C Language Extensions Architecture specification,
* __ARM_NEON is defined to a value indicating the Advanced SIMD (NEON)
* architecture supported.
*/
#if !defined(__ARM_NEON) || !defined(__ARM_NEON__)
#error "You must enable NEON instructions (e.g. -mfpu=neon) to use SSE2NEON."
#endif
#if !defined(__clang__)
#pragma GCC push_options
#pragma GCC target("fpu=neon")
#endif
#elif defined(__aarch64__)
#if !defined(__clang__)
#pragma GCC push_options
#pragma GCC target("+simd")
#endif
#else
#error "Unsupported target. Must be either ARMv7-A+NEON or ARMv8-A."
#endif
#endif
#include <arm_neon.h>
/* Rounding functions require either Aarch64 instructions or libm failback */
#if !defined(__aarch64__)
#include <math.h>
#endif
/* "__has_builtin" can be used to query support for built-in functions
* provided by gcc/clang and other compilers that support it.
*/
#ifndef __has_builtin /* GCC prior to 10 or non-clang compilers */
/* Compatibility with gcc <= 9 */
#if __GNUC__ <= 9
#define __has_builtin(x) HAS##x
#define HAS__builtin_popcount 1
#define HAS__builtin_popcountll 1
#else
#define __has_builtin(x) 0
#endif
#endif
/**
* MACRO for shuffle parameter for _mm_shuffle_ps().
* Argument fp3 is a digit[0123] that represents the fp from argument "b"
* of mm_shuffle_ps that will be placed in fp3 of result. fp2 is the same
* for fp2 in result. fp1 is a digit[0123] that represents the fp from
* argument "a" of mm_shuffle_ps that will be places in fp1 of result.
* fp0 is the same for fp0 of result.
*/
#define _MM_SHUFFLE(fp3, fp2, fp1, fp0) \
(((fp3) << 6) | ((fp2) << 4) | ((fp1) << 2) | ((fp0)))
/* Rounding mode macros. */
#define _MM_FROUND_TO_NEAREST_INT 0x00
#define _MM_FROUND_TO_NEG_INF 0x01
#define _MM_FROUND_TO_POS_INF 0x02
#define _MM_FROUND_TO_ZERO 0x03
#define _MM_FROUND_CUR_DIRECTION 0x04
#define _MM_FROUND_NO_EXC 0x08
#define _MM_ROUND_NEAREST 0x0000
#define _MM_ROUND_DOWN 0x2000
#define _MM_ROUND_UP 0x4000
#define _MM_ROUND_TOWARD_ZERO 0x6000
/* indicate immediate constant argument in a given range */
#define __constrange(a, b) const
/* A few intrinsics accept traditional data types like ints or floats, but
* most operate on data types that are specific to SSE.
* If a vector type ends in d, it contains doubles, and if it does not have
* a suffix, it contains floats. An integer vector type can contain any type
* of integer, from chars to shorts to unsigned long longs.
*/
typedef int64x1_t __m64;
typedef float32x4_t __m128; /* 128-bit vector containing 4 floats */
// On ARM 32-bit architecture, the float64x2_t is not supported.
// The data type __m128d should be represented in a different way for related
// intrinsic conversion.
#if defined(__aarch64__)
typedef float64x2_t __m128d; /* 128-bit vector containing 2 doubles */
#else
typedef float32x4_t __m128d;
#endif
typedef int64x2_t __m128i; /* 128-bit vector containing integers */
/* type-safe casting between types */
#define vreinterpretq_m128_f16(x) vreinterpretq_f32_f16(x)
#define vreinterpretq_m128_f32(x) (x)
#define vreinterpretq_m128_f64(x) vreinterpretq_f32_f64(x)
#define vreinterpretq_m128_u8(x) vreinterpretq_f32_u8(x)
#define vreinterpretq_m128_u16(x) vreinterpretq_f32_u16(x)
#define vreinterpretq_m128_u32(x) vreinterpretq_f32_u32(x)
#define vreinterpretq_m128_u64(x) vreinterpretq_f32_u64(x)
#define vreinterpretq_m128_s8(x) vreinterpretq_f32_s8(x)
#define vreinterpretq_m128_s16(x) vreinterpretq_f32_s16(x)
#define vreinterpretq_m128_s32(x) vreinterpretq_f32_s32(x)
#define vreinterpretq_m128_s64(x) vreinterpretq_f32_s64(x)
#define vreinterpretq_f16_m128(x) vreinterpretq_f16_f32(x)
#define vreinterpretq_f32_m128(x) (x)
#define vreinterpretq_f64_m128(x) vreinterpretq_f64_f32(x)
#define vreinterpretq_u8_m128(x) vreinterpretq_u8_f32(x)
#define vreinterpretq_u16_m128(x) vreinterpretq_u16_f32(x)
#define vreinterpretq_u32_m128(x) vreinterpretq_u32_f32(x)
#define vreinterpretq_u64_m128(x) vreinterpretq_u64_f32(x)
#define vreinterpretq_s8_m128(x) vreinterpretq_s8_f32(x)
#define vreinterpretq_s16_m128(x) vreinterpretq_s16_f32(x)
#define vreinterpretq_s32_m128(x) vreinterpretq_s32_f32(x)
#define vreinterpretq_s64_m128(x) vreinterpretq_s64_f32(x)
#define vreinterpretq_m128i_s8(x) vreinterpretq_s64_s8(x)
#define vreinterpretq_m128i_s16(x) vreinterpretq_s64_s16(x)
#define vreinterpretq_m128i_s32(x) vreinterpretq_s64_s32(x)
#define vreinterpretq_m128i_s64(x) (x)
#define vreinterpretq_m128i_u8(x) vreinterpretq_s64_u8(x)
#define vreinterpretq_m128i_u16(x) vreinterpretq_s64_u16(x)
#define vreinterpretq_m128i_u32(x) vreinterpretq_s64_u32(x)
#define vreinterpretq_m128i_u64(x) vreinterpretq_s64_u64(x)
#define vreinterpretq_f32_m128i(x) vreinterpretq_f32_s64(x)
#define vreinterpretq_f64_m128i(x) vreinterpretq_f64_s64(x)
#define vreinterpretq_s8_m128i(x) vreinterpretq_s8_s64(x)
#define vreinterpretq_s16_m128i(x) vreinterpretq_s16_s64(x)
#define vreinterpretq_s32_m128i(x) vreinterpretq_s32_s64(x)
#define vreinterpretq_s64_m128i(x) (x)
#define vreinterpretq_u8_m128i(x) vreinterpretq_u8_s64(x)
#define vreinterpretq_u16_m128i(x) vreinterpretq_u16_s64(x)
#define vreinterpretq_u32_m128i(x) vreinterpretq_u32_s64(x)
#define vreinterpretq_u64_m128i(x) vreinterpretq_u64_s64(x)
#define vreinterpret_m64_s8(x) vreinterpret_s64_s8(x)
#define vreinterpret_m64_s16(x) vreinterpret_s64_s16(x)
#define vreinterpret_m64_s32(x) vreinterpret_s64_s32(x)
#define vreinterpret_m64_s64(x) (x)
#define vreinterpret_m64_u8(x) vreinterpret_s64_u8(x)
#define vreinterpret_m64_u16(x) vreinterpret_s64_u16(x)
#define vreinterpret_m64_u32(x) vreinterpret_s64_u32(x)
#define vreinterpret_m64_u64(x) vreinterpret_s64_u64(x)
#define vreinterpret_m64_f16(x) vreinterpret_s64_f16(x)
#define vreinterpret_m64_f32(x) vreinterpret_s64_f32(x)
#define vreinterpret_m64_f64(x) vreinterpret_s64_f64(x)
#define vreinterpret_u8_m64(x) vreinterpret_u8_s64(x)
#define vreinterpret_u16_m64(x) vreinterpret_u16_s64(x)
#define vreinterpret_u32_m64(x) vreinterpret_u32_s64(x)
#define vreinterpret_u64_m64(x) vreinterpret_u64_s64(x)
#define vreinterpret_s8_m64(x) vreinterpret_s8_s64(x)
#define vreinterpret_s16_m64(x) vreinterpret_s16_s64(x)
#define vreinterpret_s32_m64(x) vreinterpret_s32_s64(x)
#define vreinterpret_s64_m64(x) (x)
#define vreinterpret_f32_m64(x) vreinterpret_f32_s64(x)
#if defined(__aarch64__)
#define vreinterpretq_m128d_s32(x) vreinterpretq_f64_s32(x)
#define vreinterpretq_m128d_s64(x) vreinterpretq_f64_s64(x)
#define vreinterpretq_m128d_u64(x) vreinterpretq_f64_u64(x)
#define vreinterpretq_m128d_f32(x) vreinterpretq_f64_f32(x)
#define vreinterpretq_m128d_f64(x) (x)
#define vreinterpretq_s64_m128d(x) vreinterpretq_s64_f64(x)
#define vreinterpretq_u64_m128d(x) vreinterpretq_u64_f64(x)
#define vreinterpretq_f64_m128d(x) (x)
#define vreinterpretq_f32_m128d(x) vreinterpretq_f32_f64(x)
#else
#define vreinterpretq_m128d_s32(x) vreinterpretq_f32_s32(x)
#define vreinterpretq_m128d_s64(x) vreinterpretq_f32_s64(x)
#define vreinterpretq_m128d_u32(x) vreinterpretq_f32_u32(x)
#define vreinterpretq_m128d_u64(x) vreinterpretq_f32_u64(x)
#define vreinterpretq_m128d_f32(x) (x)
#define vreinterpretq_s64_m128d(x) vreinterpretq_s64_f32(x)
#define vreinterpretq_u32_m128d(x) vreinterpretq_u32_f32(x)
#define vreinterpretq_u64_m128d(x) vreinterpretq_u64_f32(x)
#define vreinterpretq_f32_m128d(x) (x)
#endif
// A struct is defined in this header file called 'SIMDVec' which can be used
// by applications which attempt to access the contents of an _m128 struct
// directly. It is important to note that accessing the __m128 struct directly
// is bad coding practice by Microsoft: @see:
// https://msdn.microsoft.com/en-us/library/ayeb3ayc.aspx
//
// However, some legacy source code may try to access the contents of an __m128
// struct directly so the developer can use the SIMDVec as an alias for it. Any
// casting must be done manually by the developer, as you cannot cast or
// otherwise alias the base NEON data type for intrinsic operations.
//
// union intended to allow direct access to an __m128 variable using the names
// that the MSVC compiler provides. This union should really only be used when
// trying to access the members of the vector as integer values. GCC/clang
// allow native access to the float members through a simple array access
// operator (in C since 4.6, in C++ since 4.8).
//
// Ideally direct accesses to SIMD vectors should not be used since it can cause
// a performance hit. If it really is needed however, the original __m128
// variable can be aliased with a pointer to this union and used to access
// individual components. The use of this union should be hidden behind a macro
// that is used throughout the codebase to access the members instead of always
// declaring this type of variable.
typedef union ALIGN_STRUCT(16) SIMDVec {
float m128_f32[4]; // as floats - DON'T USE. Added for convenience.
int8_t m128_i8[16]; // as signed 8-bit integers.
int16_t m128_i16[8]; // as signed 16-bit integers.
int32_t m128_i32[4]; // as signed 32-bit integers.
int64_t m128_i64[2]; // as signed 64-bit integers.
uint8_t m128_u8[16]; // as unsigned 8-bit integers.
uint16_t m128_u16[8]; // as unsigned 16-bit integers.
uint32_t m128_u32[4]; // as unsigned 32-bit integers.
uint64_t m128_u64[2]; // as unsigned 64-bit integers.
} SIMDVec;
// casting using SIMDVec
#define vreinterpretq_nth_u64_m128i(x, n) (((SIMDVec *) &x)->m128_u64[n])
#define vreinterpretq_nth_u32_m128i(x, n) (((SIMDVec *) &x)->m128_u32[n])
#define vreinterpretq_nth_u8_m128i(x, n) (((SIMDVec *) &x)->m128_u8[n])
/* Backwards compatibility for compilers with lack of specific type support */
// Older gcc does not define vld1q_u8_x4 type
#if defined(__GNUC__) && !defined(__clang__) && \
((__GNUC__ == 10 && (__GNUC_MINOR__ <= 1)) || \
(__GNUC__ == 9 && (__GNUC_MINOR__ <= 3)) || \
(__GNUC__ == 8 && (__GNUC_MINOR__ <= 4)) || __GNUC__ <= 7)
FORCE_INLINE uint8x16x4_t _sse2neon_vld1q_u8_x4(const uint8_t *p)
{
uint8x16x4_t ret;
ret.val[0] = vld1q_u8(p + 0);
ret.val[1] = vld1q_u8(p + 16);
ret.val[2] = vld1q_u8(p + 32);
ret.val[3] = vld1q_u8(p + 48);
return ret;
}
#else
// Wraps vld1q_u8_x4
FORCE_INLINE uint8x16x4_t _sse2neon_vld1q_u8_x4(const uint8_t *p)
{
return vld1q_u8_x4(p);
}
#endif
/* Function Naming Conventions
* The naming convention of SSE intrinsics is straightforward. A generic SSE
* intrinsic function is given as follows:
* _mm_<name>_<data_type>
*
* The parts of this format are given as follows:
* 1. <name> describes the operation performed by the intrinsic
* 2. <data_type> identifies the data type of the function's primary arguments
*
* This last part, <data_type>, is a little complicated. It identifies the
* content of the input values, and can be set to any of the following values:
* + ps - vectors contain floats (ps stands for packed single-precision)
* + pd - vectors cantain doubles (pd stands for packed double-precision)
* + epi8/epi16/epi32/epi64 - vectors contain 8-bit/16-bit/32-bit/64-bit
* signed integers
* + epu8/epu16/epu32/epu64 - vectors contain 8-bit/16-bit/32-bit/64-bit
* unsigned integers
* + si128 - unspecified 128-bit vector or 256-bit vector
* + m128/m128i/m128d - identifies input vector types when they are different
* than the type of the returned vector
*
* For example, _mm_setzero_ps. The _mm implies that the function returns
* a 128-bit vector. The _ps at the end implies that the argument vectors
* contain floats.
*
* A complete example: Byte Shuffle - pshufb (_mm_shuffle_epi8)
* // Set packed 16-bit integers. 128 bits, 8 short, per 16 bits
* __m128i v_in = _mm_setr_epi16(1, 2, 3, 4, 5, 6, 7, 8);
* // Set packed 8-bit integers
* // 128 bits, 16 chars, per 8 bits
* __m128i v_perm = _mm_setr_epi8(1, 0, 2, 3, 8, 9, 10, 11,
* 4, 5, 12, 13, 6, 7, 14, 15);
* // Shuffle packed 8-bit integers
* __m128i v_out = _mm_shuffle_epi8(v_in, v_perm); // pshufb
*
* Data (Number, Binary, Byte Index):
+------+------+-------------+------+------+-------------+
| 1 | 2 | 3 | 4 | Number
+------+------+------+------+------+------+------+------+
| 0000 | 0001 | 0000 | 0010 | 0000 | 0011 | 0000 | 0100 | Binary
+------+------+------+------+------+------+------+------+
| 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | Index
+------+------+------+------+------+------+------+------+
+------+------+------+------+------+------+------+------+
| 5 | 6 | 7 | 8 | Number
+------+------+------+------+------+------+------+------+
| 0000 | 0101 | 0000 | 0110 | 0000 | 0111 | 0000 | 1000 | Binary
+------+------+------+------+------+------+------+------+
| 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | Index
+------+------+------+------+------+------+------+------+
* Index (Byte Index):
+------+------+------+------+------+------+------+------+
| 1 | 0 | 2 | 3 | 8 | 9 | 10 | 11 |
+------+------+------+------+------+------+------+------+
+------+------+------+------+------+------+------+------+
| 4 | 5 | 12 | 13 | 6 | 7 | 14 | 15 |
+------+------+------+------+------+------+------+------+
* Result:
+------+------+------+------+------+------+------+------+
| 1 | 0 | 2 | 3 | 8 | 9 | 10 | 11 | Index
+------+------+------+------+------+------+------+------+
| 0001 | 0000 | 0000 | 0010 | 0000 | 0101 | 0000 | 0110 | Binary
+------+------+------+------+------+------+------+------+
| 256 | 2 | 5 | 6 | Number
+------+------+------+------+------+------+------+------+
+------+------+------+------+------+------+------+------+
| 4 | 5 | 12 | 13 | 6 | 7 | 14 | 15 | Index
+------+------+------+------+------+------+------+------+
| 0000 | 0011 | 0000 | 0111 | 0000 | 0100 | 0000 | 1000 | Binary
+------+------+------+------+------+------+------+------+
| 3 | 7 | 4 | 8 | Number
+------+------+------+------+------+------+-------------+
*/
/* Set/get methods */
/* Constants for use with _mm_prefetch. */
enum _mm_hint {
_MM_HINT_NTA = 0, /* load data to L1 and L2 cache, mark it as NTA */
_MM_HINT_T0 = 1, /* load data to L1 and L2 cache */
_MM_HINT_T1 = 2, /* load data to L2 cache only */
_MM_HINT_T2 = 3, /* load data to L2 cache only, mark it as NTA */
_MM_HINT_ENTA = 4, /* exclusive version of _MM_HINT_NTA */
_MM_HINT_ET0 = 5, /* exclusive version of _MM_HINT_T0 */
_MM_HINT_ET1 = 6, /* exclusive version of _MM_HINT_T1 */
_MM_HINT_ET2 = 7 /* exclusive version of _MM_HINT_T2 */
};
// Loads one cache line of data from address p to a location closer to the
// processor. https://msdn.microsoft.com/en-us/library/84szxsww(v=vs.100).aspx
FORCE_INLINE void _mm_prefetch(const void *p, int i)
{
(void) i;
__builtin_prefetch(p);
}
// Pause the processor. This is typically used in spin-wait loops and depending
// on the x86 processor typical values are in the 40-100 cycle range. The
// 'yield' instruction isn't a good fit beacuse it's effectively a nop on most
// Arm cores. Experience with several databases has shown has shown an 'isb' is
// a reasonable approximation.
FORCE_INLINE void _mm_pause()
{
__asm__ __volatile__("isb\n");
}
// Copy the lower single-precision (32-bit) floating-point element of a to dst.
//
// dst[31:0] := a[31:0]
//
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtss_f32
FORCE_INLINE float _mm_cvtss_f32(__m128 a)
{
return vgetq_lane_f32(vreinterpretq_f32_m128(a), 0);
}
// Convert the lower single-precision (32-bit) floating-point element in b to a
// double-precision (64-bit) floating-point element, store the result in the
// lower element of dst, and copy the upper element from a to the upper element
// of dst.
//
// dst[63:0] := Convert_FP32_To_FP64(b[31:0])
// dst[127:64] := a[127:64]
//
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtss_sd
FORCE_INLINE __m128d _mm_cvtss_sd(__m128d a, __m128 b)
{
double d = (double) vgetq_lane_f32(vreinterpretq_f32_m128(b), 0);
#if defined(__aarch64__)
return vreinterpretq_m128d_f64(
vsetq_lane_f64(d, vreinterpretq_f64_m128d(a), 0));
#else
return vreinterpretq_m128d_s64(
vsetq_lane_s64(*(int64_t *) &d, vreinterpretq_s64_m128d(a), 0));
#endif
}
// Convert the lower single-precision (32-bit) floating-point element in a to a
// 32-bit integer, and store the result in dst.
//
// dst[31:0] := Convert_FP32_To_Int32(a[31:0])
//
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtss_si32
#define _mm_cvtss_si32(a) _mm_cvt_ss2si(a)
// Convert the lower single-precision (32-bit) floating-point element in a to a
// 64-bit integer, and store the result in dst.
//
// dst[63:0] := Convert_FP32_To_Int64(a[31:0])
//
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtss_si64
FORCE_INLINE int _mm_cvtss_si64(__m128 a)
{
#if defined(__aarch64__)
return vgetq_lane_s64(
vreinterpretq_s64_s32(vcvtnq_s32_f32(vreinterpretq_f32_m128(a))), 0);
#else
float32_t data = vgetq_lane_f32(vreinterpretq_f32_m128(a), 0);
float32_t diff = data - floor(data);
if (diff > 0.5)
return (int64_t) ceil(data);
if (unlikely(diff == 0.5)) {
int64_t f = (int64_t) floor(data);
int64_t c = (int64_t) ceil(data);
return c & 1 ? f : c;
}
return (int64_t) floor(data);
#endif
}
// Convert packed single-precision (32-bit) floating-point elements in a to
// packed 32-bit integers with truncation, and store the results in dst.
//
// FOR j := 0 to 1
// i := 32*j
// dst[i+31:i] := Convert_FP32_To_Int32_Truncate(a[i+31:i])
// ENDFOR
//
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_ps2pi
FORCE_INLINE __m64 _mm_cvtt_ps2pi(__m128 a)
{
return vreinterpret_m64_s32(
vget_low_s32(vcvtq_s32_f32(vreinterpretq_f32_m128(a))));
}
// Convert the lower single-precision (32-bit) floating-point element in a to a
// 32-bit integer with truncation, and store the result in dst.
//
// dst[31:0] := Convert_FP32_To_Int32_Truncate(a[31:0])
//
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvtt_ss2si
FORCE_INLINE int _mm_cvtt_ss2si(__m128 a)
{
return vgetq_lane_s32(vcvtq_s32_f32(vreinterpretq_f32_m128(a)), 0);
}
// Convert packed single-precision (32-bit) floating-point elements in a to
// packed 32-bit integers with truncation, and store the results in dst.
//
// FOR j := 0 to 1
// i := 32*j
// dst[i+31:i] := Convert_FP32_To_Int32_Truncate(a[i+31:i])
// ENDFOR
//
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttps_pi32
#define _mm_cvttps_pi32(a) _mm_cvtt_ps2pi(a)
// Convert the lower single-precision (32-bit) floating-point element in a to a
// 32-bit integer with truncation, and store the result in dst.
//
// dst[31:0] := Convert_FP32_To_Int32_Truncate(a[31:0])
//
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttss_si32
#define _mm_cvttss_si32(a) _mm_cvtt_ss2si(a)
// Convert the lower single-precision (32-bit) floating-point element in a to a
// 64-bit integer with truncation, and store the result in dst.
//
// dst[63:0] := Convert_FP32_To_Int64_Truncate(a[31:0])
//
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_cvttss_si64
FORCE_INLINE int64_t _mm_cvttss_si64(__m128 a)
{
return vgetq_lane_s64(
vmovl_s32(vget_low_s32(vcvtq_s32_f32(vreinterpretq_f32_m128(a)))), 0);
}
// Sets the 128-bit value to zero
// https://msdn.microsoft.com/en-us/library/vstudio/ys7dw0kh(v=vs.100).aspx
FORCE_INLINE __m128i _mm_setzero_si128(void)
{
return vreinterpretq_m128i_s32(vdupq_n_s32(0));
}
// Clears the four single-precision, floating-point values.
// https://msdn.microsoft.com/en-us/library/vstudio/tk1t2tbz(v=vs.100).aspx
FORCE_INLINE __m128 _mm_setzero_ps(void)
{
return vreinterpretq_m128_f32(vdupq_n_f32(0));
}
// Return vector of type __m128d with all elements set to zero.
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_setzero_pd
FORCE_INLINE __m128d _mm_setzero_pd(void)
{
#if defined(__aarch64__)
return vreinterpretq_m128d_f64(vdupq_n_f64(0));
#else
return vreinterpretq_m128d_f32(vdupq_n_f32(0));
#endif
}
// Sets the four single-precision, floating-point values to w.
//
// r0 := r1 := r2 := r3 := w
//
// https://msdn.microsoft.com/en-us/library/vstudio/2x1se8ha(v=vs.100).aspx
FORCE_INLINE __m128 _mm_set1_ps(float _w)
{
return vreinterpretq_m128_f32(vdupq_n_f32(_w));
}
// Sets the four single-precision, floating-point values to w.
// https://msdn.microsoft.com/en-us/library/vstudio/2x1se8ha(v=vs.100).aspx
FORCE_INLINE __m128 _mm_set_ps1(float _w)
{
return vreinterpretq_m128_f32(vdupq_n_f32(_w));
}
// Sets the four single-precision, floating-point values to the four inputs.
// https://msdn.microsoft.com/en-us/library/vstudio/afh0zf75(v=vs.100).aspx
FORCE_INLINE __m128 _mm_set_ps(float w, float z, float y, float x)
{
float ALIGN_STRUCT(16) data[4] = {x, y, z, w};
return vreinterpretq_m128_f32(vld1q_f32(data));
}
// Copy single-precision (32-bit) floating-point element a to the lower element
// of dst, and zero the upper 3 elements.
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_set_ss
FORCE_INLINE __m128 _mm_set_ss(float a)
{
float ALIGN_STRUCT(16) data[4] = {a, 0, 0, 0};
return vreinterpretq_m128_f32(vld1q_f32(data));
}
// Sets the four single-precision, floating-point values to the four inputs in
// reverse order.
// https://msdn.microsoft.com/en-us/library/vstudio/d2172ct3(v=vs.100).aspx
FORCE_INLINE __m128 _mm_setr_ps(float w, float z, float y, float x)
{
float ALIGN_STRUCT(16) data[4] = {w, z, y, x};
return vreinterpretq_m128_f32(vld1q_f32(data));
}
// Sets the 8 signed 16-bit integer values in reverse order.
//
// Return Value
// r0 := w0
// r1 := w1
// ...
// r7 := w7
FORCE_INLINE __m128i _mm_setr_epi16(short w0,
short w1,
short w2,
short w3,
short w4,
short w5,
short w6,
short w7)
{
int16_t ALIGN_STRUCT(16) data[8] = {w0, w1, w2, w3, w4, w5, w6, w7};
return vreinterpretq_m128i_s16(vld1q_s16((int16_t *) data));
}
// Sets the 4 signed 32-bit integer values in reverse order
// https://technet.microsoft.com/en-us/library/security/27yb3ee5(v=vs.90).aspx
FORCE_INLINE __m128i _mm_setr_epi32(int i3, int i2, int i1, int i0)
{
int32_t ALIGN_STRUCT(16) data[4] = {i3, i2, i1, i0};
return vreinterpretq_m128i_s32(vld1q_s32(data));
}
// Set packed 64-bit integers in dst with the supplied values in reverse order.
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_setr_epi64
FORCE_INLINE __m128i _mm_setr_epi64(__m64 e1, __m64 e0)
{
return vreinterpretq_m128i_s64(vcombine_s64(e1, e0));
}
// Sets the 16 signed 8-bit integer values to b.
//
// r0 := b
// r1 := b
// ...
// r15 := b
//
// https://msdn.microsoft.com/en-us/library/6e14xhyf(v=vs.100).aspx
FORCE_INLINE __m128i _mm_set1_epi8(signed char w)
{
return vreinterpretq_m128i_s8(vdupq_n_s8(w));
}
// Broadcast double-precision (64-bit) floating-point value a to all elements of
// dst.
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_set1_pd
FORCE_INLINE __m128d _mm_set1_pd(double d)
{
#if defined(__aarch64__)
return vreinterpretq_m128d_f64(vdupq_n_f64(d));
#else
return vreinterpretq_m128d_s64(vdupq_n_s64(*(int64_t *) &d));
#endif
}
// Sets the 8 signed 16-bit integer values to w.
//
// r0 := w
// r1 := w
// ...
// r7 := w
//
// https://msdn.microsoft.com/en-us/library/k0ya3x0e(v=vs.90).aspx
FORCE_INLINE __m128i _mm_set1_epi16(short w)
{
return vreinterpretq_m128i_s16(vdupq_n_s16(w));
}
// Sets the 16 signed 8-bit integer values.
// https://msdn.microsoft.com/en-us/library/x0cx8zd3(v=vs.90).aspx
FORCE_INLINE __m128i _mm_set_epi8(signed char b15,
signed char b14,
signed char b13,
signed char b12,
signed char b11,
signed char b10,
signed char b9,
signed char b8,
signed char b7,
signed char b6,
signed char b5,
signed char b4,
signed char b3,
signed char b2,
signed char b1,
signed char b0)
{
int8_t ALIGN_STRUCT(16)
data[16] = {(int8_t) b0, (int8_t) b1, (int8_t) b2, (int8_t) b3,
(int8_t) b4, (int8_t) b5, (int8_t) b6, (int8_t) b7,
(int8_t) b8, (int8_t) b9, (int8_t) b10, (int8_t) b11,
(int8_t) b12, (int8_t) b13, (int8_t) b14, (int8_t) b15};
return (__m128i) vld1q_s8(data);
}
// Sets the 8 signed 16-bit integer values.
// https://msdn.microsoft.com/en-au/library/3e0fek84(v=vs.90).aspx
FORCE_INLINE __m128i _mm_set_epi16(short i7,
short i6,
short i5,
short i4,
short i3,
short i2,
short i1,
short i0)
{
int16_t ALIGN_STRUCT(16) data[8] = {i0, i1, i2, i3, i4, i5, i6, i7};
return vreinterpretq_m128i_s16(vld1q_s16(data));
}
// Sets the 16 signed 8-bit integer values in reverse order.
// https://msdn.microsoft.com/en-us/library/2khb9c7k(v=vs.90).aspx
FORCE_INLINE __m128i _mm_setr_epi8(signed char b0,
signed char b1,
signed char b2,
signed char b3,
signed char b4,
signed char b5,
signed char b6,
signed char b7,
signed char b8,
signed char b9,
signed char b10,
signed char b11,
signed char b12,
signed char b13,
signed char b14,
signed char b15)
{
int8_t ALIGN_STRUCT(16)
data[16] = {(int8_t) b0, (int8_t) b1, (int8_t) b2, (int8_t) b3,
(int8_t) b4, (int8_t) b5, (int8_t) b6, (int8_t) b7,
(int8_t) b8, (int8_t) b9, (int8_t) b10, (int8_t) b11,
(int8_t) b12, (int8_t) b13, (int8_t) b14, (int8_t) b15};
return (__m128i) vld1q_s8(data);
}
// Sets the 4 signed 32-bit integer values to i.
//
// r0 := i
// r1 := i
// r2 := i
// r3 := I
//
// https://msdn.microsoft.com/en-us/library/vstudio/h4xscxat(v=vs.100).aspx
FORCE_INLINE __m128i _mm_set1_epi32(int _i)
{
return vreinterpretq_m128i_s32(vdupq_n_s32(_i));
}
// Sets the 2 signed 64-bit integer values to i.
// https://docs.microsoft.com/en-us/previous-versions/visualstudio/visual-studio-2010/whtfzhzk(v=vs.100)
FORCE_INLINE __m128i _mm_set1_epi64(__m64 _i)
{
return vreinterpretq_m128i_s64(vdupq_n_s64((int64_t) _i));
}
// Sets the 2 signed 64-bit integer values to i.
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_set1_epi64x
FORCE_INLINE __m128i _mm_set1_epi64x(int64_t _i)
{
return vreinterpretq_m128i_s64(vdupq_n_s64(_i));
}
// Sets the 4 signed 32-bit integer values.
// https://msdn.microsoft.com/en-us/library/vstudio/019beekt(v=vs.100).aspx
FORCE_INLINE __m128i _mm_set_epi32(int i3, int i2, int i1, int i0)
{
int32_t ALIGN_STRUCT(16) data[4] = {i0, i1, i2, i3};
return vreinterpretq_m128i_s32(vld1q_s32(data));
}
// Returns the __m128i structure with its two 64-bit integer values
// initialized to the values of the two 64-bit integers passed in.
// https://msdn.microsoft.com/en-us/library/dk2sdw0h(v=vs.120).aspx
FORCE_INLINE __m128i _mm_set_epi64x(int64_t i1, int64_t i2)
{
return vreinterpretq_m128i_s64(
vcombine_s64(vcreate_s64(i2), vcreate_s64(i1)));
}
// Returns the __m128i structure with its two 64-bit integer values
// initialized to the values of the two 64-bit integers passed in.
// https://msdn.microsoft.com/en-us/library/dk2sdw0h(v=vs.120).aspx
FORCE_INLINE __m128i _mm_set_epi64(__m64 i1, __m64 i2)
{
return _mm_set_epi64x((int64_t) i1, (int64_t) i2);
}
// Set packed double-precision (64-bit) floating-point elements in dst with the
// supplied values.
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_set_pd
FORCE_INLINE __m128d _mm_set_pd(double e1, double e0)
{
double ALIGN_STRUCT(16) data[2] = {e0, e1};
#if defined(__aarch64__)
return vreinterpretq_m128d_f64(vld1q_f64((float64_t *) data));
#else
return vreinterpretq_m128d_f32(vld1q_f32((float32_t *) data));
#endif
}
// Set packed double-precision (64-bit) floating-point elements in dst with the
// supplied values in reverse order.
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_setr_pd
FORCE_INLINE __m128d _mm_setr_pd(double e1, double e0)
{
return _mm_set_pd(e0, e1);
}
// Copy double-precision (64-bit) floating-point element a to the lower element
// of dst, and zero the upper element.
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_set_sd
FORCE_INLINE __m128d _mm_set_sd(double a)
{
return _mm_set_pd(0, a);
}
// Broadcast double-precision (64-bit) floating-point value a to all elements of
// dst.
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_set_pd1
#define _mm_set_pd1 _mm_set1_pd
// Stores four single-precision, floating-point values.
// https://msdn.microsoft.com/en-us/library/vstudio/s3h4ay6y(v=vs.100).aspx
FORCE_INLINE void _mm_store_ps(float *p, __m128 a)
{
vst1q_f32(p, vreinterpretq_f32_m128(a));
}
// Store the lower single-precision (32-bit) floating-point element from a into
// 4 contiguous elements in memory. mem_addr must be aligned on a 16-byte
// boundary or a general-protection exception may be generated.
//
// MEM[mem_addr+31:mem_addr] := a[31:0]
// MEM[mem_addr+63:mem_addr+32] := a[31:0]
// MEM[mem_addr+95:mem_addr+64] := a[31:0]
// MEM[mem_addr+127:mem_addr+96] := a[31:0]
//
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_store_ps1
FORCE_INLINE void _mm_store_ps1(float *p, __m128 a)
{
float32_t a0 = vgetq_lane_f32(vreinterpretq_f32_m128(a), 0);
vst1q_f32(p, vdupq_n_f32(a0));
}
// Store the lower single-precision (32-bit) floating-point element from a into
// 4 contiguous elements in memory. mem_addr must be aligned on a 16-byte
// boundary or a general-protection exception may be generated.
//
// MEM[mem_addr+31:mem_addr] := a[31:0]
// MEM[mem_addr+63:mem_addr+32] := a[31:0]
// MEM[mem_addr+95:mem_addr+64] := a[31:0]
// MEM[mem_addr+127:mem_addr+96] := a[31:0]
//
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_store1_ps
#define _mm_store1_ps _mm_store_ps1
// Store 4 single-precision (32-bit) floating-point elements from a into memory
// in reverse order. mem_addr must be aligned on a 16-byte boundary or a
// general-protection exception may be generated.
//
// MEM[mem_addr+31:mem_addr] := a[127:96]
// MEM[mem_addr+63:mem_addr+32] := a[95:64]
// MEM[mem_addr+95:mem_addr+64] := a[63:32]
// MEM[mem_addr+127:mem_addr+96] := a[31:0]
//
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_storer_ps
FORCE_INLINE void _mm_storer_ps(float *p, __m128 a)
{
float32x4_t tmp = vrev64q_f32(vreinterpretq_f32_m128(a));
float32x4_t rev = vextq_f32(tmp, tmp, 2);
vst1q_f32(p, rev);
}
// Stores four single-precision, floating-point values.
// https://msdn.microsoft.com/en-us/library/44e30x22(v=vs.100).aspx
FORCE_INLINE void _mm_storeu_ps(float *p, __m128 a)
{
vst1q_f32(p, vreinterpretq_f32_m128(a));
}
// Stores four 32-bit integer values as (as a __m128i value) at the address p.
// https://msdn.microsoft.com/en-us/library/vstudio/edk11s13(v=vs.100).aspx
FORCE_INLINE void _mm_store_si128(__m128i *p, __m128i a)
{
vst1q_s32((int32_t *) p, vreinterpretq_s32_m128i(a));
}
// Stores 128-bits of integer data a at the address p.
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_storeu_si128
FORCE_INLINE void _mm_storeu_si128(__m128i *p, __m128i a)
{
vst1q_s32((int32_t *) p, vreinterpretq_s32_m128i(a));
}
// Stores 64-bits of integer data a at the address p.
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_storeu_si64
FORCE_INLINE void _mm_storeu_si64(void *p, __m128i a)
{
vst1q_lane_s64((int64_t *) p, vreinterpretq_s64_m128i(a), 0);
}
// Stores 32-bits of integer data a at the address p.
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_storeu_si32
FORCE_INLINE void _mm_storeu_si32(void *p, __m128i a)
{
vst1q_lane_s32((int32_t *) p, vreinterpretq_s32_m128i(a), 0);
}
// Stores 16-bits of integer data a at the address p.
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_storeu_si16
FORCE_INLINE void _mm_storeu_si16(void *p, __m128i a)
{
vst1q_lane_s16((int16_t *) p, vreinterpretq_s16_m128i(a), 0);
}
// Stores the lower single - precision, floating - point value.
// https://msdn.microsoft.com/en-us/library/tzz10fbx(v=vs.100).aspx
FORCE_INLINE void _mm_store_ss(float *p, __m128 a)
{
vst1q_lane_f32(p, vreinterpretq_f32_m128(a), 0);
}
// Store 128-bits (composed of 2 packed double-precision (64-bit) floating-point
// elements) from a into memory. mem_addr must be aligned on a 16-byte boundary
// or a general-protection exception may be generated.
// https://software.intel.com/sites/landingpage/IntrinsicsGuide/#text=_mm_store_pd
FORCE_INLINE void _mm_store_pd(double *mem_addr, __m128d a)
{