1// Copyright 2017 The Gemmlowp Authors. All Rights Reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15// simd_wrappers_neon.h: SSE SIMD wrappers
16
17#ifndef GEMMLOWP_INTERNAL_SIMD_WRAPPERS_SSE_H_
18#define GEMMLOWP_INTERNAL_SIMD_WRAPPERS_SSE_H_
19
20#include <smmintrin.h>
21
22namespace gemmlowp {
23
24using Int32x4 = __m128i;
25using Int16x8 = __m128i;
26using Uint8x16 = __m128i;
27
28template <int ScalarCount>
29struct RegisterType<std::int32_t, ScalarCount> {
30 using Type =
31 typename std::conditional<ScalarCount >= 4, Int32x4, std::int32_t>::type;
32};
33
34template <int ScalarCount>
35struct RegisterType<std::int16_t, ScalarCount> {
36 using Type =
37 typename std::conditional<ScalarCount >= 8, Int16x8, std::int16_t>::type;
38};
39
40template <int ScalarCount>
41struct RegisterType<std::uint8_t, ScalarCount> {
42 using Type = typename std::conditional<
43 ScalarCount >= 16, Uint8x16,
44 typename std::conditional<ScalarCount >= 4, std::uint32_t,
45 std::uint8_t>::type>::type;
46};
47
48inline Int32x4 LoadInt32x4(const std::int32_t* src) {
49 return _mm_loadu_si128(reinterpret_cast<const Int32x4*>(src));
50}
51
52inline Int32x4 LoadInt16x8(const std::int16_t* src) {
53 return _mm_loadu_si128(reinterpret_cast<const Int16x8*>(src));
54}
55
56inline void StoreInt32x4(std::int32_t* dst, Int32x4 value) {
57 _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), value);
58}
59
60inline void StoreInt16x8(std::int16_t* dst, Int16x8 value) {
61 _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), value);
62}
63
64inline Uint8x16 LoadUint8x16(const std::uint8_t* src) {
65 return _mm_loadu_si128(reinterpret_cast<const Uint8x16*>(src));
66}
67
68inline void StoreUint8x16(std::uint8_t* dst, Uint8x16 value) {
69 _mm_storeu_si128(reinterpret_cast<__m128i*>(dst), value);
70}
71
72template <int Lane>
73std::int32_t GetLane(Int32x4 value) {
74 return _mm_extract_epi32(value, Lane);
75}
76
77template <int Lane>
78Int32x4 DupLane(Int32x4 value) {
79 return _mm_shuffle_epi32(value, _MM_SHUFFLE(Lane, Lane, Lane, Lane));
80}
81
82inline Int32x4 Mul(Int32x4 a, std::int32_t b) {
83 return Mul(a, Dup<Int32x4>(b));
84}
85
86inline Int32x4 Min(Int32x4 a, Int32x4 b) { return _mm_min_epi32(a, b); }
87
88inline Int32x4 Max(Int32x4 a, Int32x4 b) { return _mm_max_epi32(a, b); }
89
90inline Int32x4 SaturatingRoundingDoublingHighMul(Int32x4 a, std::int32_t b) {
91 return SaturatingRoundingDoublingHighMul(a, Dup<Int32x4>(b));
92}
93
94template <int Lane>
95Int32x4 MulByRhsLane(Int32x4 a, Int32x4 b) {
96 return Mul(a, DupLane<Lane>(b));
97}
98
99inline void MulAdd(Int32x4 lhs, Int32x4 rhs, Int32x4* acc) {
100 *acc = Add(*acc, Mul(lhs, rhs));
101}
102
103inline void MulAdd(Int32x4 lhs, std::int32_t rhs, Int32x4* acc) {
104 *acc = Add(*acc, Mul(lhs, rhs));
105}
106
107template <int Lane>
108inline void MulAddByRhsLane(Int32x4 lhs, Int32x4 rhs, Int32x4* acc) {
109 *acc = Add(*acc, MulByRhsLane<Lane>(lhs, rhs));
110}
111
112template <>
113struct LoadContiguousImpl<RegBlockUint8<8, 8>> {
114 static RegBlockUint8<8, 8> Run(const std::uint8_t* src) {
115 RegBlockUint8<8, 8> result;
116 for (int i = 0; i < 4; i++) {
117 result.buf.reg[i] = LoadUint8x16(src + 16 * i);
118 }
119 return result;
120 }
121};
122
123template <>
124struct LoadContiguousImpl<RegBlockInt32<8, 8>> {
125 static RegBlockInt32<8, 8> Run(const std::int32_t* src) {
126 RegBlockInt32<8, 8> result;
127 for (int i = 0; i < 16; i++) {
128 result.buf.reg[i] = LoadInt32x4(src + 4 * i);
129 }
130 return result;
131 }
132};
133
134template <>
135struct LoadContiguousImpl<RegBlockInt16<8, 8>> {
136 static RegBlockInt16<8, 8> Run(const std::int16_t* src) {
137 RegBlockInt16<8, 8> result;
138 for (int i = 0; i < 8; i++) {
139 result.buf.reg[i] = LoadInt16x8(src + 8 * i);
140 }
141 return result;
142 }
143};
144
145} // end namespace gemmlowp
146
147#include "simd_wrappers_common_neon_sse.h"
148
149#endif // GEMMLOWP_INTERNAL_SIMD_WRAPPERS_SSE_H_
150