1/* Copyright (C) 2011-2017 Free Software Foundation, Inc.
2 Contributed by Intel Corporation.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19#include <sysdep.h>
20
21#ifdef USE_AS_WMEMCHR
22# define MEMCHR wmemchr
23# define PCMPEQ pcmpeqd
24#else
25# define MEMCHR memchr
26# define PCMPEQ pcmpeqb
27#endif
28
29/* fast SSE2 version with using pmaxub and 64 byte loop */
30
31 .text
32ENTRY(MEMCHR)
33 movd %esi, %xmm1
34 mov %edi, %ecx
35
36#ifdef USE_AS_WMEMCHR
37 test %rdx, %rdx
38 jz L(return_null)
39 shl $2, %rdx
40#else
41 punpcklbw %xmm1, %xmm1
42 test %rdx, %rdx
43 jz L(return_null)
44 punpcklbw %xmm1, %xmm1
45#endif
46
47 and $63, %ecx
48 pshufd $0, %xmm1, %xmm1
49
50 cmp $48, %ecx
51 ja L(crosscache)
52
53 movdqu (%rdi), %xmm0
54 PCMPEQ %xmm1, %xmm0
55 pmovmskb %xmm0, %eax
56 test %eax, %eax
57
58 jnz L(matches_1)
59 sub $16, %rdx
60 jbe L(return_null)
61 add $16, %rdi
62 and $15, %ecx
63 and $-16, %rdi
64 add %rcx, %rdx
65 sub $64, %rdx
66 jbe L(exit_loop)
67 jmp L(loop_prolog)
68
69 .p2align 4
70L(crosscache):
71 and $15, %ecx
72 and $-16, %rdi
73 movdqa (%rdi), %xmm0
74
75 PCMPEQ %xmm1, %xmm0
76/* Check if there is a match. */
77 pmovmskb %xmm0, %eax
78/* Remove the leading bytes. */
79 sar %cl, %eax
80 test %eax, %eax
81 je L(unaligned_no_match)
82/* Check which byte is a match. */
83 bsf %eax, %eax
84
85 sub %rax, %rdx
86 jbe L(return_null)
87 add %rdi, %rax
88 add %rcx, %rax
89 ret
90
91 .p2align 4
92L(unaligned_no_match):
93 /* "rcx" is less than 16. Calculate "rdx + rcx - 16" by using
94 "rdx - (16 - rcx)" instead of "(rdx + rcx) - 16" to void
95 possible addition overflow. */
96 neg %rcx
97 add $16, %rcx
98 sub %rcx, %rdx
99 jbe L(return_null)
100 add $16, %rdi
101 sub $64, %rdx
102 jbe L(exit_loop)
103
104 .p2align 4
105L(loop_prolog):
106 movdqa (%rdi), %xmm0
107 PCMPEQ %xmm1, %xmm0
108 pmovmskb %xmm0, %eax
109 test %eax, %eax
110 jnz L(matches)
111
112 movdqa 16(%rdi), %xmm2
113 PCMPEQ %xmm1, %xmm2
114 pmovmskb %xmm2, %eax
115 test %eax, %eax
116 jnz L(matches16)
117
118 movdqa 32(%rdi), %xmm3
119 PCMPEQ %xmm1, %xmm3
120 pmovmskb %xmm3, %eax
121 test %eax, %eax
122 jnz L(matches32)
123
124 movdqa 48(%rdi), %xmm4
125 PCMPEQ %xmm1, %xmm4
126 add $64, %rdi
127 pmovmskb %xmm4, %eax
128 test %eax, %eax
129 jnz L(matches0)
130
131 test $0x3f, %rdi
132 jz L(align64_loop)
133
134 sub $64, %rdx
135 jbe L(exit_loop)
136
137 movdqa (%rdi), %xmm0
138 PCMPEQ %xmm1, %xmm0
139 pmovmskb %xmm0, %eax
140 test %eax, %eax
141 jnz L(matches)
142
143 movdqa 16(%rdi), %xmm2
144 PCMPEQ %xmm1, %xmm2
145 pmovmskb %xmm2, %eax
146 test %eax, %eax
147 jnz L(matches16)
148
149 movdqa 32(%rdi), %xmm3
150 PCMPEQ %xmm1, %xmm3
151 pmovmskb %xmm3, %eax
152 test %eax, %eax
153 jnz L(matches32)
154
155 movdqa 48(%rdi), %xmm3
156 PCMPEQ %xmm1, %xmm3
157 pmovmskb %xmm3, %eax
158
159 add $64, %rdi
160 test %eax, %eax
161 jnz L(matches0)
162
163 mov %rdi, %rcx
164 and $-64, %rdi
165 and $63, %ecx
166 add %rcx, %rdx
167
168 .p2align 4
169L(align64_loop):
170 sub $64, %rdx
171 jbe L(exit_loop)
172 movdqa (%rdi), %xmm0
173 movdqa 16(%rdi), %xmm2
174 movdqa 32(%rdi), %xmm3
175 movdqa 48(%rdi), %xmm4
176
177 PCMPEQ %xmm1, %xmm0
178 PCMPEQ %xmm1, %xmm2
179 PCMPEQ %xmm1, %xmm3
180 PCMPEQ %xmm1, %xmm4
181
182 pmaxub %xmm0, %xmm3
183 pmaxub %xmm2, %xmm4
184 pmaxub %xmm3, %xmm4
185 pmovmskb %xmm4, %eax
186
187 add $64, %rdi
188
189 test %eax, %eax
190 jz L(align64_loop)
191
192 sub $64, %rdi
193
194 pmovmskb %xmm0, %eax
195 test %eax, %eax
196 jnz L(matches)
197
198 pmovmskb %xmm2, %eax
199 test %eax, %eax
200 jnz L(matches16)
201
202 movdqa 32(%rdi), %xmm3
203 PCMPEQ %xmm1, %xmm3
204
205 PCMPEQ 48(%rdi), %xmm1
206 pmovmskb %xmm3, %eax
207 test %eax, %eax
208 jnz L(matches32)
209
210 pmovmskb %xmm1, %eax
211 bsf %eax, %eax
212 lea 48(%rdi, %rax), %rax
213 ret
214
215 .p2align 4
216L(exit_loop):
217 add $32, %edx
218 jle L(exit_loop_32)
219
220 movdqa (%rdi), %xmm0
221 PCMPEQ %xmm1, %xmm0
222 pmovmskb %xmm0, %eax
223 test %eax, %eax
224 jnz L(matches)
225
226 movdqa 16(%rdi), %xmm2
227 PCMPEQ %xmm1, %xmm2
228 pmovmskb %xmm2, %eax
229 test %eax, %eax
230 jnz L(matches16)
231
232 movdqa 32(%rdi), %xmm3
233 PCMPEQ %xmm1, %xmm3
234 pmovmskb %xmm3, %eax
235 test %eax, %eax
236 jnz L(matches32_1)
237 sub $16, %edx
238 jle L(return_null)
239
240 PCMPEQ 48(%rdi), %xmm1
241 pmovmskb %xmm1, %eax
242 test %eax, %eax
243 jnz L(matches48_1)
244 xor %eax, %eax
245 ret
246
247 .p2align 4
248L(exit_loop_32):
249 add $32, %edx
250 movdqa (%rdi), %xmm0
251 PCMPEQ %xmm1, %xmm0
252 pmovmskb %xmm0, %eax
253 test %eax, %eax
254 jnz L(matches_1)
255 sub $16, %edx
256 jbe L(return_null)
257
258 PCMPEQ 16(%rdi), %xmm1
259 pmovmskb %xmm1, %eax
260 test %eax, %eax
261 jnz L(matches16_1)
262 xor %eax, %eax
263 ret
264
265 .p2align 4
266L(matches0):
267 bsf %eax, %eax
268 lea -16(%rax, %rdi), %rax
269 ret
270
271 .p2align 4
272L(matches):
273 bsf %eax, %eax
274 add %rdi, %rax
275 ret
276
277 .p2align 4
278L(matches16):
279 bsf %eax, %eax
280 lea 16(%rax, %rdi), %rax
281 ret
282
283 .p2align 4
284L(matches32):
285 bsf %eax, %eax
286 lea 32(%rax, %rdi), %rax
287 ret
288
289 .p2align 4
290L(matches_1):
291 bsf %eax, %eax
292 sub %rax, %rdx
293 jbe L(return_null)
294 add %rdi, %rax
295 ret
296
297 .p2align 4
298L(matches16_1):
299 bsf %eax, %eax
300 sub %rax, %rdx
301 jbe L(return_null)
302 lea 16(%rdi, %rax), %rax
303 ret
304
305 .p2align 4
306L(matches32_1):
307 bsf %eax, %eax
308 sub %rax, %rdx
309 jbe L(return_null)
310 lea 32(%rdi, %rax), %rax
311 ret
312
313 .p2align 4
314L(matches48_1):
315 bsf %eax, %eax
316 sub %rax, %rdx
317 jbe L(return_null)
318 lea 48(%rdi, %rax), %rax
319 ret
320
321 .p2align 4
322L(return_null):
323 xor %eax, %eax
324 ret
325END(MEMCHR)
326
327#ifndef USE_AS_WMEMCHR
328strong_alias (memchr, __memchr)
329libc_hidden_builtin_def(memchr)
330#endif
331