1/* SSE2 version of strlen.
2 Copyright (C) 2012-2017 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19#include <sysdep.h>
20
21/* Long lived register in strlen(s), strnlen(s, n) are:
22
23 %xmm3 - zero
24 %rdi - s
25 %r10 (s+n) & (~(64-1))
26 %r11 s+n
27*/
28
29
30.text
31ENTRY(strlen)
32
33/* Test 64 bytes from %rax for zero. Save result as bitmask in %rdx. */
34#define FIND_ZERO \
35 pcmpeqb (%rax), %xmm0; \
36 pcmpeqb 16(%rax), %xmm1; \
37 pcmpeqb 32(%rax), %xmm2; \
38 pcmpeqb 48(%rax), %xmm3; \
39 pmovmskb %xmm0, %esi; \
40 pmovmskb %xmm1, %edx; \
41 pmovmskb %xmm2, %r8d; \
42 pmovmskb %xmm3, %ecx; \
43 salq $16, %rdx; \
44 salq $16, %rcx; \
45 orq %rsi, %rdx; \
46 orq %r8, %rcx; \
47 salq $32, %rcx; \
48 orq %rcx, %rdx;
49
50#ifdef AS_STRNLEN
51/* Do not read anything when n==0. */
52 test %rsi, %rsi
53 jne L(n_nonzero)
54 xor %rax, %rax
55 ret
56L(n_nonzero):
57
58/* Initialize long lived registers. */
59
60 add %rdi, %rsi
61 mov %rsi, %r10
62 and $-64, %r10
63 mov %rsi, %r11
64#endif
65
66 pxor %xmm0, %xmm0
67 pxor %xmm1, %xmm1
68 pxor %xmm2, %xmm2
69 pxor %xmm3, %xmm3
70 movq %rdi, %rax
71 movq %rdi, %rcx
72 andq $4095, %rcx
73/* Offsets 4032-4047 will be aligned into 4032 thus fit into page. */
74 cmpq $4047, %rcx
75/* We cannot unify this branching as it would be ~6 cycles slower. */
76 ja L(cross_page)
77
78#ifdef AS_STRNLEN
79/* Test if end is among first 64 bytes. */
80# define STRNLEN_PROLOG \
81 mov %r11, %rsi; \
82 subq %rax, %rsi; \
83 andq $-64, %rax; \
84 testq $-64, %rsi; \
85 je L(strnlen_ret)
86#else
87# define STRNLEN_PROLOG andq $-64, %rax;
88#endif
89
90/* Ignore bits in mask that come before start of string. */
91#define PROLOG(lab) \
92 movq %rdi, %rcx; \
93 xorq %rax, %rcx; \
94 STRNLEN_PROLOG; \
95 sarq %cl, %rdx; \
96 test %rdx, %rdx; \
97 je L(lab); \
98 bsfq %rdx, %rax; \
99 ret
100
101#ifdef AS_STRNLEN
102 andq $-16, %rax
103 FIND_ZERO
104#else
105 /* Test first 16 bytes unaligned. */
106 movdqu (%rax), %xmm4
107 pcmpeqb %xmm0, %xmm4
108 pmovmskb %xmm4, %edx
109 test %edx, %edx
110 je L(next48_bytes)
111 bsf %edx, %eax /* If eax is zeroed 16bit bsf can be used. */
112 ret
113
114L(next48_bytes):
115/* Same as FIND_ZERO except we do not check first 16 bytes. */
116 andq $-16, %rax
117 pcmpeqb 16(%rax), %xmm1
118 pcmpeqb 32(%rax), %xmm2
119 pcmpeqb 48(%rax), %xmm3
120 pmovmskb %xmm1, %edx
121 pmovmskb %xmm2, %r8d
122 pmovmskb %xmm3, %ecx
123 salq $16, %rdx
124 salq $16, %rcx
125 orq %r8, %rcx
126 salq $32, %rcx
127 orq %rcx, %rdx
128#endif
129
130 /* When no zero byte is found xmm1-3 are zero so we do not have to
131 zero them. */
132 PROLOG(loop)
133
134 .p2align 4
135L(cross_page):
136 andq $-64, %rax
137 FIND_ZERO
138 PROLOG(loop_init)
139
140#ifdef AS_STRNLEN
141/* We must do this check to correctly handle strnlen (s, -1). */
142L(strnlen_ret):
143 bts %rsi, %rdx
144 sarq %cl, %rdx
145 test %rdx, %rdx
146 je L(loop_init)
147 bsfq %rdx, %rax
148 ret
149#endif
150 .p2align 4
151L(loop_init):
152 pxor %xmm1, %xmm1
153 pxor %xmm2, %xmm2
154 pxor %xmm3, %xmm3
155#ifdef AS_STRNLEN
156 .p2align 4
157L(loop):
158
159 addq $64, %rax
160 cmpq %rax, %r10
161 je L(exit_end)
162
163 movdqa (%rax), %xmm0
164 pminub 16(%rax), %xmm0
165 pminub 32(%rax), %xmm0
166 pminub 48(%rax), %xmm0
167 pcmpeqb %xmm3, %xmm0
168 pmovmskb %xmm0, %edx
169 testl %edx, %edx
170 jne L(exit)
171 jmp L(loop)
172
173 .p2align 4
174L(exit_end):
175 cmp %rax, %r11
176 je L(first) /* Do not read when end is at page boundary. */
177 pxor %xmm0, %xmm0
178 FIND_ZERO
179
180L(first):
181 bts %r11, %rdx
182 bsfq %rdx, %rdx
183 addq %rdx, %rax
184 subq %rdi, %rax
185 ret
186
187 .p2align 4
188L(exit):
189 pxor %xmm0, %xmm0
190 FIND_ZERO
191
192 bsfq %rdx, %rdx
193 addq %rdx, %rax
194 subq %rdi, %rax
195 ret
196
197#else
198
199 /* Main loop. Unrolled twice to improve L2 cache performance on core2. */
200 .p2align 4
201L(loop):
202
203 movdqa 64(%rax), %xmm0
204 pminub 80(%rax), %xmm0
205 pminub 96(%rax), %xmm0
206 pminub 112(%rax), %xmm0
207 pcmpeqb %xmm3, %xmm0
208 pmovmskb %xmm0, %edx
209 testl %edx, %edx
210 jne L(exit64)
211
212 subq $-128, %rax
213
214 movdqa (%rax), %xmm0
215 pminub 16(%rax), %xmm0
216 pminub 32(%rax), %xmm0
217 pminub 48(%rax), %xmm0
218 pcmpeqb %xmm3, %xmm0
219 pmovmskb %xmm0, %edx
220 testl %edx, %edx
221 jne L(exit0)
222 jmp L(loop)
223
224 .p2align 4
225L(exit64):
226 addq $64, %rax
227L(exit0):
228 pxor %xmm0, %xmm0
229 FIND_ZERO
230
231 bsfq %rdx, %rdx
232 addq %rdx, %rax
233 subq %rdi, %rax
234 ret
235
236#endif
237
238END(strlen)
239libc_hidden_builtin_def (strlen)
240