1/* strcspn with SSE4.2 intrinsics
2 Copyright (C) 2009-2021 Free Software Foundation, Inc.
3 Contributed by Intel Corporation.
4 This file is part of the GNU C Library.
5
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
10
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
15
16 You should have received a copy of the GNU Lesser General Public
17 License along with the GNU C Library; if not, see
18 <https://www.gnu.org/licenses/>. */
19
20#include <nmmintrin.h>
21#include <string.h>
22#include "varshift.h"
23
24/* We use 0x2:
25 _SIDD_SBYTE_OPS
26 | _SIDD_CMP_EQUAL_ANY
27 | _SIDD_POSITIVE_POLARITY
28 | _SIDD_LEAST_SIGNIFICANT
29 on pcmpistri to compare xmm/mem128
30
31 0 1 2 3 4 5 6 7 8 9 A B C D E F
32 X X X X X X X X X X X X X X X X
33
34 against xmm
35
36 0 1 2 3 4 5 6 7 8 9 A B C D E F
37 A A A A A A A A A A A A A A A A
38
39 to find out if the first 16byte data element has any byte A and
40 the offset of the first byte. There are 3 cases:
41
42 1. The first 16byte data element has the byte A at the offset X.
43 2. The first 16byte data element has EOS and doesn't have the byte A.
44 3. The first 16byte data element is valid and doesn't have the byte A.
45
46 Here is the table of ECX, CFlag, ZFlag and SFlag for 2 cases:
47
48 1 X 1 0/1 0
49 2 16 0 1 0
50 3 16 0 0 0
51
52 We exit from the loop for cases 1 and 2 with jbe which branches
53 when either CFlag or ZFlag is 1. If CFlag == 1, ECX has the offset
54 X for case 1. */
55
56#ifndef STRCSPN_SSE2
57# define STRCSPN_SSE2 __strcspn_sse2
58# define STRCSPN_SSE42 __strcspn_sse42
59#endif
60
61#ifdef USE_AS_STRPBRK
62# define RETURN(val1, val2) return val1
63#else
64# define RETURN(val1, val2) return val2
65#endif
66
67extern
68#ifdef USE_AS_STRPBRK
69char *
70#else
71size_t
72#endif
73STRCSPN_SSE2 (const char *, const char *) attribute_hidden;
74
75
76#ifdef USE_AS_STRPBRK
77char *
78#else
79size_t
80#endif
81__attribute__ ((section (".text.sse4.2")))
82STRCSPN_SSE42 (const char *s, const char *a)
83{
84 if (*a == 0)
85 RETURN (NULL, strlen (s));
86
87 const char *aligned;
88 __m128i mask;
89 int offset = (int) ((size_t) a & 15);
90 if (offset != 0)
91 {
92 /* Load masks. */
93 aligned = (const char *) ((size_t) a & -16L);
94 __m128i mask0 = _mm_load_si128 ((__m128i *) aligned);
95
96 mask = __m128i_shift_right (mask0, offset);
97
98 /* Find where the NULL terminator is. */
99 int length = _mm_cmpistri (mask, mask, 0x3a);
100 if (length == 16 - offset)
101 {
102 /* There is no NULL terminator. */
103 __m128i mask1 = _mm_load_si128 ((__m128i *) (aligned + 16));
104 int index = _mm_cmpistri (mask1, mask1, 0x3a);
105 length += index;
106
107 /* Don't use SSE4.2 if the length of A > 16. */
108 if (length > 16)
109 return STRCSPN_SSE2 (s, a);
110
111 if (index != 0)
112 {
113 /* Combine mask0 and mask1. We could play games with
114 palignr, but frankly this data should be in L1 now
115 so do the merge via an unaligned load. */
116 mask = _mm_loadu_si128 ((__m128i *) a);
117 }
118 }
119 }
120 else
121 {
122 /* A is aligned. */
123 mask = _mm_load_si128 ((__m128i *) a);
124
125 /* Find where the NULL terminator is. */
126 int length = _mm_cmpistri (mask, mask, 0x3a);
127 if (length == 16)
128 {
129 /* There is no NULL terminator. Don't use SSE4.2 if the length
130 of A > 16. */
131 if (a[16] != 0)
132 return STRCSPN_SSE2 (s, a);
133 }
134 }
135
136 offset = (int) ((size_t) s & 15);
137 if (offset != 0)
138 {
139 /* Check partial string. */
140 aligned = (const char *) ((size_t) s & -16L);
141 __m128i value = _mm_load_si128 ((__m128i *) aligned);
142
143 value = __m128i_shift_right (value, offset);
144
145 int length = _mm_cmpistri (mask, value, 0x2);
146 /* No need to check ZFlag since ZFlag is always 1. */
147 int cflag = _mm_cmpistrc (mask, value, 0x2);
148 if (cflag)
149 RETURN ((char *) (s + length), length);
150 /* Find where the NULL terminator is. */
151 int index = _mm_cmpistri (value, value, 0x3a);
152 if (index < 16 - offset)
153 RETURN (NULL, index);
154 aligned += 16;
155 }
156 else
157 aligned = s;
158
159 while (1)
160 {
161 __m128i value = _mm_load_si128 ((__m128i *) aligned);
162 int index = _mm_cmpistri (mask, value, 0x2);
163 int cflag = _mm_cmpistrc (mask, value, 0x2);
164 int zflag = _mm_cmpistrz (mask, value, 0x2);
165 if (cflag)
166 RETURN ((char *) (aligned + index), (size_t) (aligned + index - s));
167 if (zflag)
168 RETURN (NULL,
169 /* Find where the NULL terminator is. */
170 (size_t) (aligned + _mm_cmpistri (value, value, 0x3a) - s));
171 aligned += 16;
172 }
173}
174