1/* strcat(dest, src) -- Append SRC on the end of DEST.
2 Optimized for x86-64.
3 Copyright (C) 2002-2023 Free Software Foundation, Inc.
4 This file is part of the GNU C Library.
5
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
10
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
15
16 You should have received a copy of the GNU Lesser General Public
17 License along with the GNU C Library; if not, see
18 <https://www.gnu.org/licenses/>. */
19
20#include <isa-level.h>
21
22#if ISA_SHOULD_BUILD (1)
23
24# include <sysdep.h>
25
26# ifndef STRCAT
27# define STRCAT __strcat_sse2
28# endif
29
30/* Will be removed when new strcpy implementation gets merged. */
31
32 .text
33ENTRY (STRCAT)
34 movq %rdi, %rcx /* Dest. register. */
35 andl $7, %ecx /* mask alignment bits */
36 movq %rdi, %rax /* Duplicate destination pointer. */
37 movq $0xfefefefefefefeff,%r8
38
39 /* First step: Find end of destination. */
40 jz 4f /* aligned => start loop */
41
42 neg %ecx /* We need to align to 8 bytes. */
43 addl $8,%ecx
44 /* Search the first bytes directly. */
450: cmpb $0x0,(%rax) /* is byte NUL? */
46 je 2f /* yes => start copy */
47 incq %rax /* increment pointer */
48 decl %ecx
49 jnz 0b
50
51
52
53 /* Now the source is aligned. Scan for NUL byte. */
54 .p2align 4
554:
56 /* First unroll. */
57 movq (%rax), %rcx /* get double word (= 8 bytes) in question */
58 addq $8,%rax /* adjust pointer for next word */
59 movq %r8, %rdx /* magic value */
60 addq %rcx, %rdx /* add the magic value to the word. We get
61 carry bits reported for each byte which
62 is *not* 0 */
63 jnc 3f /* highest byte is NUL => return pointer */
64 xorq %rcx, %rdx /* (word+magic)^word */
65 orq %r8, %rdx /* set all non-carry bits */
66 incq %rdx /* add 1: if one carry bit was *not* set
67 the addition will not result in 0. */
68 jnz 3f /* found NUL => return pointer */
69
70 /* Second unroll. */
71 movq (%rax), %rcx /* get double word (= 8 bytes) in question */
72 addq $8,%rax /* adjust pointer for next word */
73 movq %r8, %rdx /* magic value */
74 addq %rcx, %rdx /* add the magic value to the word. We get
75 carry bits reported for each byte which
76 is *not* 0 */
77 jnc 3f /* highest byte is NUL => return pointer */
78 xorq %rcx, %rdx /* (word+magic)^word */
79 orq %r8, %rdx /* set all non-carry bits */
80 incq %rdx /* add 1: if one carry bit was *not* set
81 the addition will not result in 0. */
82 jnz 3f /* found NUL => return pointer */
83
84 /* Third unroll. */
85 movq (%rax), %rcx /* get double word (= 8 bytes) in question */
86 addq $8,%rax /* adjust pointer for next word */
87 movq %r8, %rdx /* magic value */
88 addq %rcx, %rdx /* add the magic value to the word. We get
89 carry bits reported for each byte which
90 is *not* 0 */
91 jnc 3f /* highest byte is NUL => return pointer */
92 xorq %rcx, %rdx /* (word+magic)^word */
93 orq %r8, %rdx /* set all non-carry bits */
94 incq %rdx /* add 1: if one carry bit was *not* set
95 the addition will not result in 0. */
96 jnz 3f /* found NUL => return pointer */
97
98 /* Fourth unroll. */
99 movq (%rax), %rcx /* get double word (= 8 bytes) in question */
100 addq $8,%rax /* adjust pointer for next word */
101 movq %r8, %rdx /* magic value */
102 addq %rcx, %rdx /* add the magic value to the word. We get
103 carry bits reported for each byte which
104 is *not* 0 */
105 jnc 3f /* highest byte is NUL => return pointer */
106 xorq %rcx, %rdx /* (word+magic)^word */
107 orq %r8, %rdx /* set all non-carry bits */
108 incq %rdx /* add 1: if one carry bit was *not* set
109 the addition will not result in 0. */
110 jz 4b /* no NUL found => continue loop */
111
112 .p2align 4 /* Align, it's a jump target. */
1133: subq $8,%rax /* correct pointer increment. */
114
115 testb %cl, %cl /* is first byte NUL? */
116 jz 2f /* yes => return */
117 incq %rax /* increment pointer */
118
119 testb %ch, %ch /* is second byte NUL? */
120 jz 2f /* yes => return */
121 incq %rax /* increment pointer */
122
123 testl $0x00ff0000, %ecx /* is third byte NUL? */
124 jz 2f /* yes => return pointer */
125 incq %rax /* increment pointer */
126
127 testl $0xff000000, %ecx /* is fourth byte NUL? */
128 jz 2f /* yes => return pointer */
129 incq %rax /* increment pointer */
130
131 shrq $32, %rcx /* look at other half. */
132
133 testb %cl, %cl /* is first byte NUL? */
134 jz 2f /* yes => return */
135 incq %rax /* increment pointer */
136
137 testb %ch, %ch /* is second byte NUL? */
138 jz 2f /* yes => return */
139 incq %rax /* increment pointer */
140
141 testl $0xff0000, %ecx /* is third byte NUL? */
142 jz 2f /* yes => return pointer */
143 incq %rax /* increment pointer */
144
1452:
146 /* Second step: Copy source to destination. */
147
148 movq %rsi, %rcx /* duplicate */
149 andl $7,%ecx /* mask alignment bits */
150 movq %rax, %rdx /* move around */
151 jz 22f /* aligned => start loop */
152
153 neg %ecx /* align to 8 bytes. */
154 addl $8, %ecx
155 /* Align the source pointer. */
15621:
157 movb (%rsi), %al /* Fetch a byte */
158 testb %al, %al /* Is it NUL? */
159 movb %al, (%rdx) /* Store it */
160 jz 24f /* If it was NUL, done! */
161 incq %rsi
162 incq %rdx
163 decl %ecx
164 jnz 21b
165
166 /* Now the sources is aligned. Unfortunatly we cannot force
167 to have both source and destination aligned, so ignore the
168 alignment of the destination. */
169 .p2align 4
17022:
171 /* 1st unroll. */
172 movq (%rsi), %rax /* Read double word (8 bytes). */
173 addq $8, %rsi /* Adjust pointer for next word. */
174 movq %rax, %r9 /* Save a copy for NUL finding. */
175 addq %r8, %r9 /* add the magic value to the word. We get
176 carry bits reported for each byte which
177 is *not* 0 */
178 jnc 23f /* highest byte is NUL => return pointer */
179 xorq %rax, %r9 /* (word+magic)^word */
180 orq %r8, %r9 /* set all non-carry bits */
181 incq %r9 /* add 1: if one carry bit was *not* set
182 the addition will not result in 0. */
183
184 jnz 23f /* found NUL => return pointer */
185
186 movq %rax, (%rdx) /* Write value to destination. */
187 addq $8, %rdx /* Adjust pointer. */
188
189 /* 2nd unroll. */
190 movq (%rsi), %rax /* Read double word (8 bytes). */
191 addq $8, %rsi /* Adjust pointer for next word. */
192 movq %rax, %r9 /* Save a copy for NUL finding. */
193 addq %r8, %r9 /* add the magic value to the word. We get
194 carry bits reported for each byte which
195 is *not* 0 */
196 jnc 23f /* highest byte is NUL => return pointer */
197 xorq %rax, %r9 /* (word+magic)^word */
198 orq %r8, %r9 /* set all non-carry bits */
199 incq %r9 /* add 1: if one carry bit was *not* set
200 the addition will not result in 0. */
201
202 jnz 23f /* found NUL => return pointer */
203
204 movq %rax, (%rdx) /* Write value to destination. */
205 addq $8, %rdx /* Adjust pointer. */
206
207 /* 3rd unroll. */
208 movq (%rsi), %rax /* Read double word (8 bytes). */
209 addq $8, %rsi /* Adjust pointer for next word. */
210 movq %rax, %r9 /* Save a copy for NUL finding. */
211 addq %r8, %r9 /* add the magic value to the word. We get
212 carry bits reported for each byte which
213 is *not* 0 */
214 jnc 23f /* highest byte is NUL => return pointer */
215 xorq %rax, %r9 /* (word+magic)^word */
216 orq %r8, %r9 /* set all non-carry bits */
217 incq %r9 /* add 1: if one carry bit was *not* set
218 the addition will not result in 0. */
219
220 jnz 23f /* found NUL => return pointer */
221
222 movq %rax, (%rdx) /* Write value to destination. */
223 addq $8, %rdx /* Adjust pointer. */
224
225 /* 4th unroll. */
226 movq (%rsi), %rax /* Read double word (8 bytes). */
227 addq $8, %rsi /* Adjust pointer for next word. */
228 movq %rax, %r9 /* Save a copy for NUL finding. */
229 addq %r8, %r9 /* add the magic value to the word. We get
230 carry bits reported for each byte which
231 is *not* 0 */
232 jnc 23f /* highest byte is NUL => return pointer */
233 xorq %rax, %r9 /* (word+magic)^word */
234 orq %r8, %r9 /* set all non-carry bits */
235 incq %r9 /* add 1: if one carry bit was *not* set
236 the addition will not result in 0. */
237
238 jnz 23f /* found NUL => return pointer */
239
240 movq %rax, (%rdx) /* Write value to destination. */
241 addq $8, %rdx /* Adjust pointer. */
242 jmp 22b /* Next iteration. */
243
244 /* Do the last few bytes. %rax contains the value to write.
245 The loop is unrolled twice. */
246 .p2align 4
24723:
248 movb %al, (%rdx) /* 1st byte. */
249 testb %al, %al /* Is it NUL. */
250 jz 24f /* yes, finish. */
251 incq %rdx /* Increment destination. */
252 movb %ah, (%rdx) /* 2nd byte. */
253 testb %ah, %ah /* Is it NUL?. */
254 jz 24f /* yes, finish. */
255 incq %rdx /* Increment destination. */
256 shrq $16, %rax /* Shift... */
257 jmp 23b /* and look at next two bytes in %rax. */
258
259
26024:
261 movq %rdi, %rax /* Source is return value. */
262 retq
263END (STRCAT)
264#endif
265