1/* memcmp with SSSE3, wmemcmp with SSSE3
2 Copyright (C) 2011-2017 Free Software Foundation, Inc.
3 Contributed by Intel Corporation.
4 This file is part of the GNU C Library.
5
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
10
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
15
16 You should have received a copy of the GNU Lesser General Public
17 License along with the GNU C Library; if not, see
18 <http://www.gnu.org/licenses/>. */
19
20#if IS_IN (libc)
21
22# include <sysdep.h>
23
24# ifndef MEMCMP
25# define MEMCMP __memcmp_ssse3
26# endif
27
28/* Warning!
29 wmemcmp has to use SIGNED comparison for elements.
30 memcmp has to use UNSIGNED comparison for elemnts.
31*/
32
33 atom_text_section
34ENTRY (MEMCMP)
35# ifdef USE_AS_WMEMCMP
36 shl $2, %rdx
37 test %rdx, %rdx
38 jz L(equal)
39# endif
40 mov %rdx, %rcx
41 mov %rdi, %rdx
42 cmp $48, %rcx;
43 jae L(48bytesormore) /* LEN => 48 */
44
45 add %rcx, %rsi
46 add %rcx, %rdi
47 jmp L(less48bytes)
48
49 .p2align 4
50/* ECX >= 32. */
51L(48bytesormore):
52 movdqu (%rdi), %xmm3
53 movdqu (%rsi), %xmm0
54 pcmpeqb %xmm0, %xmm3
55 pmovmskb %xmm3, %edx
56 lea 16(%rdi), %rdi
57 lea 16(%rsi), %rsi
58 sub $0xffff, %edx
59 jnz L(less16bytes)
60 mov %edi, %edx
61 and $0xf, %edx
62 xor %rdx, %rdi
63 sub %rdx, %rsi
64 add %rdx, %rcx
65 mov %esi, %edx
66 and $0xf, %edx
67 jz L(shr_0)
68 xor %rdx, %rsi
69
70# ifndef USE_AS_WMEMCMP
71 cmp $8, %edx
72 jae L(next_unaligned_table)
73 cmp $0, %edx
74 je L(shr_0)
75 cmp $1, %edx
76 je L(shr_1)
77 cmp $2, %edx
78 je L(shr_2)
79 cmp $3, %edx
80 je L(shr_3)
81 cmp $4, %edx
82 je L(shr_4)
83 cmp $5, %edx
84 je L(shr_5)
85 cmp $6, %edx
86 je L(shr_6)
87 jmp L(shr_7)
88
89 .p2align 2
90L(next_unaligned_table):
91 cmp $8, %edx
92 je L(shr_8)
93 cmp $9, %edx
94 je L(shr_9)
95 cmp $10, %edx
96 je L(shr_10)
97 cmp $11, %edx
98 je L(shr_11)
99 cmp $12, %edx
100 je L(shr_12)
101 cmp $13, %edx
102 je L(shr_13)
103 cmp $14, %edx
104 je L(shr_14)
105 jmp L(shr_15)
106# else
107 cmp $0, %edx
108 je L(shr_0)
109 cmp $4, %edx
110 je L(shr_4)
111 cmp $8, %edx
112 je L(shr_8)
113 jmp L(shr_12)
114# endif
115
116 .p2align 4
117L(shr_0):
118 cmp $80, %rcx
119 lea -48(%rcx), %rcx
120 jae L(shr_0_gobble)
121 xor %eax, %eax
122 movdqa (%rsi), %xmm1
123 pcmpeqb (%rdi), %xmm1
124 movdqa 16(%rsi), %xmm2
125 pcmpeqb 16(%rdi), %xmm2
126 pand %xmm1, %xmm2
127 pmovmskb %xmm2, %edx
128 lea 32(%rdi), %rdi
129 lea 32(%rsi), %rsi
130 sub $0xffff, %edx
131 jnz L(exit)
132 add %rcx, %rsi
133 add %rcx, %rdi
134 jmp L(less48bytes)
135
136 .p2align 4
137L(shr_0_gobble):
138 movdqa (%rsi), %xmm0
139 xor %eax, %eax
140 pcmpeqb (%rdi), %xmm0
141 sub $32, %rcx
142 movdqa 16(%rsi), %xmm2
143 pcmpeqb 16(%rdi), %xmm2
144L(shr_0_gobble_loop):
145 pand %xmm0, %xmm2
146 sub $32, %rcx
147 pmovmskb %xmm2, %edx
148 movdqa %xmm0, %xmm1
149 movdqa 32(%rsi), %xmm0
150 movdqa 48(%rsi), %xmm2
151 sbb $0xffff, %edx
152 pcmpeqb 32(%rdi), %xmm0
153 pcmpeqb 48(%rdi), %xmm2
154 lea 32(%rdi), %rdi
155 lea 32(%rsi), %rsi
156 jz L(shr_0_gobble_loop)
157
158 pand %xmm0, %xmm2
159 cmp $0, %rcx
160 jge L(next)
161 inc %edx
162 add $32, %rcx
163L(next):
164 test %edx, %edx
165 jnz L(exit)
166
167 pmovmskb %xmm2, %edx
168 movdqa %xmm0, %xmm1
169 lea 32(%rdi), %rdi
170 lea 32(%rsi), %rsi
171 sub $0xffff, %edx
172 jnz L(exit)
173 add %rcx, %rsi
174 add %rcx, %rdi
175 jmp L(less48bytes)
176
177# ifndef USE_AS_WMEMCMP
178
179 .p2align 4
180L(shr_1):
181 cmp $80, %rcx
182 lea -48(%rcx), %rcx
183 mov %edx, %eax
184 jae L(shr_1_gobble)
185
186 movdqa 16(%rsi), %xmm1
187 movdqa %xmm1, %xmm2
188 palignr $1, (%rsi), %xmm1
189 pcmpeqb (%rdi), %xmm1
190
191 movdqa 32(%rsi), %xmm3
192 palignr $1, %xmm2, %xmm3
193 pcmpeqb 16(%rdi), %xmm3
194
195 pand %xmm1, %xmm3
196 pmovmskb %xmm3, %edx
197 lea 32(%rdi), %rdi
198 lea 32(%rsi), %rsi
199 sub $0xffff, %edx
200 jnz L(exit)
201 add $1, %rsi
202 add %rcx, %rsi
203 add %rcx, %rdi
204 jmp L(less48bytes)
205
206 .p2align 4
207L(shr_1_gobble):
208 sub $32, %rcx
209 movdqa 16(%rsi), %xmm0
210 palignr $1, (%rsi), %xmm0
211 pcmpeqb (%rdi), %xmm0
212
213 movdqa 32(%rsi), %xmm3
214 palignr $1, 16(%rsi), %xmm3
215 pcmpeqb 16(%rdi), %xmm3
216
217L(shr_1_gobble_loop):
218 pand %xmm0, %xmm3
219 sub $32, %rcx
220 pmovmskb %xmm3, %edx
221 movdqa %xmm0, %xmm1
222
223 movdqa 64(%rsi), %xmm3
224 palignr $1, 48(%rsi), %xmm3
225 sbb $0xffff, %edx
226 movdqa 48(%rsi), %xmm0
227 palignr $1, 32(%rsi), %xmm0
228 pcmpeqb 32(%rdi), %xmm0
229 lea 32(%rsi), %rsi
230 pcmpeqb 48(%rdi), %xmm3
231
232 lea 32(%rdi), %rdi
233 jz L(shr_1_gobble_loop)
234 pand %xmm0, %xmm3
235
236 cmp $0, %rcx
237 jge L(shr_1_gobble_next)
238 inc %edx
239 add $32, %rcx
240L(shr_1_gobble_next):
241 test %edx, %edx
242 jnz L(exit)
243
244 pmovmskb %xmm3, %edx
245 movdqa %xmm0, %xmm1
246 lea 32(%rdi), %rdi
247 lea 32(%rsi), %rsi
248 sub $0xffff, %edx
249 jnz L(exit)
250
251 lea 1(%rsi), %rsi
252 add %rcx, %rsi
253 add %rcx, %rdi
254 jmp L(less48bytes)
255
256
257 .p2align 4
258L(shr_2):
259 cmp $80, %rcx
260 lea -48(%rcx), %rcx
261 mov %edx, %eax
262 jae L(shr_2_gobble)
263
264 movdqa 16(%rsi), %xmm1
265 movdqa %xmm1, %xmm2
266 palignr $2, (%rsi), %xmm1
267 pcmpeqb (%rdi), %xmm1
268
269 movdqa 32(%rsi), %xmm3
270 palignr $2, %xmm2, %xmm3
271 pcmpeqb 16(%rdi), %xmm3
272
273 pand %xmm1, %xmm3
274 pmovmskb %xmm3, %edx
275 lea 32(%rdi), %rdi
276 lea 32(%rsi), %rsi
277 sub $0xffff, %edx
278 jnz L(exit)
279 add $2, %rsi
280 add %rcx, %rsi
281 add %rcx, %rdi
282 jmp L(less48bytes)
283
284 .p2align 4
285L(shr_2_gobble):
286 sub $32, %rcx
287 movdqa 16(%rsi), %xmm0
288 palignr $2, (%rsi), %xmm0
289 pcmpeqb (%rdi), %xmm0
290
291 movdqa 32(%rsi), %xmm3
292 palignr $2, 16(%rsi), %xmm3
293 pcmpeqb 16(%rdi), %xmm3
294
295L(shr_2_gobble_loop):
296 pand %xmm0, %xmm3
297 sub $32, %rcx
298 pmovmskb %xmm3, %edx
299 movdqa %xmm0, %xmm1
300
301 movdqa 64(%rsi), %xmm3
302 palignr $2, 48(%rsi), %xmm3
303 sbb $0xffff, %edx
304 movdqa 48(%rsi), %xmm0
305 palignr $2, 32(%rsi), %xmm0
306 pcmpeqb 32(%rdi), %xmm0
307 lea 32(%rsi), %rsi
308 pcmpeqb 48(%rdi), %xmm3
309
310 lea 32(%rdi), %rdi
311 jz L(shr_2_gobble_loop)
312 pand %xmm0, %xmm3
313
314 cmp $0, %rcx
315 jge L(shr_2_gobble_next)
316 inc %edx
317 add $32, %rcx
318L(shr_2_gobble_next):
319 test %edx, %edx
320 jnz L(exit)
321
322 pmovmskb %xmm3, %edx
323 movdqa %xmm0, %xmm1
324 lea 32(%rdi), %rdi
325 lea 32(%rsi), %rsi
326 sub $0xffff, %edx
327 jnz L(exit)
328
329 lea 2(%rsi), %rsi
330 add %rcx, %rsi
331 add %rcx, %rdi
332 jmp L(less48bytes)
333
334 .p2align 4
335L(shr_3):
336 cmp $80, %rcx
337 lea -48(%rcx), %rcx
338 mov %edx, %eax
339 jae L(shr_3_gobble)
340
341 movdqa 16(%rsi), %xmm1
342 movdqa %xmm1, %xmm2
343 palignr $3, (%rsi), %xmm1
344 pcmpeqb (%rdi), %xmm1
345
346 movdqa 32(%rsi), %xmm3
347 palignr $3, %xmm2, %xmm3
348 pcmpeqb 16(%rdi), %xmm3
349
350 pand %xmm1, %xmm3
351 pmovmskb %xmm3, %edx
352 lea 32(%rdi), %rdi
353 lea 32(%rsi), %rsi
354 sub $0xffff, %edx
355 jnz L(exit)
356 add $3, %rsi
357 add %rcx, %rsi
358 add %rcx, %rdi
359 jmp L(less48bytes)
360
361 .p2align 4
362L(shr_3_gobble):
363 sub $32, %rcx
364 movdqa 16(%rsi), %xmm0
365 palignr $3, (%rsi), %xmm0
366 pcmpeqb (%rdi), %xmm0
367
368 movdqa 32(%rsi), %xmm3
369 palignr $3, 16(%rsi), %xmm3
370 pcmpeqb 16(%rdi), %xmm3
371
372L(shr_3_gobble_loop):
373 pand %xmm0, %xmm3
374 sub $32, %rcx
375 pmovmskb %xmm3, %edx
376 movdqa %xmm0, %xmm1
377
378 movdqa 64(%rsi), %xmm3
379 palignr $3, 48(%rsi), %xmm3
380 sbb $0xffff, %edx
381 movdqa 48(%rsi), %xmm0
382 palignr $3, 32(%rsi), %xmm0
383 pcmpeqb 32(%rdi), %xmm0
384 lea 32(%rsi), %rsi
385 pcmpeqb 48(%rdi), %xmm3
386
387 lea 32(%rdi), %rdi
388 jz L(shr_3_gobble_loop)
389 pand %xmm0, %xmm3
390
391 cmp $0, %rcx
392 jge L(shr_3_gobble_next)
393 inc %edx
394 add $32, %rcx
395L(shr_3_gobble_next):
396 test %edx, %edx
397 jnz L(exit)
398
399 pmovmskb %xmm3, %edx
400 movdqa %xmm0, %xmm1
401 lea 32(%rdi), %rdi
402 lea 32(%rsi), %rsi
403 sub $0xffff, %edx
404 jnz L(exit)
405
406 lea 3(%rsi), %rsi
407 add %rcx, %rsi
408 add %rcx, %rdi
409 jmp L(less48bytes)
410
411# endif
412
413 .p2align 4
414L(shr_4):
415 cmp $80, %rcx
416 lea -48(%rcx), %rcx
417 mov %edx, %eax
418 jae L(shr_4_gobble)
419
420 movdqa 16(%rsi), %xmm1
421 movdqa %xmm1, %xmm2
422 palignr $4, (%rsi), %xmm1
423 pcmpeqb (%rdi), %xmm1
424
425 movdqa 32(%rsi), %xmm3
426 palignr $4, %xmm2, %xmm3
427 pcmpeqb 16(%rdi), %xmm3
428
429 pand %xmm1, %xmm3
430 pmovmskb %xmm3, %edx
431 lea 32(%rdi), %rdi
432 lea 32(%rsi), %rsi
433 sub $0xffff, %edx
434 jnz L(exit)
435 add $4, %rsi
436 add %rcx, %rsi
437 add %rcx, %rdi
438 jmp L(less48bytes)
439
440 .p2align 4
441L(shr_4_gobble):
442 sub $32, %rcx
443 movdqa 16(%rsi), %xmm0
444 palignr $4, (%rsi), %xmm0
445 pcmpeqb (%rdi), %xmm0
446
447 movdqa 32(%rsi), %xmm3
448 palignr $4, 16(%rsi), %xmm3
449 pcmpeqb 16(%rdi), %xmm3
450
451L(shr_4_gobble_loop):
452 pand %xmm0, %xmm3
453 sub $32, %rcx
454 pmovmskb %xmm3, %edx
455 movdqa %xmm0, %xmm1
456
457 movdqa 64(%rsi), %xmm3
458 palignr $4, 48(%rsi), %xmm3
459 sbb $0xffff, %edx
460 movdqa 48(%rsi), %xmm0
461 palignr $4, 32(%rsi), %xmm0
462 pcmpeqb 32(%rdi), %xmm0
463 lea 32(%rsi), %rsi
464 pcmpeqb 48(%rdi), %xmm3
465
466 lea 32(%rdi), %rdi
467 jz L(shr_4_gobble_loop)
468 pand %xmm0, %xmm3
469
470 cmp $0, %rcx
471 jge L(shr_4_gobble_next)
472 inc %edx
473 add $32, %rcx
474L(shr_4_gobble_next):
475 test %edx, %edx
476 jnz L(exit)
477
478 pmovmskb %xmm3, %edx
479 movdqa %xmm0, %xmm1
480 lea 32(%rdi), %rdi
481 lea 32(%rsi), %rsi
482 sub $0xffff, %edx
483 jnz L(exit)
484
485 lea 4(%rsi), %rsi
486 add %rcx, %rsi
487 add %rcx, %rdi
488 jmp L(less48bytes)
489
490# ifndef USE_AS_WMEMCMP
491
492 .p2align 4
493L(shr_5):
494 cmp $80, %rcx
495 lea -48(%rcx), %rcx
496 mov %edx, %eax
497 jae L(shr_5_gobble)
498
499 movdqa 16(%rsi), %xmm1
500 movdqa %xmm1, %xmm2
501 palignr $5, (%rsi), %xmm1
502 pcmpeqb (%rdi), %xmm1
503
504 movdqa 32(%rsi), %xmm3
505 palignr $5, %xmm2, %xmm3
506 pcmpeqb 16(%rdi), %xmm3
507
508 pand %xmm1, %xmm3
509 pmovmskb %xmm3, %edx
510 lea 32(%rdi), %rdi
511 lea 32(%rsi), %rsi
512 sub $0xffff, %edx
513 jnz L(exit)
514 add $5, %rsi
515 add %rcx, %rsi
516 add %rcx, %rdi
517 jmp L(less48bytes)
518
519 .p2align 4
520L(shr_5_gobble):
521 sub $32, %rcx
522 movdqa 16(%rsi), %xmm0
523 palignr $5, (%rsi), %xmm0
524 pcmpeqb (%rdi), %xmm0
525
526 movdqa 32(%rsi), %xmm3
527 palignr $5, 16(%rsi), %xmm3
528 pcmpeqb 16(%rdi), %xmm3
529
530L(shr_5_gobble_loop):
531 pand %xmm0, %xmm3
532 sub $32, %rcx
533 pmovmskb %xmm3, %edx
534 movdqa %xmm0, %xmm1
535
536 movdqa 64(%rsi), %xmm3
537 palignr $5, 48(%rsi), %xmm3
538 sbb $0xffff, %edx
539 movdqa 48(%rsi), %xmm0
540 palignr $5, 32(%rsi), %xmm0
541 pcmpeqb 32(%rdi), %xmm0
542 lea 32(%rsi), %rsi
543 pcmpeqb 48(%rdi), %xmm3
544
545 lea 32(%rdi), %rdi
546 jz L(shr_5_gobble_loop)
547 pand %xmm0, %xmm3
548
549 cmp $0, %rcx
550 jge L(shr_5_gobble_next)
551 inc %edx
552 add $32, %rcx
553L(shr_5_gobble_next):
554 test %edx, %edx
555 jnz L(exit)
556
557 pmovmskb %xmm3, %edx
558 movdqa %xmm0, %xmm1
559 lea 32(%rdi), %rdi
560 lea 32(%rsi), %rsi
561 sub $0xffff, %edx
562 jnz L(exit)
563
564 lea 5(%rsi), %rsi
565 add %rcx, %rsi
566 add %rcx, %rdi
567 jmp L(less48bytes)
568
569 .p2align 4
570L(shr_6):
571 cmp $80, %rcx
572 lea -48(%rcx), %rcx
573 mov %edx, %eax
574 jae L(shr_6_gobble)
575
576 movdqa 16(%rsi), %xmm1
577 movdqa %xmm1, %xmm2
578 palignr $6, (%rsi), %xmm1
579 pcmpeqb (%rdi), %xmm1
580
581 movdqa 32(%rsi), %xmm3
582 palignr $6, %xmm2, %xmm3
583 pcmpeqb 16(%rdi), %xmm3
584
585 pand %xmm1, %xmm3
586 pmovmskb %xmm3, %edx
587 lea 32(%rdi), %rdi
588 lea 32(%rsi), %rsi
589 sub $0xffff, %edx
590 jnz L(exit)
591 add $6, %rsi
592 add %rcx, %rsi
593 add %rcx, %rdi
594 jmp L(less48bytes)
595
596 .p2align 4
597L(shr_6_gobble):
598 sub $32, %rcx
599 movdqa 16(%rsi), %xmm0
600 palignr $6, (%rsi), %xmm0
601 pcmpeqb (%rdi), %xmm0
602
603 movdqa 32(%rsi), %xmm3
604 palignr $6, 16(%rsi), %xmm3
605 pcmpeqb 16(%rdi), %xmm3
606
607L(shr_6_gobble_loop):
608 pand %xmm0, %xmm3
609 sub $32, %rcx
610 pmovmskb %xmm3, %edx
611 movdqa %xmm0, %xmm1
612
613 movdqa 64(%rsi), %xmm3
614 palignr $6, 48(%rsi), %xmm3
615 sbb $0xffff, %edx
616 movdqa 48(%rsi), %xmm0
617 palignr $6, 32(%rsi), %xmm0
618 pcmpeqb 32(%rdi), %xmm0
619 lea 32(%rsi), %rsi
620 pcmpeqb 48(%rdi), %xmm3
621
622 lea 32(%rdi), %rdi
623 jz L(shr_6_gobble_loop)
624 pand %xmm0, %xmm3
625
626 cmp $0, %rcx
627 jge L(shr_6_gobble_next)
628 inc %edx
629 add $32, %rcx
630L(shr_6_gobble_next):
631 test %edx, %edx
632 jnz L(exit)
633
634 pmovmskb %xmm3, %edx
635 movdqa %xmm0, %xmm1
636 lea 32(%rdi), %rdi
637 lea 32(%rsi), %rsi
638 sub $0xffff, %edx
639 jnz L(exit)
640
641 lea 6(%rsi), %rsi
642 add %rcx, %rsi
643 add %rcx, %rdi
644 jmp L(less48bytes)
645
646 .p2align 4
647L(shr_7):
648 cmp $80, %rcx
649 lea -48(%rcx), %rcx
650 mov %edx, %eax
651 jae L(shr_7_gobble)
652
653 movdqa 16(%rsi), %xmm1
654 movdqa %xmm1, %xmm2
655 palignr $7, (%rsi), %xmm1
656 pcmpeqb (%rdi), %xmm1
657
658 movdqa 32(%rsi), %xmm3
659 palignr $7, %xmm2, %xmm3
660 pcmpeqb 16(%rdi), %xmm3
661
662 pand %xmm1, %xmm3
663 pmovmskb %xmm3, %edx
664 lea 32(%rdi), %rdi
665 lea 32(%rsi), %rsi
666 sub $0xffff, %edx
667 jnz L(exit)
668 add $7, %rsi
669 add %rcx, %rsi
670 add %rcx, %rdi
671 jmp L(less48bytes)
672
673 .p2align 4
674L(shr_7_gobble):
675 sub $32, %rcx
676 movdqa 16(%rsi), %xmm0
677 palignr $7, (%rsi), %xmm0
678 pcmpeqb (%rdi), %xmm0
679
680 movdqa 32(%rsi), %xmm3
681 palignr $7, 16(%rsi), %xmm3
682 pcmpeqb 16(%rdi), %xmm3
683
684L(shr_7_gobble_loop):
685 pand %xmm0, %xmm3
686 sub $32, %rcx
687 pmovmskb %xmm3, %edx
688 movdqa %xmm0, %xmm1
689
690 movdqa 64(%rsi), %xmm3
691 palignr $7, 48(%rsi), %xmm3
692 sbb $0xffff, %edx
693 movdqa 48(%rsi), %xmm0
694 palignr $7, 32(%rsi), %xmm0
695 pcmpeqb 32(%rdi), %xmm0
696 lea 32(%rsi), %rsi
697 pcmpeqb 48(%rdi), %xmm3
698
699 lea 32(%rdi), %rdi
700 jz L(shr_7_gobble_loop)
701 pand %xmm0, %xmm3
702
703 cmp $0, %rcx
704 jge L(shr_7_gobble_next)
705 inc %edx
706 add $32, %rcx
707L(shr_7_gobble_next):
708 test %edx, %edx
709 jnz L(exit)
710
711 pmovmskb %xmm3, %edx
712 movdqa %xmm0, %xmm1
713 lea 32(%rdi), %rdi
714 lea 32(%rsi), %rsi
715 sub $0xffff, %edx
716 jnz L(exit)
717
718 lea 7(%rsi), %rsi
719 add %rcx, %rsi
720 add %rcx, %rdi
721 jmp L(less48bytes)
722
723# endif
724
725 .p2align 4
726L(shr_8):
727 cmp $80, %rcx
728 lea -48(%rcx), %rcx
729 mov %edx, %eax
730 jae L(shr_8_gobble)
731
732 movdqa 16(%rsi), %xmm1
733 movdqa %xmm1, %xmm2
734 palignr $8, (%rsi), %xmm1
735 pcmpeqb (%rdi), %xmm1
736
737 movdqa 32(%rsi), %xmm3
738 palignr $8, %xmm2, %xmm3
739 pcmpeqb 16(%rdi), %xmm3
740
741 pand %xmm1, %xmm3
742 pmovmskb %xmm3, %edx
743 lea 32(%rdi), %rdi
744 lea 32(%rsi), %rsi
745 sub $0xffff, %edx
746 jnz L(exit)
747 add $8, %rsi
748 add %rcx, %rsi
749 add %rcx, %rdi
750 jmp L(less48bytes)
751
752 .p2align 4
753L(shr_8_gobble):
754 sub $32, %rcx
755 movdqa 16(%rsi), %xmm0
756 palignr $8, (%rsi), %xmm0
757 pcmpeqb (%rdi), %xmm0
758
759 movdqa 32(%rsi), %xmm3
760 palignr $8, 16(%rsi), %xmm3
761 pcmpeqb 16(%rdi), %xmm3
762
763L(shr_8_gobble_loop):
764 pand %xmm0, %xmm3
765 sub $32, %rcx
766 pmovmskb %xmm3, %edx
767 movdqa %xmm0, %xmm1
768
769 movdqa 64(%rsi), %xmm3
770 palignr $8, 48(%rsi), %xmm3
771 sbb $0xffff, %edx
772 movdqa 48(%rsi), %xmm0
773 palignr $8, 32(%rsi), %xmm0
774 pcmpeqb 32(%rdi), %xmm0
775 lea 32(%rsi), %rsi
776 pcmpeqb 48(%rdi), %xmm3
777
778 lea 32(%rdi), %rdi
779 jz L(shr_8_gobble_loop)
780 pand %xmm0, %xmm3
781
782 cmp $0, %rcx
783 jge L(shr_8_gobble_next)
784 inc %edx
785 add $32, %rcx
786L(shr_8_gobble_next):
787 test %edx, %edx
788 jnz L(exit)
789
790 pmovmskb %xmm3, %edx
791 movdqa %xmm0, %xmm1
792 lea 32(%rdi), %rdi
793 lea 32(%rsi), %rsi
794 sub $0xffff, %edx
795 jnz L(exit)
796
797 lea 8(%rsi), %rsi
798 add %rcx, %rsi
799 add %rcx, %rdi
800 jmp L(less48bytes)
801
802# ifndef USE_AS_WMEMCMP
803
804 .p2align 4
805L(shr_9):
806 cmp $80, %rcx
807 lea -48(%rcx), %rcx
808 mov %edx, %eax
809 jae L(shr_9_gobble)
810
811 movdqa 16(%rsi), %xmm1
812 movdqa %xmm1, %xmm2
813 palignr $9, (%rsi), %xmm1
814 pcmpeqb (%rdi), %xmm1
815
816 movdqa 32(%rsi), %xmm3
817 palignr $9, %xmm2, %xmm3
818 pcmpeqb 16(%rdi), %xmm3
819
820 pand %xmm1, %xmm3
821 pmovmskb %xmm3, %edx
822 lea 32(%rdi), %rdi
823 lea 32(%rsi), %rsi
824 sub $0xffff, %edx
825 jnz L(exit)
826 add $9, %rsi
827 add %rcx, %rsi
828 add %rcx, %rdi
829 jmp L(less48bytes)
830
831 .p2align 4
832L(shr_9_gobble):
833 sub $32, %rcx
834 movdqa 16(%rsi), %xmm0
835 palignr $9, (%rsi), %xmm0
836 pcmpeqb (%rdi), %xmm0
837
838 movdqa 32(%rsi), %xmm3
839 palignr $9, 16(%rsi), %xmm3
840 pcmpeqb 16(%rdi), %xmm3
841
842L(shr_9_gobble_loop):
843 pand %xmm0, %xmm3
844 sub $32, %rcx
845 pmovmskb %xmm3, %edx
846 movdqa %xmm0, %xmm1
847
848 movdqa 64(%rsi), %xmm3
849 palignr $9, 48(%rsi), %xmm3
850 sbb $0xffff, %edx
851 movdqa 48(%rsi), %xmm0
852 palignr $9, 32(%rsi), %xmm0
853 pcmpeqb 32(%rdi), %xmm0
854 lea 32(%rsi), %rsi
855 pcmpeqb 48(%rdi), %xmm3
856
857 lea 32(%rdi), %rdi
858 jz L(shr_9_gobble_loop)
859 pand %xmm0, %xmm3
860
861 cmp $0, %rcx
862 jge L(shr_9_gobble_next)
863 inc %edx
864 add $32, %rcx
865L(shr_9_gobble_next):
866 test %edx, %edx
867 jnz L(exit)
868
869 pmovmskb %xmm3, %edx
870 movdqa %xmm0, %xmm1
871 lea 32(%rdi), %rdi
872 lea 32(%rsi), %rsi
873 sub $0xffff, %edx
874 jnz L(exit)
875
876 lea 9(%rsi), %rsi
877 add %rcx, %rsi
878 add %rcx, %rdi
879 jmp L(less48bytes)
880
881 .p2align 4
882L(shr_10):
883 cmp $80, %rcx
884 lea -48(%rcx), %rcx
885 mov %edx, %eax
886 jae L(shr_10_gobble)
887
888 movdqa 16(%rsi), %xmm1
889 movdqa %xmm1, %xmm2
890 palignr $10, (%rsi), %xmm1
891 pcmpeqb (%rdi), %xmm1
892
893 movdqa 32(%rsi), %xmm3
894 palignr $10, %xmm2, %xmm3
895 pcmpeqb 16(%rdi), %xmm3
896
897 pand %xmm1, %xmm3
898 pmovmskb %xmm3, %edx
899 lea 32(%rdi), %rdi
900 lea 32(%rsi), %rsi
901 sub $0xffff, %edx
902 jnz L(exit)
903 add $10, %rsi
904 add %rcx, %rsi
905 add %rcx, %rdi
906 jmp L(less48bytes)
907
908 .p2align 4
909L(shr_10_gobble):
910 sub $32, %rcx
911 movdqa 16(%rsi), %xmm0
912 palignr $10, (%rsi), %xmm0
913 pcmpeqb (%rdi), %xmm0
914
915 movdqa 32(%rsi), %xmm3
916 palignr $10, 16(%rsi), %xmm3
917 pcmpeqb 16(%rdi), %xmm3
918
919L(shr_10_gobble_loop):
920 pand %xmm0, %xmm3
921 sub $32, %rcx
922 pmovmskb %xmm3, %edx
923 movdqa %xmm0, %xmm1
924
925 movdqa 64(%rsi), %xmm3
926 palignr $10, 48(%rsi), %xmm3
927 sbb $0xffff, %edx
928 movdqa 48(%rsi), %xmm0
929 palignr $10, 32(%rsi), %xmm0
930 pcmpeqb 32(%rdi), %xmm0
931 lea 32(%rsi), %rsi
932 pcmpeqb 48(%rdi), %xmm3
933
934 lea 32(%rdi), %rdi
935 jz L(shr_10_gobble_loop)
936 pand %xmm0, %xmm3
937
938 cmp $0, %rcx
939 jge L(shr_10_gobble_next)
940 inc %edx
941 add $32, %rcx
942L(shr_10_gobble_next):
943 test %edx, %edx
944 jnz L(exit)
945
946 pmovmskb %xmm3, %edx
947 movdqa %xmm0, %xmm1
948 lea 32(%rdi), %rdi
949 lea 32(%rsi), %rsi
950 sub $0xffff, %edx
951 jnz L(exit)
952
953 lea 10(%rsi), %rsi
954 add %rcx, %rsi
955 add %rcx, %rdi
956 jmp L(less48bytes)
957
958 .p2align 4
959L(shr_11):
960 cmp $80, %rcx
961 lea -48(%rcx), %rcx
962 mov %edx, %eax
963 jae L(shr_11_gobble)
964
965 movdqa 16(%rsi), %xmm1
966 movdqa %xmm1, %xmm2
967 palignr $11, (%rsi), %xmm1
968 pcmpeqb (%rdi), %xmm1
969
970 movdqa 32(%rsi), %xmm3
971 palignr $11, %xmm2, %xmm3
972 pcmpeqb 16(%rdi), %xmm3
973
974 pand %xmm1, %xmm3
975 pmovmskb %xmm3, %edx
976 lea 32(%rdi), %rdi
977 lea 32(%rsi), %rsi
978 sub $0xffff, %edx
979 jnz L(exit)
980 add $11, %rsi
981 add %rcx, %rsi
982 add %rcx, %rdi
983 jmp L(less48bytes)
984
985 .p2align 4
986L(shr_11_gobble):
987 sub $32, %rcx
988 movdqa 16(%rsi), %xmm0
989 palignr $11, (%rsi), %xmm0
990 pcmpeqb (%rdi), %xmm0
991
992 movdqa 32(%rsi), %xmm3
993 palignr $11, 16(%rsi), %xmm3
994 pcmpeqb 16(%rdi), %xmm3
995
996L(shr_11_gobble_loop):
997 pand %xmm0, %xmm3
998 sub $32, %rcx
999 pmovmskb %xmm3, %edx
1000 movdqa %xmm0, %xmm1
1001
1002 movdqa 64(%rsi), %xmm3
1003 palignr $11, 48(%rsi), %xmm3
1004 sbb $0xffff, %edx
1005 movdqa 48(%rsi), %xmm0
1006 palignr $11, 32(%rsi), %xmm0
1007 pcmpeqb 32(%rdi), %xmm0
1008 lea 32(%rsi), %rsi
1009 pcmpeqb 48(%rdi), %xmm3
1010
1011 lea 32(%rdi), %rdi
1012 jz L(shr_11_gobble_loop)
1013 pand %xmm0, %xmm3
1014
1015 cmp $0, %rcx
1016 jge L(shr_11_gobble_next)
1017 inc %edx
1018 add $32, %rcx
1019L(shr_11_gobble_next):
1020 test %edx, %edx
1021 jnz L(exit)
1022
1023 pmovmskb %xmm3, %edx
1024 movdqa %xmm0, %xmm1
1025 lea 32(%rdi), %rdi
1026 lea 32(%rsi), %rsi
1027 sub $0xffff, %edx
1028 jnz L(exit)
1029
1030 lea 11(%rsi), %rsi
1031 add %rcx, %rsi
1032 add %rcx, %rdi
1033 jmp L(less48bytes)
1034
1035# endif
1036
1037 .p2align 4
1038L(shr_12):
1039 cmp $80, %rcx
1040 lea -48(%rcx), %rcx
1041 mov %edx, %eax
1042 jae L(shr_12_gobble)
1043
1044 movdqa 16(%rsi), %xmm1
1045 movdqa %xmm1, %xmm2
1046 palignr $12, (%rsi), %xmm1
1047 pcmpeqb (%rdi), %xmm1
1048
1049 movdqa 32(%rsi), %xmm3
1050 palignr $12, %xmm2, %xmm3
1051 pcmpeqb 16(%rdi), %xmm3
1052
1053 pand %xmm1, %xmm3
1054 pmovmskb %xmm3, %edx
1055 lea 32(%rdi), %rdi
1056 lea 32(%rsi), %rsi
1057 sub $0xffff, %edx
1058 jnz L(exit)
1059 add $12, %rsi
1060 add %rcx, %rsi
1061 add %rcx, %rdi
1062 jmp L(less48bytes)
1063
1064 .p2align 4
1065L(shr_12_gobble):
1066 sub $32, %rcx
1067 movdqa 16(%rsi), %xmm0
1068 palignr $12, (%rsi), %xmm0
1069 pcmpeqb (%rdi), %xmm0
1070
1071 movdqa 32(%rsi), %xmm3
1072 palignr $12, 16(%rsi), %xmm3
1073 pcmpeqb 16(%rdi), %xmm3
1074
1075L(shr_12_gobble_loop):
1076 pand %xmm0, %xmm3
1077 sub $32, %rcx
1078 pmovmskb %xmm3, %edx
1079 movdqa %xmm0, %xmm1
1080
1081 movdqa 64(%rsi), %xmm3
1082 palignr $12, 48(%rsi), %xmm3
1083 sbb $0xffff, %edx
1084 movdqa 48(%rsi), %xmm0
1085 palignr $12, 32(%rsi), %xmm0
1086 pcmpeqb 32(%rdi), %xmm0
1087 lea 32(%rsi), %rsi
1088 pcmpeqb 48(%rdi), %xmm3
1089
1090 lea 32(%rdi), %rdi
1091 jz L(shr_12_gobble_loop)
1092 pand %xmm0, %xmm3
1093
1094 cmp $0, %rcx
1095 jge L(shr_12_gobble_next)
1096 inc %edx
1097 add $32, %rcx
1098L(shr_12_gobble_next):
1099 test %edx, %edx
1100 jnz L(exit)
1101
1102 pmovmskb %xmm3, %edx
1103 movdqa %xmm0, %xmm1
1104 lea 32(%rdi), %rdi
1105 lea 32(%rsi), %rsi
1106 sub $0xffff, %edx
1107 jnz L(exit)
1108
1109 lea 12(%rsi), %rsi
1110 add %rcx, %rsi
1111 add %rcx, %rdi
1112 jmp L(less48bytes)
1113
1114# ifndef USE_AS_WMEMCMP
1115
1116 .p2align 4
1117L(shr_13):
1118 cmp $80, %rcx
1119 lea -48(%rcx), %rcx
1120 mov %edx, %eax
1121 jae L(shr_13_gobble)
1122
1123 movdqa 16(%rsi), %xmm1
1124 movdqa %xmm1, %xmm2
1125 palignr $13, (%rsi), %xmm1
1126 pcmpeqb (%rdi), %xmm1
1127
1128 movdqa 32(%rsi), %xmm3
1129 palignr $13, %xmm2, %xmm3
1130 pcmpeqb 16(%rdi), %xmm3
1131
1132 pand %xmm1, %xmm3
1133 pmovmskb %xmm3, %edx
1134 lea 32(%rdi), %rdi
1135 lea 32(%rsi), %rsi
1136 sub $0xffff, %edx
1137 jnz L(exit)
1138 add $13, %rsi
1139 add %rcx, %rsi
1140 add %rcx, %rdi
1141 jmp L(less48bytes)
1142
1143 .p2align 4
1144L(shr_13_gobble):
1145 sub $32, %rcx
1146 movdqa 16(%rsi), %xmm0
1147 palignr $13, (%rsi), %xmm0
1148 pcmpeqb (%rdi), %xmm0
1149
1150 movdqa 32(%rsi), %xmm3
1151 palignr $13, 16(%rsi), %xmm3
1152 pcmpeqb 16(%rdi), %xmm3
1153
1154L(shr_13_gobble_loop):
1155 pand %xmm0, %xmm3
1156 sub $32, %rcx
1157 pmovmskb %xmm3, %edx
1158 movdqa %xmm0, %xmm1
1159
1160 movdqa 64(%rsi), %xmm3
1161 palignr $13, 48(%rsi), %xmm3
1162 sbb $0xffff, %edx
1163 movdqa 48(%rsi), %xmm0
1164 palignr $13, 32(%rsi), %xmm0
1165 pcmpeqb 32(%rdi), %xmm0
1166 lea 32(%rsi), %rsi
1167 pcmpeqb 48(%rdi), %xmm3
1168
1169 lea 32(%rdi), %rdi
1170 jz L(shr_13_gobble_loop)
1171 pand %xmm0, %xmm3
1172
1173 cmp $0, %rcx
1174 jge L(shr_13_gobble_next)
1175 inc %edx
1176 add $32, %rcx
1177L(shr_13_gobble_next):
1178 test %edx, %edx
1179 jnz L(exit)
1180
1181 pmovmskb %xmm3, %edx
1182 movdqa %xmm0, %xmm1
1183 lea 32(%rdi), %rdi
1184 lea 32(%rsi), %rsi
1185 sub $0xffff, %edx
1186 jnz L(exit)
1187
1188 lea 13(%rsi), %rsi
1189 add %rcx, %rsi
1190 add %rcx, %rdi
1191 jmp L(less48bytes)
1192
1193 .p2align 4
1194L(shr_14):
1195 cmp $80, %rcx
1196 lea -48(%rcx), %rcx
1197 mov %edx, %eax
1198 jae L(shr_14_gobble)
1199
1200 movdqa 16(%rsi), %xmm1
1201 movdqa %xmm1, %xmm2
1202 palignr $14, (%rsi), %xmm1
1203 pcmpeqb (%rdi), %xmm1
1204
1205 movdqa 32(%rsi), %xmm3
1206 palignr $14, %xmm2, %xmm3
1207 pcmpeqb 16(%rdi), %xmm3
1208
1209 pand %xmm1, %xmm3
1210 pmovmskb %xmm3, %edx
1211 lea 32(%rdi), %rdi
1212 lea 32(%rsi), %rsi
1213 sub $0xffff, %edx
1214 jnz L(exit)
1215 add $14, %rsi
1216 add %rcx, %rsi
1217 add %rcx, %rdi
1218 jmp L(less48bytes)
1219
1220 .p2align 4
1221L(shr_14_gobble):
1222 sub $32, %rcx
1223 movdqa 16(%rsi), %xmm0
1224 palignr $14, (%rsi), %xmm0
1225 pcmpeqb (%rdi), %xmm0
1226
1227 movdqa 32(%rsi), %xmm3
1228 palignr $14, 16(%rsi), %xmm3
1229 pcmpeqb 16(%rdi), %xmm3
1230
1231L(shr_14_gobble_loop):
1232 pand %xmm0, %xmm3
1233 sub $32, %rcx
1234 pmovmskb %xmm3, %edx
1235 movdqa %xmm0, %xmm1
1236
1237 movdqa 64(%rsi), %xmm3
1238 palignr $14, 48(%rsi), %xmm3
1239 sbb $0xffff, %edx
1240 movdqa 48(%rsi), %xmm0
1241 palignr $14, 32(%rsi), %xmm0
1242 pcmpeqb 32(%rdi), %xmm0
1243 lea 32(%rsi), %rsi
1244 pcmpeqb 48(%rdi), %xmm3
1245
1246 lea 32(%rdi), %rdi
1247 jz L(shr_14_gobble_loop)
1248 pand %xmm0, %xmm3
1249
1250 cmp $0, %rcx
1251 jge L(shr_14_gobble_next)
1252 inc %edx
1253 add $32, %rcx
1254L(shr_14_gobble_next):
1255 test %edx, %edx
1256 jnz L(exit)
1257
1258 pmovmskb %xmm3, %edx
1259 movdqa %xmm0, %xmm1
1260 lea 32(%rdi), %rdi
1261 lea 32(%rsi), %rsi
1262 sub $0xffff, %edx
1263 jnz L(exit)
1264
1265 lea 14(%rsi), %rsi
1266 add %rcx, %rsi
1267 add %rcx, %rdi
1268 jmp L(less48bytes)
1269
1270 .p2align 4
1271L(shr_15):
1272 cmp $80, %rcx
1273 lea -48(%rcx), %rcx
1274 mov %edx, %eax
1275 jae L(shr_15_gobble)
1276
1277 movdqa 16(%rsi), %xmm1
1278 movdqa %xmm1, %xmm2
1279 palignr $15, (%rsi), %xmm1
1280 pcmpeqb (%rdi), %xmm1
1281
1282 movdqa 32(%rsi), %xmm3
1283 palignr $15, %xmm2, %xmm3
1284 pcmpeqb 16(%rdi), %xmm3
1285
1286 pand %xmm1, %xmm3
1287 pmovmskb %xmm3, %edx
1288 lea 32(%rdi), %rdi
1289 lea 32(%rsi), %rsi
1290 sub $0xffff, %edx
1291 jnz L(exit)
1292 add $15, %rsi
1293 add %rcx, %rsi
1294 add %rcx, %rdi
1295 jmp L(less48bytes)
1296
1297 .p2align 4
1298L(shr_15_gobble):
1299 sub $32, %rcx
1300 movdqa 16(%rsi), %xmm0
1301 palignr $15, (%rsi), %xmm0
1302 pcmpeqb (%rdi), %xmm0
1303
1304 movdqa 32(%rsi), %xmm3
1305 palignr $15, 16(%rsi), %xmm3
1306 pcmpeqb 16(%rdi), %xmm3
1307
1308L(shr_15_gobble_loop):
1309 pand %xmm0, %xmm3
1310 sub $32, %rcx
1311 pmovmskb %xmm3, %edx
1312 movdqa %xmm0, %xmm1
1313
1314 movdqa 64(%rsi), %xmm3
1315 palignr $15, 48(%rsi), %xmm3
1316 sbb $0xffff, %edx
1317 movdqa 48(%rsi), %xmm0
1318 palignr $15, 32(%rsi), %xmm0
1319 pcmpeqb 32(%rdi), %xmm0
1320 lea 32(%rsi), %rsi
1321 pcmpeqb 48(%rdi), %xmm3
1322
1323 lea 32(%rdi), %rdi
1324 jz L(shr_15_gobble_loop)
1325 pand %xmm0, %xmm3
1326
1327 cmp $0, %rcx
1328 jge L(shr_15_gobble_next)
1329 inc %edx
1330 add $32, %rcx
1331L(shr_15_gobble_next):
1332 test %edx, %edx
1333 jnz L(exit)
1334
1335 pmovmskb %xmm3, %edx
1336 movdqa %xmm0, %xmm1
1337 lea 32(%rdi), %rdi
1338 lea 32(%rsi), %rsi
1339 sub $0xffff, %edx
1340 jnz L(exit)
1341
1342 lea 15(%rsi), %rsi
1343 add %rcx, %rsi
1344 add %rcx, %rdi
1345 jmp L(less48bytes)
1346# endif
1347 .p2align 4
1348L(exit):
1349 pmovmskb %xmm1, %r8d
1350 sub $0xffff, %r8d
1351 jz L(first16bytes)
1352 lea -16(%rsi), %rsi
1353 lea -16(%rdi), %rdi
1354 mov %r8d, %edx
1355L(first16bytes):
1356 add %rax, %rsi
1357L(less16bytes):
1358# ifndef USE_AS_WMEMCMP
1359 test %dl, %dl
1360 jz L(next_24_bytes)
1361
1362 test $0x01, %dl
1363 jnz L(Byte16)
1364
1365 test $0x02, %dl
1366 jnz L(Byte17)
1367
1368 test $0x04, %dl
1369 jnz L(Byte18)
1370
1371 test $0x08, %dl
1372 jnz L(Byte19)
1373
1374 test $0x10, %dl
1375 jnz L(Byte20)
1376
1377 test $0x20, %dl
1378 jnz L(Byte21)
1379
1380 test $0x40, %dl
1381 jnz L(Byte22)
1382
1383 movzbl -9(%rdi), %eax
1384 movzbl -9(%rsi), %edx
1385 sub %edx, %eax
1386 ret
1387
1388 .p2align 4
1389L(Byte16):
1390 movzbl -16(%rdi), %eax
1391 movzbl -16(%rsi), %edx
1392 sub %edx, %eax
1393 ret
1394
1395 .p2align 4
1396L(Byte17):
1397 movzbl -15(%rdi), %eax
1398 movzbl -15(%rsi), %edx
1399 sub %edx, %eax
1400 ret
1401
1402 .p2align 4
1403L(Byte18):
1404 movzbl -14(%rdi), %eax
1405 movzbl -14(%rsi), %edx
1406 sub %edx, %eax
1407 ret
1408
1409 .p2align 4
1410L(Byte19):
1411 movzbl -13(%rdi), %eax
1412 movzbl -13(%rsi), %edx
1413 sub %edx, %eax
1414 ret
1415
1416 .p2align 4
1417L(Byte20):
1418 movzbl -12(%rdi), %eax
1419 movzbl -12(%rsi), %edx
1420 sub %edx, %eax
1421 ret
1422
1423 .p2align 4
1424L(Byte21):
1425 movzbl -11(%rdi), %eax
1426 movzbl -11(%rsi), %edx
1427 sub %edx, %eax
1428 ret
1429
1430 .p2align 4
1431L(Byte22):
1432 movzbl -10(%rdi), %eax
1433 movzbl -10(%rsi), %edx
1434 sub %edx, %eax
1435 ret
1436
1437 .p2align 4
1438L(next_24_bytes):
1439 lea 8(%rdi), %rdi
1440 lea 8(%rsi), %rsi
1441 test $0x01, %dh
1442 jnz L(Byte16)
1443
1444 test $0x02, %dh
1445 jnz L(Byte17)
1446
1447 test $0x04, %dh
1448 jnz L(Byte18)
1449
1450 test $0x08, %dh
1451 jnz L(Byte19)
1452
1453 test $0x10, %dh
1454 jnz L(Byte20)
1455
1456 test $0x20, %dh
1457 jnz L(Byte21)
1458
1459 test $0x40, %dh
1460 jnz L(Byte22)
1461
1462 movzbl -9(%rdi), %eax
1463 movzbl -9(%rsi), %edx
1464 sub %edx, %eax
1465 ret
1466# else
1467/* special for wmemcmp */
1468 xor %eax, %eax
1469 test %dl, %dl
1470 jz L(next_two_double_words)
1471 and $15, %dl
1472 jz L(second_double_word)
1473 mov -16(%rdi), %eax
1474 cmp -16(%rsi), %eax
1475 jne L(find_diff)
1476 ret
1477
1478 .p2align 4
1479L(second_double_word):
1480 mov -12(%rdi), %eax
1481 cmp -12(%rsi), %eax
1482 jne L(find_diff)
1483 ret
1484
1485 .p2align 4
1486L(next_two_double_words):
1487 and $15, %dh
1488 jz L(fourth_double_word)
1489 mov -8(%rdi), %eax
1490 cmp -8(%rsi), %eax
1491 jne L(find_diff)
1492 ret
1493
1494 .p2align 4
1495L(fourth_double_word):
1496 mov -4(%rdi), %eax
1497 cmp -4(%rsi), %eax
1498 jne L(find_diff)
1499 ret
1500# endif
1501
1502 .p2align 4
1503L(less48bytes):
1504 cmp $8, %ecx
1505 jae L(more8bytes)
1506 cmp $0, %ecx
1507 je L(0bytes)
1508# ifndef USE_AS_WMEMCMP
1509 cmp $1, %ecx
1510 je L(1bytes)
1511 cmp $2, %ecx
1512 je L(2bytes)
1513 cmp $3, %ecx
1514 je L(3bytes)
1515 cmp $4, %ecx
1516 je L(4bytes)
1517 cmp $5, %ecx
1518 je L(5bytes)
1519 cmp $6, %ecx
1520 je L(6bytes)
1521 jmp L(7bytes)
1522# else
1523 jmp L(4bytes)
1524# endif
1525
1526 .p2align 4
1527L(more8bytes):
1528 cmp $16, %ecx
1529 jae L(more16bytes)
1530 cmp $8, %ecx
1531 je L(8bytes)
1532# ifndef USE_AS_WMEMCMP
1533 cmp $9, %ecx
1534 je L(9bytes)
1535 cmp $10, %ecx
1536 je L(10bytes)
1537 cmp $11, %ecx
1538 je L(11bytes)
1539 cmp $12, %ecx
1540 je L(12bytes)
1541 cmp $13, %ecx
1542 je L(13bytes)
1543 cmp $14, %ecx
1544 je L(14bytes)
1545 jmp L(15bytes)
1546# else
1547 jmp L(12bytes)
1548# endif
1549
1550 .p2align 4
1551L(more16bytes):
1552 cmp $24, %ecx
1553 jae L(more24bytes)
1554 cmp $16, %ecx
1555 je L(16bytes)
1556# ifndef USE_AS_WMEMCMP
1557 cmp $17, %ecx
1558 je L(17bytes)
1559 cmp $18, %ecx
1560 je L(18bytes)
1561 cmp $19, %ecx
1562 je L(19bytes)
1563 cmp $20, %ecx
1564 je L(20bytes)
1565 cmp $21, %ecx
1566 je L(21bytes)
1567 cmp $22, %ecx
1568 je L(22bytes)
1569 jmp L(23bytes)
1570# else
1571 jmp L(20bytes)
1572# endif
1573
1574 .p2align 4
1575L(more24bytes):
1576 cmp $32, %ecx
1577 jae L(more32bytes)
1578 cmp $24, %ecx
1579 je L(24bytes)
1580# ifndef USE_AS_WMEMCMP
1581 cmp $25, %ecx
1582 je L(25bytes)
1583 cmp $26, %ecx
1584 je L(26bytes)
1585 cmp $27, %ecx
1586 je L(27bytes)
1587 cmp $28, %ecx
1588 je L(28bytes)
1589 cmp $29, %ecx
1590 je L(29bytes)
1591 cmp $30, %ecx
1592 je L(30bytes)
1593 jmp L(31bytes)
1594# else
1595 jmp L(28bytes)
1596# endif
1597
1598 .p2align 4
1599L(more32bytes):
1600 cmp $40, %ecx
1601 jae L(more40bytes)
1602 cmp $32, %ecx
1603 je L(32bytes)
1604# ifndef USE_AS_WMEMCMP
1605 cmp $33, %ecx
1606 je L(33bytes)
1607 cmp $34, %ecx
1608 je L(34bytes)
1609 cmp $35, %ecx
1610 je L(35bytes)
1611 cmp $36, %ecx
1612 je L(36bytes)
1613 cmp $37, %ecx
1614 je L(37bytes)
1615 cmp $38, %ecx
1616 je L(38bytes)
1617 jmp L(39bytes)
1618# else
1619 jmp L(36bytes)
1620# endif
1621
1622 .p2align 4
1623L(more40bytes):
1624 cmp $40, %ecx
1625 je L(40bytes)
1626# ifndef USE_AS_WMEMCMP
1627 cmp $41, %ecx
1628 je L(41bytes)
1629 cmp $42, %ecx
1630 je L(42bytes)
1631 cmp $43, %ecx
1632 je L(43bytes)
1633 cmp $44, %ecx
1634 je L(44bytes)
1635 cmp $45, %ecx
1636 je L(45bytes)
1637 cmp $46, %ecx
1638 je L(46bytes)
1639 jmp L(47bytes)
1640
1641 .p2align 4
1642L(44bytes):
1643 movl -44(%rdi), %eax
1644 movl -44(%rsi), %ecx
1645 cmp %ecx, %eax
1646 jne L(find_diff)
1647L(40bytes):
1648 movl -40(%rdi), %eax
1649 movl -40(%rsi), %ecx
1650 cmp %ecx, %eax
1651 jne L(find_diff)
1652L(36bytes):
1653 movl -36(%rdi), %eax
1654 movl -36(%rsi), %ecx
1655 cmp %ecx, %eax
1656 jne L(find_diff)
1657L(32bytes):
1658 movl -32(%rdi), %eax
1659 movl -32(%rsi), %ecx
1660 cmp %ecx, %eax
1661 jne L(find_diff)
1662L(28bytes):
1663 movl -28(%rdi), %eax
1664 movl -28(%rsi), %ecx
1665 cmp %ecx, %eax
1666 jne L(find_diff)
1667L(24bytes):
1668 movl -24(%rdi), %eax
1669 movl -24(%rsi), %ecx
1670 cmp %ecx, %eax
1671 jne L(find_diff)
1672L(20bytes):
1673 movl -20(%rdi), %eax
1674 movl -20(%rsi), %ecx
1675 cmp %ecx, %eax
1676 jne L(find_diff)
1677L(16bytes):
1678 movl -16(%rdi), %eax
1679 movl -16(%rsi), %ecx
1680 cmp %ecx, %eax
1681 jne L(find_diff)
1682L(12bytes):
1683 movl -12(%rdi), %eax
1684 movl -12(%rsi), %ecx
1685 cmp %ecx, %eax
1686 jne L(find_diff)
1687L(8bytes):
1688 movl -8(%rdi), %eax
1689 movl -8(%rsi), %ecx
1690 cmp %ecx, %eax
1691 jne L(find_diff)
1692L(4bytes):
1693 movl -4(%rdi), %eax
1694 movl -4(%rsi), %ecx
1695 cmp %ecx, %eax
1696 jne L(find_diff)
1697L(0bytes):
1698 xor %eax, %eax
1699 ret
1700# else
1701 .p2align 4
1702L(44bytes):
1703 movl -44(%rdi), %eax
1704 cmp -44(%rsi), %eax
1705 jne L(find_diff)
1706L(40bytes):
1707 movl -40(%rdi), %eax
1708 cmp -40(%rsi), %eax
1709 jne L(find_diff)
1710L(36bytes):
1711 movl -36(%rdi), %eax
1712 cmp -36(%rsi), %eax
1713 jne L(find_diff)
1714L(32bytes):
1715 movl -32(%rdi), %eax
1716 cmp -32(%rsi), %eax
1717 jne L(find_diff)
1718L(28bytes):
1719 movl -28(%rdi), %eax
1720 cmp -28(%rsi), %eax
1721 jne L(find_diff)
1722L(24bytes):
1723 movl -24(%rdi), %eax
1724 cmp -24(%rsi), %eax
1725 jne L(find_diff)
1726L(20bytes):
1727 movl -20(%rdi), %eax
1728 cmp -20(%rsi), %eax
1729 jne L(find_diff)
1730L(16bytes):
1731 movl -16(%rdi), %eax
1732 cmp -16(%rsi), %eax
1733 jne L(find_diff)
1734L(12bytes):
1735 movl -12(%rdi), %eax
1736 cmp -12(%rsi), %eax
1737 jne L(find_diff)
1738L(8bytes):
1739 movl -8(%rdi), %eax
1740 cmp -8(%rsi), %eax
1741 jne L(find_diff)
1742L(4bytes):
1743 movl -4(%rdi), %eax
1744 cmp -4(%rsi), %eax
1745 jne L(find_diff)
1746L(0bytes):
1747 xor %eax, %eax
1748 ret
1749# endif
1750
1751# ifndef USE_AS_WMEMCMP
1752 .p2align 4
1753L(45bytes):
1754 movl -45(%rdi), %eax
1755 movl -45(%rsi), %ecx
1756 cmp %ecx, %eax
1757 jne L(find_diff)
1758L(41bytes):
1759 movl -41(%rdi), %eax
1760 movl -41(%rsi), %ecx
1761 cmp %ecx, %eax
1762 jne L(find_diff)
1763L(37bytes):
1764 movl -37(%rdi), %eax
1765 movl -37(%rsi), %ecx
1766 cmp %ecx, %eax
1767 jne L(find_diff)
1768L(33bytes):
1769 movl -33(%rdi), %eax
1770 movl -33(%rsi), %ecx
1771 cmp %ecx, %eax
1772 jne L(find_diff)
1773L(29bytes):
1774 movl -29(%rdi), %eax
1775 movl -29(%rsi), %ecx
1776 cmp %ecx, %eax
1777 jne L(find_diff)
1778L(25bytes):
1779 movl -25(%rdi), %eax
1780 movl -25(%rsi), %ecx
1781 cmp %ecx, %eax
1782 jne L(find_diff)
1783L(21bytes):
1784 movl -21(%rdi), %eax
1785 movl -21(%rsi), %ecx
1786 cmp %ecx, %eax
1787 jne L(find_diff)
1788L(17bytes):
1789 movl -17(%rdi), %eax
1790 movl -17(%rsi), %ecx
1791 cmp %ecx, %eax
1792 jne L(find_diff)
1793L(13bytes):
1794 movl -13(%rdi), %eax
1795 movl -13(%rsi), %ecx
1796 cmp %ecx, %eax
1797 jne L(find_diff)
1798L(9bytes):
1799 movl -9(%rdi), %eax
1800 movl -9(%rsi), %ecx
1801 cmp %ecx, %eax
1802 jne L(find_diff)
1803L(5bytes):
1804 movl -5(%rdi), %eax
1805 movl -5(%rsi), %ecx
1806 cmp %ecx, %eax
1807 jne L(find_diff)
1808L(1bytes):
1809 movzbl -1(%rdi), %eax
1810 cmpb -1(%rsi), %al
1811 jne L(set)
1812 xor %eax, %eax
1813 ret
1814
1815 .p2align 4
1816L(46bytes):
1817 movl -46(%rdi), %eax
1818 movl -46(%rsi), %ecx
1819 cmp %ecx, %eax
1820 jne L(find_diff)
1821L(42bytes):
1822 movl -42(%rdi), %eax
1823 movl -42(%rsi), %ecx
1824 cmp %ecx, %eax
1825 jne L(find_diff)
1826L(38bytes):
1827 movl -38(%rdi), %eax
1828 movl -38(%rsi), %ecx
1829 cmp %ecx, %eax
1830 jne L(find_diff)
1831L(34bytes):
1832 movl -34(%rdi), %eax
1833 movl -34(%rsi), %ecx
1834 cmp %ecx, %eax
1835 jne L(find_diff)
1836L(30bytes):
1837 movl -30(%rdi), %eax
1838 movl -30(%rsi), %ecx
1839 cmp %ecx, %eax
1840 jne L(find_diff)
1841L(26bytes):
1842 movl -26(%rdi), %eax
1843 movl -26(%rsi), %ecx
1844 cmp %ecx, %eax
1845 jne L(find_diff)
1846L(22bytes):
1847 movl -22(%rdi), %eax
1848 movl -22(%rsi), %ecx
1849 cmp %ecx, %eax
1850 jne L(find_diff)
1851L(18bytes):
1852 movl -18(%rdi), %eax
1853 movl -18(%rsi), %ecx
1854 cmp %ecx, %eax
1855 jne L(find_diff)
1856L(14bytes):
1857 movl -14(%rdi), %eax
1858 movl -14(%rsi), %ecx
1859 cmp %ecx, %eax
1860 jne L(find_diff)
1861L(10bytes):
1862 movl -10(%rdi), %eax
1863 movl -10(%rsi), %ecx
1864 cmp %ecx, %eax
1865 jne L(find_diff)
1866L(6bytes):
1867 movl -6(%rdi), %eax
1868 movl -6(%rsi), %ecx
1869 cmp %ecx, %eax
1870 jne L(find_diff)
1871L(2bytes):
1872 movzwl -2(%rdi), %eax
1873 movzwl -2(%rsi), %ecx
1874 cmpb %cl, %al
1875 jne L(set)
1876 cmp %ecx, %eax
1877 jne L(set)
1878 xor %eax, %eax
1879 ret
1880
1881 .p2align 4
1882L(47bytes):
1883 movl -47(%rdi), %eax
1884 movl -47(%rsi), %ecx
1885 cmp %ecx, %eax
1886 jne L(find_diff)
1887L(43bytes):
1888 movl -43(%rdi), %eax
1889 movl -43(%rsi), %ecx
1890 cmp %ecx, %eax
1891 jne L(find_diff)
1892L(39bytes):
1893 movl -39(%rdi), %eax
1894 movl -39(%rsi), %ecx
1895 cmp %ecx, %eax
1896 jne L(find_diff)
1897L(35bytes):
1898 movl -35(%rdi), %eax
1899 movl -35(%rsi), %ecx
1900 cmp %ecx, %eax
1901 jne L(find_diff)
1902L(31bytes):
1903 movl -31(%rdi), %eax
1904 movl -31(%rsi), %ecx
1905 cmp %ecx, %eax
1906 jne L(find_diff)
1907L(27bytes):
1908 movl -27(%rdi), %eax
1909 movl -27(%rsi), %ecx
1910 cmp %ecx, %eax
1911 jne L(find_diff)
1912L(23bytes):
1913 movl -23(%rdi), %eax
1914 movl -23(%rsi), %ecx
1915 cmp %ecx, %eax
1916 jne L(find_diff)
1917L(19bytes):
1918 movl -19(%rdi), %eax
1919 movl -19(%rsi), %ecx
1920 cmp %ecx, %eax
1921 jne L(find_diff)
1922L(15bytes):
1923 movl -15(%rdi), %eax
1924 movl -15(%rsi), %ecx
1925 cmp %ecx, %eax
1926 jne L(find_diff)
1927L(11bytes):
1928 movl -11(%rdi), %eax
1929 movl -11(%rsi), %ecx
1930 cmp %ecx, %eax
1931 jne L(find_diff)
1932L(7bytes):
1933 movl -7(%rdi), %eax
1934 movl -7(%rsi), %ecx
1935 cmp %ecx, %eax
1936 jne L(find_diff)
1937L(3bytes):
1938 movzwl -3(%rdi), %eax
1939 movzwl -3(%rsi), %ecx
1940 cmpb %cl, %al
1941 jne L(set)
1942 cmp %ecx, %eax
1943 jne L(set)
1944 movzbl -1(%rdi), %eax
1945 cmpb -1(%rsi), %al
1946 jne L(set)
1947 xor %eax, %eax
1948 ret
1949
1950 .p2align 4
1951L(find_diff):
1952 cmpb %cl, %al
1953 jne L(set)
1954 cmpw %cx, %ax
1955 jne L(set)
1956 shr $16, %eax
1957 shr $16, %ecx
1958 cmpb %cl, %al
1959 jne L(set)
1960
1961/* We get there only if we already know there is a
1962difference. */
1963
1964 cmp %ecx, %eax
1965L(set):
1966 sbb %eax, %eax
1967 sbb $-1, %eax
1968 ret
1969# else
1970
1971/* for wmemcmp */
1972 .p2align 4
1973L(find_diff):
1974 mov $1, %eax
1975 jg L(find_diff_bigger)
1976 neg %eax
1977 ret
1978
1979 .p2align 4
1980L(find_diff_bigger):
1981 ret
1982# endif
1983
1984 .p2align 4
1985L(equal):
1986 xor %eax, %eax
1987 ret
1988
1989END (MEMCMP)
1990#endif
1991