1/* memcpy with SSSE3
2 Copyright (C) 2010-2018 Free Software Foundation, Inc.
3 Contributed by Intel Corporation.
4 This file is part of the GNU C Library.
5
6 The GNU C Library is free software; you can redistribute it and/or
7 modify it under the terms of the GNU Lesser General Public
8 License as published by the Free Software Foundation; either
9 version 2.1 of the License, or (at your option) any later version.
10
11 The GNU C Library is distributed in the hope that it will be useful,
12 but WITHOUT ANY WARRANTY; without even the implied warranty of
13 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 Lesser General Public License for more details.
15
16 You should have received a copy of the GNU Lesser General Public
17 License along with the GNU C Library; if not, see
18 <http://www.gnu.org/licenses/>. */
19
20#include <sysdep.h>
21
22#if IS_IN (libc)
23
24#include "asm-syntax.h"
25
26#ifndef MEMCPY
27# define MEMCPY __memcpy_ssse3
28# define MEMCPY_CHK __memcpy_chk_ssse3
29# define MEMPCPY __mempcpy_ssse3
30# define MEMPCPY_CHK __mempcpy_chk_ssse3
31#endif
32
33#define JMPTBL(I, B) I - B
34
35/* Branch to an entry in a jump table. TABLE is a jump table with
36 relative offsets. INDEX is a register contains the index into the
37 jump table. SCALE is the scale of INDEX. */
38#define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE) \
39 lea TABLE(%rip), %r11; \
40 movslq (%r11, INDEX, SCALE), INDEX; \
41 lea (%r11, INDEX), INDEX; \
42 jmp *INDEX; \
43 ud2
44
45 .section .text.ssse3,"ax",@progbits
46#if !defined USE_AS_MEMPCPY && !defined USE_AS_MEMMOVE
47ENTRY (MEMPCPY_CHK)
48 cmpq %rdx, %rcx
49 jb HIDDEN_JUMPTARGET (__chk_fail)
50END (MEMPCPY_CHK)
51
52ENTRY (MEMPCPY)
53 movq %rdi, %rax
54 addq %rdx, %rax
55 jmp L(start)
56END (MEMPCPY)
57#endif
58
59#if !defined USE_AS_BCOPY
60ENTRY (MEMCPY_CHK)
61 cmpq %rdx, %rcx
62 jb HIDDEN_JUMPTARGET (__chk_fail)
63END (MEMCPY_CHK)
64#endif
65
66ENTRY (MEMCPY)
67 mov %rdi, %rax
68#ifdef USE_AS_MEMPCPY
69 add %rdx, %rax
70#endif
71
72#ifdef USE_AS_MEMMOVE
73 cmp %rsi, %rdi
74 jb L(copy_forward)
75 je L(write_0bytes)
76 cmp $79, %rdx
77 jbe L(copy_forward)
78 jmp L(copy_backward)
79L(copy_forward):
80#endif
81L(start):
82 cmp $79, %rdx
83 lea L(table_less_80bytes)(%rip), %r11
84 ja L(80bytesormore)
85 movslq (%r11, %rdx, 4), %r9
86 add %rdx, %rsi
87 add %rdx, %rdi
88 add %r11, %r9
89 jmp *%r9
90 ud2
91
92 .p2align 4
93L(80bytesormore):
94#ifndef USE_AS_MEMMOVE
95 cmp %dil, %sil
96 jle L(copy_backward)
97#endif
98
99 movdqu (%rsi), %xmm0
100 mov %rdi, %rcx
101 and $-16, %rdi
102 add $16, %rdi
103 mov %rcx, %r8
104 sub %rdi, %rcx
105 add %rcx, %rdx
106 sub %rcx, %rsi
107
108#ifdef SHARED_CACHE_SIZE_HALF
109 mov $SHARED_CACHE_SIZE_HALF, %RCX_LP
110#else
111 mov __x86_shared_cache_size_half(%rip), %RCX_LP
112#endif
113 cmp %rcx, %rdx
114 mov %rsi, %r9
115 ja L(large_page_fwd)
116 and $0xf, %r9
117 jz L(shl_0)
118#ifdef DATA_CACHE_SIZE_HALF
119 mov $DATA_CACHE_SIZE_HALF, %RCX_LP
120#else
121 mov __x86_data_cache_size_half(%rip), %RCX_LP
122#endif
123 BRANCH_TO_JMPTBL_ENTRY (L(shl_table), %r9, 4)
124
125 .p2align 4
126L(copy_backward):
127 movdqu -16(%rsi, %rdx), %xmm0
128 add %rdx, %rsi
129 lea -16(%rdi, %rdx), %r8
130 add %rdx, %rdi
131
132 mov %rdi, %rcx
133 and $0xf, %rcx
134 xor %rcx, %rdi
135 sub %rcx, %rdx
136 sub %rcx, %rsi
137
138#ifdef SHARED_CACHE_SIZE_HALF
139 mov $SHARED_CACHE_SIZE_HALF, %RCX_LP
140#else
141 mov __x86_shared_cache_size_half(%rip), %RCX_LP
142#endif
143
144 cmp %rcx, %rdx
145 mov %rsi, %r9
146 ja L(large_page_bwd)
147 and $0xf, %r9
148 jz L(shl_0_bwd)
149#ifdef DATA_CACHE_SIZE_HALF
150 mov $DATA_CACHE_SIZE_HALF, %RCX_LP
151#else
152 mov __x86_data_cache_size_half(%rip), %RCX_LP
153#endif
154 BRANCH_TO_JMPTBL_ENTRY (L(shl_table_bwd), %r9, 4)
155
156 .p2align 4
157L(shl_0):
158 sub $16, %rdx
159 movdqa (%rsi), %xmm1
160 add $16, %rsi
161 movdqa %xmm1, (%rdi)
162 add $16, %rdi
163 cmp $128, %rdx
164 movdqu %xmm0, (%r8)
165 ja L(shl_0_gobble)
166 cmp $64, %rdx
167 jb L(shl_0_less_64bytes)
168 movaps (%rsi), %xmm4
169 movaps 16(%rsi), %xmm1
170 movaps 32(%rsi), %xmm2
171 movaps 48(%rsi), %xmm3
172 movaps %xmm4, (%rdi)
173 movaps %xmm1, 16(%rdi)
174 movaps %xmm2, 32(%rdi)
175 movaps %xmm3, 48(%rdi)
176 sub $64, %rdx
177 add $64, %rsi
178 add $64, %rdi
179L(shl_0_less_64bytes):
180 add %rdx, %rsi
181 add %rdx, %rdi
182 BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4)
183
184 .p2align 4
185L(shl_0_gobble):
186#ifdef DATA_CACHE_SIZE_HALF
187 cmp $DATA_CACHE_SIZE_HALF, %RDX_LP
188#else
189 cmp __x86_data_cache_size_half(%rip), %RDX_LP
190#endif
191 lea -128(%rdx), %rdx
192 jae L(shl_0_gobble_mem_loop)
193L(shl_0_gobble_cache_loop):
194 movdqa (%rsi), %xmm4
195 movaps 0x10(%rsi), %xmm1
196 movaps 0x20(%rsi), %xmm2
197 movaps 0x30(%rsi), %xmm3
198
199 movdqa %xmm4, (%rdi)
200 movaps %xmm1, 0x10(%rdi)
201 movaps %xmm2, 0x20(%rdi)
202 movaps %xmm3, 0x30(%rdi)
203
204 sub $128, %rdx
205 movaps 0x40(%rsi), %xmm4
206 movaps 0x50(%rsi), %xmm5
207 movaps 0x60(%rsi), %xmm6
208 movaps 0x70(%rsi), %xmm7
209 lea 0x80(%rsi), %rsi
210 movaps %xmm4, 0x40(%rdi)
211 movaps %xmm5, 0x50(%rdi)
212 movaps %xmm6, 0x60(%rdi)
213 movaps %xmm7, 0x70(%rdi)
214 lea 0x80(%rdi), %rdi
215
216 jae L(shl_0_gobble_cache_loop)
217 cmp $-0x40, %rdx
218 lea 0x80(%rdx), %rdx
219 jl L(shl_0_cache_less_64bytes)
220
221 movdqa (%rsi), %xmm4
222 sub $0x40, %rdx
223 movdqa 0x10(%rsi), %xmm1
224
225 movdqa %xmm4, (%rdi)
226 movdqa %xmm1, 0x10(%rdi)
227
228 movdqa 0x20(%rsi), %xmm4
229 movdqa 0x30(%rsi), %xmm1
230 add $0x40, %rsi
231
232 movdqa %xmm4, 0x20(%rdi)
233 movdqa %xmm1, 0x30(%rdi)
234 add $0x40, %rdi
235L(shl_0_cache_less_64bytes):
236 add %rdx, %rsi
237 add %rdx, %rdi
238 BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4)
239
240 .p2align 4
241L(shl_0_gobble_mem_loop):
242 prefetcht0 0x1c0(%rsi)
243 prefetcht0 0x280(%rsi)
244
245 movdqa (%rsi), %xmm0
246 movdqa 0x10(%rsi), %xmm1
247 movdqa 0x20(%rsi), %xmm2
248 movdqa 0x30(%rsi), %xmm3
249 movdqa 0x40(%rsi), %xmm4
250 movdqa 0x50(%rsi), %xmm5
251 movdqa 0x60(%rsi), %xmm6
252 movdqa 0x70(%rsi), %xmm7
253 lea 0x80(%rsi), %rsi
254 sub $0x80, %rdx
255 movdqa %xmm0, (%rdi)
256 movdqa %xmm1, 0x10(%rdi)
257 movdqa %xmm2, 0x20(%rdi)
258 movdqa %xmm3, 0x30(%rdi)
259 movdqa %xmm4, 0x40(%rdi)
260 movdqa %xmm5, 0x50(%rdi)
261 movdqa %xmm6, 0x60(%rdi)
262 movdqa %xmm7, 0x70(%rdi)
263 lea 0x80(%rdi), %rdi
264
265 jae L(shl_0_gobble_mem_loop)
266 cmp $-0x40, %rdx
267 lea 0x80(%rdx), %rdx
268 jl L(shl_0_mem_less_64bytes)
269
270 movdqa (%rsi), %xmm0
271 sub $0x40, %rdx
272 movdqa 0x10(%rsi), %xmm1
273
274 movdqa %xmm0, (%rdi)
275 movdqa %xmm1, 0x10(%rdi)
276
277 movdqa 0x20(%rsi), %xmm0
278 movdqa 0x30(%rsi), %xmm1
279 add $0x40, %rsi
280
281 movdqa %xmm0, 0x20(%rdi)
282 movdqa %xmm1, 0x30(%rdi)
283 add $0x40, %rdi
284L(shl_0_mem_less_64bytes):
285 cmp $0x20, %rdx
286 jb L(shl_0_mem_less_32bytes)
287 movdqa (%rsi), %xmm0
288 sub $0x20, %rdx
289 movdqa 0x10(%rsi), %xmm1
290 add $0x20, %rsi
291 movdqa %xmm0, (%rdi)
292 movdqa %xmm1, 0x10(%rdi)
293 add $0x20, %rdi
294L(shl_0_mem_less_32bytes):
295 add %rdx, %rdi
296 add %rdx, %rsi
297 BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4)
298
299 .p2align 4
300L(shl_0_bwd):
301 sub $16, %rdx
302 movdqa -0x10(%rsi), %xmm1
303 sub $16, %rsi
304 movdqa %xmm1, -0x10(%rdi)
305 sub $16, %rdi
306 cmp $0x80, %rdx
307 movdqu %xmm0, (%r8)
308 ja L(shl_0_gobble_bwd)
309 cmp $64, %rdx
310 jb L(shl_0_less_64bytes_bwd)
311 movaps -0x10(%rsi), %xmm0
312 movaps -0x20(%rsi), %xmm1
313 movaps -0x30(%rsi), %xmm2
314 movaps -0x40(%rsi), %xmm3
315 movaps %xmm0, -0x10(%rdi)
316 movaps %xmm1, -0x20(%rdi)
317 movaps %xmm2, -0x30(%rdi)
318 movaps %xmm3, -0x40(%rdi)
319 sub $64, %rdx
320 sub $0x40, %rsi
321 sub $0x40, %rdi
322L(shl_0_less_64bytes_bwd):
323 BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4)
324
325 .p2align 4
326L(shl_0_gobble_bwd):
327#ifdef DATA_CACHE_SIZE_HALF
328 cmp $DATA_CACHE_SIZE_HALF, %RDX_LP
329#else
330 cmp __x86_data_cache_size_half(%rip), %RDX_LP
331#endif
332 lea -128(%rdx), %rdx
333 jae L(shl_0_gobble_mem_bwd_loop)
334L(shl_0_gobble_bwd_loop):
335 movdqa -0x10(%rsi), %xmm0
336 movaps -0x20(%rsi), %xmm1
337 movaps -0x30(%rsi), %xmm2
338 movaps -0x40(%rsi), %xmm3
339
340 movdqa %xmm0, -0x10(%rdi)
341 movaps %xmm1, -0x20(%rdi)
342 movaps %xmm2, -0x30(%rdi)
343 movaps %xmm3, -0x40(%rdi)
344
345 sub $0x80, %rdx
346 movaps -0x50(%rsi), %xmm4
347 movaps -0x60(%rsi), %xmm5
348 movaps -0x70(%rsi), %xmm6
349 movaps -0x80(%rsi), %xmm7
350 lea -0x80(%rsi), %rsi
351 movaps %xmm4, -0x50(%rdi)
352 movaps %xmm5, -0x60(%rdi)
353 movaps %xmm6, -0x70(%rdi)
354 movaps %xmm7, -0x80(%rdi)
355 lea -0x80(%rdi), %rdi
356
357 jae L(shl_0_gobble_bwd_loop)
358 cmp $-0x40, %rdx
359 lea 0x80(%rdx), %rdx
360 jl L(shl_0_gobble_bwd_less_64bytes)
361
362 movdqa -0x10(%rsi), %xmm0
363 sub $0x40, %rdx
364 movdqa -0x20(%rsi), %xmm1
365
366 movdqa %xmm0, -0x10(%rdi)
367 movdqa %xmm1, -0x20(%rdi)
368
369 movdqa -0x30(%rsi), %xmm0
370 movdqa -0x40(%rsi), %xmm1
371 sub $0x40, %rsi
372
373 movdqa %xmm0, -0x30(%rdi)
374 movdqa %xmm1, -0x40(%rdi)
375 sub $0x40, %rdi
376L(shl_0_gobble_bwd_less_64bytes):
377 BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4)
378
379 .p2align 4
380L(shl_0_gobble_mem_bwd_loop):
381 prefetcht0 -0x1c0(%rsi)
382 prefetcht0 -0x280(%rsi)
383 movdqa -0x10(%rsi), %xmm0
384 movdqa -0x20(%rsi), %xmm1
385 movdqa -0x30(%rsi), %xmm2
386 movdqa -0x40(%rsi), %xmm3
387 movdqa -0x50(%rsi), %xmm4
388 movdqa -0x60(%rsi), %xmm5
389 movdqa -0x70(%rsi), %xmm6
390 movdqa -0x80(%rsi), %xmm7
391 lea -0x80(%rsi), %rsi
392 sub $0x80, %rdx
393 movdqa %xmm0, -0x10(%rdi)
394 movdqa %xmm1, -0x20(%rdi)
395 movdqa %xmm2, -0x30(%rdi)
396 movdqa %xmm3, -0x40(%rdi)
397 movdqa %xmm4, -0x50(%rdi)
398 movdqa %xmm5, -0x60(%rdi)
399 movdqa %xmm6, -0x70(%rdi)
400 movdqa %xmm7, -0x80(%rdi)
401 lea -0x80(%rdi), %rdi
402
403 jae L(shl_0_gobble_mem_bwd_loop)
404 cmp $-0x40, %rdx
405 lea 0x80(%rdx), %rdx
406 jl L(shl_0_mem_bwd_less_64bytes)
407
408 movdqa -0x10(%rsi), %xmm0
409 sub $0x40, %rdx
410 movdqa -0x20(%rsi), %xmm1
411
412 movdqa %xmm0, -0x10(%rdi)
413 movdqa %xmm1, -0x20(%rdi)
414
415 movdqa -0x30(%rsi), %xmm0
416 movdqa -0x40(%rsi), %xmm1
417 sub $0x40, %rsi
418
419 movdqa %xmm0, -0x30(%rdi)
420 movdqa %xmm1, -0x40(%rdi)
421 sub $0x40, %rdi
422L(shl_0_mem_bwd_less_64bytes):
423 cmp $0x20, %rdx
424 jb L(shl_0_mem_bwd_less_32bytes)
425 movdqa -0x10(%rsi), %xmm0
426 sub $0x20, %rdx
427 movdqa -0x20(%rsi), %xmm1
428 sub $0x20, %rsi
429 movdqa %xmm0, -0x10(%rdi)
430 movdqa %xmm1, -0x20(%rdi)
431 sub $0x20, %rdi
432L(shl_0_mem_bwd_less_32bytes):
433 BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4)
434
435 .p2align 4
436L(shl_1):
437 lea (L(shl_1_loop_L1)-L(shl_1))(%r9), %r9
438 cmp %rcx, %rdx
439 movaps -0x01(%rsi), %xmm1
440 jb L(L1_fwd)
441 lea (L(shl_1_loop_L2)-L(shl_1_loop_L1))(%r9), %r9
442L(L1_fwd):
443 lea -64(%rdx), %rdx
444 jmp *%r9
445 ud2
446L(shl_1_loop_L2):
447 prefetchnta 0x1c0(%rsi)
448L(shl_1_loop_L1):
449 sub $64, %rdx
450 movaps 0x0f(%rsi), %xmm2
451 movaps 0x1f(%rsi), %xmm3
452 movaps 0x2f(%rsi), %xmm4
453 movaps 0x3f(%rsi), %xmm5
454 movdqa %xmm5, %xmm6
455 palignr $1, %xmm4, %xmm5
456 lea 64(%rsi), %rsi
457 palignr $1, %xmm3, %xmm4
458 palignr $1, %xmm2, %xmm3
459 lea 64(%rdi), %rdi
460 palignr $1, %xmm1, %xmm2
461 movdqa %xmm6, %xmm1
462 movdqa %xmm2, -0x40(%rdi)
463 movaps %xmm3, -0x30(%rdi)
464 jb L(shl_1_end)
465 movaps %xmm4, -0x20(%rdi)
466 movaps %xmm5, -0x10(%rdi)
467 jmp *%r9
468 ud2
469L(shl_1_end):
470 movaps %xmm4, -0x20(%rdi)
471 lea 64(%rdx), %rdx
472 movaps %xmm5, -0x10(%rdi)
473 add %rdx, %rdi
474 movdqu %xmm0, (%r8)
475 add %rdx, %rsi
476 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
477
478 .p2align 4
479L(shl_1_bwd):
480 lea (L(shl_1_bwd_loop_L1)-L(shl_1_bwd))(%r9), %r9
481 cmp %rcx, %rdx
482 movaps -0x01(%rsi), %xmm1
483 jb L(L1_bwd)
484 lea (L(shl_1_bwd_loop_L2)-L(shl_1_bwd_loop_L1))(%r9), %r9
485L(L1_bwd):
486 lea -64(%rdx), %rdx
487 jmp *%r9
488 ud2
489L(shl_1_bwd_loop_L2):
490 prefetchnta -0x1c0(%rsi)
491L(shl_1_bwd_loop_L1):
492 movaps -0x11(%rsi), %xmm2
493 sub $0x40, %rdx
494 movaps -0x21(%rsi), %xmm3
495 movaps -0x31(%rsi), %xmm4
496 movaps -0x41(%rsi), %xmm5
497 lea -0x40(%rsi), %rsi
498 palignr $1, %xmm2, %xmm1
499 palignr $1, %xmm3, %xmm2
500 palignr $1, %xmm4, %xmm3
501 palignr $1, %xmm5, %xmm4
502
503 movaps %xmm1, -0x10(%rdi)
504 movaps %xmm5, %xmm1
505
506 movaps %xmm2, -0x20(%rdi)
507 lea -0x40(%rdi), %rdi
508
509 movaps %xmm3, 0x10(%rdi)
510 jb L(shl_1_bwd_end)
511 movaps %xmm4, (%rdi)
512 jmp *%r9
513 ud2
514L(shl_1_bwd_end):
515 movaps %xmm4, (%rdi)
516 lea 64(%rdx), %rdx
517 movdqu %xmm0, (%r8)
518 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
519
520 .p2align 4
521L(shl_2):
522 lea (L(shl_2_loop_L1)-L(shl_2))(%r9), %r9
523 cmp %rcx, %rdx
524 movaps -0x02(%rsi), %xmm1
525 jb L(L2_fwd)
526 lea (L(shl_2_loop_L2)-L(shl_2_loop_L1))(%r9), %r9
527L(L2_fwd):
528 lea -64(%rdx), %rdx
529 jmp *%r9
530 ud2
531L(shl_2_loop_L2):
532 prefetchnta 0x1c0(%rsi)
533L(shl_2_loop_L1):
534 sub $64, %rdx
535 movaps 0x0e(%rsi), %xmm2
536 movaps 0x1e(%rsi), %xmm3
537 movaps 0x2e(%rsi), %xmm4
538 movaps 0x3e(%rsi), %xmm5
539 movdqa %xmm5, %xmm6
540 palignr $2, %xmm4, %xmm5
541 lea 64(%rsi), %rsi
542 palignr $2, %xmm3, %xmm4
543 palignr $2, %xmm2, %xmm3
544 lea 64(%rdi), %rdi
545 palignr $2, %xmm1, %xmm2
546 movdqa %xmm6, %xmm1
547 movdqa %xmm2, -0x40(%rdi)
548 movaps %xmm3, -0x30(%rdi)
549 jb L(shl_2_end)
550 movaps %xmm4, -0x20(%rdi)
551 movaps %xmm5, -0x10(%rdi)
552 jmp *%r9
553 ud2
554L(shl_2_end):
555 movaps %xmm4, -0x20(%rdi)
556 lea 64(%rdx), %rdx
557 movaps %xmm5, -0x10(%rdi)
558 add %rdx, %rdi
559 movdqu %xmm0, (%r8)
560 add %rdx, %rsi
561 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
562
563 .p2align 4
564L(shl_2_bwd):
565 lea (L(shl_2_bwd_loop_L1)-L(shl_2_bwd))(%r9), %r9
566 cmp %rcx, %rdx
567 movaps -0x02(%rsi), %xmm1
568 jb L(L2_bwd)
569 lea (L(shl_2_bwd_loop_L2)-L(shl_2_bwd_loop_L1))(%r9), %r9
570L(L2_bwd):
571 lea -64(%rdx), %rdx
572 jmp *%r9
573 ud2
574L(shl_2_bwd_loop_L2):
575 prefetchnta -0x1c0(%rsi)
576L(shl_2_bwd_loop_L1):
577 movaps -0x12(%rsi), %xmm2
578 sub $0x40, %rdx
579 movaps -0x22(%rsi), %xmm3
580 movaps -0x32(%rsi), %xmm4
581 movaps -0x42(%rsi), %xmm5
582 lea -0x40(%rsi), %rsi
583 palignr $2, %xmm2, %xmm1
584 palignr $2, %xmm3, %xmm2
585 palignr $2, %xmm4, %xmm3
586 palignr $2, %xmm5, %xmm4
587
588 movaps %xmm1, -0x10(%rdi)
589 movaps %xmm5, %xmm1
590
591 movaps %xmm2, -0x20(%rdi)
592 lea -0x40(%rdi), %rdi
593
594 movaps %xmm3, 0x10(%rdi)
595 jb L(shl_2_bwd_end)
596 movaps %xmm4, (%rdi)
597 jmp *%r9
598 ud2
599L(shl_2_bwd_end):
600 movaps %xmm4, (%rdi)
601 lea 64(%rdx), %rdx
602 movdqu %xmm0, (%r8)
603 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
604
605 .p2align 4
606L(shl_3):
607 lea (L(shl_3_loop_L1)-L(shl_3))(%r9), %r9
608 cmp %rcx, %rdx
609 movaps -0x03(%rsi), %xmm1
610 jb L(L3_fwd)
611 lea (L(shl_3_loop_L2)-L(shl_3_loop_L1))(%r9), %r9
612L(L3_fwd):
613 lea -64(%rdx), %rdx
614 jmp *%r9
615 ud2
616L(shl_3_loop_L2):
617 prefetchnta 0x1c0(%rsi)
618L(shl_3_loop_L1):
619 sub $64, %rdx
620 movaps 0x0d(%rsi), %xmm2
621 movaps 0x1d(%rsi), %xmm3
622 movaps 0x2d(%rsi), %xmm4
623 movaps 0x3d(%rsi), %xmm5
624 movdqa %xmm5, %xmm6
625 palignr $3, %xmm4, %xmm5
626 lea 64(%rsi), %rsi
627 palignr $3, %xmm3, %xmm4
628 palignr $3, %xmm2, %xmm3
629 lea 64(%rdi), %rdi
630 palignr $3, %xmm1, %xmm2
631 movdqa %xmm6, %xmm1
632 movdqa %xmm2, -0x40(%rdi)
633 movaps %xmm3, -0x30(%rdi)
634 jb L(shl_3_end)
635 movaps %xmm4, -0x20(%rdi)
636 movaps %xmm5, -0x10(%rdi)
637 jmp *%r9
638 ud2
639L(shl_3_end):
640 movaps %xmm4, -0x20(%rdi)
641 lea 64(%rdx), %rdx
642 movaps %xmm5, -0x10(%rdi)
643 add %rdx, %rdi
644 movdqu %xmm0, (%r8)
645 add %rdx, %rsi
646 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
647
648 .p2align 4
649L(shl_3_bwd):
650 lea (L(shl_3_bwd_loop_L1)-L(shl_3_bwd))(%r9), %r9
651 cmp %rcx, %rdx
652 movaps -0x03(%rsi), %xmm1
653 jb L(L3_bwd)
654 lea (L(shl_3_bwd_loop_L2)-L(shl_3_bwd_loop_L1))(%r9), %r9
655L(L3_bwd):
656 lea -64(%rdx), %rdx
657 jmp *%r9
658 ud2
659L(shl_3_bwd_loop_L2):
660 prefetchnta -0x1c0(%rsi)
661L(shl_3_bwd_loop_L1):
662 movaps -0x13(%rsi), %xmm2
663 sub $0x40, %rdx
664 movaps -0x23(%rsi), %xmm3
665 movaps -0x33(%rsi), %xmm4
666 movaps -0x43(%rsi), %xmm5
667 lea -0x40(%rsi), %rsi
668 palignr $3, %xmm2, %xmm1
669 palignr $3, %xmm3, %xmm2
670 palignr $3, %xmm4, %xmm3
671 palignr $3, %xmm5, %xmm4
672
673 movaps %xmm1, -0x10(%rdi)
674 movaps %xmm5, %xmm1
675
676 movaps %xmm2, -0x20(%rdi)
677 lea -0x40(%rdi), %rdi
678
679 movaps %xmm3, 0x10(%rdi)
680 jb L(shl_3_bwd_end)
681 movaps %xmm4, (%rdi)
682 jmp *%r9
683 ud2
684L(shl_3_bwd_end):
685 movaps %xmm4, (%rdi)
686 lea 64(%rdx), %rdx
687 movdqu %xmm0, (%r8)
688 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
689
690 .p2align 4
691L(shl_4):
692 lea (L(shl_4_loop_L1)-L(shl_4))(%r9), %r9
693 cmp %rcx, %rdx
694 movaps -0x04(%rsi), %xmm1
695 jb L(L4_fwd)
696 lea (L(shl_4_loop_L2)-L(shl_4_loop_L1))(%r9), %r9
697L(L4_fwd):
698 lea -64(%rdx), %rdx
699 jmp *%r9
700 ud2
701L(shl_4_loop_L2):
702 prefetchnta 0x1c0(%rsi)
703L(shl_4_loop_L1):
704 sub $64, %rdx
705 movaps 0x0c(%rsi), %xmm2
706 movaps 0x1c(%rsi), %xmm3
707 movaps 0x2c(%rsi), %xmm4
708 movaps 0x3c(%rsi), %xmm5
709 movdqa %xmm5, %xmm6
710 palignr $4, %xmm4, %xmm5
711 lea 64(%rsi), %rsi
712 palignr $4, %xmm3, %xmm4
713 palignr $4, %xmm2, %xmm3
714 lea 64(%rdi), %rdi
715 palignr $4, %xmm1, %xmm2
716 movdqa %xmm6, %xmm1
717 movdqa %xmm2, -0x40(%rdi)
718 movaps %xmm3, -0x30(%rdi)
719 jb L(shl_4_end)
720 movaps %xmm4, -0x20(%rdi)
721 movaps %xmm5, -0x10(%rdi)
722 jmp *%r9
723 ud2
724L(shl_4_end):
725 movaps %xmm4, -0x20(%rdi)
726 lea 64(%rdx), %rdx
727 movaps %xmm5, -0x10(%rdi)
728 add %rdx, %rdi
729 movdqu %xmm0, (%r8)
730 add %rdx, %rsi
731 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
732
733 .p2align 4
734L(shl_4_bwd):
735 lea (L(shl_4_bwd_loop_L1)-L(shl_4_bwd))(%r9), %r9
736 cmp %rcx, %rdx
737 movaps -0x04(%rsi), %xmm1
738 jb L(L4_bwd)
739 lea (L(shl_4_bwd_loop_L2)-L(shl_4_bwd_loop_L1))(%r9), %r9
740L(L4_bwd):
741 lea -64(%rdx), %rdx
742 jmp *%r9
743 ud2
744L(shl_4_bwd_loop_L2):
745 prefetchnta -0x1c0(%rsi)
746L(shl_4_bwd_loop_L1):
747 movaps -0x14(%rsi), %xmm2
748 sub $0x40, %rdx
749 movaps -0x24(%rsi), %xmm3
750 movaps -0x34(%rsi), %xmm4
751 movaps -0x44(%rsi), %xmm5
752 lea -0x40(%rsi), %rsi
753 palignr $4, %xmm2, %xmm1
754 palignr $4, %xmm3, %xmm2
755 palignr $4, %xmm4, %xmm3
756 palignr $4, %xmm5, %xmm4
757
758 movaps %xmm1, -0x10(%rdi)
759 movaps %xmm5, %xmm1
760
761 movaps %xmm2, -0x20(%rdi)
762 lea -0x40(%rdi), %rdi
763
764 movaps %xmm3, 0x10(%rdi)
765 jb L(shl_4_bwd_end)
766 movaps %xmm4, (%rdi)
767 jmp *%r9
768 ud2
769L(shl_4_bwd_end):
770 movaps %xmm4, (%rdi)
771 lea 64(%rdx), %rdx
772 movdqu %xmm0, (%r8)
773 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
774
775 .p2align 4
776L(shl_5):
777 lea (L(shl_5_loop_L1)-L(shl_5))(%r9), %r9
778 cmp %rcx, %rdx
779 movaps -0x05(%rsi), %xmm1
780 jb L(L5_fwd)
781 lea (L(shl_5_loop_L2)-L(shl_5_loop_L1))(%r9), %r9
782L(L5_fwd):
783 lea -64(%rdx), %rdx
784 jmp *%r9
785 ud2
786L(shl_5_loop_L2):
787 prefetchnta 0x1c0(%rsi)
788L(shl_5_loop_L1):
789 sub $64, %rdx
790 movaps 0x0b(%rsi), %xmm2
791 movaps 0x1b(%rsi), %xmm3
792 movaps 0x2b(%rsi), %xmm4
793 movaps 0x3b(%rsi), %xmm5
794 movdqa %xmm5, %xmm6
795 palignr $5, %xmm4, %xmm5
796 lea 64(%rsi), %rsi
797 palignr $5, %xmm3, %xmm4
798 palignr $5, %xmm2, %xmm3
799 lea 64(%rdi), %rdi
800 palignr $5, %xmm1, %xmm2
801 movdqa %xmm6, %xmm1
802 movdqa %xmm2, -0x40(%rdi)
803 movaps %xmm3, -0x30(%rdi)
804 jb L(shl_5_end)
805 movaps %xmm4, -0x20(%rdi)
806 movaps %xmm5, -0x10(%rdi)
807 jmp *%r9
808 ud2
809L(shl_5_end):
810 movaps %xmm4, -0x20(%rdi)
811 lea 64(%rdx), %rdx
812 movaps %xmm5, -0x10(%rdi)
813 add %rdx, %rdi
814 movdqu %xmm0, (%r8)
815 add %rdx, %rsi
816 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
817
818 .p2align 4
819L(shl_5_bwd):
820 lea (L(shl_5_bwd_loop_L1)-L(shl_5_bwd))(%r9), %r9
821 cmp %rcx, %rdx
822 movaps -0x05(%rsi), %xmm1
823 jb L(L5_bwd)
824 lea (L(shl_5_bwd_loop_L2)-L(shl_5_bwd_loop_L1))(%r9), %r9
825L(L5_bwd):
826 lea -64(%rdx), %rdx
827 jmp *%r9
828 ud2
829L(shl_5_bwd_loop_L2):
830 prefetchnta -0x1c0(%rsi)
831L(shl_5_bwd_loop_L1):
832 movaps -0x15(%rsi), %xmm2
833 sub $0x40, %rdx
834 movaps -0x25(%rsi), %xmm3
835 movaps -0x35(%rsi), %xmm4
836 movaps -0x45(%rsi), %xmm5
837 lea -0x40(%rsi), %rsi
838 palignr $5, %xmm2, %xmm1
839 palignr $5, %xmm3, %xmm2
840 palignr $5, %xmm4, %xmm3
841 palignr $5, %xmm5, %xmm4
842
843 movaps %xmm1, -0x10(%rdi)
844 movaps %xmm5, %xmm1
845
846 movaps %xmm2, -0x20(%rdi)
847 lea -0x40(%rdi), %rdi
848
849 movaps %xmm3, 0x10(%rdi)
850 jb L(shl_5_bwd_end)
851 movaps %xmm4, (%rdi)
852 jmp *%r9
853 ud2
854L(shl_5_bwd_end):
855 movaps %xmm4, (%rdi)
856 lea 64(%rdx), %rdx
857 movdqu %xmm0, (%r8)
858 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
859
860 .p2align 4
861L(shl_6):
862 lea (L(shl_6_loop_L1)-L(shl_6))(%r9), %r9
863 cmp %rcx, %rdx
864 movaps -0x06(%rsi), %xmm1
865 jb L(L6_fwd)
866 lea (L(shl_6_loop_L2)-L(shl_6_loop_L1))(%r9), %r9
867L(L6_fwd):
868 lea -64(%rdx), %rdx
869 jmp *%r9
870 ud2
871L(shl_6_loop_L2):
872 prefetchnta 0x1c0(%rsi)
873L(shl_6_loop_L1):
874 sub $64, %rdx
875 movaps 0x0a(%rsi), %xmm2
876 movaps 0x1a(%rsi), %xmm3
877 movaps 0x2a(%rsi), %xmm4
878 movaps 0x3a(%rsi), %xmm5
879 movdqa %xmm5, %xmm6
880 palignr $6, %xmm4, %xmm5
881 lea 64(%rsi), %rsi
882 palignr $6, %xmm3, %xmm4
883 palignr $6, %xmm2, %xmm3
884 lea 64(%rdi), %rdi
885 palignr $6, %xmm1, %xmm2
886 movdqa %xmm6, %xmm1
887 movdqa %xmm2, -0x40(%rdi)
888 movaps %xmm3, -0x30(%rdi)
889 jb L(shl_6_end)
890 movaps %xmm4, -0x20(%rdi)
891 movaps %xmm5, -0x10(%rdi)
892 jmp *%r9
893 ud2
894L(shl_6_end):
895 movaps %xmm4, -0x20(%rdi)
896 lea 64(%rdx), %rdx
897 movaps %xmm5, -0x10(%rdi)
898 add %rdx, %rdi
899 movdqu %xmm0, (%r8)
900 add %rdx, %rsi
901 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
902
903 .p2align 4
904L(shl_6_bwd):
905 lea (L(shl_6_bwd_loop_L1)-L(shl_6_bwd))(%r9), %r9
906 cmp %rcx, %rdx
907 movaps -0x06(%rsi), %xmm1
908 jb L(L6_bwd)
909 lea (L(shl_6_bwd_loop_L2)-L(shl_6_bwd_loop_L1))(%r9), %r9
910L(L6_bwd):
911 lea -64(%rdx), %rdx
912 jmp *%r9
913 ud2
914L(shl_6_bwd_loop_L2):
915 prefetchnta -0x1c0(%rsi)
916L(shl_6_bwd_loop_L1):
917 movaps -0x16(%rsi), %xmm2
918 sub $0x40, %rdx
919 movaps -0x26(%rsi), %xmm3
920 movaps -0x36(%rsi), %xmm4
921 movaps -0x46(%rsi), %xmm5
922 lea -0x40(%rsi), %rsi
923 palignr $6, %xmm2, %xmm1
924 palignr $6, %xmm3, %xmm2
925 palignr $6, %xmm4, %xmm3
926 palignr $6, %xmm5, %xmm4
927
928 movaps %xmm1, -0x10(%rdi)
929 movaps %xmm5, %xmm1
930
931 movaps %xmm2, -0x20(%rdi)
932 lea -0x40(%rdi), %rdi
933
934 movaps %xmm3, 0x10(%rdi)
935 jb L(shl_6_bwd_end)
936 movaps %xmm4, (%rdi)
937 jmp *%r9
938 ud2
939L(shl_6_bwd_end):
940 movaps %xmm4, (%rdi)
941 lea 64(%rdx), %rdx
942 movdqu %xmm0, (%r8)
943 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
944
945 .p2align 4
946L(shl_7):
947 lea (L(shl_7_loop_L1)-L(shl_7))(%r9), %r9
948 cmp %rcx, %rdx
949 movaps -0x07(%rsi), %xmm1
950 jb L(L7_fwd)
951 lea (L(shl_7_loop_L2)-L(shl_7_loop_L1))(%r9), %r9
952L(L7_fwd):
953 lea -64(%rdx), %rdx
954 jmp *%r9
955 ud2
956L(shl_7_loop_L2):
957 prefetchnta 0x1c0(%rsi)
958L(shl_7_loop_L1):
959 sub $64, %rdx
960 movaps 0x09(%rsi), %xmm2
961 movaps 0x19(%rsi), %xmm3
962 movaps 0x29(%rsi), %xmm4
963 movaps 0x39(%rsi), %xmm5
964 movdqa %xmm5, %xmm6
965 palignr $7, %xmm4, %xmm5
966 lea 64(%rsi), %rsi
967 palignr $7, %xmm3, %xmm4
968 palignr $7, %xmm2, %xmm3
969 lea 64(%rdi), %rdi
970 palignr $7, %xmm1, %xmm2
971 movdqa %xmm6, %xmm1
972 movdqa %xmm2, -0x40(%rdi)
973 movaps %xmm3, -0x30(%rdi)
974 jb L(shl_7_end)
975 movaps %xmm4, -0x20(%rdi)
976 movaps %xmm5, -0x10(%rdi)
977 jmp *%r9
978 ud2
979L(shl_7_end):
980 movaps %xmm4, -0x20(%rdi)
981 lea 64(%rdx), %rdx
982 movaps %xmm5, -0x10(%rdi)
983 add %rdx, %rdi
984 movdqu %xmm0, (%r8)
985 add %rdx, %rsi
986 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
987
988 .p2align 4
989L(shl_7_bwd):
990 lea (L(shl_7_bwd_loop_L1)-L(shl_7_bwd))(%r9), %r9
991 cmp %rcx, %rdx
992 movaps -0x07(%rsi), %xmm1
993 jb L(L7_bwd)
994 lea (L(shl_7_bwd_loop_L2)-L(shl_7_bwd_loop_L1))(%r9), %r9
995L(L7_bwd):
996 lea -64(%rdx), %rdx
997 jmp *%r9
998 ud2
999L(shl_7_bwd_loop_L2):
1000 prefetchnta -0x1c0(%rsi)
1001L(shl_7_bwd_loop_L1):
1002 movaps -0x17(%rsi), %xmm2
1003 sub $0x40, %rdx
1004 movaps -0x27(%rsi), %xmm3
1005 movaps -0x37(%rsi), %xmm4
1006 movaps -0x47(%rsi), %xmm5
1007 lea -0x40(%rsi), %rsi
1008 palignr $7, %xmm2, %xmm1
1009 palignr $7, %xmm3, %xmm2
1010 palignr $7, %xmm4, %xmm3
1011 palignr $7, %xmm5, %xmm4
1012
1013 movaps %xmm1, -0x10(%rdi)
1014 movaps %xmm5, %xmm1
1015
1016 movaps %xmm2, -0x20(%rdi)
1017 lea -0x40(%rdi), %rdi
1018
1019 movaps %xmm3, 0x10(%rdi)
1020 jb L(shl_7_bwd_end)
1021 movaps %xmm4, (%rdi)
1022 jmp *%r9
1023 ud2
1024L(shl_7_bwd_end):
1025 movaps %xmm4, (%rdi)
1026 lea 64(%rdx), %rdx
1027 movdqu %xmm0, (%r8)
1028 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
1029
1030 .p2align 4
1031L(shl_8):
1032 lea (L(shl_8_loop_L1)-L(shl_8))(%r9), %r9
1033 cmp %rcx, %rdx
1034 movaps -0x08(%rsi), %xmm1
1035 jb L(L8_fwd)
1036 lea (L(shl_8_loop_L2)-L(shl_8_loop_L1))(%r9), %r9
1037L(L8_fwd):
1038 lea -64(%rdx), %rdx
1039 jmp *%r9
1040L(shl_8_loop_L2):
1041 prefetchnta 0x1c0(%rsi)
1042L(shl_8_loop_L1):
1043 sub $64, %rdx
1044 movaps 0x08(%rsi), %xmm2
1045 movaps 0x18(%rsi), %xmm3
1046 movaps 0x28(%rsi), %xmm4
1047 movaps 0x38(%rsi), %xmm5
1048 movdqa %xmm5, %xmm6
1049 palignr $8, %xmm4, %xmm5
1050 lea 64(%rsi), %rsi
1051 palignr $8, %xmm3, %xmm4
1052 palignr $8, %xmm2, %xmm3
1053 lea 64(%rdi), %rdi
1054 palignr $8, %xmm1, %xmm2
1055 movdqa %xmm6, %xmm1
1056 movdqa %xmm2, -0x40(%rdi)
1057 movaps %xmm3, -0x30(%rdi)
1058 jb L(shl_8_end)
1059 movaps %xmm4, -0x20(%rdi)
1060 movaps %xmm5, -0x10(%rdi)
1061 jmp *%r9
1062 ud2
1063 .p2align 4
1064L(shl_8_end):
1065 lea 64(%rdx), %rdx
1066 movaps %xmm4, -0x20(%rdi)
1067 add %rdx, %rsi
1068 movaps %xmm5, -0x10(%rdi)
1069 add %rdx, %rdi
1070 movdqu %xmm0, (%r8)
1071 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
1072
1073 .p2align 4
1074L(shl_8_bwd):
1075 lea (L(shl_8_bwd_loop_L1)-L(shl_8_bwd))(%r9), %r9
1076 cmp %rcx, %rdx
1077 movaps -0x08(%rsi), %xmm1
1078 jb L(L8_bwd)
1079 lea (L(shl_8_bwd_loop_L2)-L(shl_8_bwd_loop_L1))(%r9), %r9
1080L(L8_bwd):
1081 lea -64(%rdx), %rdx
1082 jmp *%r9
1083 ud2
1084L(shl_8_bwd_loop_L2):
1085 prefetchnta -0x1c0(%rsi)
1086L(shl_8_bwd_loop_L1):
1087 movaps -0x18(%rsi), %xmm2
1088 sub $0x40, %rdx
1089 movaps -0x28(%rsi), %xmm3
1090 movaps -0x38(%rsi), %xmm4
1091 movaps -0x48(%rsi), %xmm5
1092 lea -0x40(%rsi), %rsi
1093 palignr $8, %xmm2, %xmm1
1094 palignr $8, %xmm3, %xmm2
1095 palignr $8, %xmm4, %xmm3
1096 palignr $8, %xmm5, %xmm4
1097
1098 movaps %xmm1, -0x10(%rdi)
1099 movaps %xmm5, %xmm1
1100
1101 movaps %xmm2, -0x20(%rdi)
1102 lea -0x40(%rdi), %rdi
1103
1104 movaps %xmm3, 0x10(%rdi)
1105 jb L(shl_8_bwd_end)
1106 movaps %xmm4, (%rdi)
1107 jmp *%r9
1108 ud2
1109L(shl_8_bwd_end):
1110 movaps %xmm4, (%rdi)
1111 lea 64(%rdx), %rdx
1112 movdqu %xmm0, (%r8)
1113 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
1114
1115 .p2align 4
1116L(shl_9):
1117 lea (L(shl_9_loop_L1)-L(shl_9))(%r9), %r9
1118 cmp %rcx, %rdx
1119 movaps -0x09(%rsi), %xmm1
1120 jb L(L9_fwd)
1121 lea (L(shl_9_loop_L2)-L(shl_9_loop_L1))(%r9), %r9
1122L(L9_fwd):
1123 lea -64(%rdx), %rdx
1124 jmp *%r9
1125 ud2
1126L(shl_9_loop_L2):
1127 prefetchnta 0x1c0(%rsi)
1128L(shl_9_loop_L1):
1129 sub $64, %rdx
1130 movaps 0x07(%rsi), %xmm2
1131 movaps 0x17(%rsi), %xmm3
1132 movaps 0x27(%rsi), %xmm4
1133 movaps 0x37(%rsi), %xmm5
1134 movdqa %xmm5, %xmm6
1135 palignr $9, %xmm4, %xmm5
1136 lea 64(%rsi), %rsi
1137 palignr $9, %xmm3, %xmm4
1138 palignr $9, %xmm2, %xmm3
1139 lea 64(%rdi), %rdi
1140 palignr $9, %xmm1, %xmm2
1141 movdqa %xmm6, %xmm1
1142 movdqa %xmm2, -0x40(%rdi)
1143 movaps %xmm3, -0x30(%rdi)
1144 jb L(shl_9_end)
1145 movaps %xmm4, -0x20(%rdi)
1146 movaps %xmm5, -0x10(%rdi)
1147 jmp *%r9
1148 ud2
1149L(shl_9_end):
1150 movaps %xmm4, -0x20(%rdi)
1151 lea 64(%rdx), %rdx
1152 movaps %xmm5, -0x10(%rdi)
1153 add %rdx, %rdi
1154 movdqu %xmm0, (%r8)
1155 add %rdx, %rsi
1156 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
1157
1158 .p2align 4
1159L(shl_9_bwd):
1160 lea (L(shl_9_bwd_loop_L1)-L(shl_9_bwd))(%r9), %r9
1161 cmp %rcx, %rdx
1162 movaps -0x09(%rsi), %xmm1
1163 jb L(L9_bwd)
1164 lea (L(shl_9_bwd_loop_L2)-L(shl_9_bwd_loop_L1))(%r9), %r9
1165L(L9_bwd):
1166 lea -64(%rdx), %rdx
1167 jmp *%r9
1168 ud2
1169L(shl_9_bwd_loop_L2):
1170 prefetchnta -0x1c0(%rsi)
1171L(shl_9_bwd_loop_L1):
1172 movaps -0x19(%rsi), %xmm2
1173 sub $0x40, %rdx
1174 movaps -0x29(%rsi), %xmm3
1175 movaps -0x39(%rsi), %xmm4
1176 movaps -0x49(%rsi), %xmm5
1177 lea -0x40(%rsi), %rsi
1178 palignr $9, %xmm2, %xmm1
1179 palignr $9, %xmm3, %xmm2
1180 palignr $9, %xmm4, %xmm3
1181 palignr $9, %xmm5, %xmm4
1182
1183 movaps %xmm1, -0x10(%rdi)
1184 movaps %xmm5, %xmm1
1185
1186 movaps %xmm2, -0x20(%rdi)
1187 lea -0x40(%rdi), %rdi
1188
1189 movaps %xmm3, 0x10(%rdi)
1190 jb L(shl_9_bwd_end)
1191 movaps %xmm4, (%rdi)
1192 jmp *%r9
1193 ud2
1194L(shl_9_bwd_end):
1195 movaps %xmm4, (%rdi)
1196 lea 64(%rdx), %rdx
1197 movdqu %xmm0, (%r8)
1198 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
1199
1200 .p2align 4
1201L(shl_10):
1202 lea (L(shl_10_loop_L1)-L(shl_10))(%r9), %r9
1203 cmp %rcx, %rdx
1204 movaps -0x0a(%rsi), %xmm1
1205 jb L(L10_fwd)
1206 lea (L(shl_10_loop_L2)-L(shl_10_loop_L1))(%r9), %r9
1207L(L10_fwd):
1208 lea -64(%rdx), %rdx
1209 jmp *%r9
1210 ud2
1211L(shl_10_loop_L2):
1212 prefetchnta 0x1c0(%rsi)
1213L(shl_10_loop_L1):
1214 sub $64, %rdx
1215 movaps 0x06(%rsi), %xmm2
1216 movaps 0x16(%rsi), %xmm3
1217 movaps 0x26(%rsi), %xmm4
1218 movaps 0x36(%rsi), %xmm5
1219 movdqa %xmm5, %xmm6
1220 palignr $10, %xmm4, %xmm5
1221 lea 64(%rsi), %rsi
1222 palignr $10, %xmm3, %xmm4
1223 palignr $10, %xmm2, %xmm3
1224 lea 64(%rdi), %rdi
1225 palignr $10, %xmm1, %xmm2
1226 movdqa %xmm6, %xmm1
1227 movdqa %xmm2, -0x40(%rdi)
1228 movaps %xmm3, -0x30(%rdi)
1229 jb L(shl_10_end)
1230 movaps %xmm4, -0x20(%rdi)
1231 movaps %xmm5, -0x10(%rdi)
1232 jmp *%r9
1233 ud2
1234L(shl_10_end):
1235 movaps %xmm4, -0x20(%rdi)
1236 lea 64(%rdx), %rdx
1237 movaps %xmm5, -0x10(%rdi)
1238 add %rdx, %rdi
1239 movdqu %xmm0, (%r8)
1240 add %rdx, %rsi
1241 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
1242
1243 .p2align 4
1244L(shl_10_bwd):
1245 lea (L(shl_10_bwd_loop_L1)-L(shl_10_bwd))(%r9), %r9
1246 cmp %rcx, %rdx
1247 movaps -0x0a(%rsi), %xmm1
1248 jb L(L10_bwd)
1249 lea (L(shl_10_bwd_loop_L2)-L(shl_10_bwd_loop_L1))(%r9), %r9
1250L(L10_bwd):
1251 lea -64(%rdx), %rdx
1252 jmp *%r9
1253 ud2
1254L(shl_10_bwd_loop_L2):
1255 prefetchnta -0x1c0(%rsi)
1256L(shl_10_bwd_loop_L1):
1257 movaps -0x1a(%rsi), %xmm2
1258 sub $0x40, %rdx
1259 movaps -0x2a(%rsi), %xmm3
1260 movaps -0x3a(%rsi), %xmm4
1261 movaps -0x4a(%rsi), %xmm5
1262 lea -0x40(%rsi), %rsi
1263 palignr $10, %xmm2, %xmm1
1264 palignr $10, %xmm3, %xmm2
1265 palignr $10, %xmm4, %xmm3
1266 palignr $10, %xmm5, %xmm4
1267
1268 movaps %xmm1, -0x10(%rdi)
1269 movaps %xmm5, %xmm1
1270
1271 movaps %xmm2, -0x20(%rdi)
1272 lea -0x40(%rdi), %rdi
1273
1274 movaps %xmm3, 0x10(%rdi)
1275 jb L(shl_10_bwd_end)
1276 movaps %xmm4, (%rdi)
1277 jmp *%r9
1278 ud2
1279L(shl_10_bwd_end):
1280 movaps %xmm4, (%rdi)
1281 lea 64(%rdx), %rdx
1282 movdqu %xmm0, (%r8)
1283 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
1284
1285 .p2align 4
1286L(shl_11):
1287 lea (L(shl_11_loop_L1)-L(shl_11))(%r9), %r9
1288 cmp %rcx, %rdx
1289 movaps -0x0b(%rsi), %xmm1
1290 jb L(L11_fwd)
1291 lea (L(shl_11_loop_L2)-L(shl_11_loop_L1))(%r9), %r9
1292L(L11_fwd):
1293 lea -64(%rdx), %rdx
1294 jmp *%r9
1295 ud2
1296L(shl_11_loop_L2):
1297 prefetchnta 0x1c0(%rsi)
1298L(shl_11_loop_L1):
1299 sub $64, %rdx
1300 movaps 0x05(%rsi), %xmm2
1301 movaps 0x15(%rsi), %xmm3
1302 movaps 0x25(%rsi), %xmm4
1303 movaps 0x35(%rsi), %xmm5
1304 movdqa %xmm5, %xmm6
1305 palignr $11, %xmm4, %xmm5
1306 lea 64(%rsi), %rsi
1307 palignr $11, %xmm3, %xmm4
1308 palignr $11, %xmm2, %xmm3
1309 lea 64(%rdi), %rdi
1310 palignr $11, %xmm1, %xmm2
1311 movdqa %xmm6, %xmm1
1312 movdqa %xmm2, -0x40(%rdi)
1313 movaps %xmm3, -0x30(%rdi)
1314 jb L(shl_11_end)
1315 movaps %xmm4, -0x20(%rdi)
1316 movaps %xmm5, -0x10(%rdi)
1317 jmp *%r9
1318 ud2
1319L(shl_11_end):
1320 movaps %xmm4, -0x20(%rdi)
1321 lea 64(%rdx), %rdx
1322 movaps %xmm5, -0x10(%rdi)
1323 add %rdx, %rdi
1324 movdqu %xmm0, (%r8)
1325 add %rdx, %rsi
1326 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
1327
1328 .p2align 4
1329L(shl_11_bwd):
1330 lea (L(shl_11_bwd_loop_L1)-L(shl_11_bwd))(%r9), %r9
1331 cmp %rcx, %rdx
1332 movaps -0x0b(%rsi), %xmm1
1333 jb L(L11_bwd)
1334 lea (L(shl_11_bwd_loop_L2)-L(shl_11_bwd_loop_L1))(%r9), %r9
1335L(L11_bwd):
1336 lea -64(%rdx), %rdx
1337 jmp *%r9
1338 ud2
1339L(shl_11_bwd_loop_L2):
1340 prefetchnta -0x1c0(%rsi)
1341L(shl_11_bwd_loop_L1):
1342 movaps -0x1b(%rsi), %xmm2
1343 sub $0x40, %rdx
1344 movaps -0x2b(%rsi), %xmm3
1345 movaps -0x3b(%rsi), %xmm4
1346 movaps -0x4b(%rsi), %xmm5
1347 lea -0x40(%rsi), %rsi
1348 palignr $11, %xmm2, %xmm1
1349 palignr $11, %xmm3, %xmm2
1350 palignr $11, %xmm4, %xmm3
1351 palignr $11, %xmm5, %xmm4
1352
1353 movaps %xmm1, -0x10(%rdi)
1354 movaps %xmm5, %xmm1
1355
1356 movaps %xmm2, -0x20(%rdi)
1357 lea -0x40(%rdi), %rdi
1358
1359 movaps %xmm3, 0x10(%rdi)
1360 jb L(shl_11_bwd_end)
1361 movaps %xmm4, (%rdi)
1362 jmp *%r9
1363 ud2
1364L(shl_11_bwd_end):
1365 movaps %xmm4, (%rdi)
1366 lea 64(%rdx), %rdx
1367 movdqu %xmm0, (%r8)
1368 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
1369
1370 .p2align 4
1371L(shl_12):
1372 lea (L(shl_12_loop_L1)-L(shl_12))(%r9), %r9
1373 cmp %rcx, %rdx
1374 movaps -0x0c(%rsi), %xmm1
1375 jb L(L12_fwd)
1376 lea (L(shl_12_loop_L2)-L(shl_12_loop_L1))(%r9), %r9
1377L(L12_fwd):
1378 lea -64(%rdx), %rdx
1379 jmp *%r9
1380 ud2
1381L(shl_12_loop_L2):
1382 prefetchnta 0x1c0(%rsi)
1383L(shl_12_loop_L1):
1384 sub $64, %rdx
1385 movaps 0x04(%rsi), %xmm2
1386 movaps 0x14(%rsi), %xmm3
1387 movaps 0x24(%rsi), %xmm4
1388 movaps 0x34(%rsi), %xmm5
1389 movdqa %xmm5, %xmm6
1390 palignr $12, %xmm4, %xmm5
1391 lea 64(%rsi), %rsi
1392 palignr $12, %xmm3, %xmm4
1393 palignr $12, %xmm2, %xmm3
1394 lea 64(%rdi), %rdi
1395 palignr $12, %xmm1, %xmm2
1396 movdqa %xmm6, %xmm1
1397 movdqa %xmm2, -0x40(%rdi)
1398 movaps %xmm3, -0x30(%rdi)
1399 jb L(shl_12_end)
1400 movaps %xmm4, -0x20(%rdi)
1401 movaps %xmm5, -0x10(%rdi)
1402 jmp *%r9
1403 ud2
1404L(shl_12_end):
1405 movaps %xmm4, -0x20(%rdi)
1406 lea 64(%rdx), %rdx
1407 movaps %xmm5, -0x10(%rdi)
1408 add %rdx, %rdi
1409 movdqu %xmm0, (%r8)
1410 add %rdx, %rsi
1411 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
1412
1413 .p2align 4
1414L(shl_12_bwd):
1415 lea (L(shl_12_bwd_loop_L1)-L(shl_12_bwd))(%r9), %r9
1416 cmp %rcx, %rdx
1417 movaps -0x0c(%rsi), %xmm1
1418 jb L(L12_bwd)
1419 lea (L(shl_12_bwd_loop_L2)-L(shl_12_bwd_loop_L1))(%r9), %r9
1420L(L12_bwd):
1421 lea -64(%rdx), %rdx
1422 jmp *%r9
1423 ud2
1424L(shl_12_bwd_loop_L2):
1425 prefetchnta -0x1c0(%rsi)
1426L(shl_12_bwd_loop_L1):
1427 movaps -0x1c(%rsi), %xmm2
1428 sub $0x40, %rdx
1429 movaps -0x2c(%rsi), %xmm3
1430 movaps -0x3c(%rsi), %xmm4
1431 movaps -0x4c(%rsi), %xmm5
1432 lea -0x40(%rsi), %rsi
1433 palignr $12, %xmm2, %xmm1
1434 palignr $12, %xmm3, %xmm2
1435 palignr $12, %xmm4, %xmm3
1436 palignr $12, %xmm5, %xmm4
1437
1438 movaps %xmm1, -0x10(%rdi)
1439 movaps %xmm5, %xmm1
1440
1441 movaps %xmm2, -0x20(%rdi)
1442 lea -0x40(%rdi), %rdi
1443
1444 movaps %xmm3, 0x10(%rdi)
1445 jb L(shl_12_bwd_end)
1446 movaps %xmm4, (%rdi)
1447 jmp *%r9
1448 ud2
1449L(shl_12_bwd_end):
1450 movaps %xmm4, (%rdi)
1451 lea 64(%rdx), %rdx
1452 movdqu %xmm0, (%r8)
1453 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
1454
1455 .p2align 4
1456L(shl_13):
1457 lea (L(shl_13_loop_L1)-L(shl_13))(%r9), %r9
1458 cmp %rcx, %rdx
1459 movaps -0x0d(%rsi), %xmm1
1460 jb L(L13_fwd)
1461 lea (L(shl_13_loop_L2)-L(shl_13_loop_L1))(%r9), %r9
1462L(L13_fwd):
1463 lea -64(%rdx), %rdx
1464 jmp *%r9
1465 ud2
1466L(shl_13_loop_L2):
1467 prefetchnta 0x1c0(%rsi)
1468L(shl_13_loop_L1):
1469 sub $64, %rdx
1470 movaps 0x03(%rsi), %xmm2
1471 movaps 0x13(%rsi), %xmm3
1472 movaps 0x23(%rsi), %xmm4
1473 movaps 0x33(%rsi), %xmm5
1474 movdqa %xmm5, %xmm6
1475 palignr $13, %xmm4, %xmm5
1476 lea 64(%rsi), %rsi
1477 palignr $13, %xmm3, %xmm4
1478 palignr $13, %xmm2, %xmm3
1479 lea 64(%rdi), %rdi
1480 palignr $13, %xmm1, %xmm2
1481 movdqa %xmm6, %xmm1
1482 movdqa %xmm2, -0x40(%rdi)
1483 movaps %xmm3, -0x30(%rdi)
1484 jb L(shl_13_end)
1485 movaps %xmm4, -0x20(%rdi)
1486 movaps %xmm5, -0x10(%rdi)
1487 jmp *%r9
1488 ud2
1489L(shl_13_end):
1490 movaps %xmm4, -0x20(%rdi)
1491 lea 64(%rdx), %rdx
1492 movaps %xmm5, -0x10(%rdi)
1493 add %rdx, %rdi
1494 movdqu %xmm0, (%r8)
1495 add %rdx, %rsi
1496 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
1497
1498 .p2align 4
1499L(shl_13_bwd):
1500 lea (L(shl_13_bwd_loop_L1)-L(shl_13_bwd))(%r9), %r9
1501 cmp %rcx, %rdx
1502 movaps -0x0d(%rsi), %xmm1
1503 jb L(L13_bwd)
1504 lea (L(shl_13_bwd_loop_L2)-L(shl_13_bwd_loop_L1))(%r9), %r9
1505L(L13_bwd):
1506 lea -64(%rdx), %rdx
1507 jmp *%r9
1508 ud2
1509L(shl_13_bwd_loop_L2):
1510 prefetchnta -0x1c0(%rsi)
1511L(shl_13_bwd_loop_L1):
1512 movaps -0x1d(%rsi), %xmm2
1513 sub $0x40, %rdx
1514 movaps -0x2d(%rsi), %xmm3
1515 movaps -0x3d(%rsi), %xmm4
1516 movaps -0x4d(%rsi), %xmm5
1517 lea -0x40(%rsi), %rsi
1518 palignr $13, %xmm2, %xmm1
1519 palignr $13, %xmm3, %xmm2
1520 palignr $13, %xmm4, %xmm3
1521 palignr $13, %xmm5, %xmm4
1522
1523 movaps %xmm1, -0x10(%rdi)
1524 movaps %xmm5, %xmm1
1525
1526 movaps %xmm2, -0x20(%rdi)
1527 lea -0x40(%rdi), %rdi
1528
1529 movaps %xmm3, 0x10(%rdi)
1530 jb L(shl_13_bwd_end)
1531 movaps %xmm4, (%rdi)
1532 jmp *%r9
1533 ud2
1534L(shl_13_bwd_end):
1535 movaps %xmm4, (%rdi)
1536 lea 64(%rdx), %rdx
1537 movdqu %xmm0, (%r8)
1538 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
1539
1540 .p2align 4
1541L(shl_14):
1542 lea (L(shl_14_loop_L1)-L(shl_14))(%r9), %r9
1543 cmp %rcx, %rdx
1544 movaps -0x0e(%rsi), %xmm1
1545 jb L(L14_fwd)
1546 lea (L(shl_14_loop_L2)-L(shl_14_loop_L1))(%r9), %r9
1547L(L14_fwd):
1548 lea -64(%rdx), %rdx
1549 jmp *%r9
1550 ud2
1551L(shl_14_loop_L2):
1552 prefetchnta 0x1c0(%rsi)
1553L(shl_14_loop_L1):
1554 sub $64, %rdx
1555 movaps 0x02(%rsi), %xmm2
1556 movaps 0x12(%rsi), %xmm3
1557 movaps 0x22(%rsi), %xmm4
1558 movaps 0x32(%rsi), %xmm5
1559 movdqa %xmm5, %xmm6
1560 palignr $14, %xmm4, %xmm5
1561 lea 64(%rsi), %rsi
1562 palignr $14, %xmm3, %xmm4
1563 palignr $14, %xmm2, %xmm3
1564 lea 64(%rdi), %rdi
1565 palignr $14, %xmm1, %xmm2
1566 movdqa %xmm6, %xmm1
1567 movdqa %xmm2, -0x40(%rdi)
1568 movaps %xmm3, -0x30(%rdi)
1569 jb L(shl_14_end)
1570 movaps %xmm4, -0x20(%rdi)
1571 movaps %xmm5, -0x10(%rdi)
1572 jmp *%r9
1573 ud2
1574L(shl_14_end):
1575 movaps %xmm4, -0x20(%rdi)
1576 lea 64(%rdx), %rdx
1577 movaps %xmm5, -0x10(%rdi)
1578 add %rdx, %rdi
1579 movdqu %xmm0, (%r8)
1580 add %rdx, %rsi
1581 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
1582
1583 .p2align 4
1584L(shl_14_bwd):
1585 lea (L(shl_14_bwd_loop_L1)-L(shl_14_bwd))(%r9), %r9
1586 cmp %rcx, %rdx
1587 movaps -0x0e(%rsi), %xmm1
1588 jb L(L14_bwd)
1589 lea (L(shl_14_bwd_loop_L2)-L(shl_14_bwd_loop_L1))(%r9), %r9
1590L(L14_bwd):
1591 lea -64(%rdx), %rdx
1592 jmp *%r9
1593 ud2
1594L(shl_14_bwd_loop_L2):
1595 prefetchnta -0x1c0(%rsi)
1596L(shl_14_bwd_loop_L1):
1597 movaps -0x1e(%rsi), %xmm2
1598 sub $0x40, %rdx
1599 movaps -0x2e(%rsi), %xmm3
1600 movaps -0x3e(%rsi), %xmm4
1601 movaps -0x4e(%rsi), %xmm5
1602 lea -0x40(%rsi), %rsi
1603 palignr $14, %xmm2, %xmm1
1604 palignr $14, %xmm3, %xmm2
1605 palignr $14, %xmm4, %xmm3
1606 palignr $14, %xmm5, %xmm4
1607
1608 movaps %xmm1, -0x10(%rdi)
1609 movaps %xmm5, %xmm1
1610
1611 movaps %xmm2, -0x20(%rdi)
1612 lea -0x40(%rdi), %rdi
1613
1614 movaps %xmm3, 0x10(%rdi)
1615 jb L(shl_14_bwd_end)
1616 movaps %xmm4, (%rdi)
1617 jmp *%r9
1618 ud2
1619L(shl_14_bwd_end):
1620 movaps %xmm4, (%rdi)
1621 lea 64(%rdx), %rdx
1622 movdqu %xmm0, (%r8)
1623 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
1624
1625 .p2align 4
1626L(shl_15):
1627 lea (L(shl_15_loop_L1)-L(shl_15))(%r9), %r9
1628 cmp %rcx, %rdx
1629 movaps -0x0f(%rsi), %xmm1
1630 jb L(L15_fwd)
1631 lea (L(shl_15_loop_L2)-L(shl_15_loop_L1))(%r9), %r9
1632L(L15_fwd):
1633 lea -64(%rdx), %rdx
1634 jmp *%r9
1635 ud2
1636L(shl_15_loop_L2):
1637 prefetchnta 0x1c0(%rsi)
1638L(shl_15_loop_L1):
1639 sub $64, %rdx
1640 movaps 0x01(%rsi), %xmm2
1641 movaps 0x11(%rsi), %xmm3
1642 movaps 0x21(%rsi), %xmm4
1643 movaps 0x31(%rsi), %xmm5
1644 movdqa %xmm5, %xmm6
1645 palignr $15, %xmm4, %xmm5
1646 lea 64(%rsi), %rsi
1647 palignr $15, %xmm3, %xmm4
1648 palignr $15, %xmm2, %xmm3
1649 lea 64(%rdi), %rdi
1650 palignr $15, %xmm1, %xmm2
1651 movdqa %xmm6, %xmm1
1652 movdqa %xmm2, -0x40(%rdi)
1653 movaps %xmm3, -0x30(%rdi)
1654 jb L(shl_15_end)
1655 movaps %xmm4, -0x20(%rdi)
1656 movaps %xmm5, -0x10(%rdi)
1657 jmp *%r9
1658 ud2
1659L(shl_15_end):
1660 movaps %xmm4, -0x20(%rdi)
1661 lea 64(%rdx), %rdx
1662 movaps %xmm5, -0x10(%rdi)
1663 add %rdx, %rdi
1664 movdqu %xmm0, (%r8)
1665 add %rdx, %rsi
1666 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
1667
1668 .p2align 4
1669L(shl_15_bwd):
1670 lea (L(shl_15_bwd_loop_L1)-L(shl_15_bwd))(%r9), %r9
1671 cmp %rcx, %rdx
1672 movaps -0x0f(%rsi), %xmm1
1673 jb L(L15_bwd)
1674 lea (L(shl_15_bwd_loop_L2)-L(shl_15_bwd_loop_L1))(%r9), %r9
1675L(L15_bwd):
1676 lea -64(%rdx), %rdx
1677 jmp *%r9
1678 ud2
1679L(shl_15_bwd_loop_L2):
1680 prefetchnta -0x1c0(%rsi)
1681L(shl_15_bwd_loop_L1):
1682 movaps -0x1f(%rsi), %xmm2
1683 sub $0x40, %rdx
1684 movaps -0x2f(%rsi), %xmm3
1685 movaps -0x3f(%rsi), %xmm4
1686 movaps -0x4f(%rsi), %xmm5
1687 lea -0x40(%rsi), %rsi
1688 palignr $15, %xmm2, %xmm1
1689 palignr $15, %xmm3, %xmm2
1690 palignr $15, %xmm4, %xmm3
1691 palignr $15, %xmm5, %xmm4
1692
1693 movaps %xmm1, -0x10(%rdi)
1694 movaps %xmm5, %xmm1
1695
1696 movaps %xmm2, -0x20(%rdi)
1697 lea -0x40(%rdi), %rdi
1698
1699 movaps %xmm3, 0x10(%rdi)
1700 jb L(shl_15_bwd_end)
1701 movaps %xmm4, (%rdi)
1702 jmp *%r9
1703 ud2
1704L(shl_15_bwd_end):
1705 movaps %xmm4, (%rdi)
1706 lea 64(%rdx), %rdx
1707 movdqu %xmm0, (%r8)
1708 BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4)
1709
1710 .p2align 4
1711L(write_72bytes):
1712 movdqu -72(%rsi), %xmm0
1713 movdqu -56(%rsi), %xmm1
1714 mov -40(%rsi), %r8
1715 mov -32(%rsi), %r9
1716 mov -24(%rsi), %r10
1717 mov -16(%rsi), %r11
1718 mov -8(%rsi), %rcx
1719 movdqu %xmm0, -72(%rdi)
1720 movdqu %xmm1, -56(%rdi)
1721 mov %r8, -40(%rdi)
1722 mov %r9, -32(%rdi)
1723 mov %r10, -24(%rdi)
1724 mov %r11, -16(%rdi)
1725 mov %rcx, -8(%rdi)
1726 ret
1727
1728 .p2align 4
1729L(write_64bytes):
1730 movdqu -64(%rsi), %xmm0
1731 mov -48(%rsi), %rcx
1732 mov -40(%rsi), %r8
1733 mov -32(%rsi), %r9
1734 mov -24(%rsi), %r10
1735 mov -16(%rsi), %r11
1736 mov -8(%rsi), %rdx
1737 movdqu %xmm0, -64(%rdi)
1738 mov %rcx, -48(%rdi)
1739 mov %r8, -40(%rdi)
1740 mov %r9, -32(%rdi)
1741 mov %r10, -24(%rdi)
1742 mov %r11, -16(%rdi)
1743 mov %rdx, -8(%rdi)
1744 ret
1745
1746 .p2align 4
1747L(write_56bytes):
1748 movdqu -56(%rsi), %xmm0
1749 mov -40(%rsi), %r8
1750 mov -32(%rsi), %r9
1751 mov -24(%rsi), %r10
1752 mov -16(%rsi), %r11
1753 mov -8(%rsi), %rcx
1754 movdqu %xmm0, -56(%rdi)
1755 mov %r8, -40(%rdi)
1756 mov %r9, -32(%rdi)
1757 mov %r10, -24(%rdi)
1758 mov %r11, -16(%rdi)
1759 mov %rcx, -8(%rdi)
1760 ret
1761
1762 .p2align 4
1763L(write_48bytes):
1764 mov -48(%rsi), %rcx
1765 mov -40(%rsi), %r8
1766 mov -32(%rsi), %r9
1767 mov -24(%rsi), %r10
1768 mov -16(%rsi), %r11
1769 mov -8(%rsi), %rdx
1770 mov %rcx, -48(%rdi)
1771 mov %r8, -40(%rdi)
1772 mov %r9, -32(%rdi)
1773 mov %r10, -24(%rdi)
1774 mov %r11, -16(%rdi)
1775 mov %rdx, -8(%rdi)
1776 ret
1777
1778 .p2align 4
1779L(write_40bytes):
1780 mov -40(%rsi), %r8
1781 mov -32(%rsi), %r9
1782 mov -24(%rsi), %r10
1783 mov -16(%rsi), %r11
1784 mov -8(%rsi), %rdx
1785 mov %r8, -40(%rdi)
1786 mov %r9, -32(%rdi)
1787 mov %r10, -24(%rdi)
1788 mov %r11, -16(%rdi)
1789 mov %rdx, -8(%rdi)
1790 ret
1791
1792 .p2align 4
1793L(write_32bytes):
1794 mov -32(%rsi), %r9
1795 mov -24(%rsi), %r10
1796 mov -16(%rsi), %r11
1797 mov -8(%rsi), %rdx
1798 mov %r9, -32(%rdi)
1799 mov %r10, -24(%rdi)
1800 mov %r11, -16(%rdi)
1801 mov %rdx, -8(%rdi)
1802 ret
1803
1804 .p2align 4
1805L(write_24bytes):
1806 mov -24(%rsi), %r10
1807 mov -16(%rsi), %r11
1808 mov -8(%rsi), %rdx
1809 mov %r10, -24(%rdi)
1810 mov %r11, -16(%rdi)
1811 mov %rdx, -8(%rdi)
1812 ret
1813
1814 .p2align 4
1815L(write_16bytes):
1816 mov -16(%rsi), %r11
1817 mov -8(%rsi), %rdx
1818 mov %r11, -16(%rdi)
1819 mov %rdx, -8(%rdi)
1820 ret
1821
1822 .p2align 4
1823L(write_8bytes):
1824 mov -8(%rsi), %rdx
1825 mov %rdx, -8(%rdi)
1826L(write_0bytes):
1827 ret
1828
1829 .p2align 4
1830L(write_73bytes):
1831 movdqu -73(%rsi), %xmm0
1832 movdqu -57(%rsi), %xmm1
1833 mov -41(%rsi), %rcx
1834 mov -33(%rsi), %r9
1835 mov -25(%rsi), %r10
1836 mov -17(%rsi), %r11
1837 mov -9(%rsi), %r8
1838 mov -4(%rsi), %edx
1839 movdqu %xmm0, -73(%rdi)
1840 movdqu %xmm1, -57(%rdi)
1841 mov %rcx, -41(%rdi)
1842 mov %r9, -33(%rdi)
1843 mov %r10, -25(%rdi)
1844 mov %r11, -17(%rdi)
1845 mov %r8, -9(%rdi)
1846 mov %edx, -4(%rdi)
1847 ret
1848
1849 .p2align 4
1850L(write_65bytes):
1851 movdqu -65(%rsi), %xmm0
1852 movdqu -49(%rsi), %xmm1
1853 mov -33(%rsi), %r9
1854 mov -25(%rsi), %r10
1855 mov -17(%rsi), %r11
1856 mov -9(%rsi), %rcx
1857 mov -4(%rsi), %edx
1858 movdqu %xmm0, -65(%rdi)
1859 movdqu %xmm1, -49(%rdi)
1860 mov %r9, -33(%rdi)
1861 mov %r10, -25(%rdi)
1862 mov %r11, -17(%rdi)
1863 mov %rcx, -9(%rdi)
1864 mov %edx, -4(%rdi)
1865 ret
1866
1867 .p2align 4
1868L(write_57bytes):
1869 movdqu -57(%rsi), %xmm0
1870 mov -41(%rsi), %r8
1871 mov -33(%rsi), %r9
1872 mov -25(%rsi), %r10
1873 mov -17(%rsi), %r11
1874 mov -9(%rsi), %rcx
1875 mov -4(%rsi), %edx
1876 movdqu %xmm0, -57(%rdi)
1877 mov %r8, -41(%rdi)
1878 mov %r9, -33(%rdi)
1879 mov %r10, -25(%rdi)
1880 mov %r11, -17(%rdi)
1881 mov %rcx, -9(%rdi)
1882 mov %edx, -4(%rdi)
1883 ret
1884
1885 .p2align 4
1886L(write_49bytes):
1887 movdqu -49(%rsi), %xmm0
1888 mov -33(%rsi), %r9
1889 mov -25(%rsi), %r10
1890 mov -17(%rsi), %r11
1891 mov -9(%rsi), %rcx
1892 mov -4(%rsi), %edx
1893 movdqu %xmm0, -49(%rdi)
1894 mov %r9, -33(%rdi)
1895 mov %r10, -25(%rdi)
1896 mov %r11, -17(%rdi)
1897 mov %rcx, -9(%rdi)
1898 mov %edx, -4(%rdi)
1899 ret
1900
1901 .p2align 4
1902L(write_41bytes):
1903 mov -41(%rsi), %r8
1904 mov -33(%rsi), %r9
1905 mov -25(%rsi), %r10
1906 mov -17(%rsi), %r11
1907 mov -9(%rsi), %rcx
1908 mov -1(%rsi), %dl
1909 mov %r8, -41(%rdi)
1910 mov %r9, -33(%rdi)
1911 mov %r10, -25(%rdi)
1912 mov %r11, -17(%rdi)
1913 mov %rcx, -9(%rdi)
1914 mov %dl, -1(%rdi)
1915 ret
1916
1917 .p2align 4
1918L(write_33bytes):
1919 mov -33(%rsi), %r9
1920 mov -25(%rsi), %r10
1921 mov -17(%rsi), %r11
1922 mov -9(%rsi), %rcx
1923 mov -1(%rsi), %dl
1924 mov %r9, -33(%rdi)
1925 mov %r10, -25(%rdi)
1926 mov %r11, -17(%rdi)
1927 mov %rcx, -9(%rdi)
1928 mov %dl, -1(%rdi)
1929 ret
1930
1931 .p2align 4
1932L(write_25bytes):
1933 mov -25(%rsi), %r10
1934 mov -17(%rsi), %r11
1935 mov -9(%rsi), %rcx
1936 mov -1(%rsi), %dl
1937 mov %r10, -25(%rdi)
1938 mov %r11, -17(%rdi)
1939 mov %rcx, -9(%rdi)
1940 mov %dl, -1(%rdi)
1941 ret
1942
1943 .p2align 4
1944L(write_17bytes):
1945 mov -17(%rsi), %r11
1946 mov -9(%rsi), %rcx
1947 mov -4(%rsi), %edx
1948 mov %r11, -17(%rdi)
1949 mov %rcx, -9(%rdi)
1950 mov %edx, -4(%rdi)
1951 ret
1952
1953 .p2align 4
1954L(write_9bytes):
1955 mov -9(%rsi), %rcx
1956 mov -4(%rsi), %edx
1957 mov %rcx, -9(%rdi)
1958 mov %edx, -4(%rdi)
1959 ret
1960
1961 .p2align 4
1962L(write_1bytes):
1963 mov -1(%rsi), %dl
1964 mov %dl, -1(%rdi)
1965 ret
1966
1967 .p2align 4
1968L(write_74bytes):
1969 movdqu -74(%rsi), %xmm0
1970 movdqu -58(%rsi), %xmm1
1971 mov -42(%rsi), %r8
1972 mov -34(%rsi), %r9
1973 mov -26(%rsi), %r10
1974 mov -18(%rsi), %r11
1975 mov -10(%rsi), %rcx
1976 mov -4(%rsi), %edx
1977 movdqu %xmm0, -74(%rdi)
1978 movdqu %xmm1, -58(%rdi)
1979 mov %r8, -42(%rdi)
1980 mov %r9, -34(%rdi)
1981 mov %r10, -26(%rdi)
1982 mov %r11, -18(%rdi)
1983 mov %rcx, -10(%rdi)
1984 mov %edx, -4(%rdi)
1985 ret
1986
1987 .p2align 4
1988L(write_66bytes):
1989 movdqu -66(%rsi), %xmm0
1990 movdqu -50(%rsi), %xmm1
1991 mov -42(%rsi), %r8
1992 mov -34(%rsi), %r9
1993 mov -26(%rsi), %r10
1994 mov -18(%rsi), %r11
1995 mov -10(%rsi), %rcx
1996 mov -4(%rsi), %edx
1997 movdqu %xmm0, -66(%rdi)
1998 movdqu %xmm1, -50(%rdi)
1999 mov %r8, -42(%rdi)
2000 mov %r9, -34(%rdi)
2001 mov %r10, -26(%rdi)
2002 mov %r11, -18(%rdi)
2003 mov %rcx, -10(%rdi)
2004 mov %edx, -4(%rdi)
2005 ret
2006
2007 .p2align 4
2008L(write_58bytes):
2009 movdqu -58(%rsi), %xmm1
2010 mov -42(%rsi), %r8
2011 mov -34(%rsi), %r9
2012 mov -26(%rsi), %r10
2013 mov -18(%rsi), %r11
2014 mov -10(%rsi), %rcx
2015 mov -4(%rsi), %edx
2016 movdqu %xmm1, -58(%rdi)
2017 mov %r8, -42(%rdi)
2018 mov %r9, -34(%rdi)
2019 mov %r10, -26(%rdi)
2020 mov %r11, -18(%rdi)
2021 mov %rcx, -10(%rdi)
2022 mov %edx, -4(%rdi)
2023 ret
2024
2025 .p2align 4
2026L(write_50bytes):
2027 movdqu -50(%rsi), %xmm0
2028 mov -34(%rsi), %r9
2029 mov -26(%rsi), %r10
2030 mov -18(%rsi), %r11
2031 mov -10(%rsi), %rcx
2032 mov -4(%rsi), %edx
2033 movdqu %xmm0, -50(%rdi)
2034 mov %r9, -34(%rdi)
2035 mov %r10, -26(%rdi)
2036 mov %r11, -18(%rdi)
2037 mov %rcx, -10(%rdi)
2038 mov %edx, -4(%rdi)
2039 ret
2040
2041 .p2align 4
2042L(write_42bytes):
2043 mov -42(%rsi), %r8
2044 mov -34(%rsi), %r9
2045 mov -26(%rsi), %r10
2046 mov -18(%rsi), %r11
2047 mov -10(%rsi), %rcx
2048 mov -4(%rsi), %edx
2049 mov %r8, -42(%rdi)
2050 mov %r9, -34(%rdi)
2051 mov %r10, -26(%rdi)
2052 mov %r11, -18(%rdi)
2053 mov %rcx, -10(%rdi)
2054 mov %edx, -4(%rdi)
2055 ret
2056
2057 .p2align 4
2058L(write_34bytes):
2059 mov -34(%rsi), %r9
2060 mov -26(%rsi), %r10
2061 mov -18(%rsi), %r11
2062 mov -10(%rsi), %rcx
2063 mov -4(%rsi), %edx
2064 mov %r9, -34(%rdi)
2065 mov %r10, -26(%rdi)
2066 mov %r11, -18(%rdi)
2067 mov %rcx, -10(%rdi)
2068 mov %edx, -4(%rdi)
2069 ret
2070
2071 .p2align 4
2072L(write_26bytes):
2073 mov -26(%rsi), %r10
2074 mov -18(%rsi), %r11
2075 mov -10(%rsi), %rcx
2076 mov -4(%rsi), %edx
2077 mov %r10, -26(%rdi)
2078 mov %r11, -18(%rdi)
2079 mov %rcx, -10(%rdi)
2080 mov %edx, -4(%rdi)
2081 ret
2082
2083 .p2align 4
2084L(write_18bytes):
2085 mov -18(%rsi), %r11
2086 mov -10(%rsi), %rcx
2087 mov -4(%rsi), %edx
2088 mov %r11, -18(%rdi)
2089 mov %rcx, -10(%rdi)
2090 mov %edx, -4(%rdi)
2091 ret
2092
2093 .p2align 4
2094L(write_10bytes):
2095 mov -10(%rsi), %rcx
2096 mov -4(%rsi), %edx
2097 mov %rcx, -10(%rdi)
2098 mov %edx, -4(%rdi)
2099 ret
2100
2101 .p2align 4
2102L(write_2bytes):
2103 mov -2(%rsi), %dx
2104 mov %dx, -2(%rdi)
2105 ret
2106
2107 .p2align 4
2108L(write_75bytes):
2109 movdqu -75(%rsi), %xmm0
2110 movdqu -59(%rsi), %xmm1
2111 mov -43(%rsi), %r8
2112 mov -35(%rsi), %r9
2113 mov -27(%rsi), %r10
2114 mov -19(%rsi), %r11
2115 mov -11(%rsi), %rcx
2116 mov -4(%rsi), %edx
2117 movdqu %xmm0, -75(%rdi)
2118 movdqu %xmm1, -59(%rdi)
2119 mov %r8, -43(%rdi)
2120 mov %r9, -35(%rdi)
2121 mov %r10, -27(%rdi)
2122 mov %r11, -19(%rdi)
2123 mov %rcx, -11(%rdi)
2124 mov %edx, -4(%rdi)
2125 ret
2126
2127 .p2align 4
2128L(write_67bytes):
2129 movdqu -67(%rsi), %xmm0
2130 movdqu -59(%rsi), %xmm1
2131 mov -43(%rsi), %r8
2132 mov -35(%rsi), %r9
2133 mov -27(%rsi), %r10
2134 mov -19(%rsi), %r11
2135 mov -11(%rsi), %rcx
2136 mov -4(%rsi), %edx
2137 movdqu %xmm0, -67(%rdi)
2138 movdqu %xmm1, -59(%rdi)
2139 mov %r8, -43(%rdi)
2140 mov %r9, -35(%rdi)
2141 mov %r10, -27(%rdi)
2142 mov %r11, -19(%rdi)
2143 mov %rcx, -11(%rdi)
2144 mov %edx, -4(%rdi)
2145 ret
2146
2147 .p2align 4
2148L(write_59bytes):
2149 movdqu -59(%rsi), %xmm0
2150 mov -43(%rsi), %r8
2151 mov -35(%rsi), %r9
2152 mov -27(%rsi), %r10
2153 mov -19(%rsi), %r11
2154 mov -11(%rsi), %rcx
2155 mov -4(%rsi), %edx
2156 movdqu %xmm0, -59(%rdi)
2157 mov %r8, -43(%rdi)
2158 mov %r9, -35(%rdi)
2159 mov %r10, -27(%rdi)
2160 mov %r11, -19(%rdi)
2161 mov %rcx, -11(%rdi)
2162 mov %edx, -4(%rdi)
2163 ret
2164
2165 .p2align 4
2166L(write_51bytes):
2167 movdqu -51(%rsi), %xmm0
2168 mov -35(%rsi), %r9
2169 mov -27(%rsi), %r10
2170 mov -19(%rsi), %r11
2171 mov -11(%rsi), %rcx
2172 mov -4(%rsi), %edx
2173 movdqu %xmm0, -51(%rdi)
2174 mov %r9, -35(%rdi)
2175 mov %r10, -27(%rdi)
2176 mov %r11, -19(%rdi)
2177 mov %rcx, -11(%rdi)
2178 mov %edx, -4(%rdi)
2179 ret
2180
2181 .p2align 4
2182L(write_43bytes):
2183 mov -43(%rsi), %r8
2184 mov -35(%rsi), %r9
2185 mov -27(%rsi), %r10
2186 mov -19(%rsi), %r11
2187 mov -11(%rsi), %rcx
2188 mov -4(%rsi), %edx
2189 mov %r8, -43(%rdi)
2190 mov %r9, -35(%rdi)
2191 mov %r10, -27(%rdi)
2192 mov %r11, -19(%rdi)
2193 mov %rcx, -11(%rdi)
2194 mov %edx, -4(%rdi)
2195 ret
2196
2197 .p2align 4
2198L(write_35bytes):
2199 mov -35(%rsi), %r9
2200 mov -27(%rsi), %r10
2201 mov -19(%rsi), %r11
2202 mov -11(%rsi), %rcx
2203 mov -4(%rsi), %edx
2204 mov %r9, -35(%rdi)
2205 mov %r10, -27(%rdi)
2206 mov %r11, -19(%rdi)
2207 mov %rcx, -11(%rdi)
2208 mov %edx, -4(%rdi)
2209 ret
2210
2211 .p2align 4
2212L(write_27bytes):
2213 mov -27(%rsi), %r10
2214 mov -19(%rsi), %r11
2215 mov -11(%rsi), %rcx
2216 mov -4(%rsi), %edx
2217 mov %r10, -27(%rdi)
2218 mov %r11, -19(%rdi)
2219 mov %rcx, -11(%rdi)
2220 mov %edx, -4(%rdi)
2221 ret
2222
2223 .p2align 4
2224L(write_19bytes):
2225 mov -19(%rsi), %r11
2226 mov -11(%rsi), %rcx
2227 mov -4(%rsi), %edx
2228 mov %r11, -19(%rdi)
2229 mov %rcx, -11(%rdi)
2230 mov %edx, -4(%rdi)
2231 ret
2232
2233 .p2align 4
2234L(write_11bytes):
2235 mov -11(%rsi), %rcx
2236 mov -4(%rsi), %edx
2237 mov %rcx, -11(%rdi)
2238 mov %edx, -4(%rdi)
2239 ret
2240
2241 .p2align 4
2242L(write_3bytes):
2243 mov -3(%rsi), %dx
2244 mov -2(%rsi), %cx
2245 mov %dx, -3(%rdi)
2246 mov %cx, -2(%rdi)
2247 ret
2248
2249 .p2align 4
2250L(write_76bytes):
2251 movdqu -76(%rsi), %xmm0
2252 movdqu -60(%rsi), %xmm1
2253 mov -44(%rsi), %r8
2254 mov -36(%rsi), %r9
2255 mov -28(%rsi), %r10
2256 mov -20(%rsi), %r11
2257 mov -12(%rsi), %rcx
2258 mov -4(%rsi), %edx
2259 movdqu %xmm0, -76(%rdi)
2260 movdqu %xmm1, -60(%rdi)
2261 mov %r8, -44(%rdi)
2262 mov %r9, -36(%rdi)
2263 mov %r10, -28(%rdi)
2264 mov %r11, -20(%rdi)
2265 mov %rcx, -12(%rdi)
2266 mov %edx, -4(%rdi)
2267 ret
2268
2269 .p2align 4
2270L(write_68bytes):
2271 movdqu -68(%rsi), %xmm0
2272 movdqu -52(%rsi), %xmm1
2273 mov -36(%rsi), %r9
2274 mov -28(%rsi), %r10
2275 mov -20(%rsi), %r11
2276 mov -12(%rsi), %rcx
2277 mov -4(%rsi), %edx
2278 movdqu %xmm0, -68(%rdi)
2279 movdqu %xmm1, -52(%rdi)
2280 mov %r9, -36(%rdi)
2281 mov %r10, -28(%rdi)
2282 mov %r11, -20(%rdi)
2283 mov %rcx, -12(%rdi)
2284 mov %edx, -4(%rdi)
2285 ret
2286
2287 .p2align 4
2288L(write_60bytes):
2289 movdqu -60(%rsi), %xmm0
2290 mov -44(%rsi), %r8
2291 mov -36(%rsi), %r9
2292 mov -28(%rsi), %r10
2293 mov -20(%rsi), %r11
2294 mov -12(%rsi), %rcx
2295 mov -4(%rsi), %edx
2296 movdqu %xmm0, -60(%rdi)
2297 mov %r8, -44(%rdi)
2298 mov %r9, -36(%rdi)
2299 mov %r10, -28(%rdi)
2300 mov %r11, -20(%rdi)
2301 mov %rcx, -12(%rdi)
2302 mov %edx, -4(%rdi)
2303 ret
2304
2305 .p2align 4
2306L(write_52bytes):
2307 movdqu -52(%rsi), %xmm0
2308 mov -36(%rsi), %r9
2309 mov -28(%rsi), %r10
2310 mov -20(%rsi), %r11
2311 mov -12(%rsi), %rcx
2312 mov -4(%rsi), %edx
2313 movdqu %xmm0, -52(%rdi)
2314 mov %r9, -36(%rdi)
2315 mov %r10, -28(%rdi)
2316 mov %r11, -20(%rdi)
2317 mov %rcx, -12(%rdi)
2318 mov %edx, -4(%rdi)
2319 ret
2320
2321 .p2align 4
2322L(write_44bytes):
2323 mov -44(%rsi), %r8
2324 mov -36(%rsi), %r9
2325 mov -28(%rsi), %r10
2326 mov -20(%rsi), %r11
2327 mov -12(%rsi), %rcx
2328 mov -4(%rsi), %edx
2329 mov %r8, -44(%rdi)
2330 mov %r9, -36(%rdi)
2331 mov %r10, -28(%rdi)
2332 mov %r11, -20(%rdi)
2333 mov %rcx, -12(%rdi)
2334 mov %edx, -4(%rdi)
2335 ret
2336
2337 .p2align 4
2338L(write_36bytes):
2339 mov -36(%rsi), %r9
2340 mov -28(%rsi), %r10
2341 mov -20(%rsi), %r11
2342 mov -12(%rsi), %rcx
2343 mov -4(%rsi), %edx
2344 mov %r9, -36(%rdi)
2345 mov %r10, -28(%rdi)
2346 mov %r11, -20(%rdi)
2347 mov %rcx, -12(%rdi)
2348 mov %edx, -4(%rdi)
2349 ret
2350
2351 .p2align 4
2352L(write_28bytes):
2353 mov -28(%rsi), %r10
2354 mov -20(%rsi), %r11
2355 mov -12(%rsi), %rcx
2356 mov -4(%rsi), %edx
2357 mov %r10, -28(%rdi)
2358 mov %r11, -20(%rdi)
2359 mov %rcx, -12(%rdi)
2360 mov %edx, -4(%rdi)
2361 ret
2362
2363 .p2align 4
2364L(write_20bytes):
2365 mov -20(%rsi), %r11
2366 mov -12(%rsi), %rcx
2367 mov -4(%rsi), %edx
2368 mov %r11, -20(%rdi)
2369 mov %rcx, -12(%rdi)
2370 mov %edx, -4(%rdi)
2371 ret
2372
2373 .p2align 4
2374L(write_12bytes):
2375 mov -12(%rsi), %rcx
2376 mov -4(%rsi), %edx
2377 mov %rcx, -12(%rdi)
2378 mov %edx, -4(%rdi)
2379 ret
2380
2381 .p2align 4
2382L(write_4bytes):
2383 mov -4(%rsi), %edx
2384 mov %edx, -4(%rdi)
2385 ret
2386
2387 .p2align 4
2388L(write_77bytes):
2389 movdqu -77(%rsi), %xmm0
2390 movdqu -61(%rsi), %xmm1
2391 mov -45(%rsi), %r8
2392 mov -37(%rsi), %r9
2393 mov -29(%rsi), %r10
2394 mov -21(%rsi), %r11
2395 mov -13(%rsi), %rcx
2396 mov -8(%rsi), %rdx
2397 movdqu %xmm0, -77(%rdi)
2398 movdqu %xmm1, -61(%rdi)
2399 mov %r8, -45(%rdi)
2400 mov %r9, -37(%rdi)
2401 mov %r10, -29(%rdi)
2402 mov %r11, -21(%rdi)
2403 mov %rcx, -13(%rdi)
2404 mov %rdx, -8(%rdi)
2405 ret
2406
2407 .p2align 4
2408L(write_69bytes):
2409 movdqu -69(%rsi), %xmm0
2410 movdqu -53(%rsi), %xmm1
2411 mov -37(%rsi), %r9
2412 mov -29(%rsi), %r10
2413 mov -21(%rsi), %r11
2414 mov -13(%rsi), %rcx
2415 mov -8(%rsi), %rdx
2416 movdqu %xmm0, -69(%rdi)
2417 movdqu %xmm1, -53(%rdi)
2418 mov %r9, -37(%rdi)
2419 mov %r10, -29(%rdi)
2420 mov %r11, -21(%rdi)
2421 mov %rcx, -13(%rdi)
2422 mov %rdx, -8(%rdi)
2423 ret
2424
2425 .p2align 4
2426L(write_61bytes):
2427 movdqu -61(%rsi), %xmm0
2428 mov -45(%rsi), %r8
2429 mov -37(%rsi), %r9
2430 mov -29(%rsi), %r10
2431 mov -21(%rsi), %r11
2432 mov -13(%rsi), %rcx
2433 mov -8(%rsi), %rdx
2434 movdqu %xmm0, -61(%rdi)
2435 mov %r8, -45(%rdi)
2436 mov %r9, -37(%rdi)
2437 mov %r10, -29(%rdi)
2438 mov %r11, -21(%rdi)
2439 mov %rcx, -13(%rdi)
2440 mov %rdx, -8(%rdi)
2441 ret
2442
2443 .p2align 4
2444L(write_53bytes):
2445 movdqu -53(%rsi), %xmm0
2446 mov -45(%rsi), %r8
2447 mov -37(%rsi), %r9
2448 mov -29(%rsi), %r10
2449 mov -21(%rsi), %r11
2450 mov -13(%rsi), %rcx
2451 mov -8(%rsi), %rdx
2452 movdqu %xmm0, -53(%rdi)
2453 mov %r9, -37(%rdi)
2454 mov %r10, -29(%rdi)
2455 mov %r11, -21(%rdi)
2456 mov %rcx, -13(%rdi)
2457 mov %rdx, -8(%rdi)
2458 ret
2459
2460 .p2align 4
2461L(write_45bytes):
2462 mov -45(%rsi), %r8
2463 mov -37(%rsi), %r9
2464 mov -29(%rsi), %r10
2465 mov -21(%rsi), %r11
2466 mov -13(%rsi), %rcx
2467 mov -8(%rsi), %rdx
2468 mov %r8, -45(%rdi)
2469 mov %r9, -37(%rdi)
2470 mov %r10, -29(%rdi)
2471 mov %r11, -21(%rdi)
2472 mov %rcx, -13(%rdi)
2473 mov %rdx, -8(%rdi)
2474 ret
2475
2476 .p2align 4
2477L(write_37bytes):
2478 mov -37(%rsi), %r9
2479 mov -29(%rsi), %r10
2480 mov -21(%rsi), %r11
2481 mov -13(%rsi), %rcx
2482 mov -8(%rsi), %rdx
2483 mov %r9, -37(%rdi)
2484 mov %r10, -29(%rdi)
2485 mov %r11, -21(%rdi)
2486 mov %rcx, -13(%rdi)
2487 mov %rdx, -8(%rdi)
2488 ret
2489
2490 .p2align 4
2491L(write_29bytes):
2492 mov -29(%rsi), %r10
2493 mov -21(%rsi), %r11
2494 mov -13(%rsi), %rcx
2495 mov -8(%rsi), %rdx
2496 mov %r10, -29(%rdi)
2497 mov %r11, -21(%rdi)
2498 mov %rcx, -13(%rdi)
2499 mov %rdx, -8(%rdi)
2500 ret
2501
2502 .p2align 4
2503L(write_21bytes):
2504 mov -21(%rsi), %r11
2505 mov -13(%rsi), %rcx
2506 mov -8(%rsi), %rdx
2507 mov %r11, -21(%rdi)
2508 mov %rcx, -13(%rdi)
2509 mov %rdx, -8(%rdi)
2510 ret
2511
2512 .p2align 4
2513L(write_13bytes):
2514 mov -13(%rsi), %rcx
2515 mov -8(%rsi), %rdx
2516 mov %rcx, -13(%rdi)
2517 mov %rdx, -8(%rdi)
2518 ret
2519
2520 .p2align 4
2521L(write_5bytes):
2522 mov -5(%rsi), %edx
2523 mov -4(%rsi), %ecx
2524 mov %edx, -5(%rdi)
2525 mov %ecx, -4(%rdi)
2526 ret
2527
2528 .p2align 4
2529L(write_78bytes):
2530 movdqu -78(%rsi), %xmm0
2531 movdqu -62(%rsi), %xmm1
2532 mov -46(%rsi), %r8
2533 mov -38(%rsi), %r9
2534 mov -30(%rsi), %r10
2535 mov -22(%rsi), %r11
2536 mov -14(%rsi), %rcx
2537 mov -8(%rsi), %rdx
2538 movdqu %xmm0, -78(%rdi)
2539 movdqu %xmm1, -62(%rdi)
2540 mov %r8, -46(%rdi)
2541 mov %r9, -38(%rdi)
2542 mov %r10, -30(%rdi)
2543 mov %r11, -22(%rdi)
2544 mov %rcx, -14(%rdi)
2545 mov %rdx, -8(%rdi)
2546 ret
2547
2548 .p2align 4
2549L(write_70bytes):
2550 movdqu -70(%rsi), %xmm0
2551 movdqu -54(%rsi), %xmm1
2552 mov -38(%rsi), %r9
2553 mov -30(%rsi), %r10
2554 mov -22(%rsi), %r11
2555 mov -14(%rsi), %rcx
2556 mov -8(%rsi), %rdx
2557 movdqu %xmm0, -70(%rdi)
2558 movdqu %xmm1, -54(%rdi)
2559 mov %r9, -38(%rdi)
2560 mov %r10, -30(%rdi)
2561 mov %r11, -22(%rdi)
2562 mov %rcx, -14(%rdi)
2563 mov %rdx, -8(%rdi)
2564 ret
2565
2566 .p2align 4
2567L(write_62bytes):
2568 movdqu -62(%rsi), %xmm0
2569 mov -46(%rsi), %r8
2570 mov -38(%rsi), %r9
2571 mov -30(%rsi), %r10
2572 mov -22(%rsi), %r11
2573 mov -14(%rsi), %rcx
2574 mov -8(%rsi), %rdx
2575 movdqu %xmm0, -62(%rdi)
2576 mov %r8, -46(%rdi)
2577 mov %r9, -38(%rdi)
2578 mov %r10, -30(%rdi)
2579 mov %r11, -22(%rdi)
2580 mov %rcx, -14(%rdi)
2581 mov %rdx, -8(%rdi)
2582 ret
2583
2584 .p2align 4
2585L(write_54bytes):
2586 movdqu -54(%rsi), %xmm0
2587 mov -38(%rsi), %r9
2588 mov -30(%rsi), %r10
2589 mov -22(%rsi), %r11
2590 mov -14(%rsi), %rcx
2591 mov -8(%rsi), %rdx
2592 movdqu %xmm0, -54(%rdi)
2593 mov %r9, -38(%rdi)
2594 mov %r10, -30(%rdi)
2595 mov %r11, -22(%rdi)
2596 mov %rcx, -14(%rdi)
2597 mov %rdx, -8(%rdi)
2598 ret
2599
2600 .p2align 4
2601L(write_46bytes):
2602 mov -46(%rsi), %r8
2603 mov -38(%rsi), %r9
2604 mov -30(%rsi), %r10
2605 mov -22(%rsi), %r11
2606 mov -14(%rsi), %rcx
2607 mov -8(%rsi), %rdx
2608 mov %r8, -46(%rdi)
2609 mov %r9, -38(%rdi)
2610 mov %r10, -30(%rdi)
2611 mov %r11, -22(%rdi)
2612 mov %rcx, -14(%rdi)
2613 mov %rdx, -8(%rdi)
2614 ret
2615
2616 .p2align 4
2617L(write_38bytes):
2618 mov -38(%rsi), %r9
2619 mov -30(%rsi), %r10
2620 mov -22(%rsi), %r11
2621 mov -14(%rsi), %rcx
2622 mov -8(%rsi), %rdx
2623 mov %r9, -38(%rdi)
2624 mov %r10, -30(%rdi)
2625 mov %r11, -22(%rdi)
2626 mov %rcx, -14(%rdi)
2627 mov %rdx, -8(%rdi)
2628 ret
2629
2630 .p2align 4
2631L(write_30bytes):
2632 mov -30(%rsi), %r10
2633 mov -22(%rsi), %r11
2634 mov -14(%rsi), %rcx
2635 mov -8(%rsi), %rdx
2636 mov %r10, -30(%rdi)
2637 mov %r11, -22(%rdi)
2638 mov %rcx, -14(%rdi)
2639 mov %rdx, -8(%rdi)
2640 ret
2641
2642 .p2align 4
2643L(write_22bytes):
2644 mov -22(%rsi), %r11
2645 mov -14(%rsi), %rcx
2646 mov -8(%rsi), %rdx
2647 mov %r11, -22(%rdi)
2648 mov %rcx, -14(%rdi)
2649 mov %rdx, -8(%rdi)
2650 ret
2651
2652 .p2align 4
2653L(write_14bytes):
2654 mov -14(%rsi), %rcx
2655 mov -8(%rsi), %rdx
2656 mov %rcx, -14(%rdi)
2657 mov %rdx, -8(%rdi)
2658 ret
2659
2660 .p2align 4
2661L(write_6bytes):
2662 mov -6(%rsi), %edx
2663 mov -4(%rsi), %ecx
2664 mov %edx, -6(%rdi)
2665 mov %ecx, -4(%rdi)
2666 ret
2667
2668 .p2align 4
2669L(write_79bytes):
2670 movdqu -79(%rsi), %xmm0
2671 movdqu -63(%rsi), %xmm1
2672 mov -47(%rsi), %r8
2673 mov -39(%rsi), %r9
2674 mov -31(%rsi), %r10
2675 mov -23(%rsi), %r11
2676 mov -15(%rsi), %rcx
2677 mov -8(%rsi), %rdx
2678 movdqu %xmm0, -79(%rdi)
2679 movdqu %xmm1, -63(%rdi)
2680 mov %r8, -47(%rdi)
2681 mov %r9, -39(%rdi)
2682 mov %r10, -31(%rdi)
2683 mov %r11, -23(%rdi)
2684 mov %rcx, -15(%rdi)
2685 mov %rdx, -8(%rdi)
2686 ret
2687
2688 .p2align 4
2689L(write_71bytes):
2690 movdqu -71(%rsi), %xmm0
2691 movdqu -55(%rsi), %xmm1
2692 mov -39(%rsi), %r9
2693 mov -31(%rsi), %r10
2694 mov -23(%rsi), %r11
2695 mov -15(%rsi), %rcx
2696 mov -8(%rsi), %rdx
2697 movdqu %xmm0, -71(%rdi)
2698 movdqu %xmm1, -55(%rdi)
2699 mov %r9, -39(%rdi)
2700 mov %r10, -31(%rdi)
2701 mov %r11, -23(%rdi)
2702 mov %rcx, -15(%rdi)
2703 mov %rdx, -8(%rdi)
2704 ret
2705
2706 .p2align 4
2707L(write_63bytes):
2708 movdqu -63(%rsi), %xmm0
2709 mov -47(%rsi), %r8
2710 mov -39(%rsi), %r9
2711 mov -31(%rsi), %r10
2712 mov -23(%rsi), %r11
2713 mov -15(%rsi), %rcx
2714 mov -8(%rsi), %rdx
2715 movdqu %xmm0, -63(%rdi)
2716 mov %r8, -47(%rdi)
2717 mov %r9, -39(%rdi)
2718 mov %r10, -31(%rdi)
2719 mov %r11, -23(%rdi)
2720 mov %rcx, -15(%rdi)
2721 mov %rdx, -8(%rdi)
2722 ret
2723
2724 .p2align 4
2725L(write_55bytes):
2726 movdqu -55(%rsi), %xmm0
2727 mov -39(%rsi), %r9
2728 mov -31(%rsi), %r10
2729 mov -23(%rsi), %r11
2730 mov -15(%rsi), %rcx
2731 mov -8(%rsi), %rdx
2732 movdqu %xmm0, -55(%rdi)
2733 mov %r9, -39(%rdi)
2734 mov %r10, -31(%rdi)
2735 mov %r11, -23(%rdi)
2736 mov %rcx, -15(%rdi)
2737 mov %rdx, -8(%rdi)
2738 ret
2739
2740 .p2align 4
2741L(write_47bytes):
2742 mov -47(%rsi), %r8
2743 mov -39(%rsi), %r9
2744 mov -31(%rsi), %r10
2745 mov -23(%rsi), %r11
2746 mov -15(%rsi), %rcx
2747 mov -8(%rsi), %rdx
2748 mov %r8, -47(%rdi)
2749 mov %r9, -39(%rdi)
2750 mov %r10, -31(%rdi)
2751 mov %r11, -23(%rdi)
2752 mov %rcx, -15(%rdi)
2753 mov %rdx, -8(%rdi)
2754 ret
2755
2756 .p2align 4
2757L(write_39bytes):
2758 mov -39(%rsi), %r9
2759 mov -31(%rsi), %r10
2760 mov -23(%rsi), %r11
2761 mov -15(%rsi), %rcx
2762 mov -8(%rsi), %rdx
2763 mov %r9, -39(%rdi)
2764 mov %r10, -31(%rdi)
2765 mov %r11, -23(%rdi)
2766 mov %rcx, -15(%rdi)
2767 mov %rdx, -8(%rdi)
2768 ret
2769
2770 .p2align 4
2771L(write_31bytes):
2772 mov -31(%rsi), %r10
2773 mov -23(%rsi), %r11
2774 mov -15(%rsi), %rcx
2775 mov -8(%rsi), %rdx
2776 mov %r10, -31(%rdi)
2777 mov %r11, -23(%rdi)
2778 mov %rcx, -15(%rdi)
2779 mov %rdx, -8(%rdi)
2780 ret
2781
2782 .p2align 4
2783L(write_23bytes):
2784 mov -23(%rsi), %r11
2785 mov -15(%rsi), %rcx
2786 mov -8(%rsi), %rdx
2787 mov %r11, -23(%rdi)
2788 mov %rcx, -15(%rdi)
2789 mov %rdx, -8(%rdi)
2790 ret
2791
2792 .p2align 4
2793L(write_15bytes):
2794 mov -15(%rsi), %rcx
2795 mov -8(%rsi), %rdx
2796 mov %rcx, -15(%rdi)
2797 mov %rdx, -8(%rdi)
2798 ret
2799
2800 .p2align 4
2801L(write_7bytes):
2802 mov -7(%rsi), %edx
2803 mov -4(%rsi), %ecx
2804 mov %edx, -7(%rdi)
2805 mov %ecx, -4(%rdi)
2806 ret
2807
2808 .p2align 4
2809L(large_page_fwd):
2810 movdqu (%rsi), %xmm1
2811 lea 16(%rsi), %rsi
2812 movdqu %xmm0, (%r8)
2813 movntdq %xmm1, (%rdi)
2814 lea 16(%rdi), %rdi
2815 lea -0x90(%rdx), %rdx
2816#ifdef USE_AS_MEMMOVE
2817 mov %rsi, %r9
2818 sub %rdi, %r9
2819 cmp %rdx, %r9
2820 jae L(memmove_is_memcpy_fwd)
2821 shl $2, %rcx
2822 cmp %rcx, %rdx
2823 jb L(ll_cache_copy_fwd_start)
2824L(memmove_is_memcpy_fwd):
2825#endif
2826L(large_page_loop):
2827 movdqu (%rsi), %xmm0
2828 movdqu 0x10(%rsi), %xmm1
2829 movdqu 0x20(%rsi), %xmm2
2830 movdqu 0x30(%rsi), %xmm3
2831 movdqu 0x40(%rsi), %xmm4
2832 movdqu 0x50(%rsi), %xmm5
2833 movdqu 0x60(%rsi), %xmm6
2834 movdqu 0x70(%rsi), %xmm7
2835 lea 0x80(%rsi), %rsi
2836
2837 sub $0x80, %rdx
2838 movntdq %xmm0, (%rdi)
2839 movntdq %xmm1, 0x10(%rdi)
2840 movntdq %xmm2, 0x20(%rdi)
2841 movntdq %xmm3, 0x30(%rdi)
2842 movntdq %xmm4, 0x40(%rdi)
2843 movntdq %xmm5, 0x50(%rdi)
2844 movntdq %xmm6, 0x60(%rdi)
2845 movntdq %xmm7, 0x70(%rdi)
2846 lea 0x80(%rdi), %rdi
2847 jae L(large_page_loop)
2848 cmp $-0x40, %rdx
2849 lea 0x80(%rdx), %rdx
2850 jl L(large_page_less_64bytes)
2851
2852 movdqu (%rsi), %xmm0
2853 movdqu 0x10(%rsi), %xmm1
2854 movdqu 0x20(%rsi), %xmm2
2855 movdqu 0x30(%rsi), %xmm3
2856 lea 0x40(%rsi), %rsi
2857
2858 movntdq %xmm0, (%rdi)
2859 movntdq %xmm1, 0x10(%rdi)
2860 movntdq %xmm2, 0x20(%rdi)
2861 movntdq %xmm3, 0x30(%rdi)
2862 lea 0x40(%rdi), %rdi
2863 sub $0x40, %rdx
2864L(large_page_less_64bytes):
2865 add %rdx, %rsi
2866 add %rdx, %rdi
2867 sfence
2868 BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4)
2869
2870#ifdef USE_AS_MEMMOVE
2871 .p2align 4
2872L(ll_cache_copy_fwd_start):
2873 prefetcht0 0x1c0(%rsi)
2874 prefetcht0 0x200(%rsi)
2875 movdqu (%rsi), %xmm0
2876 movdqu 0x10(%rsi), %xmm1
2877 movdqu 0x20(%rsi), %xmm2
2878 movdqu 0x30(%rsi), %xmm3
2879 movdqu 0x40(%rsi), %xmm4
2880 movdqu 0x50(%rsi), %xmm5
2881 movdqu 0x60(%rsi), %xmm6
2882 movdqu 0x70(%rsi), %xmm7
2883 lea 0x80(%rsi), %rsi
2884
2885 sub $0x80, %rdx
2886 movaps %xmm0, (%rdi)
2887 movaps %xmm1, 0x10(%rdi)
2888 movaps %xmm2, 0x20(%rdi)
2889 movaps %xmm3, 0x30(%rdi)
2890 movaps %xmm4, 0x40(%rdi)
2891 movaps %xmm5, 0x50(%rdi)
2892 movaps %xmm6, 0x60(%rdi)
2893 movaps %xmm7, 0x70(%rdi)
2894 lea 0x80(%rdi), %rdi
2895 jae L(ll_cache_copy_fwd_start)
2896 cmp $-0x40, %rdx
2897 lea 0x80(%rdx), %rdx
2898 jl L(large_page_ll_less_fwd_64bytes)
2899
2900 movdqu (%rsi), %xmm0
2901 movdqu 0x10(%rsi), %xmm1
2902 movdqu 0x20(%rsi), %xmm2
2903 movdqu 0x30(%rsi), %xmm3
2904 lea 0x40(%rsi), %rsi
2905
2906 movaps %xmm0, (%rdi)
2907 movaps %xmm1, 0x10(%rdi)
2908 movaps %xmm2, 0x20(%rdi)
2909 movaps %xmm3, 0x30(%rdi)
2910 lea 0x40(%rdi), %rdi
2911 sub $0x40, %rdx
2912L(large_page_ll_less_fwd_64bytes):
2913 add %rdx, %rsi
2914 add %rdx, %rdi
2915 BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4)
2916
2917#endif
2918 .p2align 4
2919L(large_page_bwd):
2920 movdqu -0x10(%rsi), %xmm1
2921 lea -16(%rsi), %rsi
2922 movdqu %xmm0, (%r8)
2923 movdqa %xmm1, -0x10(%rdi)
2924 lea -16(%rdi), %rdi
2925 lea -0x90(%rdx), %rdx
2926#ifdef USE_AS_MEMMOVE
2927 mov %rdi, %r9
2928 sub %rsi, %r9
2929 cmp %rdx, %r9
2930 jae L(memmove_is_memcpy_bwd)
2931 cmp %rcx, %r9
2932 jb L(ll_cache_copy_bwd_start)
2933L(memmove_is_memcpy_bwd):
2934#endif
2935L(large_page_bwd_loop):
2936 movdqu -0x10(%rsi), %xmm0
2937 movdqu -0x20(%rsi), %xmm1
2938 movdqu -0x30(%rsi), %xmm2
2939 movdqu -0x40(%rsi), %xmm3
2940 movdqu -0x50(%rsi), %xmm4
2941 movdqu -0x60(%rsi), %xmm5
2942 movdqu -0x70(%rsi), %xmm6
2943 movdqu -0x80(%rsi), %xmm7
2944 lea -0x80(%rsi), %rsi
2945
2946 sub $0x80, %rdx
2947 movntdq %xmm0, -0x10(%rdi)
2948 movntdq %xmm1, -0x20(%rdi)
2949 movntdq %xmm2, -0x30(%rdi)
2950 movntdq %xmm3, -0x40(%rdi)
2951 movntdq %xmm4, -0x50(%rdi)
2952 movntdq %xmm5, -0x60(%rdi)
2953 movntdq %xmm6, -0x70(%rdi)
2954 movntdq %xmm7, -0x80(%rdi)
2955 lea -0x80(%rdi), %rdi
2956 jae L(large_page_bwd_loop)
2957 cmp $-0x40, %rdx
2958 lea 0x80(%rdx), %rdx
2959 jl L(large_page_less_bwd_64bytes)
2960
2961 movdqu -0x10(%rsi), %xmm0
2962 movdqu -0x20(%rsi), %xmm1
2963 movdqu -0x30(%rsi), %xmm2
2964 movdqu -0x40(%rsi), %xmm3
2965 lea -0x40(%rsi), %rsi
2966
2967 movntdq %xmm0, -0x10(%rdi)
2968 movntdq %xmm1, -0x20(%rdi)
2969 movntdq %xmm2, -0x30(%rdi)
2970 movntdq %xmm3, -0x40(%rdi)
2971 lea -0x40(%rdi), %rdi
2972 sub $0x40, %rdx
2973L(large_page_less_bwd_64bytes):
2974 sfence
2975 BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4)
2976
2977#ifdef USE_AS_MEMMOVE
2978 .p2align 4
2979L(ll_cache_copy_bwd_start):
2980 prefetcht0 -0x1c0(%rsi)
2981 prefetcht0 -0x200(%rsi)
2982 movdqu -0x10(%rsi), %xmm0
2983 movdqu -0x20(%rsi), %xmm1
2984 movdqu -0x30(%rsi), %xmm2
2985 movdqu -0x40(%rsi), %xmm3
2986 movdqu -0x50(%rsi), %xmm4
2987 movdqu -0x60(%rsi), %xmm5
2988 movdqu -0x70(%rsi), %xmm6
2989 movdqu -0x80(%rsi), %xmm7
2990 lea -0x80(%rsi), %rsi
2991
2992 sub $0x80, %rdx
2993 movaps %xmm0, -0x10(%rdi)
2994 movaps %xmm1, -0x20(%rdi)
2995 movaps %xmm2, -0x30(%rdi)
2996 movaps %xmm3, -0x40(%rdi)
2997 movaps %xmm4, -0x50(%rdi)
2998 movaps %xmm5, -0x60(%rdi)
2999 movaps %xmm6, -0x70(%rdi)
3000 movaps %xmm7, -0x80(%rdi)
3001 lea -0x80(%rdi), %rdi
3002 jae L(ll_cache_copy_bwd_start)
3003 cmp $-0x40, %rdx
3004 lea 0x80(%rdx), %rdx
3005 jl L(large_page_ll_less_bwd_64bytes)
3006
3007 movdqu -0x10(%rsi), %xmm0
3008 movdqu -0x20(%rsi), %xmm1
3009 movdqu -0x30(%rsi), %xmm2
3010 movdqu -0x40(%rsi), %xmm3
3011 lea -0x40(%rsi), %rsi
3012
3013 movaps %xmm0, -0x10(%rdi)
3014 movaps %xmm1, -0x20(%rdi)
3015 movaps %xmm2, -0x30(%rdi)
3016 movaps %xmm3, -0x40(%rdi)
3017 lea -0x40(%rdi), %rdi
3018 sub $0x40, %rdx
3019L(large_page_ll_less_bwd_64bytes):
3020 BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4)
3021#endif
3022
3023END (MEMCPY)
3024
3025 .section .rodata.ssse3,"a",@progbits
3026 .p2align 3
3027L(table_less_80bytes):
3028 .int JMPTBL (L(write_0bytes), L(table_less_80bytes))
3029 .int JMPTBL (L(write_1bytes), L(table_less_80bytes))
3030 .int JMPTBL (L(write_2bytes), L(table_less_80bytes))
3031 .int JMPTBL (L(write_3bytes), L(table_less_80bytes))
3032 .int JMPTBL (L(write_4bytes), L(table_less_80bytes))
3033 .int JMPTBL (L(write_5bytes), L(table_less_80bytes))
3034 .int JMPTBL (L(write_6bytes), L(table_less_80bytes))
3035 .int JMPTBL (L(write_7bytes), L(table_less_80bytes))
3036 .int JMPTBL (L(write_8bytes), L(table_less_80bytes))
3037 .int JMPTBL (L(write_9bytes), L(table_less_80bytes))
3038 .int JMPTBL (L(write_10bytes), L(table_less_80bytes))
3039 .int JMPTBL (L(write_11bytes), L(table_less_80bytes))
3040 .int JMPTBL (L(write_12bytes), L(table_less_80bytes))
3041 .int JMPTBL (L(write_13bytes), L(table_less_80bytes))
3042 .int JMPTBL (L(write_14bytes), L(table_less_80bytes))
3043 .int JMPTBL (L(write_15bytes), L(table_less_80bytes))
3044 .int JMPTBL (L(write_16bytes), L(table_less_80bytes))
3045 .int JMPTBL (L(write_17bytes), L(table_less_80bytes))
3046 .int JMPTBL (L(write_18bytes), L(table_less_80bytes))
3047 .int JMPTBL (L(write_19bytes), L(table_less_80bytes))
3048 .int JMPTBL (L(write_20bytes), L(table_less_80bytes))
3049 .int JMPTBL (L(write_21bytes), L(table_less_80bytes))
3050 .int JMPTBL (L(write_22bytes), L(table_less_80bytes))
3051 .int JMPTBL (L(write_23bytes), L(table_less_80bytes))
3052 .int JMPTBL (L(write_24bytes), L(table_less_80bytes))
3053 .int JMPTBL (L(write_25bytes), L(table_less_80bytes))
3054 .int JMPTBL (L(write_26bytes), L(table_less_80bytes))
3055 .int JMPTBL (L(write_27bytes), L(table_less_80bytes))
3056 .int JMPTBL (L(write_28bytes), L(table_less_80bytes))
3057 .int JMPTBL (L(write_29bytes), L(table_less_80bytes))
3058 .int JMPTBL (L(write_30bytes), L(table_less_80bytes))
3059 .int JMPTBL (L(write_31bytes), L(table_less_80bytes))
3060 .int JMPTBL (L(write_32bytes), L(table_less_80bytes))
3061 .int JMPTBL (L(write_33bytes), L(table_less_80bytes))
3062 .int JMPTBL (L(write_34bytes), L(table_less_80bytes))
3063 .int JMPTBL (L(write_35bytes), L(table_less_80bytes))
3064 .int JMPTBL (L(write_36bytes), L(table_less_80bytes))
3065 .int JMPTBL (L(write_37bytes), L(table_less_80bytes))
3066 .int JMPTBL (L(write_38bytes), L(table_less_80bytes))
3067 .int JMPTBL (L(write_39bytes), L(table_less_80bytes))
3068 .int JMPTBL (L(write_40bytes), L(table_less_80bytes))
3069 .int JMPTBL (L(write_41bytes), L(table_less_80bytes))
3070 .int JMPTBL (L(write_42bytes), L(table_less_80bytes))
3071 .int JMPTBL (L(write_43bytes), L(table_less_80bytes))
3072 .int JMPTBL (L(write_44bytes), L(table_less_80bytes))
3073 .int JMPTBL (L(write_45bytes), L(table_less_80bytes))
3074 .int JMPTBL (L(write_46bytes), L(table_less_80bytes))
3075 .int JMPTBL (L(write_47bytes), L(table_less_80bytes))
3076 .int JMPTBL (L(write_48bytes), L(table_less_80bytes))
3077 .int JMPTBL (L(write_49bytes), L(table_less_80bytes))
3078 .int JMPTBL (L(write_50bytes), L(table_less_80bytes))
3079 .int JMPTBL (L(write_51bytes), L(table_less_80bytes))
3080 .int JMPTBL (L(write_52bytes), L(table_less_80bytes))
3081 .int JMPTBL (L(write_53bytes), L(table_less_80bytes))
3082 .int JMPTBL (L(write_54bytes), L(table_less_80bytes))
3083 .int JMPTBL (L(write_55bytes), L(table_less_80bytes))
3084 .int JMPTBL (L(write_56bytes), L(table_less_80bytes))
3085 .int JMPTBL (L(write_57bytes), L(table_less_80bytes))
3086 .int JMPTBL (L(write_58bytes), L(table_less_80bytes))
3087 .int JMPTBL (L(write_59bytes), L(table_less_80bytes))
3088 .int JMPTBL (L(write_60bytes), L(table_less_80bytes))
3089 .int JMPTBL (L(write_61bytes), L(table_less_80bytes))
3090 .int JMPTBL (L(write_62bytes), L(table_less_80bytes))
3091 .int JMPTBL (L(write_63bytes), L(table_less_80bytes))
3092 .int JMPTBL (L(write_64bytes), L(table_less_80bytes))
3093 .int JMPTBL (L(write_65bytes), L(table_less_80bytes))
3094 .int JMPTBL (L(write_66bytes), L(table_less_80bytes))
3095 .int JMPTBL (L(write_67bytes), L(table_less_80bytes))
3096 .int JMPTBL (L(write_68bytes), L(table_less_80bytes))
3097 .int JMPTBL (L(write_69bytes), L(table_less_80bytes))
3098 .int JMPTBL (L(write_70bytes), L(table_less_80bytes))
3099 .int JMPTBL (L(write_71bytes), L(table_less_80bytes))
3100 .int JMPTBL (L(write_72bytes), L(table_less_80bytes))
3101 .int JMPTBL (L(write_73bytes), L(table_less_80bytes))
3102 .int JMPTBL (L(write_74bytes), L(table_less_80bytes))
3103 .int JMPTBL (L(write_75bytes), L(table_less_80bytes))
3104 .int JMPTBL (L(write_76bytes), L(table_less_80bytes))
3105 .int JMPTBL (L(write_77bytes), L(table_less_80bytes))
3106 .int JMPTBL (L(write_78bytes), L(table_less_80bytes))
3107 .int JMPTBL (L(write_79bytes), L(table_less_80bytes))
3108
3109 .p2align 3
3110L(shl_table):
3111 .int JMPTBL (L(shl_0), L(shl_table))
3112 .int JMPTBL (L(shl_1), L(shl_table))
3113 .int JMPTBL (L(shl_2), L(shl_table))
3114 .int JMPTBL (L(shl_3), L(shl_table))
3115 .int JMPTBL (L(shl_4), L(shl_table))
3116 .int JMPTBL (L(shl_5), L(shl_table))
3117 .int JMPTBL (L(shl_6), L(shl_table))
3118 .int JMPTBL (L(shl_7), L(shl_table))
3119 .int JMPTBL (L(shl_8), L(shl_table))
3120 .int JMPTBL (L(shl_9), L(shl_table))
3121 .int JMPTBL (L(shl_10), L(shl_table))
3122 .int JMPTBL (L(shl_11), L(shl_table))
3123 .int JMPTBL (L(shl_12), L(shl_table))
3124 .int JMPTBL (L(shl_13), L(shl_table))
3125 .int JMPTBL (L(shl_14), L(shl_table))
3126 .int JMPTBL (L(shl_15), L(shl_table))
3127
3128 .p2align 3
3129L(shl_table_bwd):
3130 .int JMPTBL (L(shl_0_bwd), L(shl_table_bwd))
3131 .int JMPTBL (L(shl_1_bwd), L(shl_table_bwd))
3132 .int JMPTBL (L(shl_2_bwd), L(shl_table_bwd))
3133 .int JMPTBL (L(shl_3_bwd), L(shl_table_bwd))
3134 .int JMPTBL (L(shl_4_bwd), L(shl_table_bwd))
3135 .int JMPTBL (L(shl_5_bwd), L(shl_table_bwd))
3136 .int JMPTBL (L(shl_6_bwd), L(shl_table_bwd))
3137 .int JMPTBL (L(shl_7_bwd), L(shl_table_bwd))
3138 .int JMPTBL (L(shl_8_bwd), L(shl_table_bwd))
3139 .int JMPTBL (L(shl_9_bwd), L(shl_table_bwd))
3140 .int JMPTBL (L(shl_10_bwd), L(shl_table_bwd))
3141 .int JMPTBL (L(shl_11_bwd), L(shl_table_bwd))
3142 .int JMPTBL (L(shl_12_bwd), L(shl_table_bwd))
3143 .int JMPTBL (L(shl_13_bwd), L(shl_table_bwd))
3144 .int JMPTBL (L(shl_14_bwd), L(shl_table_bwd))
3145 .int JMPTBL (L(shl_15_bwd), L(shl_table_bwd))
3146
3147#endif
3148