diff options
Diffstat (limited to 'lib/libc/amd64')
| -rw-r--r-- | lib/libc/amd64/string/Makefile.inc | 1 | ||||
| -rw-r--r-- | lib/libc/amd64/string/bcopy.S | 103 | ||||
| -rw-r--r-- | lib/libc/amd64/string/bcopy.c | 15 | ||||
| -rw-r--r-- | lib/libc/amd64/string/bzero.S | 45 | ||||
| -rw-r--r-- | lib/libc/amd64/string/memcpy.S | 4 | ||||
| -rw-r--r-- | lib/libc/amd64/string/memmove.S | 273 | ||||
| -rw-r--r-- | lib/libc/amd64/string/memset.S | 166 |
7 files changed, 406 insertions, 201 deletions
diff --git a/lib/libc/amd64/string/Makefile.inc b/lib/libc/amd64/string/Makefile.inc index 425de40d66bfd..a8a07a1eb5872 100644 --- a/lib/libc/amd64/string/Makefile.inc +++ b/lib/libc/amd64/string/Makefile.inc @@ -2,7 +2,6 @@ MDSRCS+= \ bcmp.S \ - bcopy.S \ bzero.S \ memcmp.S \ memcpy.S \ diff --git a/lib/libc/amd64/string/bcopy.S b/lib/libc/amd64/string/bcopy.S deleted file mode 100644 index 9446e75b8053b..0000000000000 --- a/lib/libc/amd64/string/bcopy.S +++ /dev/null @@ -1,103 +0,0 @@ -/*- - * Copyright (c) 1990 The Regents of the University of California. - * All rights reserved. - * - * This code is derived from locore.s. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include <machine/asm.h> -__FBSDID("$FreeBSD$"); - -#if 0 - RCSID("$NetBSD: bcopy.S,v 1.2 2003/08/07 16:42:36 agc Exp $") -#endif - - /* - * (ov)bcopy (src,dst,cnt) - * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800 - */ - -#ifdef MEMCOPY -ENTRY(memcpy) -#else -#ifdef MEMMOVE -ENTRY(memmove) -#else -ENTRY(bcopy) -#endif -#endif -#if defined(MEMCOPY) || defined(MEMMOVE) - movq %rdi,%rax /* return dst */ -#else - xchgq %rdi,%rsi -#endif - movq %rdx,%rcx - movq %rdi,%r8 - subq %rsi,%r8 - cmpq %rcx,%r8 /* overlapping? */ - jb 1f - shrq $3,%rcx /* copy by words */ - rep - movsq - movq %rdx,%rcx - andq $7,%rcx /* any bytes left? */ - jne 2f - ret -2: - rep - movsb - ret -1: - addq %rcx,%rdi /* copy backwards. */ - addq %rcx,%rsi - std - decq %rdi - decq %rsi - andq $7,%rcx /* any fractional bytes? */ - je 3f - rep - movsb -3: - movq %rdx,%rcx /* copy remainder by words */ - shrq $3,%rcx - subq $7,%rsi - subq $7,%rdi - rep - movsq - cld - ret -#ifdef MEMCOPY -END(memcpy) -#else -#ifdef MEMMOVE -END(memmove) -#else -END(bcopy) -#endif -#endif - - .section .note.GNU-stack,"",%progbits diff --git a/lib/libc/amd64/string/bcopy.c b/lib/libc/amd64/string/bcopy.c new file mode 100644 index 0000000000000..9e0c4187e439f --- /dev/null +++ b/lib/libc/amd64/string/bcopy.c @@ -0,0 +1,15 @@ +/*- + * Public domain. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <string.h> + +void +bcopy(const void *src, void *dst, size_t len) +{ + + memmove(dst, src, len); +} diff --git a/lib/libc/amd64/string/bzero.S b/lib/libc/amd64/string/bzero.S index cf46a2a317b76..123d9c4a7efbd 100644 --- a/lib/libc/amd64/string/bzero.S +++ b/lib/libc/amd64/string/bzero.S @@ -1,46 +1,7 @@ -/* - * Written by J.T. Conklin <jtc@NetBSD.org>. - * Public domain. - * Adapted for NetBSD/x86_64 by Frank van der Linden <fvdl@wasabisystems.com> - */ +/* $FreeBSD */ #include <machine/asm.h> __FBSDID("$FreeBSD$"); -#if 0 - RCSID("$NetBSD: bzero.S,v 1.2 2003/07/26 19:24:38 salo Exp $") -#endif - -ENTRY(bzero) - cld /* set fill direction forward */ - xorq %rax,%rax /* set fill data to 0 */ - - /* - * if the string is too short, it's really not worth the overhead - * of aligning to word boundries, etc. So we jump to a plain - * unaligned set. - */ - cmpq $16,%rsi - jb L1 - - movq %rdi,%rcx /* compute misalignment */ - negq %rcx - andq $7,%rcx - subq %rcx,%rsi - rep /* zero until word aligned */ - stosb - - movq %rsi,%rcx /* zero by words */ - shrq $3,%rcx - andq $7,%rsi - rep - stosq - -L1: movq %rsi,%rcx /* zero remainder by bytes */ - rep - stosb - - ret -END(bzero) - - .section .note.GNU-stack,"",%progbits +#define BZERO +#include "memset.S" diff --git a/lib/libc/amd64/string/memcpy.S b/lib/libc/amd64/string/memcpy.S index bd1e8422ad847..2b6c73abeb982 100644 --- a/lib/libc/amd64/string/memcpy.S +++ b/lib/libc/amd64/string/memcpy.S @@ -1,5 +1,5 @@ /* $NetBSD: memcpy.S,v 1.1 2001/06/19 00:25:05 fvdl Exp $ */ /* $FreeBSD$ */ -#define MEMCOPY -#include "bcopy.S" +#define MEMCPY +#include "memmove.S" diff --git a/lib/libc/amd64/string/memmove.S b/lib/libc/amd64/string/memmove.S index 85beb262f8d9a..9d6fa8a7e2962 100644 --- a/lib/libc/amd64/string/memmove.S +++ b/lib/libc/amd64/string/memmove.S @@ -1,5 +1,270 @@ -/* $NetBSD: memmove.S,v 1.1 2001/06/19 00:25:05 fvdl Exp $ */ -/* $FreeBSD$ */ +/*- + * Copyright (c) 2018 The FreeBSD Foundation + * + * This software was developed by Mateusz Guzik <mjg@FreeBSD.org> + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ -#define MEMMOVE -#include "bcopy.S" +#include <machine/asm.h> +__FBSDID("$FreeBSD$"); + +#define ALIGN_TEXT .p2align 4,0x90 /* 16-byte alignment, nop filled */ + +/* + * memmove(dst, src, cnt) + * rdi, rsi, rdx + * Contains parts of bcopy written by: + * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800 + */ + +/* + * Register state at entry is supposed to be as follows: + * rdi - destination + * rsi - source + * rdx - count + * + * The macro possibly clobbers the above and: rcx, r8. + * It does not clobber rax, r10 nor r11. + */ +.macro MEMMOVE erms overlap begin end + \begin +.if \overlap == 1 + movq %rdi,%r8 + subq %rsi,%r8 + cmpq %rcx,%r8 /* overlapping && src < dst? */ + jb 2f +.endif + + cmpq $32,%rcx + jb 1016f + + cmpq $256,%rcx + ja 1256f + +1032: + movq (%rsi),%rdx + movq %rdx,(%rdi) + movq 8(%rsi),%rdx + movq %rdx,8(%rdi) + movq 16(%rsi),%rdx + movq %rdx,16(%rdi) + movq 24(%rsi),%rdx + movq %rdx,24(%rdi) + leaq 32(%rsi),%rsi + leaq 32(%rdi),%rdi + subq $32,%rcx + cmpq $32,%rcx + jae 1032b + cmpb $0,%cl + jne 1016f + \end + ret + ALIGN_TEXT +1016: + cmpb $16,%cl + jl 1008f + movq (%rsi),%rdx + movq %rdx,(%rdi) + movq 8(%rsi),%rdx + movq %rdx,8(%rdi) + subb $16,%cl + jz 1000f + leaq 16(%rsi),%rsi + leaq 16(%rdi),%rdi +1008: + cmpb $8,%cl + jl 1004f + movq (%rsi),%rdx + movq %rdx,(%rdi) + subb $8,%cl + jz 1000f + leaq 8(%rsi),%rsi + leaq 8(%rdi),%rdi +1004: + cmpb $4,%cl + jl 1002f + movl (%rsi),%edx + movl %edx,(%rdi) + subb $4,%cl + jz 1000f + leaq 4(%rsi),%rsi + leaq 4(%rdi),%rdi +1002: + cmpb $2,%cl + jl 1001f + movw (%rsi),%dx + movw %dx,(%rdi) + subb $2,%cl + jz 1000f + leaq 2(%rsi),%rsi + leaq 2(%rdi),%rdi +1001: + cmpb $1,%cl + jl 1000f + movb (%rsi),%dl + movb %dl,(%rdi) +1000: + \end + ret + + ALIGN_TEXT +1256: +.if \erms == 1 + rep + movsb +.else + shrq $3,%rcx /* copy by 64-bit words */ + rep + movsq + movq %rdx,%rcx + andb $7,%cl /* any bytes left? */ + jne 1004b +.endif + \end + ret + +.if \overlap == 1 + /* + * Copy backwards. + */ + ALIGN_TEXT +2: + addq %rcx,%rdi + addq %rcx,%rsi + + cmpq $32,%rcx + jb 2016f + + cmpq $256,%rcx + ja 2256f + +2032: + movq -8(%rsi),%rdx + movq %rdx,-8(%rdi) + movq -16(%rsi),%rdx + movq %rdx,-16(%rdi) + movq -24(%rsi),%rdx + movq %rdx,-24(%rdi) + movq -32(%rsi),%rdx + movq %rdx,-32(%rdi) + leaq -32(%rsi),%rsi + leaq -32(%rdi),%rdi + subq $32,%rcx + cmpq $32,%rcx + jae 2032b + cmpb $0,%cl + jne 2016f + \end + ret + ALIGN_TEXT +2016: + cmpb $16,%cl + jl 2008f + movq -8(%rsi),%rdx + movq %rdx,-8(%rdi) + movq -16(%rsi),%rdx + movq %rdx,-16(%rdi) + subb $16,%cl + jz 2000f + leaq -16(%rsi),%rsi + leaq -16(%rdi),%rdi +2008: + cmpb $8,%cl + jl 2004f + movq -8(%rsi),%rdx + movq %rdx,-8(%rdi) + subb $8,%cl + jz 2000f + leaq -8(%rsi),%rsi + leaq -8(%rdi),%rdi +2004: + cmpb $4,%cl + jl 2002f + movl -4(%rsi),%edx + movl %edx,-4(%rdi) + subb $4,%cl + jz 2000f + leaq -4(%rsi),%rsi + leaq -4(%rdi),%rdi +2002: + cmpb $2,%cl + jl 2001f + movw -2(%rsi),%dx + movw %dx,-2(%rdi) + subb $2,%cl + jz 2000f + leaq -2(%rsi),%rsi + leaq -2(%rdi),%rdi +2001: + cmpb $1,%cl + jl 2000f + movb -1(%rsi),%dl + movb %dl,-1(%rdi) +2000: + \end + ret + ALIGN_TEXT +2256: + decq %rdi + decq %rsi + std +.if \erms == 1 + rep + movsb +.else + andq $7,%rcx /* any fractional bytes? */ + je 3f + rep + movsb +3: + movq %rdx,%rcx /* copy remainder by 32-bit words */ + shrq $3,%rcx + subq $7,%rsi + subq $7,%rdi + rep + movsq +.endif + cld + \end + ret +.endif +.endm + +.macro MEMMOVE_BEGIN + movq %rdi,%rax + movq %rdx,%rcx +.endm + +.macro MEMMOVE_END +.endm + +#ifndef MEMCPY +ENTRY(memmove) + MEMMOVE erms=0 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END +END(memmove) +#else +ENTRY(memcpy) + MEMMOVE erms=0 overlap=1 begin=MEMMOVE_BEGIN end=MEMMOVE_END +END(memcpy) +#endif diff --git a/lib/libc/amd64/string/memset.S b/lib/libc/amd64/string/memset.S index 84d15628766af..b2a871b0038e4 100644 --- a/lib/libc/amd64/string/memset.S +++ b/lib/libc/amd64/string/memset.S @@ -1,63 +1,131 @@ -/* - * Written by J.T. Conklin <jtc@NetBSD.org>. - * Public domain. - * Adapted for NetBSD/x86_64 by Frank van der Linden <fvdl@wasabisystems.com> +/*- + * Copyright (c) 2018 The FreeBSD Foundation + * + * This software was developed by Mateusz Guzik <mjg@FreeBSD.org> + * under sponsorship from the FreeBSD Foundation. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ */ #include <machine/asm.h> __FBSDID("$FreeBSD$"); -#if 0 - RCSID("$NetBSD: memset.S,v 1.3 2004/02/26 20:50:06 drochner Exp $") -#endif - -ENTRY(memset) - movq %rsi,%rax - andq $0xff,%rax +.macro MEMSET bzero erms +.if \bzero == 1 + movq %rsi,%rcx + movq %rsi,%rdx + xorl %eax,%eax +.else + movq %rdi,%r9 movq %rdx,%rcx - movq %rdi,%r11 - - cld /* set fill direction forward */ - - /* - * if the string is too short, it's really not worth the overhead - * of aligning to word boundries, etc. So we jump to a plain - * unaligned set. - */ - cmpq $0x0f,%rcx - jle L1 - - movb %al,%ah /* copy char to all bytes in word */ - movl %eax,%edx - sall $16,%eax - orl %edx,%eax - - movl %eax,%edx - salq $32,%rax - orq %rdx,%rax - - movq %rdi,%rdx /* compute misalignment */ - negq %rdx - andq $7,%rdx - movq %rcx,%r8 - subq %rdx,%r8 - - movq %rdx,%rcx /* set until word aligned */ + movzbq %sil,%r8 + movabs $0x0101010101010101,%rax + imulq %r8,%rax +.endif + + cmpq $32,%rcx + jb 1016f + + cmpq $256,%rcx + ja 1256f + +1032: + movq %rax,(%rdi) + movq %rax,8(%rdi) + movq %rax,16(%rdi) + movq %rax,24(%rdi) + leaq 32(%rdi),%rdi + subq $32,%rcx + cmpq $32,%rcx + jae 1032b + cmpb $0,%cl + je 1000f +1016: + cmpb $16,%cl + jl 1008f + movq %rax,(%rdi) + movq %rax,8(%rdi) + subb $16,%cl + jz 1000f + leaq 16(%rdi),%rdi +1008: + cmpb $8,%cl + jl 1004f + movq %rax,(%rdi) + subb $8,%cl + jz 1000f + leaq 8(%rdi),%rdi +1004: + cmpb $4,%cl + jl 1002f + movl %eax,(%rdi) + subb $4,%cl + jz 1000f + leaq 4(%rdi),%rdi +1002: + cmpb $2,%cl + jl 1001f + movw %ax,(%rdi) + subb $2,%cl + jz 1000f + leaq 2(%rdi),%rdi +1001: + cmpb $1,%cl + jl 1000f + movb %al,(%rdi) +1000: +.if \bzero == 0 + movq %r9,%rax +.endif + ret + +1256: +.if \erms == 1 rep stosb - - movq %r8,%rcx - shrq $3,%rcx /* set by words */ +.else + shrq $3,%rcx rep stosq - - movq %r8,%rcx /* set remainder by bytes */ - andq $7,%rcx -L1: rep - stosb - movq %r11,%rax - + movq %rdx,%rcx + andb $7,%cl + jne 1004b +.endif +.if \bzero == 0 + movq %r9,%rax +.endif ret +.endm + +#ifndef BZERO +ENTRY(memset) + MEMSET bzero=0 erms=0 END(memset) +#else +ENTRY(bzero) + MEMSET bzero=1 erms=0 +END(bzero) +#endif .section .note.GNU-stack,"",%progbits |