diff options
40 files changed, 805 insertions, 1511 deletions
diff --git a/sys/amd64/acpica/acpi_machdep.c b/sys/amd64/acpica/acpi_machdep.c index ad5f85489a92..16fa057da62c 100644 --- a/sys/amd64/acpica/acpi_machdep.c +++ b/sys/amd64/acpica/acpi_machdep.c @@ -32,6 +32,7 @@ __FBSDID("$FreeBSD$"); #include <sys/kernel.h> #include <sys/module.h> #include <sys/sysctl.h> + #include <vm/vm.h> #include <vm/pmap.h> @@ -71,7 +72,6 @@ acpi_machdep_init(device_t dev) STAILQ_INSERT_TAIL(&sc->apm_cdevs, &acpi_clone, entries); ACPI_UNLOCK(acpi); sc->acpi_clone = &acpi_clone; - acpi_install_wakeup_handler(sc); if (intr_model != ACPI_INTR_PIC) acpi_SetIntrModel(intr_model); @@ -363,13 +363,20 @@ nexus_acpi_probe(device_t dev) static int nexus_acpi_attach(device_t dev) { + device_t acpi_dev; + int error; nexus_init_resources(); bus_generic_probe(dev); - if (BUS_ADD_CHILD(dev, 10, "acpi", 0) == NULL) + acpi_dev = BUS_ADD_CHILD(dev, 10, "acpi", 0); + if (acpi_dev == NULL) panic("failed to add acpi0 device"); - return (bus_generic_attach(dev)); + error = bus_generic_attach(dev); + if (error == 0) + acpi_install_wakeup_handler(device_get_softc(acpi_dev)); + + return (error); } static device_method_t nexus_acpi_methods[] = { diff --git a/sys/amd64/acpica/acpi_switch.S b/sys/amd64/acpica/acpi_switch.S index 892dd11c47a5..2b2887bfb31c 100644 --- a/sys/amd64/acpica/acpi_switch.S +++ b/sys/amd64/acpica/acpi_switch.S @@ -1,7 +1,7 @@ /*- * Copyright (c) 2001 Takanori Watanabe <takawata@jp.freebsd.org> * Copyright (c) 2001 Mitsuru IWASAKI <iwasaki@jp.freebsd.org> - * Copyright (c) 2008-2009 Jung-uk Kim <jkim@FreeBSD.org> + * Copyright (c) 2008-2010 Jung-uk Kim <jkim@FreeBSD.org> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -34,26 +34,11 @@ #include "acpi_wakedata.h" #include "assym.s" -#define WAKEUP_DECL(member) \ - .set WAKEUP_ ## member, wakeup_ ## member - wakeup_ctx - - WAKEUP_DECL(xpcb) - WAKEUP_DECL(gdt) - WAKEUP_DECL(efer) - WAKEUP_DECL(pat) - WAKEUP_DECL(star) - WAKEUP_DECL(lstar) - WAKEUP_DECL(cstar) - WAKEUP_DECL(sfmask) - WAKEUP_DECL(cpu) - -#define WAKEUP_CTX(member) WAKEUP_ ## member (%rdi) -#define WAKEUP_PCB(member) PCB_ ## member(%r11) -#define WAKEUP_XPCB(member) XPCB_ ## member(%r11) +#define WAKEUP_CTX(member) wakeup_ ## member - wakeup_ctx(%rsi) ENTRY(acpi_restorecpu) /* Switch to KPML4phys. */ - movq %rsi, %rax + movq %rdi, %rax movq %rax, %cr3 /* Restore GDT. */ @@ -62,7 +47,7 @@ ENTRY(acpi_restorecpu) 1: /* Fetch PCB. */ - movq WAKEUP_CTX(xpcb), %r11 + movq WAKEUP_CTX(pcb), %rdi /* Force kernel segment registers. */ movl $KDSEL, %eax @@ -75,16 +60,16 @@ ENTRY(acpi_restorecpu) movw %ax, %gs movl $MSR_FSBASE, %ecx - movl WAKEUP_PCB(FSBASE), %eax - movl 4 + WAKEUP_PCB(FSBASE), %edx + movl PCB_FSBASE(%rdi), %eax + movl 4 + PCB_FSBASE(%rdi), %edx wrmsr movl $MSR_GSBASE, %ecx - movl WAKEUP_PCB(GSBASE), %eax - movl 4 + WAKEUP_PCB(GSBASE), %edx + movl PCB_GSBASE(%rdi), %eax + movl 4 + PCB_GSBASE(%rdi), %edx wrmsr movl $MSR_KGSBASE, %ecx - movl WAKEUP_XPCB(KGSBASE), %eax - movl 4 + WAKEUP_XPCB(KGSBASE), %edx + movl PCB_KGSBASE(%rdi), %eax + movl 4 + PCB_KGSBASE(%rdi), %edx wrmsr /* Restore EFER. */ @@ -115,17 +100,21 @@ ENTRY(acpi_restorecpu) movl WAKEUP_CTX(sfmask), %eax wrmsr - /* Restore CR0, CR2 and CR4. */ - movq WAKEUP_XPCB(CR0), %rax + /* Restore CR0 except for FPU mode. */ + movq PCB_CR0(%rdi), %rax + movq %rax, %rcx + andq $~(CR0_EM | CR0_TS), %rax movq %rax, %cr0 - movq WAKEUP_XPCB(CR2), %rax + + /* Restore CR2 and CR4. */ + movq PCB_CR2(%rdi), %rax movq %rax, %cr2 - movq WAKEUP_XPCB(CR4), %rax + movq PCB_CR4(%rdi), %rax movq %rax, %cr4 /* Restore descriptor tables. */ - lidt WAKEUP_XPCB(IDT) - lldt WAKEUP_XPCB(LDT) + lidt PCB_IDT(%rdi) + lldt PCB_LDT(%rdi) #define SDT_SYSTSS 9 #define SDT_SYSBSY 11 @@ -133,37 +122,44 @@ ENTRY(acpi_restorecpu) /* Clear "task busy" bit and reload TR. */ movq PCPU(TSS), %rax andb $(~SDT_SYSBSY | SDT_SYSTSS), 5(%rax) - movw WAKEUP_XPCB(TR), %ax + movw PCB_TR(%rdi), %ax ltr %ax #undef SDT_SYSTSS #undef SDT_SYSBSY /* Restore other callee saved registers. */ - movq WAKEUP_PCB(R15), %r15 - movq WAKEUP_PCB(R14), %r14 - movq WAKEUP_PCB(R13), %r13 - movq WAKEUP_PCB(R12), %r12 - movq WAKEUP_PCB(RBP), %rbp - movq WAKEUP_PCB(RSP), %rsp - movq WAKEUP_PCB(RBX), %rbx + movq PCB_R15(%rdi), %r15 + movq PCB_R14(%rdi), %r14 + movq PCB_R13(%rdi), %r13 + movq PCB_R12(%rdi), %r12 + movq PCB_RBP(%rdi), %rbp + movq PCB_RSP(%rdi), %rsp + movq PCB_RBX(%rdi), %rbx /* Restore debug registers. */ - movq WAKEUP_PCB(DR0), %rax + movq PCB_DR0(%rdi), %rax movq %rax, %dr0 - movq WAKEUP_PCB(DR1), %rax + movq PCB_DR1(%rdi), %rax movq %rax, %dr1 - movq WAKEUP_PCB(DR2), %rax + movq PCB_DR2(%rdi), %rax movq %rax, %dr2 - movq WAKEUP_PCB(DR3), %rax + movq PCB_DR3(%rdi), %rax movq %rax, %dr3 - movq WAKEUP_PCB(DR6), %rax + movq PCB_DR6(%rdi), %rax movq %rax, %dr6 - movq WAKEUP_PCB(DR7), %rax + movq PCB_DR7(%rdi), %rax movq %rax, %dr7 + /* Restore FPU state. */ + fninit + fxrstor PCB_USERFPU(%rdi) + + /* Reload CR0. */ + movq %rcx, %cr0 + /* Restore return address. */ - movq WAKEUP_PCB(RIP), %rax + movq PCB_RIP(%rdi), %rax movq %rax, (%rsp) /* Indicate the CPU is resumed. */ @@ -172,19 +168,3 @@ ENTRY(acpi_restorecpu) ret END(acpi_restorecpu) - -ENTRY(acpi_savecpu) - /* Fetch XPCB and save CPU context. */ - movq %rdi, %r10 - call savectx2 - movq %r10, %r11 - - /* Patch caller's return address and stack pointer. */ - movq (%rsp), %rax - movq %rax, WAKEUP_PCB(RIP) - movq %rsp, %rax - movq %rax, WAKEUP_PCB(RSP) - - movl $1, %eax - ret -END(acpi_savecpu) diff --git a/sys/amd64/acpica/acpi_wakecode.S b/sys/amd64/acpica/acpi_wakecode.S index 8ec0a380985e..24465b0e091b 100644 --- a/sys/amd64/acpica/acpi_wakecode.S +++ b/sys/amd64/acpica/acpi_wakecode.S @@ -2,7 +2,7 @@ * Copyright (c) 2001 Takanori Watanabe <takawata@jp.freebsd.org> * Copyright (c) 2001 Mitsuru IWASAKI <iwasaki@jp.freebsd.org> * Copyright (c) 2003 Peter Wemm - * Copyright (c) 2008-2009 Jung-uk Kim <jkim@FreeBSD.org> + * Copyright (c) 2008-2010 Jung-uk Kim <jkim@FreeBSD.org> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -52,18 +52,17 @@ .data /* So we can modify it */ ALIGN_TEXT -wakeup_start: .code16 +wakeup_start: /* * Set up segment registers for real mode, a small stack for * any calls we make, and clear any flags. */ cli /* make sure no interrupts */ - cld mov %cs, %ax /* copy %cs to %ds. Remember these */ mov %ax, %ds /* are offsets rather than selectors */ mov %ax, %ss - movw $PAGE_SIZE - 8, %sp + movw $PAGE_SIZE, %sp xorw %ax, %ax pushw %ax popfw @@ -127,6 +126,7 @@ wakeup_sw32: /* * At this point, we are running in 32 bit legacy protected mode. */ + ALIGN_TEXT .code32 wakeup_32: @@ -205,8 +205,8 @@ wakeup_64: mov %ax, %ds /* Restore arguments and return. */ - movq wakeup_ctx - wakeup_start(%rbx), %rdi - movq wakeup_kpml4 - wakeup_start(%rbx), %rsi + movq wakeup_kpml4 - wakeup_start(%rbx), %rdi + movq wakeup_ctx - wakeup_start(%rbx), %rsi movq wakeup_retaddr - wakeup_start(%rbx), %rax jmp *%rax @@ -260,7 +260,7 @@ wakeup_kpml4: wakeup_ctx: .quad 0 -wakeup_xpcb: +wakeup_pcb: .quad 0 wakeup_gdt: .word 0 diff --git a/sys/amd64/acpica/acpi_wakeup.c b/sys/amd64/acpica/acpi_wakeup.c index 3fee384752ca..6bdf6583e134 100644 --- a/sys/amd64/acpica/acpi_wakeup.c +++ b/sys/amd64/acpica/acpi_wakeup.c @@ -2,7 +2,7 @@ * Copyright (c) 2001 Takanori Watanabe <takawata@jp.freebsd.org> * Copyright (c) 2001 Mitsuru IWASAKI <iwasaki@jp.freebsd.org> * Copyright (c) 2003 Peter Wemm - * Copyright (c) 2008-2009 Jung-uk Kim <jkim@FreeBSD.org> + * Copyright (c) 2008-2010 Jung-uk Kim <jkim@FreeBSD.org> * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -31,13 +31,11 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> -#include <sys/systm.h> #include <sys/bus.h> #include <sys/kernel.h> #include <sys/malloc.h> #include <sys/memrange.h> #include <sys/smp.h> -#include <sys/types.h> #include <vm/vm.h> #include <vm/pmap.h> @@ -47,11 +45,11 @@ __FBSDID("$FreeBSD$"); #include <machine/pcb.h> #include <machine/pmap.h> #include <machine/specialreg.h> -#include <machine/vmparam.h> #ifdef SMP #include <machine/apicreg.h> #include <machine/smp.h> +#include <machine/vmparam.h> #endif #include <contrib/dev/acpica/include/acpi.h> @@ -64,23 +62,18 @@ __FBSDID("$FreeBSD$"); /* Make sure the code is less than a page and leave room for the stack. */ CTASSERT(sizeof(wakecode) < PAGE_SIZE - 1024); -#ifndef _SYS_CDEFS_H_ -#error this file needs sys/cdefs.h as a prerequisite -#endif - extern int acpi_resume_beep; extern int acpi_reset_video; #ifdef SMP -extern struct xpcb *stopxpcbs; +extern struct pcb **susppcbs; #else -static struct xpcb *stopxpcbs; +static struct pcb **susppcbs; #endif -int acpi_restorecpu(struct xpcb *, vm_offset_t); -int acpi_savecpu(struct xpcb *); +int acpi_restorecpu(struct pcb *, vm_offset_t); -static void acpi_alloc_wakeup_handler(void); +static void *acpi_alloc_wakeup_handler(void); static void acpi_stop_beep(void *); #ifdef SMP @@ -111,10 +104,10 @@ acpi_wakeup_ap(struct acpi_softc *sc, int cpu) int apic_id = cpu_apic_ids[cpu]; int ms; - WAKECODE_FIXUP(wakeup_xpcb, struct xpcb *, &stopxpcbs[cpu]); - WAKECODE_FIXUP(wakeup_gdt, uint16_t, stopxpcbs[cpu].xpcb_gdt.rd_limit); + WAKECODE_FIXUP(wakeup_pcb, struct pcb *, susppcbs[cpu]); + WAKECODE_FIXUP(wakeup_gdt, uint16_t, susppcbs[cpu]->pcb_gdt.rd_limit); WAKECODE_FIXUP(wakeup_gdt + 2, uint64_t, - stopxpcbs[cpu].xpcb_gdt.rd_base); + susppcbs[cpu]->pcb_gdt.rd_base); WAKECODE_FIXUP(wakeup_cpu, int, cpu); /* do an INIT IPI: assert RESET */ @@ -222,7 +215,6 @@ acpi_wakeup_cpus(struct acpi_softc *sc, cpumask_t wakeup_cpus) int acpi_sleep_machdep(struct acpi_softc *sc, int state) { - struct savefpu *stopfpu; #ifdef SMP cpumask_t wakeup_cpus; #endif @@ -252,10 +244,7 @@ acpi_sleep_machdep(struct acpi_softc *sc, int state) cr3 = rcr3(); load_cr3(KPML4phys); - stopfpu = &stopxpcbs[0].xpcb_pcb.pcb_save; - if (acpi_savecpu(&stopxpcbs[0])) { - fpugetregs(curthread, stopfpu); - + if (savectx(susppcbs[0])) { #ifdef SMP if (wakeup_cpus != 0 && suspend_cpus(wakeup_cpus) == 0) { device_printf(sc->acpi_dev, @@ -268,11 +257,11 @@ acpi_sleep_machdep(struct acpi_softc *sc, int state) WAKECODE_FIXUP(resume_beep, uint8_t, (acpi_resume_beep != 0)); WAKECODE_FIXUP(reset_video, uint8_t, (acpi_reset_video != 0)); - WAKECODE_FIXUP(wakeup_xpcb, struct xpcb *, &stopxpcbs[0]); + WAKECODE_FIXUP(wakeup_pcb, struct pcb *, susppcbs[0]); WAKECODE_FIXUP(wakeup_gdt, uint16_t, - stopxpcbs[0].xpcb_gdt.rd_limit); + susppcbs[0]->pcb_gdt.rd_limit); WAKECODE_FIXUP(wakeup_gdt + 2, uint64_t, - stopxpcbs[0].xpcb_gdt.rd_base); + susppcbs[0]->pcb_gdt.rd_base); WAKECODE_FIXUP(wakeup_cpu, int, 0); /* Call ACPICA to enter the desired sleep state */ @@ -291,7 +280,6 @@ acpi_sleep_machdep(struct acpi_softc *sc, int state) for (;;) ia32_pause(); } else { - fpusetregs(curthread, stopfpu); #ifdef SMP if (wakeup_cpus != 0) acpi_wakeup_cpus(sc, wakeup_cpus); @@ -324,49 +312,48 @@ out: return (ret); } -static vm_offset_t acpi_wakeaddr; - -static void +static void * acpi_alloc_wakeup_handler(void) { void *wakeaddr; - - if (!cold) - return; + int i; /* * Specify the region for our wakeup code. We want it in the low 1 MB - * region, excluding video memory and above (0xa0000). We ask for - * it to be page-aligned, just to be safe. + * region, excluding real mode IVT (0-0x3ff), BDA (0x400-0x4ff), EBDA + * (less than 128KB, below 0xa0000, must be excluded by SMAP and DSDT), + * and ROM area (0xa0000 and above). The temporary page tables must be + * page-aligned. */ - wakeaddr = contigmalloc(4 * PAGE_SIZE, M_DEVBUF, M_NOWAIT, 0, 0x9ffff, - PAGE_SIZE, 0ul); + wakeaddr = contigmalloc(4 * PAGE_SIZE, M_DEVBUF, M_NOWAIT, 0x500, + 0xa0000, PAGE_SIZE, 0ul); if (wakeaddr == NULL) { printf("%s: can't alloc wake memory\n", __func__); - return; - } - stopxpcbs = malloc(mp_ncpus * sizeof(*stopxpcbs), M_DEVBUF, M_NOWAIT); - if (stopxpcbs == NULL) { - contigfree(wakeaddr, 4 * PAGE_SIZE, M_DEVBUF); - printf("%s: can't alloc CPU state memory\n", __func__); - return; + return (NULL); } - acpi_wakeaddr = (vm_offset_t)wakeaddr; -} + susppcbs = malloc(mp_ncpus * sizeof(*susppcbs), M_DEVBUF, M_WAITOK); + for (i = 0; i < mp_ncpus; i++) + susppcbs[i] = malloc(sizeof(**susppcbs), M_DEVBUF, M_WAITOK); -SYSINIT(acpiwakeup, SI_SUB_KMEM, SI_ORDER_ANY, acpi_alloc_wakeup_handler, 0); + return (wakeaddr); +} void acpi_install_wakeup_handler(struct acpi_softc *sc) { + static void *wakeaddr = NULL; uint64_t *pt4, *pt3, *pt2; int i; - if (acpi_wakeaddr == 0ul) + if (wakeaddr != NULL) + return; + + wakeaddr = acpi_alloc_wakeup_handler(); + if (wakeaddr == NULL) return; - sc->acpi_wakeaddr = acpi_wakeaddr; - sc->acpi_wakephys = vtophys(acpi_wakeaddr); + sc->acpi_wakeaddr = (vm_offset_t)wakeaddr; + sc->acpi_wakephys = vtophys(wakeaddr); bcopy(wakecode, (void *)WAKECODE_VADDR(sc), sizeof(wakecode)); @@ -392,7 +379,7 @@ acpi_install_wakeup_handler(struct acpi_softc *sc) WAKECODE_FIXUP(wakeup_sfmask, uint64_t, rdmsr(MSR_SF_MASK)); /* Build temporary page tables below realmode code. */ - pt4 = (uint64_t *)acpi_wakeaddr; + pt4 = wakeaddr; pt3 = pt4 + (PAGE_SIZE) / sizeof(uint64_t); pt2 = pt3 + (PAGE_SIZE) / sizeof(uint64_t); diff --git a/sys/amd64/amd64/cpu_switch.S b/sys/amd64/amd64/cpu_switch.S index eb135cb5ae6b..952a70b87abf 100644 --- a/sys/amd64/amd64/cpu_switch.S +++ b/sys/amd64/amd64/cpu_switch.S @@ -113,7 +113,7 @@ done_store_dr: /* have we used fp, and need a save? */ cmpq %rdi,PCPU(FPCURTHREAD) jne 1f - addq $PCB_SAVEFPU,%r8 + movq PCB_SAVEFPU(%r8),%r8 clts fxsave (%r8) smsw %ax @@ -302,121 +302,62 @@ END(cpu_switch) * Update pcb, saving current processor state. */ ENTRY(savectx) - /* Fetch PCB. */ - movq %rdi,%rcx - - /* Save caller's return address. */ - movq (%rsp),%rax - movq %rax,PCB_RIP(%rcx) - - movq %cr3,%rax - movq %rax,PCB_CR3(%rcx) - - movq %rbx,PCB_RBX(%rcx) - movq %rsp,PCB_RSP(%rcx) - movq %rbp,PCB_RBP(%rcx) - movq %r12,PCB_R12(%rcx) - movq %r13,PCB_R13(%rcx) - movq %r14,PCB_R14(%rcx) - movq %r15,PCB_R15(%rcx) - - /* - * If fpcurthread == NULL, then the fpu h/w state is irrelevant and the - * state had better already be in the pcb. This is true for forks - * but not for dumps (the old book-keeping with FP flags in the pcb - * always lost for dumps because the dump pcb has 0 flags). - * - * If fpcurthread != NULL, then we have to save the fpu h/w state to - * fpcurthread's pcb and copy it to the requested pcb, or save to the - * requested pcb and reload. Copying is easier because we would - * have to handle h/w bugs for reloading. We used to lose the - * parent's fpu state for forks by forgetting to reload. - */ - pushfq - cli - movq PCPU(FPCURTHREAD),%rax - testq %rax,%rax - je 1f - - movq TD_PCB(%rax),%rdi - leaq PCB_SAVEFPU(%rdi),%rdi - clts - fxsave (%rdi) - smsw %ax - orb $CR0_TS,%al - lmsw %ax - - movq $PCB_SAVEFPU_SIZE,%rdx /* arg 3 */ - leaq PCB_SAVEFPU(%rcx),%rsi /* arg 2 */ - /* arg 1 (%rdi) already loaded */ - call bcopy -1: - popfq - - ret -END(savectx) - -/* - * savectx2(xpcb) - * Update xpcb, saving current processor state. - */ -ENTRY(savectx2) - /* Fetch XPCB. */ - movq %rdi,%r8 - /* Save caller's return address. */ movq (%rsp),%rax - movq %rax,PCB_RIP(%r8) + movq %rax,PCB_RIP(%rdi) - movq %rbx,PCB_RBX(%r8) - movq %rsp,PCB_RSP(%r8) - movq %rbp,PCB_RBP(%r8) - movq %r12,PCB_R12(%r8) - movq %r13,PCB_R13(%r8) - movq %r14,PCB_R14(%r8) - movq %r15,PCB_R15(%r8) + movq %rbx,PCB_RBX(%rdi) + movq %rsp,PCB_RSP(%rdi) + movq %rbp,PCB_RBP(%rdi) + movq %r12,PCB_R12(%rdi) + movq %r13,PCB_R13(%rdi) + movq %r14,PCB_R14(%rdi) + movq %r15,PCB_R15(%rdi) - movq %cr0,%rax - movq %rax,XPCB_CR0(%r8) + movq %cr0,%rsi + movq %rsi,PCB_CR0(%rdi) movq %cr2,%rax - movq %rax,XPCB_CR2(%r8) + movq %rax,PCB_CR2(%rdi) + movq %cr3,%rax + movq %rax,PCB_CR3(%rdi) movq %cr4,%rax - movq %rax,XPCB_CR4(%r8) + movq %rax,PCB_CR4(%rdi) movq %dr0,%rax - movq %rax,PCB_DR0(%r8) + movq %rax,PCB_DR0(%rdi) movq %dr1,%rax - movq %rax,PCB_DR1(%r8) + movq %rax,PCB_DR1(%rdi) movq %dr2,%rax - movq %rax,PCB_DR2(%r8) + movq %rax,PCB_DR2(%rdi) movq %dr3,%rax - movq %rax,PCB_DR3(%r8) + movq %rax,PCB_DR3(%rdi) movq %dr6,%rax - movq %rax,PCB_DR6(%r8) + movq %rax,PCB_DR6(%rdi) movq %dr7,%rax - movq %rax,PCB_DR7(%r8) - - sgdt XPCB_GDT(%r8) - sidt XPCB_IDT(%r8) - sldt XPCB_LDT(%r8) - str XPCB_TR(%r8) + movq %rax,PCB_DR7(%rdi) movl $MSR_FSBASE,%ecx rdmsr - shlq $32,%rdx - leaq (%rax,%rdx),%rax - movq %rax,PCB_FSBASE(%r8) + movl %eax,PCB_FSBASE(%rdi) + movl %edx,PCB_FSBASE+4(%rdi) movl $MSR_GSBASE,%ecx rdmsr - shlq $32,%rdx - leaq (%rax,%rdx),%rax - movq %rax,PCB_GSBASE(%r8) + movl %eax,PCB_GSBASE(%rdi) + movl %edx,PCB_GSBASE+4(%rdi) movl $MSR_KGSBASE,%ecx rdmsr - shlq $32,%rdx - leaq (%rax,%rdx),%rax - movq %rax,XPCB_KGSBASE(%r8) + movl %eax,PCB_KGSBASE(%rdi) + movl %edx,PCB_KGSBASE+4(%rdi) - movl $1, %eax + sgdt PCB_GDT(%rdi) + sidt PCB_IDT(%rdi) + sldt PCB_LDT(%rdi) + str PCB_TR(%rdi) + + clts + fxsave PCB_USERFPU(%rdi) + movq %rsi,%cr0 /* The previous %cr0 is saved in %rsi. */ + + movl $1,%eax ret -END(savectx2) +END(savectx) diff --git a/sys/amd64/amd64/fpu.c b/sys/amd64/amd64/fpu.c index 10cb6c2a0b71..d75df64943fb 100644 --- a/sys/amd64/amd64/fpu.c +++ b/sys/amd64/amd64/fpu.c @@ -65,34 +65,36 @@ __FBSDID("$FreeBSD$"); #if defined(__GNUCLIKE_ASM) && !defined(lint) -#define fldcw(addr) __asm("fldcw %0" : : "m" (*(addr))) -#define fnclex() __asm("fnclex") -#define fninit() __asm("fninit") +#define fldcw(cw) __asm __volatile("fldcw %0" : : "m" (cw)) +#define fnclex() __asm __volatile("fnclex") +#define fninit() __asm __volatile("fninit") #define fnstcw(addr) __asm __volatile("fnstcw %0" : "=m" (*(addr))) -#define fnstsw(addr) __asm __volatile("fnstsw %0" : "=m" (*(addr))) -#define fxrstor(addr) __asm("fxrstor %0" : : "m" (*(addr))) +#define fnstsw(addr) __asm __volatile("fnstsw %0" : "=am" (*(addr))) +#define fxrstor(addr) __asm __volatile("fxrstor %0" : : "m" (*(addr))) #define fxsave(addr) __asm __volatile("fxsave %0" : "=m" (*(addr))) -#define ldmxcsr(r) __asm __volatile("ldmxcsr %0" : : "m" (r)) -#define start_emulating() __asm("smsw %%ax; orb %0,%%al; lmsw %%ax" \ - : : "n" (CR0_TS) : "ax") -#define stop_emulating() __asm("clts") +#define ldmxcsr(csr) __asm __volatile("ldmxcsr %0" : : "m" (csr)) +#define start_emulating() __asm __volatile( \ + "smsw %%ax; orb %0,%%al; lmsw %%ax" \ + : : "n" (CR0_TS) : "ax") +#define stop_emulating() __asm __volatile("clts") #else /* !(__GNUCLIKE_ASM && !lint) */ -void fldcw(caddr_t addr); +void fldcw(u_short cw); void fnclex(void); void fninit(void); void fnstcw(caddr_t addr); void fnstsw(caddr_t addr); void fxsave(caddr_t addr); void fxrstor(caddr_t addr); +void ldmxcsr(u_int csr); void start_emulating(void); void stop_emulating(void); #endif /* __GNUCLIKE_ASM && !lint */ -#define GET_FPU_CW(thread) ((thread)->td_pcb->pcb_save.sv_env.en_cw) -#define GET_FPU_SW(thread) ((thread)->td_pcb->pcb_save.sv_env.en_sw) +#define GET_FPU_CW(thread) ((thread)->td_pcb->pcb_save->sv_env.en_cw) +#define GET_FPU_SW(thread) ((thread)->td_pcb->pcb_save->sv_env.en_sw) typedef u_char bool_t; @@ -111,15 +113,18 @@ static struct savefpu fpu_initialstate; void fpuinit(void) { - register_t savecrit; + register_t saveintr; u_int mxcsr; u_short control; - savecrit = intr_disable(); + /* + * It is too early for critical_enter() to work on AP. + */ + saveintr = intr_disable(); stop_emulating(); fninit(); control = __INITIAL_FPUCW__; - fldcw(&control); + fldcw(control); mxcsr = __INITIAL_MXCSR__; ldmxcsr(mxcsr); if (PCPU_GET(cpuid) == 0) { @@ -132,7 +137,7 @@ fpuinit(void) bzero(fpu_initialstate.sv_xmm, sizeof(fpu_initialstate.sv_xmm)); } start_emulating(); - intr_restore(savecrit); + intr_restore(saveintr); } /* @@ -141,16 +146,15 @@ fpuinit(void) void fpuexit(struct thread *td) { - register_t savecrit; - savecrit = intr_disable(); + critical_enter(); if (curthread == PCPU_GET(fpcurthread)) { stop_emulating(); - fxsave(&PCPU_GET(curpcb)->pcb_save); + fxsave(PCPU_GET(curpcb)->pcb_save); start_emulating(); PCPU_SET(fpcurthread, 0); } - intr_restore(savecrit); + critical_exit(); } int @@ -351,10 +355,9 @@ static char fpetable[128] = { int fputrap() { - register_t savecrit; u_short control, status; - savecrit = intr_disable(); + critical_enter(); /* * Interrupt handling (for another interrupt) may have pushed the @@ -371,7 +374,7 @@ fputrap() if (PCPU_GET(fpcurthread) == curthread) fnclex(); - intr_restore(savecrit); + critical_exit(); return (fpetable[status & ((~control & 0x3f) | 0x40)]); } @@ -389,12 +392,13 @@ void fpudna(void) { struct pcb *pcb; - register_t s; + critical_enter(); if (PCPU_GET(fpcurthread) == curthread) { printf("fpudna: fpcurthread == curthread %d times\n", ++err_count); stop_emulating(); + critical_exit(); return; } if (PCPU_GET(fpcurthread) != NULL) { @@ -404,7 +408,6 @@ fpudna(void) curthread, curthread->td_proc->p_pid); panic("fpudna"); } - s = intr_disable(); stop_emulating(); /* * Record new context early in case frstor causes a trap. @@ -422,23 +425,23 @@ fpudna(void) */ fxrstor(&fpu_initialstate); if (pcb->pcb_initial_fpucw != __INITIAL_FPUCW__) - fldcw(&pcb->pcb_initial_fpucw); + fldcw(pcb->pcb_initial_fpucw); pcb->pcb_flags |= PCB_FPUINITDONE; + if (PCB_USER_FPU(pcb)) + pcb->pcb_flags |= PCB_USERFPUINITDONE; } else - fxrstor(&pcb->pcb_save); - intr_restore(s); + fxrstor(pcb->pcb_save); + critical_exit(); } -/* - * This should be called with interrupts disabled and only when the owning - * FPU thread is non-null. - */ void fpudrop() { struct thread *td; td = PCPU_GET(fpcurthread); + KASSERT(td == curthread, ("fpudrop: fpcurthread != curthread")); + CRITICAL_ASSERT(td); PCPU_SET(fpcurthread, NULL); td->td_pcb->pcb_flags &= ~PCB_FPUINITDONE; start_emulating(); @@ -449,23 +452,47 @@ fpudrop() * It returns the FPU ownership status. */ int +fpugetuserregs(struct thread *td, struct savefpu *addr) +{ + struct pcb *pcb; + + pcb = td->td_pcb; + if ((pcb->pcb_flags & PCB_USERFPUINITDONE) == 0) { + bcopy(&fpu_initialstate, addr, sizeof(fpu_initialstate)); + addr->sv_env.en_cw = pcb->pcb_initial_fpucw; + return (_MC_FPOWNED_NONE); + } + critical_enter(); + if (td == PCPU_GET(fpcurthread) && PCB_USER_FPU(pcb)) { + fxsave(addr); + critical_exit(); + return (_MC_FPOWNED_FPU); + } else { + critical_exit(); + bcopy(&pcb->pcb_user_save, addr, sizeof(*addr)); + return (_MC_FPOWNED_PCB); + } +} + +int fpugetregs(struct thread *td, struct savefpu *addr) { - register_t s; + struct pcb *pcb; - if ((td->td_pcb->pcb_flags & PCB_FPUINITDONE) == 0) { + pcb = td->td_pcb; + if ((pcb->pcb_flags & PCB_FPUINITDONE) == 0) { bcopy(&fpu_initialstate, addr, sizeof(fpu_initialstate)); - addr->sv_env.en_cw = td->td_pcb->pcb_initial_fpucw; + addr->sv_env.en_cw = pcb->pcb_initial_fpucw; return (_MC_FPOWNED_NONE); } - s = intr_disable(); + critical_enter(); if (td == PCPU_GET(fpcurthread)) { fxsave(addr); - intr_restore(s); + critical_exit(); return (_MC_FPOWNED_FPU); } else { - intr_restore(s); - bcopy(&td->td_pcb->pcb_save, addr, sizeof(*addr)); + critical_exit(); + bcopy(pcb->pcb_save, addr, sizeof(*addr)); return (_MC_FPOWNED_PCB); } } @@ -474,19 +501,42 @@ fpugetregs(struct thread *td, struct savefpu *addr) * Set the state of the FPU. */ void +fpusetuserregs(struct thread *td, struct savefpu *addr) +{ + struct pcb *pcb; + + pcb = td->td_pcb; + critical_enter(); + if (td == PCPU_GET(fpcurthread) && PCB_USER_FPU(pcb)) { + fxrstor(addr); + critical_exit(); + pcb->pcb_flags |= PCB_FPUINITDONE | PCB_USERFPUINITDONE; + } else { + critical_exit(); + bcopy(addr, &td->td_pcb->pcb_user_save, sizeof(*addr)); + if (PCB_USER_FPU(pcb)) + pcb->pcb_flags |= PCB_FPUINITDONE; + pcb->pcb_flags |= PCB_USERFPUINITDONE; + } +} + +void fpusetregs(struct thread *td, struct savefpu *addr) { - register_t s; + struct pcb *pcb; - s = intr_disable(); + pcb = td->td_pcb; + critical_enter(); if (td == PCPU_GET(fpcurthread)) { fxrstor(addr); - intr_restore(s); + critical_exit(); } else { - intr_restore(s); - bcopy(addr, &td->td_pcb->pcb_save, sizeof(*addr)); + critical_exit(); + bcopy(addr, td->td_pcb->pcb_save, sizeof(*addr)); } - curthread->td_pcb->pcb_flags |= PCB_FPUINITDONE; + if (PCB_USER_FPU(pcb)) + pcb->pcb_flags |= PCB_USERFPUINITDONE; + pcb->pcb_flags |= PCB_FPUINITDONE; } /* @@ -575,3 +625,73 @@ static devclass_t fpupnp_devclass; DRIVER_MODULE(fpupnp, acpi, fpupnp_driver, fpupnp_devclass, 0, 0); #endif /* DEV_ISA */ + +int +fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags) +{ + struct pcb *pcb; + + pcb = td->td_pcb; + KASSERT(!PCB_USER_FPU(pcb) || pcb->pcb_save == &pcb->pcb_user_save, + ("mangled pcb_save")); + ctx->flags = 0; + if ((pcb->pcb_flags & PCB_FPUINITDONE) != 0) + ctx->flags |= FPU_KERN_CTX_FPUINITDONE; + fpuexit(td); + ctx->prev = pcb->pcb_save; + pcb->pcb_save = &ctx->hwstate; + pcb->pcb_flags |= PCB_KERNFPU; + pcb->pcb_flags &= ~PCB_FPUINITDONE; + return (0); +} + +int +fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx) +{ + struct pcb *pcb; + + pcb = td->td_pcb; + critical_enter(); + if (curthread == PCPU_GET(fpcurthread)) + fpudrop(); + critical_exit(); + pcb->pcb_save = ctx->prev; + if (pcb->pcb_save == &pcb->pcb_user_save) { + if ((pcb->pcb_flags & PCB_USERFPUINITDONE) != 0) + pcb->pcb_flags |= PCB_FPUINITDONE; + else + pcb->pcb_flags &= ~PCB_FPUINITDONE; + pcb->pcb_flags &= ~PCB_KERNFPU; + } else { + if ((ctx->flags & FPU_KERN_CTX_FPUINITDONE) != 0) + pcb->pcb_flags |= PCB_FPUINITDONE; + else + pcb->pcb_flags &= ~PCB_FPUINITDONE; + KASSERT(!PCB_USER_FPU(pcb), ("unpaired fpu_kern_leave")); + } + return (0); +} + +int +fpu_kern_thread(u_int flags) +{ + struct pcb *pcb; + + pcb = PCPU_GET(curpcb); + KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0, + ("Only kthread may use fpu_kern_thread")); + KASSERT(pcb->pcb_save == &pcb->pcb_user_save, ("mangled pcb_save")); + KASSERT(PCB_USER_FPU(pcb), ("recursive call")); + + pcb->pcb_flags |= PCB_KERNFPU; + return (0); +} + +int +is_fpu_kern_thread(u_int flags) +{ + + if ((curthread->td_pflags & TDP_KTHREAD) == 0) + return (0); + return ((PCPU_GET(curpcb)->pcb_flags & PCB_KERNFPU) != 0); +} diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c index 9d783095b38b..c7d4c2cc9dcc 100644 --- a/sys/amd64/amd64/genassym.c +++ b/sys/amd64/amd64/genassym.c @@ -123,7 +123,7 @@ ASSYM(KERNBASE, KERNBASE); ASSYM(DMAP_MIN_ADDRESS, DMAP_MIN_ADDRESS); ASSYM(DMAP_MAX_ADDRESS, DMAP_MAX_ADDRESS); ASSYM(MCLBYTES, MCLBYTES); -ASSYM(PCB_CR3, offsetof(struct pcb, pcb_cr3)); + ASSYM(PCB_R15, offsetof(struct pcb, pcb_r15)); ASSYM(PCB_R14, offsetof(struct pcb, pcb_r14)); ASSYM(PCB_R13, offsetof(struct pcb, pcb_r13)); @@ -134,39 +134,35 @@ ASSYM(PCB_RBX, offsetof(struct pcb, pcb_rbx)); ASSYM(PCB_RIP, offsetof(struct pcb, pcb_rip)); ASSYM(PCB_FSBASE, offsetof(struct pcb, pcb_fsbase)); ASSYM(PCB_GSBASE, offsetof(struct pcb, pcb_gsbase)); +ASSYM(PCB_KGSBASE, offsetof(struct pcb, pcb_kgsbase)); +ASSYM(PCB_FLAGS, offsetof(struct pcb, pcb_flags)); +ASSYM(PCB_CR0, offsetof(struct pcb, pcb_cr0)); +ASSYM(PCB_CR2, offsetof(struct pcb, pcb_cr2)); +ASSYM(PCB_CR3, offsetof(struct pcb, pcb_cr3)); +ASSYM(PCB_CR4, offsetof(struct pcb, pcb_cr4)); ASSYM(PCB_DR0, offsetof(struct pcb, pcb_dr0)); ASSYM(PCB_DR1, offsetof(struct pcb, pcb_dr1)); ASSYM(PCB_DR2, offsetof(struct pcb, pcb_dr2)); ASSYM(PCB_DR3, offsetof(struct pcb, pcb_dr3)); ASSYM(PCB_DR6, offsetof(struct pcb, pcb_dr6)); ASSYM(PCB_DR7, offsetof(struct pcb, pcb_dr7)); +ASSYM(PCB_ONFAULT, offsetof(struct pcb, pcb_onfault)); +ASSYM(PCB_GS32SD, offsetof(struct pcb, pcb_gs32sd)); ASSYM(PCB_TSSP, offsetof(struct pcb, pcb_tssp)); +ASSYM(PCB_SAVEFPU, offsetof(struct pcb, pcb_save)); +ASSYM(PCB_SAVEFPU_SIZE, sizeof(struct savefpu)); ASSYM(PCB_FULL_IRET, offsetof(struct pcb, pcb_full_iret)); +ASSYM(PCB_GDT, offsetof(struct pcb, pcb_gdt)); +ASSYM(PCB_IDT, offsetof(struct pcb, pcb_idt)); +ASSYM(PCB_LDT, offsetof(struct pcb, pcb_ldt)); +ASSYM(PCB_TR, offsetof(struct pcb, pcb_tr)); +ASSYM(PCB_USERFPU, offsetof(struct pcb, pcb_user_save)); +ASSYM(PCB_SIZE, sizeof(struct pcb)); ASSYM(PCB_DBREGS, PCB_DBREGS); ASSYM(PCB_32BIT, PCB_32BIT); ASSYM(PCB_GS32BIT, PCB_GS32BIT); ASSYM(PCB_FULLCTX, PCB_FULLCTX); -ASSYM(PCB_FLAGS, offsetof(struct pcb, pcb_flags)); -ASSYM(PCB_SAVEFPU, offsetof(struct pcb, pcb_save)); -ASSYM(PCB_SAVEFPU_SIZE, sizeof(struct savefpu)); -ASSYM(PCB_ONFAULT, offsetof(struct pcb, pcb_onfault)); -ASSYM(PCB_GS32SD, offsetof(struct pcb, pcb_gs32sd)); - -ASSYM(PCB_SIZE, sizeof(struct pcb)); - -ASSYM(XPCB_PCB, offsetof(struct xpcb, xpcb_pcb)); -ASSYM(XPCB_CR0, offsetof(struct xpcb, xpcb_cr0)); -ASSYM(XPCB_CR2, offsetof(struct xpcb, xpcb_cr2)); -ASSYM(XPCB_CR4, offsetof(struct xpcb, xpcb_cr4)); -ASSYM(XPCB_KGSBASE, offsetof(struct xpcb, xpcb_kgsbase)); -ASSYM(XPCB_GDT, offsetof(struct xpcb, xpcb_gdt)); -ASSYM(XPCB_IDT, offsetof(struct xpcb, xpcb_idt)); -ASSYM(XPCB_LDT, offsetof(struct xpcb, xpcb_ldt)); -ASSYM(XPCB_TR, offsetof(struct xpcb, xpcb_tr)); - -ASSYM(XPCB_SIZE, sizeof(struct xpcb)); - ASSYM(COMMON_TSS_RSP0, offsetof(struct amd64tss, tss_rsp0)); ASSYM(TF_R15, offsetof(struct trapframe, tf_r15)); diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c index 3a9f4ad50c4e..f90c3162417d 100644 --- a/sys/amd64/amd64/machdep.c +++ b/sys/amd64/amd64/machdep.c @@ -1964,7 +1964,7 @@ int fill_fpregs(struct thread *td, struct fpreg *fpregs) { - fill_fpregs_xmm(&td->td_pcb->pcb_save, fpregs); + fill_fpregs_xmm(&td->td_pcb->pcb_user_save, fpregs); return (0); } @@ -1973,7 +1973,7 @@ int set_fpregs(struct thread *td, struct fpreg *fpregs) { - set_fpregs_xmm(fpregs, &td->td_pcb->pcb_save); + set_fpregs_xmm(fpregs, &td->td_pcb->pcb_user_save); return (0); } @@ -2088,7 +2088,8 @@ static void get_fpcontext(struct thread *td, mcontext_t *mcp) { - mcp->mc_ownedfp = fpugetregs(td, (struct savefpu *)&mcp->mc_fpstate); + mcp->mc_ownedfp = fpugetuserregs(td, + (struct savefpu *)&mcp->mc_fpstate); mcp->mc_fpformat = fpuformat(); } @@ -2106,14 +2107,9 @@ set_fpcontext(struct thread *td, const mcontext_t *mcp) fpstate_drop(td); else if (mcp->mc_ownedfp == _MC_FPOWNED_FPU || mcp->mc_ownedfp == _MC_FPOWNED_PCB) { - /* - * XXX we violate the dubious requirement that fpusetregs() - * be called with interrupts disabled. - * XXX obsolete on trap-16 systems? - */ fpstate = (struct savefpu *)&mcp->mc_fpstate; fpstate->sv_env.en_mxcsr &= cpu_mxcsr_mask; - fpusetregs(td, fpstate); + fpusetuserregs(td, fpstate); } else return (EINVAL); return (0); @@ -2122,9 +2118,9 @@ set_fpcontext(struct thread *td, const mcontext_t *mcp) void fpstate_drop(struct thread *td) { - register_t s; - s = intr_disable(); + KASSERT(PCB_USER_FPU(td->td_pcb), ("fpstate_drop: kernel-owned fpu")); + critical_enter(); if (PCPU_GET(fpcurthread) == td) fpudrop(); /* @@ -2137,8 +2133,9 @@ fpstate_drop(struct thread *td) * sendsig() is the only caller of fpugetregs()... perhaps we just * have too many layers. */ - curthread->td_pcb->pcb_flags &= ~PCB_FPUINITDONE; - intr_restore(s); + curthread->td_pcb->pcb_flags &= ~(PCB_FPUINITDONE | + PCB_USERFPUINITDONE); + critical_exit(); } int diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index 4386404d66de..bbce1bee0212 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -105,7 +105,7 @@ extern pt_entry_t *KPTphys; extern pt_entry_t *SMPpt; struct pcb stoppcbs[MAXCPU]; -struct xpcb *stopxpcbs = NULL; +struct pcb **susppcbs = NULL; /* Variables needed for SMP tlb shootdown. */ vm_offset_t smp_tlb_addr1; @@ -1314,7 +1314,6 @@ cpustop_handler(void) void cpususpend_handler(void) { - struct savefpu *stopfpu; cpumask_t cpumask; register_t cr3, rf; u_int cpu; @@ -1324,13 +1323,11 @@ cpususpend_handler(void) rf = intr_disable(); cr3 = rcr3(); - stopfpu = &stopxpcbs[cpu].xpcb_pcb.pcb_save; - if (savectx2(&stopxpcbs[cpu])) { - fpugetregs(curthread, stopfpu); + + if (savectx(susppcbs[cpu])) { wbinvd(); atomic_set_int(&stopped_cpus, cpumask); - } else - fpusetregs(curthread, stopfpu); + } /* Wait for resume */ while (!(started_cpus & cpumask)) diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c index bd37ff77c43c..80c1a1e90b2f 100644 --- a/sys/amd64/amd64/trap.c +++ b/sys/amd64/amd64/trap.c @@ -425,6 +425,8 @@ trap(struct trapframe *frame) case T_DNA: /* transparent fault (due to context switch "late") */ + KASSERT(PCB_USER_FPU(td->td_pcb), + ("kernel FPU ctx has leaked")); fpudna(); goto userout; @@ -449,13 +451,19 @@ trap(struct trapframe *frame) goto out; case T_DNA: + KASSERT(!PCB_USER_FPU(td->td_pcb), + ("Unregistered use of FPU in kernel")); + fpudna(); + goto out; + + case T_ARITHTRAP: /* arithmetic trap */ + case T_XMMFLT: /* SIMD floating-point exception */ + case T_FPOPFLT: /* FPU operand fetch fault */ /* - * The kernel is apparently using fpu for copying. - * XXX this should be fatal unless the kernel has - * registered such use. + * XXXKIB for now disable any FPU traps in kernel + * handler registration seems to be overkill */ - fpudna(); - printf("fpudna in kernel mode!\n"); + trap_fatal(frame, 0); goto out; case T_STKFLT: /* stack fault */ @@ -600,6 +608,8 @@ trap(struct trapframe *frame) user: userret(td, frame); mtx_assert(&Giant, MA_NOTOWNED); + KASSERT(PCB_USER_FPU(td->td_pcb), + ("Return from trap with kernel FPU ctx leaked")); userout: out: return; @@ -890,5 +900,12 @@ syscall(struct trapframe *frame) trapsignal(td, &ksi); } + KASSERT(PCB_USER_FPU(td->td_pcb), + ("System call %s returing with kernel FPU ctx leaked", + syscallname(td->td_proc, sa.code))); + KASSERT(td->td_pcb->pcb_save == &td->td_pcb->pcb_user_save, + ("System call %s returning with mangled pcb_save", + syscallname(td->td_proc, sa.code))); + syscallret(td, error, &sa); } diff --git a/sys/amd64/amd64/vm_machdep.c b/sys/amd64/amd64/vm_machdep.c index 05a2652626eb..fd7727ac1809 100644 --- a/sys/amd64/amd64/vm_machdep.c +++ b/sys/amd64/amd64/vm_machdep.c @@ -122,7 +122,7 @@ cpu_fork(td1, p2, td2, flags) return; } - /* Ensure that p1's pcb is up to date. */ + /* Ensure that td1's pcb is up to date. */ fpuexit(td1); /* Point the pcb to the top of the stack */ @@ -130,9 +130,12 @@ cpu_fork(td1, p2, td2, flags) td2->td_kstack_pages * PAGE_SIZE) - 1; td2->td_pcb = pcb2; - /* Copy p1's pcb */ + /* Copy td1's pcb */ bcopy(td1->td_pcb, pcb2, sizeof(*pcb2)); + /* Properly initialize pcb_save */ + pcb2->pcb_save = &pcb2->pcb_user_save; + /* Point mdproc and then copy over td1's contents */ mdp2 = &p2->p_md; bcopy(&p1->p_md, mdp2, sizeof(*mdp2)); @@ -262,8 +265,10 @@ cpu_thread_exit(struct thread *td) { struct pcb *pcb; + critical_enter(); if (td == PCPU_GET(fpcurthread)) fpudrop(); + critical_exit(); pcb = td->td_pcb; @@ -308,6 +313,7 @@ cpu_thread_alloc(struct thread *td) td->td_pcb = (struct pcb *)(td->td_kstack + td->td_kstack_pages * PAGE_SIZE) - 1; td->td_frame = (struct trapframe *)td->td_pcb - 1; + td->td_pcb->pcb_save = &td->td_pcb->pcb_user_save; } void @@ -381,7 +387,8 @@ cpu_set_upcall(struct thread *td, struct thread *td0) * values here. */ bcopy(td0->td_pcb, pcb2, sizeof(*pcb2)); - pcb2->pcb_flags &= ~PCB_FPUINITDONE; + pcb2->pcb_flags &= ~(PCB_FPUINITDONE | PCB_USERFPUINITDONE); + pcb2->pcb_save = &pcb2->pcb_user_save; pcb2->pcb_full_iret = 1; /* diff --git a/sys/amd64/ia32/ia32_reg.c b/sys/amd64/ia32/ia32_reg.c index da5190f0db98..30fcffbd4e5e 100644 --- a/sys/amd64/ia32/ia32_reg.c +++ b/sys/amd64/ia32/ia32_reg.c @@ -147,7 +147,7 @@ fill_fpregs32(struct thread *td, struct fpreg32 *regs) { struct save87 *sv_87 = (struct save87 *)regs; struct env87 *penv_87 = &sv_87->sv_env; - struct savefpu *sv_fpu = &td->td_pcb->pcb_save; + struct savefpu *sv_fpu = &td->td_pcb->pcb_user_save; struct envxmm *penv_xmm = &sv_fpu->sv_env; int i; @@ -182,7 +182,7 @@ set_fpregs32(struct thread *td, struct fpreg32 *regs) { struct save87 *sv_87 = (struct save87 *)regs; struct env87 *penv_87 = &sv_87->sv_env; - struct savefpu *sv_fpu = &td->td_pcb->pcb_save; + struct savefpu *sv_fpu = &td->td_pcb->pcb_user_save; struct envxmm *penv_xmm = &sv_fpu->sv_env; int i; diff --git a/sys/amd64/ia32/ia32_signal.c b/sys/amd64/ia32/ia32_signal.c index 95f77c9c70f0..7e6868d1cb75 100644 --- a/sys/amd64/ia32/ia32_signal.c +++ b/sys/amd64/ia32/ia32_signal.c @@ -92,7 +92,14 @@ static void ia32_get_fpcontext(struct thread *td, struct ia32_mcontext *mcp) { - mcp->mc_ownedfp = fpugetregs(td, (struct savefpu *)&mcp->mc_fpstate); + /* + * XXX Format of 64bit and 32bit FXSAVE areas differs. FXSAVE + * in 32bit mode saves %cs and %ds, while on 64bit it saves + * 64bit instruction and data pointers. Ignore the difference + * for now, it should be irrelevant for most applications. + */ + mcp->mc_ownedfp = fpugetuserregs(td, + (struct savefpu *)&mcp->mc_fpstate); mcp->mc_fpformat = fpuformat(); } @@ -109,11 +116,7 @@ ia32_set_fpcontext(struct thread *td, const struct ia32_mcontext *mcp) fpstate_drop(td); else if (mcp->mc_ownedfp == _MC_FPOWNED_FPU || mcp->mc_ownedfp == _MC_FPOWNED_PCB) { - /* - * XXX we violate the dubious requirement that fpusetregs() - * be called with interrupts disabled. - */ - fpusetregs(td, (struct savefpu *)&mcp->mc_fpstate); + fpusetuserregs(td, (struct savefpu *)&mcp->mc_fpstate); } else return (EINVAL); return (0); diff --git a/sys/amd64/include/fpu.h b/sys/amd64/include/fpu.h index 272f94a7c6a4..ca0ac8f6486b 100644 --- a/sys/amd64/include/fpu.h +++ b/sys/amd64/include/fpu.h @@ -73,6 +73,17 @@ struct savefpu { u_char sv_pad[96]; } __aligned(16); +#ifdef _KERNEL +struct fpu_kern_ctx { + struct savefpu hwstate; + struct savefpu *prev; + uint32_t flags; +}; +#define FPU_KERN_CTX_FPUINITDONE 0x01 + +#define PCB_USER_FPU(pcb) (((pcb)->pcb_flags & PCB_KERNFPU) == 0) +#endif + /* * The hardware default control word for i387's and later coprocessors is * 0x37F, giving: @@ -102,9 +113,22 @@ void fpudrop(void); void fpuexit(struct thread *td); int fpuformat(void); int fpugetregs(struct thread *td, struct savefpu *addr); +int fpugetuserregs(struct thread *td, struct savefpu *addr); void fpuinit(void); void fpusetregs(struct thread *td, struct savefpu *addr); +void fpusetuserregs(struct thread *td, struct savefpu *addr); int fputrap(void); +int fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, + u_int flags); +int fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx); +int fpu_kern_thread(u_int flags); +int is_fpu_kern_thread(u_int flags); + +/* + * Flags for fpu_kern_enter() and fpu_kern_thread(). + */ +#define FPU_KERN_NORMAL 0x0000 + #endif #endif /* !_MACHINE_FPU_H_ */ diff --git a/sys/amd64/include/pcb.h b/sys/amd64/include/pcb.h index b26188a5bf70..4cc024054627 100644 --- a/sys/amd64/include/pcb.h +++ b/sys/amd64/include/pcb.h @@ -44,7 +44,6 @@ #include <machine/segments.h> struct pcb { - register_t pcb_cr3; register_t pcb_r15; register_t pcb_r14; register_t pcb_r13; @@ -55,50 +54,51 @@ struct pcb { register_t pcb_rip; register_t pcb_fsbase; register_t pcb_gsbase; + register_t pcb_kgsbase; u_long pcb_flags; #define PCB_DBREGS 0x02 /* process using debug registers */ +#define PCB_KERNFPU 0x04 /* kernel uses fpu */ #define PCB_FPUINITDONE 0x08 /* fpu state is initialized */ +#define PCB_USERFPUINITDONE 0x10 /* fpu user state is initialized */ #define PCB_GS32BIT 0x20 /* linux gs switch */ #define PCB_32BIT 0x40 /* process has 32 bit context (segs etc) */ #define PCB_FULLCTX 0x80 /* full context restore on sysret */ - u_int64_t pcb_dr0; - u_int64_t pcb_dr1; - u_int64_t pcb_dr2; - u_int64_t pcb_dr3; - u_int64_t pcb_dr6; - u_int64_t pcb_dr7; + register_t pcb_cr0; + register_t pcb_cr2; + register_t pcb_cr3; + register_t pcb_cr4; + register_t pcb_dr0; + register_t pcb_dr1; + register_t pcb_dr2; + register_t pcb_dr3; + register_t pcb_dr6; + register_t pcb_dr7; - struct savefpu pcb_save; uint16_t pcb_initial_fpucw; caddr_t pcb_onfault; /* copyin/out fault recovery */ /* 32-bit segment descriptor */ - struct user_segment_descriptor pcb_gs32sd; + struct user_segment_descriptor pcb_gs32sd; /* local tss, with i/o bitmap; NULL for common */ struct amd64tss *pcb_tssp; + struct savefpu *pcb_save; char pcb_full_iret; -}; -struct xpcb { - struct pcb xpcb_pcb; - register_t xpcb_cr0; - register_t xpcb_cr2; - register_t xpcb_cr4; - register_t xpcb_kgsbase; - struct region_descriptor xpcb_gdt; - struct region_descriptor xpcb_idt; - struct region_descriptor xpcb_ldt; - uint16_t xpcb_tr; + struct region_descriptor pcb_gdt; + struct region_descriptor pcb_idt; + struct region_descriptor pcb_ldt; + uint16_t pcb_tr; + + struct savefpu pcb_user_save; }; #ifdef _KERNEL struct trapframe; void makectx(struct trapframe *, struct pcb *); -void savectx(struct pcb *); -int savectx2(struct xpcb *); +int savectx(struct pcb *); #endif #endif /* _AMD64_PCB_H_ */ diff --git a/sys/crypto/via/padlock.c b/sys/crypto/via/padlock.c index ccb0595ef6fc..77e059bf1eb2 100644 --- a/sys/crypto/via/padlock.c +++ b/sys/crypto/via/padlock.c @@ -169,6 +169,7 @@ padlock_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri) struct padlock_softc *sc = device_get_softc(dev); struct padlock_session *ses = NULL; struct cryptoini *encini, *macini; + struct thread *td; int error; if (sidp == NULL || cri == NULL) @@ -236,7 +237,12 @@ padlock_newsession(device_t dev, uint32_t *sidp, struct cryptoini *cri) } if (macini != NULL) { - error = padlock_hash_setup(ses, macini); + td = curthread; + error = fpu_kern_enter(td, &ses->ses_fpu_ctx, FPU_KERN_NORMAL); + if (error == 0) { + error = padlock_hash_setup(ses, macini); + fpu_kern_leave(td, &ses->ses_fpu_ctx); + } if (error != 0) { padlock_freesession_one(sc, ses, 0); return (error); diff --git a/sys/crypto/via/padlock.h b/sys/crypto/via/padlock.h index 7d928ba1719c..c8ee9bd46153 100644 --- a/sys/crypto/via/padlock.h +++ b/sys/crypto/via/padlock.h @@ -32,6 +32,12 @@ #include <opencrypto/cryptodev.h> #include <crypto/rijndael/rijndael.h> +#if defined(__i386__) +#include <machine/npx.h> +#elif defined(__amd64__) +#include <machine/fpu.h> +#endif + union padlock_cw { uint64_t raw; struct { @@ -70,6 +76,7 @@ struct padlock_session { int ses_used; uint32_t ses_id; TAILQ_ENTRY(padlock_session) ses_next; + struct fpu_kern_ctx ses_fpu_ctx; }; #define PADLOCK_ALIGN(p) (void *)(roundup2((uintptr_t)(p), 16)) diff --git a/sys/crypto/via/padlock_cipher.c b/sys/crypto/via/padlock_cipher.c index 8195584c71e7..0ae26c8a6985 100644 --- a/sys/crypto/via/padlock_cipher.c +++ b/sys/crypto/via/padlock_cipher.c @@ -53,6 +53,7 @@ __FBSDID("$FreeBSD$"); #include <sys/module.h> #include <sys/malloc.h> #include <sys/libkern.h> +#include <sys/pcpu.h> #include <sys/uio.h> #include <opencrypto/cryptodev.h> @@ -201,9 +202,10 @@ padlock_cipher_process(struct padlock_session *ses, struct cryptodesc *enccrd, struct cryptop *crp) { union padlock_cw *cw; + struct thread *td; u_char *buf, *abuf; uint32_t *key; - int allocated; + int allocated, error; buf = padlock_cipher_alloc(enccrd, crp, &allocated); if (buf == NULL) @@ -247,9 +249,16 @@ padlock_cipher_process(struct padlock_session *ses, struct cryptodesc *enccrd, enccrd->crd_len, abuf); } + td = curthread; + error = fpu_kern_enter(td, &ses->ses_fpu_ctx, FPU_KERN_NORMAL); + if (error != 0) + goto out; + padlock_cbc(abuf, abuf, enccrd->crd_len / AES_BLOCK_LEN, key, cw, ses->ses_iv); + fpu_kern_leave(td, &ses->ses_fpu_ctx); + if (allocated) { crypto_copyback(crp->crp_flags, crp->crp_buf, enccrd->crd_skip, enccrd->crd_len, abuf); @@ -262,9 +271,10 @@ padlock_cipher_process(struct padlock_session *ses, struct cryptodesc *enccrd, AES_BLOCK_LEN, ses->ses_iv); } + out: if (allocated) { bzero(buf, enccrd->crd_len + 16); free(buf, M_PADLOCK); } - return (0); + return (error); } diff --git a/sys/crypto/via/padlock_hash.c b/sys/crypto/via/padlock_hash.c index 71f956e52908..58c58b2243fc 100644 --- a/sys/crypto/via/padlock_hash.c +++ b/sys/crypto/via/padlock_hash.c @@ -34,12 +34,14 @@ __FBSDID("$FreeBSD$"); #include <sys/malloc.h> #include <sys/libkern.h> #include <sys/endian.h> +#include <sys/pcpu.h> #if defined(__amd64__) || (defined(__i386__) && !defined(PC98)) #include <machine/cpufunc.h> #include <machine/cputypes.h> #include <machine/md_var.h> #include <machine/specialreg.h> #endif +#include <machine/pcb.h> #include <opencrypto/cryptodev.h> #include <opencrypto/cryptosoft.h> /* for hmac_ipad_buffer and hmac_opad_buffer */ @@ -363,12 +365,18 @@ int padlock_hash_process(struct padlock_session *ses, struct cryptodesc *maccrd, struct cryptop *crp) { + struct thread *td; int error; + td = curthread; + error = fpu_kern_enter(td, &ses->ses_fpu_ctx, FPU_KERN_NORMAL); + if (error != 0) + return (error); if ((maccrd->crd_flags & CRD_F_KEY_EXPLICIT) != 0) padlock_hash_key_setup(ses, maccrd->crd_key, maccrd->crd_klen); error = padlock_authcompute(ses, maccrd, crp->crp_buf, crp->crp_flags); + fpu_kern_leave(td, &ses->ses_fpu_ctx); return (error); } diff --git a/sys/dev/fb/fbreg.h b/sys/dev/fb/fbreg.h index 0be91d3abd4b..3847241af4f7 100644 --- a/sys/dev/fb/fbreg.h +++ b/sys/dev/fb/fbreg.h @@ -34,16 +34,7 @@ #define V_MAX_ADAPTERS 8 /* XXX */ /* some macros */ -#ifdef __i386__ -#define bcopy_io(s, d, c) generic_bcopy((void *)(s), (void *)(d), (c)) -#define bcopy_toio(s, d, c) generic_bcopy((void *)(s), (void *)(d), (c)) -#define bcopy_fromio(s, d, c) generic_bcopy((void *)(s), (void *)(d), (c)) -#define bzero_io(d, c) generic_bzero((void *)(d), (c)) -#define fill_io(p, d, c) fill((p), (void *)(d), (c)) -#define fillw_io(p, d, c) fillw((p), (void *)(d), (c)) -void generic_bcopy(const void *s, void *d, size_t c); -void generic_bzero(void *d, size_t c); -#elif defined(__amd64__) +#if defined(__amd64__) || defined(__i386__) #define bcopy_io(s, d, c) bcopy((void *)(s), (void *)(d), (c)) #define bcopy_toio(s, d, c) bcopy((void *)(s), (void *)(d), (c)) #define bcopy_fromio(s, d, c) bcopy((void *)(s), (void *)(d), (c)) diff --git a/sys/dev/random/nehemiah.c b/sys/dev/random/nehemiah.c index e34cdfa03ba1..f91e228e1df5 100644 --- a/sys/dev/random/nehemiah.c +++ b/sys/dev/random/nehemiah.c @@ -35,6 +35,8 @@ __FBSDID("$FreeBSD$"); #include <sys/selinfo.h> #include <sys/systm.h> +#include <machine/pcb.h> + #include <dev/random/randomdev.h> #define RANDOM_BLOCK_SIZE 256 @@ -82,6 +84,8 @@ static uint8_t out[RANDOM_BLOCK_SIZE+7] __aligned(16); static union VIA_ACE_CW acw __aligned(16); +static struct fpu_kern_ctx fpu_ctx_save; + static struct mtx random_nehemiah_mtx; /* ARGSUSED */ @@ -142,11 +146,16 @@ random_nehemiah_deinit(void) static int random_nehemiah_read(void *buf, int c) { - int i; + int i, error; size_t count, ret; uint8_t *p; mtx_lock(&random_nehemiah_mtx); + error = fpu_kern_enter(curthread, &fpu_ctx_save, FPU_KERN_NORMAL); + if (error != 0) { + mtx_unlock(&random_nehemiah_mtx); + return (0); + } /* Get a random AES key */ count = 0; @@ -187,6 +196,7 @@ random_nehemiah_read(void *buf, int c) c = MIN(RANDOM_BLOCK_SIZE, c); memcpy(buf, out, (size_t)c); + fpu_kern_leave(curthread, &fpu_ctx_save); mtx_unlock(&random_nehemiah_mtx); return (c); } diff --git a/sys/i386/i386/identcpu.c b/sys/i386/i386/identcpu.c index cfc29dd9ad58..afdedc209b7c 100644 --- a/sys/i386/i386/identcpu.c +++ b/sys/i386/i386/identcpu.c @@ -634,7 +634,6 @@ printcpuinfo(void) #if defined(I486_CPU) case CPUCLASS_486: printf("486"); - bzero_vector = i486_bzero; break; #endif #if defined(I586_CPU) diff --git a/sys/i386/i386/initcpu.c b/sys/i386/i386/initcpu.c index 97197829d044..902076532a9e 100644 --- a/sys/i386/i386/initcpu.c +++ b/sys/i386/i386/initcpu.c @@ -787,14 +787,14 @@ void enable_K5_wt_alloc(void) { u_int64_t msr; - register_t savecrit; + register_t saveintr; /* * Write allocate is supported only on models 1, 2, and 3, with * a stepping of 4 or greater. */ if (((cpu_id & 0xf0) > 0) && ((cpu_id & 0x0f) > 3)) { - savecrit = intr_disable(); + saveintr = intr_disable(); msr = rdmsr(0x83); /* HWCR */ wrmsr(0x83, msr & !(0x10)); @@ -825,7 +825,7 @@ enable_K5_wt_alloc(void) msr=rdmsr(0x83); wrmsr(0x83, msr|0x10); /* enable write allocate */ - intr_restore(savecrit); + intr_restore(saveintr); } } diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c index 9114fcb0081d..1ce9b69ddc26 100644 --- a/sys/i386/i386/machdep.c +++ b/sys/i386/i386/machdep.c @@ -3188,12 +3188,12 @@ fill_fpregs(struct thread *td, struct fpreg *fpregs) { #ifdef CPU_ENABLE_SSE if (cpu_fxsr) { - fill_fpregs_xmm(&td->td_pcb->pcb_save.sv_xmm, + fill_fpregs_xmm(&td->td_pcb->pcb_user_save.sv_xmm, (struct save87 *)fpregs); return (0); } #endif /* CPU_ENABLE_SSE */ - bcopy(&td->td_pcb->pcb_save.sv_87, fpregs, sizeof *fpregs); + bcopy(&td->td_pcb->pcb_user_save.sv_87, fpregs, sizeof *fpregs); return (0); } @@ -3203,11 +3203,11 @@ set_fpregs(struct thread *td, struct fpreg *fpregs) #ifdef CPU_ENABLE_SSE if (cpu_fxsr) { set_fpregs_xmm((struct save87 *)fpregs, - &td->td_pcb->pcb_save.sv_xmm); + &td->td_pcb->pcb_user_save.sv_xmm); return (0); } #endif /* CPU_ENABLE_SSE */ - bcopy(fpregs, &td->td_pcb->pcb_save.sv_87, sizeof *fpregs); + bcopy(fpregs, &td->td_pcb->pcb_user_save.sv_87, sizeof *fpregs); return (0); } @@ -3334,7 +3334,7 @@ get_fpcontext(struct thread *td, mcontext_t *mcp) addr = (void *)((char *)addr + 4); while ((uintptr_t)(void *)addr & 0xF); } - mcp->mc_ownedfp = npxgetregs(td, addr); + mcp->mc_ownedfp = npxgetuserregs(td, addr); if (addr != (union savefpu *)&mcp->mc_fpstate) { bcopy(addr, &mcp->mc_fpstate, sizeof(mcp->mc_fpstate)); bzero(&mcp->mc_spare2, sizeof(mcp->mc_spare2)); @@ -3379,7 +3379,7 @@ set_fpcontext(struct thread *td, const mcontext_t *mcp) * XXX we violate the dubious requirement that npxsetregs() * be called with interrupts disabled. */ - npxsetregs(td, addr); + npxsetuserregs(td, addr); #endif /* * Don't bother putting things back where they were in the @@ -3394,9 +3394,9 @@ set_fpcontext(struct thread *td, const mcontext_t *mcp) static void fpstate_drop(struct thread *td) { - register_t s; - s = intr_disable(); + KASSERT(PCB_USER_FPU(td->td_pcb), ("fpstate_drop: kernel-owned fpu")); + critical_enter(); #ifdef DEV_NPX if (PCPU_GET(fpcurthread) == td) npxdrop(); @@ -3411,8 +3411,9 @@ fpstate_drop(struct thread *td) * sendsig() is the only caller of npxgetregs()... perhaps we just * have too many layers. */ - curthread->td_pcb->pcb_flags &= ~PCB_NPXINITDONE; - intr_restore(s); + curthread->td_pcb->pcb_flags &= ~(PCB_NPXINITDONE | + PCB_NPXUSERINITDONE); + critical_exit(); } int diff --git a/sys/i386/i386/perfmon.c b/sys/i386/i386/perfmon.c index d085261547c2..c61e8fe04067 100644 --- a/sys/i386/i386/perfmon.c +++ b/sys/i386/i386/perfmon.c @@ -128,18 +128,18 @@ perfmon_avail(void) int perfmon_setup(int pmc, unsigned int control) { - register_t savecrit; + register_t saveintr; if (pmc < 0 || pmc >= NPMC) return EINVAL; perfmon_inuse |= (1 << pmc); control &= ~(PMCF_SYS_FLAGS << 16); - savecrit = intr_disable(); + saveintr = intr_disable(); ctl_shadow[pmc] = control; writectl(pmc); wrmsr(msr_pmc[pmc], pmc_shadow[pmc] = 0); - intr_restore(savecrit); + intr_restore(saveintr); return 0; } @@ -174,17 +174,17 @@ perfmon_fini(int pmc) int perfmon_start(int pmc) { - register_t savecrit; + register_t saveintr; if (pmc < 0 || pmc >= NPMC) return EINVAL; if (perfmon_inuse & (1 << pmc)) { - savecrit = intr_disable(); + saveintr = intr_disable(); ctl_shadow[pmc] |= (PMCF_EN << 16); wrmsr(msr_pmc[pmc], pmc_shadow[pmc]); writectl(pmc); - intr_restore(savecrit); + intr_restore(saveintr); return 0; } return EBUSY; @@ -193,17 +193,17 @@ perfmon_start(int pmc) int perfmon_stop(int pmc) { - register_t savecrit; + register_t saveintr; if (pmc < 0 || pmc >= NPMC) return EINVAL; if (perfmon_inuse & (1 << pmc)) { - savecrit = intr_disable(); + saveintr = intr_disable(); pmc_shadow[pmc] = rdmsr(msr_pmc[pmc]) & 0xffffffffffULL; ctl_shadow[pmc] &= ~(PMCF_EN << 16); writectl(pmc); - intr_restore(savecrit); + intr_restore(saveintr); return 0; } return EBUSY; diff --git a/sys/i386/i386/ptrace_machdep.c b/sys/i386/i386/ptrace_machdep.c index 409db160892b..4608c9b939d4 100644 --- a/sys/i386/i386/ptrace_machdep.c +++ b/sys/i386/i386/ptrace_machdep.c @@ -51,7 +51,7 @@ cpu_ptrace(struct thread *td, int req, void *addr, int data) if (!cpu_fxsr) return (EINVAL); - fpstate = &td->td_pcb->pcb_save.sv_xmm; + fpstate = &td->td_pcb->pcb_user_save.sv_xmm; switch (req) { case PT_GETXMMREGS: error = copyout(fpstate, addr, sizeof(*fpstate)); diff --git a/sys/i386/i386/support.s b/sys/i386/i386/support.s index cbf0d8280739..4927b50e6f5b 100644 --- a/sys/i386/i386/support.s +++ b/sys/i386/i386/support.s @@ -42,23 +42,6 @@ #define IDXSHIFT 10 .data - .globl bcopy_vector -bcopy_vector: - .long generic_bcopy - .globl bzero_vector -bzero_vector: - .long generic_bzero - .globl copyin_vector -copyin_vector: - .long generic_copyin - .globl copyout_vector -copyout_vector: - .long generic_copyout -#if defined(I586_CPU) && defined(DEV_NPX) -kernel_fpu_lock: - .byte 0xfe - .space 3 -#endif ALIGN_DATA .globl intrcnt, eintrcnt intrcnt: @@ -76,13 +59,7 @@ eintrnames: * bcopy family * void bzero(void *buf, u_int len) */ - ENTRY(bzero) - MEXITCOUNT - jmp *bzero_vector -END(bzero) - -ENTRY(generic_bzero) pushl %edi movl 8(%esp),%edi movl 12(%esp),%ecx @@ -97,270 +74,8 @@ ENTRY(generic_bzero) stosb popl %edi ret -END(generic_bzero) +END(bzero) -#ifdef I486_CPU -ENTRY(i486_bzero) - movl 4(%esp),%edx - movl 8(%esp),%ecx - xorl %eax,%eax -/* - * do 64 byte chunks first - * - * XXX this is probably over-unrolled at least for DX2's - */ -2: - cmpl $64,%ecx - jb 3f - movl %eax,(%edx) - movl %eax,4(%edx) - movl %eax,8(%edx) - movl %eax,12(%edx) - movl %eax,16(%edx) - movl %eax,20(%edx) - movl %eax,24(%edx) - movl %eax,28(%edx) - movl %eax,32(%edx) - movl %eax,36(%edx) - movl %eax,40(%edx) - movl %eax,44(%edx) - movl %eax,48(%edx) - movl %eax,52(%edx) - movl %eax,56(%edx) - movl %eax,60(%edx) - addl $64,%edx - subl $64,%ecx - jnz 2b - ret - -/* - * do 16 byte chunks - */ - SUPERALIGN_TEXT -3: - cmpl $16,%ecx - jb 4f - movl %eax,(%edx) - movl %eax,4(%edx) - movl %eax,8(%edx) - movl %eax,12(%edx) - addl $16,%edx - subl $16,%ecx - jnz 3b - ret - -/* - * do 4 byte chunks - */ - SUPERALIGN_TEXT -4: - cmpl $4,%ecx - jb 5f - movl %eax,(%edx) - addl $4,%edx - subl $4,%ecx - jnz 4b - ret - -/* - * do 1 byte chunks - * a jump table seems to be faster than a loop or more range reductions - * - * XXX need a const section for non-text - */ - .data -jtab: - .long do0 - .long do1 - .long do2 - .long do3 - - .text - SUPERALIGN_TEXT -5: - jmp *jtab(,%ecx,4) - - SUPERALIGN_TEXT -do3: - movw %ax,(%edx) - movb %al,2(%edx) - ret - - SUPERALIGN_TEXT -do2: - movw %ax,(%edx) - ret - - SUPERALIGN_TEXT -do1: - movb %al,(%edx) - ret - - SUPERALIGN_TEXT -do0: - ret -END(i486_bzero) -#endif - -#if defined(I586_CPU) && defined(DEV_NPX) -ENTRY(i586_bzero) - movl 4(%esp),%edx - movl 8(%esp),%ecx - - /* - * The FPU register method is twice as fast as the integer register - * method unless the target is in the L1 cache and we pre-allocate a - * cache line for it (then the integer register method is 4-5 times - * faster). However, we never pre-allocate cache lines, since that - * would make the integer method 25% or more slower for the common - * case when the target isn't in either the L1 cache or the L2 cache. - * Thus we normally use the FPU register method unless the overhead - * would be too large. - */ - cmpl $256,%ecx /* empirical; clts, fninit, smsw cost a lot */ - jb intreg_i586_bzero - - /* - * The FPU registers may belong to an application or to fastmove() - * or to another invocation of bcopy() or ourself in a higher level - * interrupt or trap handler. Preserving the registers is - * complicated since we avoid it if possible at all levels. We - * want to localize the complications even when that increases them. - * Here the extra work involves preserving CR0_TS in TS. - * `fpcurthread != NULL' is supposed to be the condition that all the - * FPU resources belong to an application, but fpcurthread and CR0_TS - * aren't set atomically enough for this condition to work in - * interrupt handlers. - * - * Case 1: FPU registers belong to the application: we must preserve - * the registers if we use them, so we only use the FPU register - * method if the target size is large enough to amortize the extra - * overhead for preserving them. CR0_TS must be preserved although - * it is very likely to end up as set. - * - * Case 2: FPU registers belong to fastmove(): fastmove() currently - * makes the registers look like they belong to an application so - * that cpu_switch() and savectx() don't have to know about it, so - * this case reduces to case 1. - * - * Case 3: FPU registers belong to the kernel: don't use the FPU - * register method. This case is unlikely, and supporting it would - * be more complicated and might take too much stack. - * - * Case 4: FPU registers don't belong to anyone: the FPU registers - * don't need to be preserved, so we always use the FPU register - * method. CR0_TS must be preserved although it is very likely to - * always end up as clear. - */ - cmpl $0,PCPU(FPCURTHREAD) - je i586_bz1 - - /* - * XXX don't use the FPU for cases 1 and 2, since preemptive - * scheduling of ithreads broke these cases. Note that we can - * no longer get here from an interrupt handler, since the - * context sitch to the interrupt handler will have saved the - * FPU state. - */ - jmp intreg_i586_bzero - - cmpl $256+184,%ecx /* empirical; not quite 2*108 more */ - jb intreg_i586_bzero - sarb $1,kernel_fpu_lock - jc intreg_i586_bzero - smsw %ax - clts - subl $108,%esp - fnsave 0(%esp) - jmp i586_bz2 - -i586_bz1: - sarb $1,kernel_fpu_lock - jc intreg_i586_bzero - smsw %ax - clts - fninit /* XXX should avoid needing this */ -i586_bz2: - fldz - - /* - * Align to an 8 byte boundary (misalignment in the main loop would - * cost a factor of >= 2). Avoid jumps (at little cost if it is - * already aligned) by always zeroing 8 bytes and using the part up - * to the _next_ alignment position. - */ - fstl 0(%edx) - addl %edx,%ecx /* part of %ecx -= new_%edx - %edx */ - addl $8,%edx - andl $~7,%edx - subl %edx,%ecx - - /* - * Similarly align `len' to a multiple of 8. - */ - fstl -8(%edx,%ecx) - decl %ecx - andl $~7,%ecx - - /* - * This wouldn't be any faster if it were unrolled, since the loop - * control instructions are much faster than the fstl and/or done - * in parallel with it so their overhead is insignificant. - */ -fpureg_i586_bzero_loop: - fstl 0(%edx) - addl $8,%edx - subl $8,%ecx - cmpl $8,%ecx - jae fpureg_i586_bzero_loop - - cmpl $0,PCPU(FPCURTHREAD) - je i586_bz3 - - /* XXX check that the condition for cases 1-2 stayed false. */ -i586_bzero_oops: - int $3 - jmp i586_bzero_oops - - frstor 0(%esp) - addl $108,%esp - lmsw %ax - movb $0xfe,kernel_fpu_lock - ret - -i586_bz3: - fstp %st(0) - lmsw %ax - movb $0xfe,kernel_fpu_lock - ret - -intreg_i586_bzero: - /* - * `rep stos' seems to be the best method in practice for small - * counts. Fancy methods usually take too long to start up due - * to cache and BTB misses. - */ - pushl %edi - movl %edx,%edi - xorl %eax,%eax - shrl $2,%ecx - cld - rep - stosl - movl 12(%esp),%ecx - andl $3,%ecx - jne 1f - popl %edi - ret - -1: - rep - stosb - popl %edi - ret -END(i586_bzero) -#endif /* I586_CPU && defined(DEV_NPX) */ - ENTRY(sse2_pagezero) pushl %ebx movl 8(%esp),%ecx @@ -473,16 +188,11 @@ ENTRY(bcopyb) ret END(bcopyb) -ENTRY(bcopy) - MEXITCOUNT - jmp *bcopy_vector -END(bcopy) - /* - * generic_bcopy(src, dst, cnt) + * bcopy(src, dst, cnt) * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800 */ -ENTRY(generic_bcopy) +ENTRY(bcopy) pushl %esi pushl %edi movl 12(%esp),%esi @@ -526,157 +236,7 @@ ENTRY(generic_bcopy) popl %esi cld ret -END(generic_bcopy) - -#if defined(I586_CPU) && defined(DEV_NPX) -ENTRY(i586_bcopy) - pushl %esi - pushl %edi - movl 12(%esp),%esi - movl 16(%esp),%edi - movl 20(%esp),%ecx - - movl %edi,%eax - subl %esi,%eax - cmpl %ecx,%eax /* overlapping && src < dst? */ - jb 1f - - cmpl $1024,%ecx - jb small_i586_bcopy - - sarb $1,kernel_fpu_lock - jc small_i586_bcopy - cmpl $0,PCPU(FPCURTHREAD) - je i586_bc1 - - /* XXX turn off handling of cases 1-2, as above. */ - movb $0xfe,kernel_fpu_lock - jmp small_i586_bcopy - - smsw %dx - clts - subl $108,%esp - fnsave 0(%esp) - jmp 4f - -i586_bc1: - smsw %dx - clts - fninit /* XXX should avoid needing this */ - - ALIGN_TEXT -4: - pushl %ecx -#define DCACHE_SIZE 8192 - cmpl $(DCACHE_SIZE-512)/2,%ecx - jbe 2f - movl $(DCACHE_SIZE-512)/2,%ecx -2: - subl %ecx,0(%esp) - cmpl $256,%ecx - jb 5f /* XXX should prefetch if %ecx >= 32 */ - pushl %esi - pushl %ecx - ALIGN_TEXT -3: - movl 0(%esi),%eax - movl 32(%esi),%eax - movl 64(%esi),%eax - movl 96(%esi),%eax - movl 128(%esi),%eax - movl 160(%esi),%eax - movl 192(%esi),%eax - movl 224(%esi),%eax - addl $256,%esi - subl $256,%ecx - cmpl $256,%ecx - jae 3b - popl %ecx - popl %esi -5: - ALIGN_TEXT -large_i586_bcopy_loop: - fildq 0(%esi) - fildq 8(%esi) - fildq 16(%esi) - fildq 24(%esi) - fildq 32(%esi) - fildq 40(%esi) - fildq 48(%esi) - fildq 56(%esi) - fistpq 56(%edi) - fistpq 48(%edi) - fistpq 40(%edi) - fistpq 32(%edi) - fistpq 24(%edi) - fistpq 16(%edi) - fistpq 8(%edi) - fistpq 0(%edi) - addl $64,%esi - addl $64,%edi - subl $64,%ecx - cmpl $64,%ecx - jae large_i586_bcopy_loop - popl %eax - addl %eax,%ecx - cmpl $64,%ecx - jae 4b - - cmpl $0,PCPU(FPCURTHREAD) - je i586_bc2 - - /* XXX check that the condition for cases 1-2 stayed false. */ -i586_bcopy_oops: - int $3 - jmp i586_bcopy_oops - - frstor 0(%esp) - addl $108,%esp -i586_bc2: - lmsw %dx - movb $0xfe,kernel_fpu_lock - -/* - * This is a duplicate of the main part of generic_bcopy. See the comments - * there. Jumping into generic_bcopy would cost a whole 0-1 cycles and - * would mess up high resolution profiling. - */ - ALIGN_TEXT -small_i586_bcopy: - shrl $2,%ecx - cld - rep - movsl - movl 20(%esp),%ecx - andl $3,%ecx - rep - movsb - popl %edi - popl %esi - ret - - ALIGN_TEXT -1: - addl %ecx,%edi - addl %ecx,%esi - decl %edi - decl %esi - andl $3,%ecx - std - rep - movsb - movl 20(%esp),%ecx - shrl $2,%ecx - subl $3,%esi - subl $3,%edi - rep - movsl - popl %edi - popl %esi - cld - ret -END(i586_bcopy) -#endif /* I586_CPU && defined(DEV_NPX) */ +END(bcopy) /* * Note: memcpy does not support overlapping copies @@ -723,11 +283,6 @@ END(memcpy) * copyout(from_kernel, to_user, len) - MP SAFE */ ENTRY(copyout) - MEXITCOUNT - jmp *copyout_vector -END(copyout) - -ENTRY(generic_copyout) movl PCPU(CURPCB),%eax movl $copyout_fault,PCB_ONFAULT(%eax) pushl %esi @@ -764,10 +319,6 @@ ENTRY(generic_copyout) /* bcopy(%esi, %edi, %ebx) */ movl %ebx,%ecx -#if defined(I586_CPU) && defined(DEV_NPX) - ALIGN_TEXT -slow_copyout: -#endif shrl $2,%ecx cld rep @@ -785,7 +336,7 @@ done_copyout: movl PCPU(CURPCB),%edx movl %eax,PCB_ONFAULT(%edx) ret -END(generic_copyout) +END(copyout) ALIGN_TEXT copyout_fault: @@ -797,70 +348,10 @@ copyout_fault: movl $EFAULT,%eax ret -#if defined(I586_CPU) && defined(DEV_NPX) -ENTRY(i586_copyout) - /* - * Duplicated from generic_copyout. Could be done a bit better. - */ - movl PCPU(CURPCB),%eax - movl $copyout_fault,PCB_ONFAULT(%eax) - pushl %esi - pushl %edi - pushl %ebx - movl 16(%esp),%esi - movl 20(%esp),%edi - movl 24(%esp),%ebx - testl %ebx,%ebx /* anything to do? */ - jz done_copyout - - /* - * Check explicitly for non-user addresses. If 486 write protection - * is being used, this check is essential because we are in kernel - * mode so the h/w does not provide any protection against writing - * kernel addresses. - */ - - /* - * First, prevent address wrapping. - */ - movl %edi,%eax - addl %ebx,%eax - jc copyout_fault -/* - * XXX STOP USING VM_MAXUSER_ADDRESS. - * It is an end address, not a max, so every time it is used correctly it - * looks like there is an off by one error, and of course it caused an off - * by one error in several places. - */ - cmpl $VM_MAXUSER_ADDRESS,%eax - ja copyout_fault - - /* bcopy(%esi, %edi, %ebx) */ -3: - movl %ebx,%ecx - /* - * End of duplicated code. - */ - - cmpl $1024,%ecx - jb slow_copyout - - pushl %ecx - call fastmove - addl $4,%esp - jmp done_copyout -END(i586_copyout) -#endif /* I586_CPU && defined(DEV_NPX) */ - /* * copyin(from_user, to_kernel, len) - MP SAFE */ ENTRY(copyin) - MEXITCOUNT - jmp *copyin_vector -END(copyin) - -ENTRY(generic_copyin) movl PCPU(CURPCB),%eax movl $copyin_fault,PCB_ONFAULT(%eax) pushl %esi @@ -878,10 +369,6 @@ ENTRY(generic_copyin) cmpl $VM_MAXUSER_ADDRESS,%edx ja copyin_fault -#if defined(I586_CPU) && defined(DEV_NPX) - ALIGN_TEXT -slow_copyin: -#endif movb %cl,%al shrl $2,%ecx /* copy longword-wise */ cld @@ -892,17 +379,13 @@ slow_copyin: rep movsb -#if defined(I586_CPU) && defined(DEV_NPX) - ALIGN_TEXT -done_copyin: -#endif popl %edi popl %esi xorl %eax,%eax movl PCPU(CURPCB),%edx movl %eax,PCB_ONFAULT(%edx) ret -END(generic_copyin) +END(copyin) ALIGN_TEXT copyin_fault: @@ -913,250 +396,6 @@ copyin_fault: movl $EFAULT,%eax ret -#if defined(I586_CPU) && defined(DEV_NPX) -ENTRY(i586_copyin) - /* - * Duplicated from generic_copyin. Could be done a bit better. - */ - movl PCPU(CURPCB),%eax - movl $copyin_fault,PCB_ONFAULT(%eax) - pushl %esi - pushl %edi - movl 12(%esp),%esi /* caddr_t from */ - movl 16(%esp),%edi /* caddr_t to */ - movl 20(%esp),%ecx /* size_t len */ - - /* - * make sure address is valid - */ - movl %esi,%edx - addl %ecx,%edx - jc copyin_fault - cmpl $VM_MAXUSER_ADDRESS,%edx - ja copyin_fault - /* - * End of duplicated code. - */ - - cmpl $1024,%ecx - jb slow_copyin - - pushl %ebx /* XXX prepare for fastmove_fault */ - pushl %ecx - call fastmove - addl $8,%esp - jmp done_copyin -END(i586_copyin) -#endif /* I586_CPU && defined(DEV_NPX) */ - -#if defined(I586_CPU) && defined(DEV_NPX) -/* fastmove(src, dst, len) - src in %esi - dst in %edi - len in %ecx XXX changed to on stack for profiling - uses %eax and %edx for tmp. storage - */ -/* XXX use ENTRY() to get profiling. fastmove() is actually a non-entry. */ -ENTRY(fastmove) - pushl %ebp - movl %esp,%ebp - subl $PCB_SAVEFPU_SIZE+3*4,%esp - - movl 8(%ebp),%ecx - cmpl $63,%ecx - jbe fastmove_tail - - testl $7,%esi /* check if src addr is multiple of 8 */ - jnz fastmove_tail - - testl $7,%edi /* check if dst addr is multiple of 8 */ - jnz fastmove_tail - - /* XXX grab FPU context atomically. */ - cli - -/* if (fpcurthread != NULL) { */ - cmpl $0,PCPU(FPCURTHREAD) - je 6f -/* fnsave(&curpcb->pcb_savefpu); */ - movl PCPU(CURPCB),%eax - fnsave PCB_SAVEFPU(%eax) -/* FPCURTHREAD = NULL; */ - movl $0,PCPU(FPCURTHREAD) -/* } */ -6: -/* now we own the FPU. */ - -/* - * The process' FP state is saved in the pcb, but if we get - * switched, the cpu_switch() will store our FP state in the - * pcb. It should be possible to avoid all the copying for - * this, e.g., by setting a flag to tell cpu_switch() to - * save the state somewhere else. - */ -/* tmp = curpcb->pcb_savefpu; */ - movl %ecx,-12(%ebp) - movl %esi,-8(%ebp) - movl %edi,-4(%ebp) - movl %esp,%edi - movl PCPU(CURPCB),%esi - addl $PCB_SAVEFPU,%esi - cld - movl $PCB_SAVEFPU_SIZE>>2,%ecx - rep - movsl - movl -12(%ebp),%ecx - movl -8(%ebp),%esi - movl -4(%ebp),%edi -/* stop_emulating(); */ - clts -/* fpcurthread = curthread; */ - movl PCPU(CURTHREAD),%eax - movl %eax,PCPU(FPCURTHREAD) - movl PCPU(CURPCB),%eax - - /* XXX end of atomic FPU context grab. */ - sti - - movl $fastmove_fault,PCB_ONFAULT(%eax) -4: - movl %ecx,-12(%ebp) - cmpl $1792,%ecx - jbe 2f - movl $1792,%ecx -2: - subl %ecx,-12(%ebp) - cmpl $256,%ecx - jb 5f - movl %ecx,-8(%ebp) - movl %esi,-4(%ebp) - ALIGN_TEXT -3: - movl 0(%esi),%eax - movl 32(%esi),%eax - movl 64(%esi),%eax - movl 96(%esi),%eax - movl 128(%esi),%eax - movl 160(%esi),%eax - movl 192(%esi),%eax - movl 224(%esi),%eax - addl $256,%esi - subl $256,%ecx - cmpl $256,%ecx - jae 3b - movl -8(%ebp),%ecx - movl -4(%ebp),%esi -5: - ALIGN_TEXT -fastmove_loop: - fildq 0(%esi) - fildq 8(%esi) - fildq 16(%esi) - fildq 24(%esi) - fildq 32(%esi) - fildq 40(%esi) - fildq 48(%esi) - fildq 56(%esi) - fistpq 56(%edi) - fistpq 48(%edi) - fistpq 40(%edi) - fistpq 32(%edi) - fistpq 24(%edi) - fistpq 16(%edi) - fistpq 8(%edi) - fistpq 0(%edi) - addl $-64,%ecx - addl $64,%esi - addl $64,%edi - cmpl $63,%ecx - ja fastmove_loop - movl -12(%ebp),%eax - addl %eax,%ecx - cmpl $64,%ecx - jae 4b - - /* XXX ungrab FPU context atomically. */ - cli - -/* curpcb->pcb_savefpu = tmp; */ - movl %ecx,-12(%ebp) - movl %esi,-8(%ebp) - movl %edi,-4(%ebp) - movl PCPU(CURPCB),%edi - addl $PCB_SAVEFPU,%edi - movl %esp,%esi - cld - movl $PCB_SAVEFPU_SIZE>>2,%ecx - rep - movsl - movl -12(%ebp),%ecx - movl -8(%ebp),%esi - movl -4(%ebp),%edi - -/* start_emulating(); */ - smsw %ax - orb $CR0_TS,%al - lmsw %ax -/* fpcurthread = NULL; */ - movl $0,PCPU(FPCURTHREAD) - - /* XXX end of atomic FPU context ungrab. */ - sti - - ALIGN_TEXT -fastmove_tail: - movl PCPU(CURPCB),%eax - movl $fastmove_tail_fault,PCB_ONFAULT(%eax) - - movb %cl,%al - shrl $2,%ecx /* copy longword-wise */ - cld - rep - movsl - movb %al,%cl - andb $3,%cl /* copy remaining bytes */ - rep - movsb - - movl %ebp,%esp - popl %ebp - ret - - ALIGN_TEXT -fastmove_fault: - /* XXX ungrab FPU context atomically. */ - cli - - movl PCPU(CURPCB),%edi - addl $PCB_SAVEFPU,%edi - movl %esp,%esi - cld - movl $PCB_SAVEFPU_SIZE>>2,%ecx - rep - movsl - - smsw %ax - orb $CR0_TS,%al - lmsw %ax - movl $0,PCPU(FPCURTHREAD) - - /* XXX end of atomic FPU context ungrab. */ - sti - -fastmove_tail_fault: - movl %ebp,%esp - popl %ebp - addl $8,%esp - popl %ebx - popl %edi - popl %esi - movl PCPU(CURPCB),%edx - movl $0,PCB_ONFAULT(%edx) - movl $EFAULT,%eax - ret -END(fastmove) -#endif /* I586_CPU && defined(DEV_NPX) */ - /* * casuword. Compare and set user word. Returns -1 or the current value. */ diff --git a/sys/i386/i386/swtch.s b/sys/i386/i386/swtch.s index 0c0787119d03..1dee5f2516da 100644 --- a/sys/i386/i386/swtch.s +++ b/sys/i386/i386/swtch.s @@ -156,8 +156,7 @@ ENTRY(cpu_switch) /* have we used fp, and need a save? */ cmpl %ecx,PCPU(FPCURTHREAD) jne 1f - addl $PCB_SAVEFPU,%edx /* h/w bugs make saving complicated */ - pushl %edx + pushl PCB_SAVEFPU(%edx) /* h/w bugs make saving complicated */ call npxsave /* do it in a big C function */ popl %eax 1: @@ -408,7 +407,7 @@ ENTRY(savectx) pushl %ecx movl TD_PCB(%eax),%eax - leal PCB_SAVEFPU(%eax),%eax + movl PCB_SAVEFPU(%eax),%eax pushl %eax pushl %eax call npxsave @@ -417,7 +416,7 @@ ENTRY(savectx) popl %ecx pushl $PCB_SAVEFPU_SIZE - leal PCB_SAVEFPU(%ecx),%ecx + movl PCB_SAVEFPU(%ecx),%ecx pushl %ecx pushl %eax call bcopy diff --git a/sys/i386/i386/trap.c b/sys/i386/i386/trap.c index 4980c5c0befa..b412151c3aba 100644 --- a/sys/i386/i386/trap.c +++ b/sys/i386/i386/trap.c @@ -501,6 +501,8 @@ trap(struct trapframe *frame) case T_DNA: #ifdef DEV_NPX + KASSERT(PCB_USER_FPU(td->td_pcb), + ("kernel FPU ctx has leaked")); /* transparent fault (due to context switch "late") */ if (npxdna()) goto userout; @@ -533,16 +535,23 @@ trap(struct trapframe *frame) case T_DNA: #ifdef DEV_NPX - /* - * The kernel is apparently using npx for copying. - * XXX this should be fatal unless the kernel has - * registered such use. - */ + KASSERT(!PCB_USER_FPU(td->td_pcb), + ("Unregistered use of FPU in kernel")); if (npxdna()) goto out; #endif break; + case T_ARITHTRAP: /* arithmetic trap */ + case T_XMMFLT: /* SIMD floating-point exception */ + case T_FPOPFLT: /* FPU operand fetch fault */ + /* + * XXXKIB for now disable any FPU traps in kernel + * handler registration seems to be overkill + */ + trap_fatal(frame, 0); + goto out; + /* * The following two traps can happen in * vm86 mode, and, if so, we want to handle @@ -748,6 +757,8 @@ trap(struct trapframe *frame) user: userret(td, frame); mtx_assert(&Giant, MA_NOTOWNED); + KASSERT(PCB_USER_FPU(td->td_pcb), + ("Return from trap with kernel FPU ctx leaked")); userout: out: return; @@ -1062,5 +1073,12 @@ syscall(struct trapframe *frame) trapsignal(td, &ksi); } + KASSERT(PCB_USER_FPU(td->td_pcb), + ("System call %s returning with kernel FPU ctx leaked", + syscallname(td->td_proc, sa.code))); + KASSERT(td->td_pcb->pcb_save == &td->td_pcb->pcb_user_save, + ("System call %s returning with mangled pcb_save", + syscallname(td->td_proc, sa.code))); + syscallret(td, error, &sa); } diff --git a/sys/i386/i386/vm_machdep.c b/sys/i386/i386/vm_machdep.c index 3eb2d08bab6c..a2148daac9a9 100644 --- a/sys/i386/i386/vm_machdep.c +++ b/sys/i386/i386/vm_machdep.c @@ -150,9 +150,6 @@ cpu_fork(td1, p2, td2, flags) register struct proc *p1; struct pcb *pcb2; struct mdproc *mdp2; -#ifdef DEV_NPX - register_t savecrit; -#endif p1 = td1->td_proc; if ((flags & RFPROC) == 0) { @@ -176,14 +173,14 @@ cpu_fork(td1, p2, td2, flags) return; } - /* Ensure that p1's pcb is up to date. */ + /* Ensure that td1's pcb is up to date. */ if (td1 == curthread) td1->td_pcb->pcb_gs = rgs(); #ifdef DEV_NPX - savecrit = intr_disable(); + critical_enter(); if (PCPU_GET(fpcurthread) == td1) - npxsave(&td1->td_pcb->pcb_save); - intr_restore(savecrit); + npxsave(td1->td_pcb->pcb_save); + critical_exit(); #endif /* Point the pcb to the top of the stack */ @@ -191,9 +188,12 @@ cpu_fork(td1, p2, td2, flags) td2->td_kstack_pages * PAGE_SIZE) - 1; td2->td_pcb = pcb2; - /* Copy p1's pcb */ + /* Copy td1's pcb */ bcopy(td1->td_pcb, pcb2, sizeof(*pcb2)); + /* Properly initialize pcb_save */ + pcb2->pcb_save = &pcb2->pcb_user_save; + /* Point mdproc and then copy over td1's contents */ mdp2 = &p2->p_md; bcopy(&p1->p_md, mdp2, sizeof(*mdp2)); @@ -325,8 +325,10 @@ cpu_thread_exit(struct thread *td) { #ifdef DEV_NPX + critical_enter(); if (td == PCPU_GET(fpcurthread)) npxdrop(); + critical_exit(); #endif /* Disable any hardware breakpoints. */ @@ -372,6 +374,7 @@ cpu_thread_alloc(struct thread *td) td->td_kstack_pages * PAGE_SIZE) - 1; td->td_frame = (struct trapframe *)((caddr_t)td->td_pcb - 16) - 1; td->td_pcb->pcb_ext = NULL; + td->td_pcb->pcb_save = &td->td_pcb->pcb_user_save; } void @@ -437,7 +440,8 @@ cpu_set_upcall(struct thread *td, struct thread *td0) * values here. */ bcopy(td0->td_pcb, pcb2, sizeof(*pcb2)); - pcb2->pcb_flags &= ~(PCB_NPXTRAP|PCB_NPXINITDONE); + pcb2->pcb_flags &= ~(PCB_NPXINITDONE | PCB_NPXUSERINITDONE); + pcb2->pcb_save = &pcb2->pcb_user_save; /* * Create a new fresh stack for the new thread. diff --git a/sys/i386/include/md_var.h b/sys/i386/include/md_var.h index 44eb8a6e2aba..168ee6298f45 100644 --- a/sys/i386/include/md_var.h +++ b/sys/i386/include/md_var.h @@ -36,11 +36,6 @@ * Miscellaneous machine-dependent declarations. */ -extern void (*bcopy_vector)(const void *from, void *to, size_t len); -extern void (*bzero_vector)(void *buf, size_t len); -extern int (*copyin_vector)(const void *udaddr, void *kaddr, size_t len); -extern int (*copyout_vector)(const void *kaddr, void *udaddr, size_t len); - extern long Maxmem; extern u_int basemem; /* PA of original top of base memory */ extern int busdma_swi_pending; @@ -98,11 +93,6 @@ void dump_add_page(vm_paddr_t); void dump_drop_page(vm_paddr_t); void enable_sse(void); void fillw(int /*u_short*/ pat, void *base, size_t cnt); -void i486_bzero(void *buf, size_t len); -void i586_bcopy(const void *from, void *to, size_t len); -void i586_bzero(void *buf, size_t len); -int i586_copyin(const void *udaddr, void *kaddr, size_t len); -int i586_copyout(const void *kaddr, void *udaddr, size_t len); void i686_pagezero(void *addr); void sse2_pagezero(void *addr); void init_AMD_Elan_sc520(void); diff --git a/sys/i386/include/npx.h b/sys/i386/include/npx.h index e9811b3a2e8c..71073a627e1e 100644 --- a/sys/i386/include/npx.h +++ b/sys/i386/include/npx.h @@ -138,23 +138,36 @@ union savefpu { #ifdef _KERNEL -#define IO_NPX 0x0F0 /* Numeric Coprocessor */ -#define IO_NPXSIZE 16 /* 80387/80487 NPX registers */ - -#define IRQ_NPX 13 +struct fpu_kern_ctx { + union savefpu hwstate; + union savefpu *prev; + uint32_t flags; +}; +#define FPU_KERN_CTX_NPXINITDONE 0x01 -/* full reset on some systems, NOP on others */ -#define npx_full_reset() outb(IO_NPX + 1, 0) +#define PCB_USER_FPU(pcb) (((pcb)->pcb_flags & PCB_KERNNPX) == 0) int npxdna(void); void npxdrop(void); void npxexit(struct thread *td); int npxformat(void); int npxgetregs(struct thread *td, union savefpu *addr); +int npxgetuserregs(struct thread *td, union savefpu *addr); void npxinit(void); void npxsave(union savefpu *addr); void npxsetregs(struct thread *td, union savefpu *addr); +void npxsetuserregs(struct thread *td, union savefpu *addr); int npxtrap(void); +int fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, + u_int flags); +int fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx); +int fpu_kern_thread(u_int flags); +int is_fpu_kern_thread(u_int flags); + +/* + * Flags for fpu_kern_enter() and fpu_kern_thread(). + */ +#define FPU_KERN_NORMAL 0x0000 #endif diff --git a/sys/i386/include/pcb.h b/sys/i386/include/pcb.h index 17c84861ba9d..1311ab1ed21e 100644 --- a/sys/i386/include/pcb.h +++ b/sys/i386/include/pcb.h @@ -60,14 +60,15 @@ struct pcb { int pcb_dr6; int pcb_dr7; - union savefpu pcb_save; + union savefpu pcb_user_save; uint16_t pcb_initial_npxcw; u_int pcb_flags; #define FP_SOFTFP 0x01 /* process using software fltng pnt emulator */ #define PCB_DBREGS 0x02 /* process using debug registers */ -#define PCB_NPXTRAP 0x04 /* npx trap pending */ #define PCB_NPXINITDONE 0x08 /* fpu state is initialized */ #define PCB_VM86CALL 0x10 /* in vm86 call */ +#define PCB_NPXUSERINITDONE 0x20 /* user fpu state is initialized */ +#define PCB_KERNNPX 0x40 /* kernel uses npx */ caddr_t pcb_onfault; /* copyin/out fault recovery */ int pcb_gs; @@ -76,6 +77,7 @@ struct pcb { struct pcb_ext *pcb_ext; /* optional pcb extension */ int pcb_psl; /* process status long */ u_long pcb_vm86[2]; /* vm86bios scratch space */ + union savefpu *pcb_save; }; #ifdef _KERNEL diff --git a/sys/i386/isa/npx.c b/sys/i386/isa/npx.c index 6da4b4c1db7e..f8dc11c6cb9b 100644 --- a/sys/i386/isa/npx.c +++ b/sys/i386/isa/npx.c @@ -85,37 +85,33 @@ __FBSDID("$FreeBSD$"); * 387 and 287 Numeric Coprocessor Extension (NPX) Driver. */ -/* Configuration flags. */ -#define NPX_DISABLE_I586_OPTIMIZED_BCOPY (1 << 0) -#define NPX_DISABLE_I586_OPTIMIZED_BZERO (1 << 1) -#define NPX_DISABLE_I586_OPTIMIZED_COPYIO (1 << 2) - #if defined(__GNUCLIKE_ASM) && !defined(lint) -#define fldcw(addr) __asm("fldcw %0" : : "m" (*(addr))) -#define fnclex() __asm("fnclex") -#define fninit() __asm("fninit") +#define fldcw(cw) __asm __volatile("fldcw %0" : : "m" (cw)) +#define fnclex() __asm __volatile("fnclex") +#define fninit() __asm __volatile("fninit") #define fnsave(addr) __asm __volatile("fnsave %0" : "=m" (*(addr))) #define fnstcw(addr) __asm __volatile("fnstcw %0" : "=m" (*(addr))) -#define fnstsw(addr) __asm __volatile("fnstsw %0" : "=m" (*(addr))) -#define fp_divide_by_0() __asm("fldz; fld1; fdiv %st,%st(1); fnop") -#define frstor(addr) __asm("frstor %0" : : "m" (*(addr))) +#define fnstsw(addr) __asm __volatile("fnstsw %0" : "=am" (*(addr))) +#define fp_divide_by_0() __asm __volatile( \ + "fldz; fld1; fdiv %st,%st(1); fnop") +#define frstor(addr) __asm __volatile("frstor %0" : : "m" (*(addr))) #ifdef CPU_ENABLE_SSE -#define fxrstor(addr) __asm("fxrstor %0" : : "m" (*(addr))) +#define fxrstor(addr) __asm __volatile("fxrstor %0" : : "m" (*(addr))) #define fxsave(addr) __asm __volatile("fxsave %0" : "=m" (*(addr))) -#define ldmxcsr(__csr) __asm __volatile("ldmxcsr %0" : : "m" (__csr)) #endif #ifdef XEN -#define start_emulating() (HYPERVISOR_fpu_taskswitch(1)) -#define stop_emulating() (HYPERVISOR_fpu_taskswitch(0)) +#define start_emulating() (HYPERVISOR_fpu_taskswitch(1)) +#define stop_emulating() (HYPERVISOR_fpu_taskswitch(0)) #else -#define start_emulating() __asm("smsw %%ax; orb %0,%%al; lmsw %%ax" \ - : : "n" (CR0_TS) : "ax") -#define stop_emulating() __asm("clts") +#define start_emulating() __asm __volatile( \ + "smsw %%ax; orb %0,%%al; lmsw %%ax" \ + : : "n" (CR0_TS) : "ax") +#define stop_emulating() __asm __volatile("clts") #endif #else /* !(__GNUCLIKE_ASM && !lint) */ -void fldcw(caddr_t addr); +void fldcw(u_short cw); void fnclex(void); void fninit(void); void fnsave(caddr_t addr); @@ -135,12 +131,12 @@ void stop_emulating(void); #ifdef CPU_ENABLE_SSE #define GET_FPU_CW(thread) \ (cpu_fxsr ? \ - (thread)->td_pcb->pcb_save.sv_xmm.sv_env.en_cw : \ - (thread)->td_pcb->pcb_save.sv_87.sv_env.en_cw) + (thread)->td_pcb->pcb_save->sv_xmm.sv_env.en_cw : \ + (thread)->td_pcb->pcb_save->sv_87.sv_env.en_cw) #define GET_FPU_SW(thread) \ (cpu_fxsr ? \ - (thread)->td_pcb->pcb_save.sv_xmm.sv_env.en_sw : \ - (thread)->td_pcb->pcb_save.sv_87.sv_env.en_sw) + (thread)->td_pcb->pcb_save->sv_xmm.sv_env.en_sw : \ + (thread)->td_pcb->pcb_save->sv_87.sv_env.en_sw) #define SET_FPU_CW(savefpu, value) do { \ if (cpu_fxsr) \ (savefpu)->sv_xmm.sv_env.en_cw = (value); \ @@ -149,9 +145,9 @@ void stop_emulating(void); } while (0) #else /* CPU_ENABLE_SSE */ #define GET_FPU_CW(thread) \ - (thread->td_pcb->pcb_save.sv_87.sv_env.en_cw) + (thread->td_pcb->pcb_save->sv_87.sv_env.en_cw) #define GET_FPU_SW(thread) \ - (thread->td_pcb->pcb_save.sv_87.sv_env.en_sw) + (thread->td_pcb->pcb_save->sv_87.sv_env.en_sw) #define SET_FPU_CW(savefpu, value) \ (savefpu)->sv_87.sv_env.en_cw = (value) #endif /* CPU_ENABLE_SSE */ @@ -166,25 +162,15 @@ static void fpusave(union savefpu *); static void fpurstor(union savefpu *); static int npx_attach(device_t dev); static void npx_identify(driver_t *driver, device_t parent); -static int npx_intr(void *); static int npx_probe(device_t dev); -#ifdef I586_CPU_XXX -static long timezero(const char *funcname, - void (*func)(void *buf, size_t len)); -#endif /* I586_CPU */ -int hw_float; /* XXX currently just alias for npx_exists */ +int hw_float; SYSCTL_INT(_hw, HW_FLOATINGPT, floatingpoint, CTLFLAG_RD, &hw_float, 0, "Floating point instructions executed in hardware"); -static volatile u_int npx_intrs_while_probing; static volatile u_int npx_traps_while_probing; - static union savefpu npx_initialstate; -static bool_t npx_ex16; -static bool_t npx_exists; -static bool_t npx_irq13; alias_for_inthand_t probetrap; __asm(" \n\ @@ -214,58 +200,14 @@ npx_identify(driver, parent) } /* - * Do minimal handling of npx interrupts to convert them to traps. - */ -static int -npx_intr(dummy) - void *dummy; -{ - struct thread *td; - - npx_intrs_while_probing++; - - /* - * The BUSY# latch must be cleared in all cases so that the next - * unmasked npx exception causes an interrupt. - */ - outb(IO_NPX, 0); - - /* - * fpcurthread is normally non-null here. In that case, schedule an - * AST to finish the exception handling in the correct context - * (this interrupt may occur after the thread has entered the - * kernel via a syscall or an interrupt). Otherwise, the npx - * state of the thread that caused this interrupt must have been - * pushed to the thread's pcb, and clearing of the busy latch - * above has finished the (essentially null) handling of this - * interrupt. Control will eventually return to the instruction - * that caused it and it will repeat. We will eventually (usually - * soon) win the race to handle the interrupt properly. - */ - td = PCPU_GET(fpcurthread); - if (td != NULL) { - td->td_pcb->pcb_flags |= PCB_NPXTRAP; - thread_lock(td); - td->td_flags |= TDF_ASTPENDING; - thread_unlock(td); - } - return (FILTER_HANDLED); -} - -/* * Probe routine. Set flags to tell npxattach() what to do. Set up an * interrupt handler if npx needs to use interrupts. */ static int -npx_probe(dev) - device_t dev; +npx_probe(device_t dev) { struct gate_descriptor save_idt_npxtrap; - struct resource *ioport_res, *irq_res; - void *irq_cookie; - int ioport_rid, irq_num, irq_rid; - u_short control; - u_short status; + u_short control, status; device_set_desc(dev, "math processor"); @@ -275,8 +217,7 @@ npx_probe(dev) * common case right away. */ if (cpu_feature & CPUID_FPU) { - hw_float = npx_exists = 1; - npx_ex16 = 1; + hw_float = 1; device_quiet(dev); return (0); } @@ -284,28 +225,6 @@ npx_probe(dev) save_idt_npxtrap = idt[IDT_MF]; setidt(IDT_MF, probetrap, SDT_SYS386TGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); - ioport_rid = 0; - ioport_res = bus_alloc_resource(dev, SYS_RES_IOPORT, &ioport_rid, - IO_NPX, IO_NPX + IO_NPXSIZE - 1, IO_NPXSIZE, RF_ACTIVE); - if (ioport_res == NULL) - panic("npx: can't get ports"); - if (resource_int_value("npx", 0, "irq", &irq_num) != 0) - irq_num = IRQ_NPX; - irq_rid = 0; - irq_res = bus_alloc_resource(dev, SYS_RES_IRQ, &irq_rid, irq_num, - irq_num, 1, RF_ACTIVE); - if (irq_res != NULL) { - if (bus_setup_intr(dev, irq_res, INTR_TYPE_MISC, - npx_intr, NULL, NULL, &irq_cookie) != 0) - panic("npx: can't create intr"); - } - - /* - * Partially reset the coprocessor, if any. Some BIOS's don't reset - * it after a warm boot. - */ - npx_full_reset(); - outb(IO_NPX, 0); /* * Don't trap while we're probing. @@ -326,9 +245,6 @@ npx_probe(dev) */ DELAY(1000); /* wait for any IRQ13 */ #ifdef DIAGNOSTIC - if (npx_intrs_while_probing != 0) - printf("fninit caused %u bogus npx interrupt(s)\n", - npx_intrs_while_probing); if (npx_traps_while_probing != 0) printf("fninit caused %u bogus npx trap(s)\n", npx_traps_while_probing); @@ -345,83 +261,57 @@ npx_probe(dev) control = 0x5a5a; fnstcw(&control); if ((control & 0x1f3f) == 0x033f) { - hw_float = npx_exists = 1; /* * We have an npx, now divide by 0 to see if exception * 16 works. */ control &= ~(1 << 2); /* enable divide by 0 trap */ - fldcw(&control); + fldcw(control); #ifdef FPU_ERROR_BROKEN /* * FPU error signal doesn't work on some CPU * accelerator board. */ - npx_ex16 = 1; + hw_float = 1; return (0); #endif - npx_traps_while_probing = npx_intrs_while_probing = 0; + npx_traps_while_probing = 0; fp_divide_by_0(); - DELAY(1000); /* wait for any IRQ13 */ if (npx_traps_while_probing != 0) { /* * Good, exception 16 works. */ - npx_ex16 = 1; - goto no_irq13; + hw_float = 1; + goto cleanup; } - if (npx_intrs_while_probing != 0) { - /* - * Bad, we are stuck with IRQ13. - */ - npx_irq13 = 1; - idt[IDT_MF] = save_idt_npxtrap; -#ifdef SMP - if (mp_ncpus > 1) - panic("npx0 cannot use IRQ 13 on an SMP system"); -#endif - return (0); - } - /* - * Worse, even IRQ13 is broken. - */ + device_printf(dev, + "FPU does not use exception 16 for error reporting\n"); + goto cleanup; } } - /* Probe failed. Floating point simply won't work. */ + /* + * Probe failed. Floating point simply won't work. + * Notify user and disable FPU/MMX/SSE instruction execution. + */ device_printf(dev, "WARNING: no FPU!\n"); + __asm __volatile("smsw %%ax; orb %0,%%al; lmsw %%ax" : : + "n" (CR0_EM | CR0_MP) : "ax"); - /* FALLTHROUGH */ -no_irq13: +cleanup: idt[IDT_MF] = save_idt_npxtrap; - if (irq_res != NULL) { - bus_teardown_intr(dev, irq_res, irq_cookie); - bus_release_resource(dev, SYS_RES_IRQ, irq_rid, irq_res); - } - bus_release_resource(dev, SYS_RES_IOPORT, ioport_rid, ioport_res); - return (npx_exists ? 0 : ENXIO); + return (hw_float ? 0 : ENXIO); } /* * Attach routine - announce which it is, and wire into system */ static int -npx_attach(dev) - device_t dev; +npx_attach(device_t dev) { - int flags; - register_t s; - - flags = device_get_flags(dev); - - if (npx_irq13) - device_printf(dev, "IRQ 13 interface\n"); - else if (!device_is_quiet(dev) || bootverbose) - device_printf(dev, "INT 16 interface\n"); npxinit(); - - s = intr_disable(); + critical_enter(); stop_emulating(); fpusave(&npx_initialstate); start_emulating(); @@ -441,23 +331,9 @@ npx_attach(dev) #endif bzero(npx_initialstate.sv_87.sv_ac, sizeof(npx_initialstate.sv_87.sv_ac)); - intr_restore(s); -#ifdef I586_CPU_XXX - if (cpu_class == CPUCLASS_586 && npx_ex16 && - timezero("i586_bzero()", i586_bzero) < - timezero("bzero()", bzero) * 4 / 5) { - if (!(flags & NPX_DISABLE_I586_OPTIMIZED_BCOPY)) - bcopy_vector = i586_bcopy; - if (!(flags & NPX_DISABLE_I586_OPTIMIZED_BZERO)) - bzero_vector = i586_bzero; - if (!(flags & NPX_DISABLE_I586_OPTIMIZED_COPYIO)) { - copyin_vector = i586_copyin; - copyout_vector = i586_copyout; - } - } -#endif + critical_exit(); - return (0); /* XXX unused */ + return (0); } /* @@ -467,17 +343,19 @@ void npxinit(void) { static union savefpu dummy; - register_t savecrit; + register_t saveintr; u_short control; - if (!npx_exists) + if (!hw_float) return; /* * fninit has the same h/w bugs as fnsave. Use the detoxified * fnsave to throw away any junk in the fpu. npxsave() initializes * the fpu and sets fpcurthread = NULL as important side effects. + * + * It is too early for critical_enter() to work on AP. */ - savecrit = intr_disable(); + saveintr = intr_disable(); npxsave(&dummy); stop_emulating(); #ifdef CPU_ENABLE_SSE @@ -486,9 +364,9 @@ npxinit(void) fninit(); #endif control = __INITIAL_NPXCW__; - fldcw(&control); + fldcw(control); start_emulating(); - intr_restore(savecrit); + intr_restore(saveintr); } /* @@ -498,14 +376,13 @@ void npxexit(td) struct thread *td; { - register_t savecrit; - savecrit = intr_disable(); + critical_enter(); if (curthread == PCPU_GET(fpcurthread)) - npxsave(&PCPU_GET(curpcb)->pcb_save); - intr_restore(savecrit); + npxsave(PCPU_GET(curpcb)->pcb_save); + critical_exit(); #ifdef NPX_DEBUG - if (npx_exists) { + if (hw_float) { u_int masked_exceptions; masked_exceptions = GET_FPU_CW(td) & GET_FPU_SW(td) & 0x7f; @@ -526,7 +403,7 @@ int npxformat() { - if (!npx_exists) + if (!hw_float) return (_MC_FPFMT_NODEV); #ifdef CPU_ENABLE_SSE if (cpu_fxsr) @@ -726,15 +603,14 @@ static char fpetable[128] = { int npxtrap() { - register_t savecrit; u_short control, status; - if (!npx_exists) { - printf("npxtrap: fpcurthread = %p, curthread = %p, npx_exists = %d\n", - PCPU_GET(fpcurthread), curthread, npx_exists); + if (!hw_float) { + printf("npxtrap: fpcurthread = %p, curthread = %p, hw_float = %d\n", + PCPU_GET(fpcurthread), curthread, hw_float); panic("npxtrap from nowhere"); } - savecrit = intr_disable(); + critical_enter(); /* * Interrupt handling (for another interrupt) may have pushed the @@ -751,7 +627,7 @@ npxtrap() if (PCPU_GET(fpcurthread) == curthread) fnclex(); - intr_restore(savecrit); + critical_exit(); return (fpetable[status & ((~control & 0x3f) | 0x40)]); } @@ -769,14 +645,15 @@ int npxdna(void) { struct pcb *pcb; - register_t s; - if (!npx_exists) + if (!hw_float) return (0); + critical_enter(); if (PCPU_GET(fpcurthread) == curthread) { printf("npxdna: fpcurthread == curthread %d times\n", ++err_count); stop_emulating(); + critical_exit(); return (1); } if (PCPU_GET(fpcurthread) != NULL) { @@ -786,7 +663,6 @@ npxdna(void) curthread, curthread->td_proc->p_pid); panic("npxdna"); } - s = intr_disable(); stop_emulating(); /* * Record new context early in case frstor causes an IRQ13. @@ -807,8 +683,10 @@ npxdna(void) */ fpurstor(&npx_initialstate); if (pcb->pcb_initial_npxcw != __INITIAL_NPXCW__) - fldcw(&pcb->pcb_initial_npxcw); + fldcw(pcb->pcb_initial_npxcw); pcb->pcb_flags |= PCB_NPXINITDONE; + if (PCB_USER_FPU(pcb)) + pcb->pcb_flags |= PCB_NPXUSERINITDONE; } else { /* * The following fpurstor() may cause an IRQ13 when the @@ -824,9 +702,9 @@ npxdna(void) * fnclex if it is the first FPU instruction after a context * switch. */ - fpurstor(&pcb->pcb_save); + fpurstor(pcb->pcb_save); } - intr_restore(s); + critical_exit(); return (1); } @@ -866,10 +744,6 @@ npxsave(addr) PCPU_SET(fpcurthread, NULL); } -/* - * This should be called with interrupts disabled and only when the owning - * FPU thread is non-null. - */ void npxdrop() { @@ -885,6 +759,8 @@ npxdrop() fnclex(); td = PCPU_GET(fpcurthread); + KASSERT(td == curthread, ("fpudrop: fpcurthread != curthread")); + CRITICAL_ASSERT(td); PCPU_SET(fpcurthread, NULL); td->td_pcb->pcb_flags &= ~PCB_NPXINITDONE; start_emulating(); @@ -895,21 +771,20 @@ npxdrop() * It returns the FPU ownership status. */ int -npxgetregs(td, addr) - struct thread *td; - union savefpu *addr; +npxgetregs(struct thread *td, union savefpu *addr) { - register_t s; + struct pcb *pcb; - if (!npx_exists) + if (!hw_float) return (_MC_FPOWNED_NONE); - if ((td->td_pcb->pcb_flags & PCB_NPXINITDONE) == 0) { + pcb = td->td_pcb; + if ((pcb->pcb_flags & PCB_NPXINITDONE) == 0) { bcopy(&npx_initialstate, addr, sizeof(npx_initialstate)); - SET_FPU_CW(addr, td->td_pcb->pcb_initial_npxcw); + SET_FPU_CW(addr, pcb->pcb_initial_npxcw); return (_MC_FPOWNED_NONE); } - s = intr_disable(); + critical_enter(); if (td == PCPU_GET(fpcurthread)) { fpusave(addr); #ifdef CPU_ENABLE_SSE @@ -921,11 +796,46 @@ npxgetregs(td, addr) * starts with a clean state next time. */ npxdrop(); - intr_restore(s); + critical_exit(); return (_MC_FPOWNED_FPU); } else { - intr_restore(s); - bcopy(&td->td_pcb->pcb_save, addr, sizeof(*addr)); + critical_exit(); + bcopy(pcb->pcb_save, addr, sizeof(*addr)); + return (_MC_FPOWNED_PCB); + } +} + +int +npxgetuserregs(struct thread *td, union savefpu *addr) +{ + struct pcb *pcb; + + if (!hw_float) + return (_MC_FPOWNED_NONE); + + pcb = td->td_pcb; + if ((pcb->pcb_flags & PCB_NPXUSERINITDONE) == 0) { + bcopy(&npx_initialstate, addr, sizeof(npx_initialstate)); + SET_FPU_CW(addr, pcb->pcb_initial_npxcw); + return (_MC_FPOWNED_NONE); + } + critical_enter(); + if (td == PCPU_GET(fpcurthread) && PCB_USER_FPU(pcb)) { + fpusave(addr); +#ifdef CPU_ENABLE_SSE + if (!cpu_fxsr) +#endif + /* + * fnsave initializes the FPU and destroys whatever + * context it contains. Make sure the FPU owner + * starts with a clean state next time. + */ + npxdrop(); + critical_exit(); + return (_MC_FPOWNED_FPU); + } else { + critical_exit(); + bcopy(&pcb->pcb_user_save, addr, sizeof(*addr)); return (_MC_FPOWNED_PCB); } } @@ -934,28 +844,56 @@ npxgetregs(td, addr) * Set the state of the FPU. */ void -npxsetregs(td, addr) - struct thread *td; - union savefpu *addr; +npxsetregs(struct thread *td, union savefpu *addr) { - register_t s; + struct pcb *pcb; - if (!npx_exists) + if (!hw_float) return; - s = intr_disable(); + pcb = td->td_pcb; + critical_enter(); if (td == PCPU_GET(fpcurthread)) { #ifdef CPU_ENABLE_SSE if (!cpu_fxsr) #endif fnclex(); /* As in npxdrop(). */ fpurstor(addr); - intr_restore(s); + critical_exit(); + } else { + critical_exit(); + bcopy(addr, pcb->pcb_save, sizeof(*addr)); + } + if (PCB_USER_FPU(pcb)) + pcb->pcb_flags |= PCB_NPXUSERINITDONE; + pcb->pcb_flags |= PCB_NPXINITDONE; +} + +void +npxsetuserregs(struct thread *td, union savefpu *addr) +{ + struct pcb *pcb; + + if (!hw_float) + return; + + pcb = td->td_pcb; + critical_enter(); + if (td == PCPU_GET(fpcurthread) && PCB_USER_FPU(pcb)) { +#ifdef CPU_ENABLE_SSE + if (!cpu_fxsr) +#endif + fnclex(); /* As in npxdrop(). */ + fpurstor(addr); + critical_exit(); + pcb->pcb_flags |= PCB_NPXUSERINITDONE | PCB_NPXINITDONE; } else { - intr_restore(s); - bcopy(addr, &td->td_pcb->pcb_save, sizeof(*addr)); + critical_exit(); + bcopy(addr, &pcb->pcb_user_save, sizeof(*addr)); + if (PCB_USER_FPU(pcb)) + pcb->pcb_flags |= PCB_NPXINITDONE; + pcb->pcb_flags |= PCB_NPXUSERINITDONE; } - curthread->td_pcb->pcb_flags |= PCB_NPXINITDONE; } static void @@ -1017,36 +955,6 @@ fpurstor(addr) frstor(addr); } -#ifdef I586_CPU_XXX -static long -timezero(funcname, func) - const char *funcname; - void (*func)(void *buf, size_t len); - -{ - void *buf; -#define BUFSIZE 1048576 - long usec; - struct timeval finish, start; - - buf = malloc(BUFSIZE, M_TEMP, M_NOWAIT); - if (buf == NULL) - return (BUFSIZE); - microtime(&start); - (*func)(buf, BUFSIZE); - microtime(&finish); - usec = 1000000 * (finish.tv_sec - start.tv_sec) + - finish.tv_usec - start.tv_usec; - if (usec <= 0) - usec = 1; - if (bootverbose) - printf("%s bandwidth = %u kBps\n", funcname, - (u_int32_t)(((BUFSIZE >> 10) * 1000000) / usec)); - free(buf, M_TEMP); - return (usec); -} -#endif /* I586_CPU */ - static device_method_t npx_methods[] = { /* Device interface */ DEVMETHOD(device_identify, npx_identify), @@ -1124,3 +1032,73 @@ DRIVER_MODULE(npxisa, isa, npxisa_driver, npxisa_devclass, 0, 0); DRIVER_MODULE(npxisa, acpi, npxisa_driver, npxisa_devclass, 0, 0); #endif #endif /* DEV_ISA */ + +int +fpu_kern_enter(struct thread *td, struct fpu_kern_ctx *ctx, u_int flags) +{ + struct pcb *pcb; + + pcb = td->td_pcb; + KASSERT(!PCB_USER_FPU(pcb) || pcb->pcb_save == &pcb->pcb_user_save, + ("mangled pcb_save")); + ctx->flags = 0; + if ((pcb->pcb_flags & PCB_NPXINITDONE) != 0) + ctx->flags |= FPU_KERN_CTX_NPXINITDONE; + npxexit(td); + ctx->prev = pcb->pcb_save; + pcb->pcb_save = &ctx->hwstate; + pcb->pcb_flags |= PCB_KERNNPX; + pcb->pcb_flags &= ~PCB_NPXINITDONE; + return (0); +} + +int +fpu_kern_leave(struct thread *td, struct fpu_kern_ctx *ctx) +{ + struct pcb *pcb; + + pcb = td->td_pcb; + critical_enter(); + if (curthread == PCPU_GET(fpcurthread)) + npxdrop(); + critical_exit(); + pcb->pcb_save = ctx->prev; + if (pcb->pcb_save == &pcb->pcb_user_save) { + if ((pcb->pcb_flags & PCB_NPXUSERINITDONE) != 0) + pcb->pcb_flags |= PCB_NPXINITDONE; + else + pcb->pcb_flags &= ~PCB_NPXINITDONE; + pcb->pcb_flags &= ~PCB_KERNNPX; + } else { + if ((ctx->flags & FPU_KERN_CTX_NPXINITDONE) != 0) + pcb->pcb_flags |= PCB_NPXINITDONE; + else + pcb->pcb_flags &= ~PCB_NPXINITDONE; + KASSERT(!PCB_USER_FPU(pcb), ("unpaired fpu_kern_leave")); + } + return (0); +} + +int +fpu_kern_thread(u_int flags) +{ + struct pcb *pcb; + + pcb = PCPU_GET(curpcb); + KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0, + ("Only kthread may use fpu_kern_thread")); + KASSERT(pcb->pcb_save == &pcb->pcb_user_save, ("mangled pcb_save")); + KASSERT(PCB_USER_FPU(pcb), ("recursive call")); + + pcb->pcb_flags |= PCB_KERNNPX; + return (0); +} + +int +is_fpu_kern_thread(u_int flags) +{ + + if ((curthread->td_pflags & TDP_KTHREAD) == 0) + return (0); + return ((PCPU_GET(curpcb)->pcb_flags & PCB_KERNNPX) != 0); +} diff --git a/sys/i386/linux/linux_ptrace.c b/sys/i386/linux/linux_ptrace.c index daee9e55b69b..e9559f81c7c2 100644 --- a/sys/i386/linux/linux_ptrace.c +++ b/sys/i386/linux/linux_ptrace.c @@ -224,7 +224,7 @@ linux_proc_read_fpxregs(struct thread *td, struct linux_pt_fpxreg *fpxregs) PROC_LOCK_ASSERT(td->td_proc, MA_OWNED); if (cpu_fxsr == 0 || (td->td_proc->p_flag & P_INMEM) == 0) return (EIO); - bcopy(&td->td_pcb->pcb_save.sv_xmm, fpxregs, sizeof(*fpxregs)); + bcopy(&td->td_pcb->pcb_user_save.sv_xmm, fpxregs, sizeof(*fpxregs)); return (0); } @@ -235,7 +235,7 @@ linux_proc_write_fpxregs(struct thread *td, struct linux_pt_fpxreg *fpxregs) PROC_LOCK_ASSERT(td->td_proc, MA_OWNED); if (cpu_fxsr == 0 || (td->td_proc->p_flag & P_INMEM) == 0) return (EIO); - bcopy(fpxregs, &td->td_pcb->pcb_save.sv_xmm, sizeof(*fpxregs)); + bcopy(fpxregs, &td->td_pcb->pcb_user_save.sv_xmm, sizeof(*fpxregs)); return (0); } #endif diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c index cbedad039d23..c4b1e2bd3811 100644 --- a/sys/kern/subr_trap.c +++ b/sys/kern/subr_trap.c @@ -46,9 +46,6 @@ __FBSDID("$FreeBSD$"); #include "opt_ktrace.h" #include "opt_kdtrace.h" -#ifdef __i386__ -#include "opt_npx.h" -#endif #include "opt_sched.h" #include <sys/param.h> @@ -75,7 +72,6 @@ __FBSDID("$FreeBSD$"); #include <security/audit/audit.h> #include <machine/cpu.h> -#include <machine/pcb.h> #ifdef XEN #include <vm/vm.h> @@ -146,10 +142,6 @@ ast(struct trapframe *framep) struct proc *p; int flags; int sig; -#if defined(DEV_NPX) && !defined(SMP) - int ucode; - ksiginfo_t ksi; -#endif td = curthread; p = td->td_proc; @@ -189,19 +181,6 @@ ast(struct trapframe *framep) psignal(p, SIGVTALRM); PROC_UNLOCK(p); } -#if defined(DEV_NPX) && !defined(SMP) - if (PCPU_GET(curpcb)->pcb_flags & PCB_NPXTRAP) { - atomic_clear_int(&PCPU_GET(curpcb)->pcb_flags, - PCB_NPXTRAP); - ucode = npxtrap(); - if (ucode != -1) { - ksiginfo_init_trap(&ksi); - ksi.ksi_signo = SIGFPE; - ksi.ksi_code = ucode; - trapsignal(td, &ksi); - } - } -#endif if (flags & TDF_PROFPEND) { PROC_LOCK(p); psignal(p, SIGPROF); diff --git a/sys/opencrypto/crypto.c b/sys/opencrypto/crypto.c index 58107802fd7a..53c11e648fac 100644 --- a/sys/opencrypto/crypto.c +++ b/sys/opencrypto/crypto.c @@ -82,6 +82,10 @@ __FBSDID("$FreeBSD$"); #include <sys/bus.h> #include "cryptodev_if.h" +#if defined(__i386__) || defined(__amd64__) +#include <machine/pcb.h> +#endif + SDT_PROVIDER_DEFINE(opencrypto); /* @@ -1241,6 +1245,10 @@ crypto_proc(void) u_int32_t hid; int result, hint; +#if defined(__i386__) || defined(__amd64__) + fpu_kern_thread(FPU_KERN_NORMAL); +#endif + CRYPTO_Q_LOCK(); for (;;) { /* diff --git a/sys/pc98/include/npx.h b/sys/pc98/include/npx.h index c1c0712a26aa..67c64027c08c 100644 --- a/sys/pc98/include/npx.h +++ b/sys/pc98/include/npx.h @@ -1,49 +1,6 @@ /*- - * Copyright (C) 2005 TAKAHASHI Yoshihiro. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ + * This file is in the public domain. */ - -#ifndef _PC98_INCLUDE_NPX_H_ -#define _PC98_INCLUDE_NPX_H_ +/* $FreeBSD$ */ #include <i386/npx.h> - -#ifdef _KERNEL - -#undef IO_NPX -#define IO_NPX 0x0F8 /* Numeric Coprocessor */ -#undef IO_NPXSIZE -#define IO_NPXSIZE 8 /* 80387/80487 NPX registers */ - -#undef IRQ_NPX -#define IRQ_NPX 8 - -/* full reset of npx: not needed on pc98 */ -#undef npx_full_reset -#define npx_full_reset() - -#endif /* _KERNEL */ - -#endif /* _PC98_INCLUDE_NPX_H_ */ diff --git a/sys/pc98/pc98/machdep.c b/sys/pc98/pc98/machdep.c index 7eb2734e3e7d..f5395a1d61c1 100644 --- a/sys/pc98/pc98/machdep.c +++ b/sys/pc98/pc98/machdep.c @@ -2528,12 +2528,12 @@ fill_fpregs(struct thread *td, struct fpreg *fpregs) { #ifdef CPU_ENABLE_SSE if (cpu_fxsr) { - fill_fpregs_xmm(&td->td_pcb->pcb_save.sv_xmm, + fill_fpregs_xmm(&td->td_pcb->pcb_save->sv_xmm, (struct save87 *)fpregs); return (0); } #endif /* CPU_ENABLE_SSE */ - bcopy(&td->td_pcb->pcb_save.sv_87, fpregs, sizeof *fpregs); + bcopy(&td->td_pcb->pcb_save->sv_87, fpregs, sizeof *fpregs); return (0); } @@ -2543,11 +2543,11 @@ set_fpregs(struct thread *td, struct fpreg *fpregs) #ifdef CPU_ENABLE_SSE if (cpu_fxsr) { set_fpregs_xmm((struct save87 *)fpregs, - &td->td_pcb->pcb_save.sv_xmm); + &td->td_pcb->pcb_save->sv_xmm); return (0); } #endif /* CPU_ENABLE_SSE */ - bcopy(fpregs, &td->td_pcb->pcb_save.sv_87, sizeof *fpregs); + bcopy(fpregs, &td->td_pcb->pcb_save->sv_87, sizeof *fpregs); return (0); } @@ -2734,9 +2734,8 @@ set_fpcontext(struct thread *td, const mcontext_t *mcp) static void fpstate_drop(struct thread *td) { - register_t s; - s = intr_disable(); + critical_enter(); #ifdef DEV_NPX if (PCPU_GET(fpcurthread) == td) npxdrop(); @@ -2752,7 +2751,7 @@ fpstate_drop(struct thread *td) * have too many layers. */ curthread->td_pcb->pcb_flags &= ~PCB_NPXINITDONE; - intr_restore(s); + critical_exit(); } int diff --git a/sys/x86/x86/local_apic.c b/sys/x86/x86/local_apic.c index 9cc056d2a071..55a04a81a3a5 100644 --- a/sys/x86/x86/local_apic.c +++ b/sys/x86/x86/local_apic.c @@ -307,12 +307,12 @@ lapic_setup(int boot) { struct lapic *la; u_int32_t maxlvt; - register_t eflags; + register_t saveintr; char buf[MAXCOMLEN + 1]; la = &lapics[lapic_id()]; KASSERT(la->la_present, ("missing APIC structure")); - eflags = intr_disable(); + saveintr = intr_disable(); maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; /* Initialize the TPR to allow all interrupts. */ @@ -355,7 +355,7 @@ lapic_setup(int boot) if (maxlvt >= LVT_CMCI) lapic->lvt_cmci = lvt_mode(la, LVT_CMCI, lapic->lvt_cmci); - intr_restore(eflags); + intr_restore(saveintr); } void @@ -1390,7 +1390,7 @@ lapic_ipi_wait(int delay) void lapic_ipi_raw(register_t icrlo, u_int dest) { - register_t value, eflags; + register_t value, saveintr; /* XXX: Need more sanity checking of icrlo? */ KASSERT(lapic != NULL, ("%s called too early", __func__)); @@ -1400,7 +1400,7 @@ lapic_ipi_raw(register_t icrlo, u_int dest) ("%s: reserved bits set in ICR LO register", __func__)); /* Set destination in ICR HI register if it is being used. */ - eflags = intr_disable(); + saveintr = intr_disable(); if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) { value = lapic->icr_hi; value &= ~APIC_ID_MASK; @@ -1413,7 +1413,7 @@ lapic_ipi_raw(register_t icrlo, u_int dest) value &= APIC_ICRLO_RESV_MASK; value |= icrlo; lapic->icr_lo = value; - intr_restore(eflags); + intr_restore(saveintr); } #define BEFORE_SPIN 1000000 |