diff options
Diffstat (limited to 'sys/arm64/vmm')
| -rw-r--r-- | sys/arm64/vmm/arm64.h | 10 | ||||
| -rw-r--r-- | sys/arm64/vmm/io/vgic_v3.c | 3 | ||||
| -rw-r--r-- | sys/arm64/vmm/io/vtimer.c | 38 | ||||
| -rw-r--r-- | sys/arm64/vmm/vmm.c | 487 | ||||
| -rw-r--r-- | sys/arm64/vmm/vmm_arm64.c | 369 | ||||
| -rw-r--r-- | sys/arm64/vmm/vmm_call.S | 3 | ||||
| -rw-r--r-- | sys/arm64/vmm/vmm_dev.c | 1054 | ||||
| -rw-r--r-- | sys/arm64/vmm/vmm_dev_machdep.c | 138 | ||||
| -rw-r--r-- | sys/arm64/vmm/vmm_handlers.c | 113 | ||||
| -rw-r--r-- | sys/arm64/vmm/vmm_handlers.h | 48 | ||||
| -rw-r--r-- | sys/arm64/vmm/vmm_hyp.c | 253 | ||||
| -rw-r--r-- | sys/arm64/vmm/vmm_hyp_el2.S | 7 | ||||
| -rw-r--r-- | sys/arm64/vmm/vmm_hyp_exception.S | 118 | ||||
| -rw-r--r-- | sys/arm64/vmm/vmm_ktr.h | 69 | ||||
| -rw-r--r-- | sys/arm64/vmm/vmm_mmu.c | 2 | ||||
| -rw-r--r-- | sys/arm64/vmm/vmm_nvhe.c | 118 | ||||
| -rw-r--r-- | sys/arm64/vmm/vmm_nvhe_exception.S | 120 | ||||
| -rw-r--r-- | sys/arm64/vmm/vmm_reset.c | 11 | ||||
| -rw-r--r-- | sys/arm64/vmm/vmm_stat.c | 165 | ||||
| -rw-r--r-- | sys/arm64/vmm/vmm_stat.h | 100 | ||||
| -rw-r--r-- | sys/arm64/vmm/vmm_vhe.c | 39 | ||||
| -rw-r--r-- | sys/arm64/vmm/vmm_vhe_exception.S | 31 |
22 files changed, 1127 insertions, 2169 deletions
diff --git a/sys/arm64/vmm/arm64.h b/sys/arm64/vmm/arm64.h index 43459d14e143..6a0c4c78e568 100644 --- a/sys/arm64/vmm/arm64.h +++ b/sys/arm64/vmm/arm64.h @@ -39,6 +39,9 @@ struct vgic_v3; struct vgic_v3_cpu; +/* + * Per-vCPU hypervisor state. + */ struct hypctx { struct trapframe tf; @@ -91,6 +94,7 @@ struct hypctx { /* EL2 control registers */ uint64_t cptr_el2; /* Architectural Feature Trap Register */ uint64_t hcr_el2; /* Hypervisor Configuration Register */ + uint64_t hcrx_el2; /* Extended Hypervisor Configuration Register */ uint64_t mdcr_el2; /* Monitor Debug Configuration Register */ uint64_t vpidr_el2; /* Virtualization Processor ID Register */ uint64_t vmpidr_el2; /* Virtualization Multiprocessor ID Register */ @@ -104,6 +108,12 @@ struct hypctx { struct vtimer_cpu vtimer_cpu; + uint64_t setcaps; /* Currently enabled capabilities. */ + + /* vCPU state used to handle guest debugging. */ + uint64_t debug_spsr; /* Saved guest SPSR */ + uint64_t debug_mdscr; /* Saved guest MDSCR */ + struct vgic_v3_regs vgic_v3_regs; struct vgic_v3_cpu *vgic_cpu; bool has_exception; diff --git a/sys/arm64/vmm/io/vgic_v3.c b/sys/arm64/vmm/io/vgic_v3.c index 7ed591c409ba..67afb3374815 100644 --- a/sys/arm64/vmm/io/vgic_v3.c +++ b/sys/arm64/vmm/io/vgic_v3.c @@ -68,6 +68,7 @@ #include <arm64/vmm/hyp.h> #include <arm64/vmm/mmu.h> #include <arm64/vmm/arm64.h> +#include <arm64/vmm/vmm_handlers.h> #include "vgic.h" #include "vgic_v3.h" @@ -2252,7 +2253,7 @@ vgic_v3_init(device_t dev) uint64_t ich_vtr_el2; uint32_t pribits, prebits; - ich_vtr_el2 = vmm_call_hyp(HYP_READ_REGISTER, HYP_REG_ICH_VTR); + ich_vtr_el2 = vmm_read_reg(HYP_REG_ICH_VTR); /* TODO: These fields are common with the vgicv2 driver */ pribits = ICH_VTR_EL2_PRIBITS(ich_vtr_el2); diff --git a/sys/arm64/vmm/io/vtimer.c b/sys/arm64/vmm/io/vtimer.c index aa0b3ff1588e..f59d7ebc1ad4 100644 --- a/sys/arm64/vmm/io/vtimer.c +++ b/sys/arm64/vmm/io/vtimer.c @@ -129,14 +129,42 @@ vtimer_vminit(struct hyp *hyp) { uint64_t now; + hyp->vtimer.cnthctl_el2 = cnthctl_el2_reg; + /* * Configure the Counter-timer Hypervisor Control Register for the VM. - * - * CNTHCTL_EL1PCEN: trap access to CNTP_{CTL, CVAL, TVAL}_EL0 from EL1 - * CNTHCTL_EL1PCTEN: trap access to CNTPCT_EL0 */ - hyp->vtimer.cnthctl_el2 = cnthctl_el2_reg & ~CNTHCTL_EL1PCEN; - hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_EL1PCTEN; + if (in_vhe()) { + /* + * CNTHCTL_E2H_EL0PCTEN: trap EL0 access to CNTP{CT,CTSS}_EL0 + * CNTHCTL_E2H_EL1VCTEN: don't trap EL0 access to + * CNTV{CT,CTSS}_EL0 + * CNTHCTL_E2H_EL0VTEN: don't trap EL0 access to + * CNTV_{CTL,CVAL,TVAL}_EL0 + * CNTHCTL_E2H_EL0PTEN: trap EL0 access to + * CNTP_{CTL,CVAL,TVAL}_EL0 + * CNTHCTL_E2H_EL1PCEN: trap EL1 access to + CNTP_{CTL,CVAL,TVAL}_EL0 + * CNTHCTL_E2H_EL1PCTEN: trap access to CNTPCT_EL0 + * + * TODO: Don't trap when FEAT_ECV is present + */ + hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_E2H_EL0PCTEN; + hyp->vtimer.cnthctl_el2 |= CNTHCTL_E2H_EL0VCTEN; + hyp->vtimer.cnthctl_el2 |= CNTHCTL_E2H_EL0VTEN; + hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_E2H_EL0PTEN; + + hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_E2H_EL1PTEN; + hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_E2H_EL1PCTEN; + } else { + /* + * CNTHCTL_EL1PCEN: trap access to CNTP_{CTL, CVAL, TVAL}_EL0 + * from EL1 + * CNTHCTL_EL1PCTEN: trap access to CNTPCT_EL0 + */ + hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_EL1PCEN; + hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_EL1PCTEN; + } now = READ_SPECIALREG(cntpct_el0); hyp->vtimer.cntvoff_el2 = now; diff --git a/sys/arm64/vmm/vmm.c b/sys/arm64/vmm/vmm.c index a2cc63448f19..3082d2941221 100644 --- a/sys/arm64/vmm/vmm.c +++ b/sys/arm64/vmm/vmm.c @@ -60,13 +60,14 @@ #include <machine/vm.h> #include <machine/vmparam.h> #include <machine/vmm.h> -#include <machine/vmm_dev.h> #include <machine/vmm_instruction_emul.h> #include <dev/pci/pcireg.h> +#include <dev/vmm/vmm_dev.h> +#include <dev/vmm/vmm_ktr.h> +#include <dev/vmm/vmm_mem.h> +#include <dev/vmm/vmm_stat.h> -#include "vmm_ktr.h" -#include "vmm_stat.h" #include "arm64.h" #include "mmu.h" @@ -94,25 +95,6 @@ struct vcpu { #define vcpu_unlock(v) mtx_unlock_spin(&((v)->mtx)) #define vcpu_assert_locked(v) mtx_assert(&((v)->mtx), MA_OWNED) -struct mem_seg { - uint64_t gpa; - size_t len; - bool wired; - bool sysmem; - vm_object_t object; -}; -#define VM_MAX_MEMSEGS 3 - -struct mem_map { - vm_paddr_t gpa; - size_t len; - vm_ooffset_t segoff; - int segid; - int prot; - int flags; -}; -#define VM_MAX_MEMMAPS 4 - struct vmm_mmio_region { uint64_t start; uint64_t end; @@ -141,11 +123,11 @@ struct vm { volatile cpuset_t active_cpus; /* (i) active vcpus */ volatile cpuset_t debug_cpus; /* (i) vcpus stopped for debug */ int suspend; /* (i) stop VM execution */ + bool dying; /* (o) is dying */ volatile cpuset_t suspended_cpus; /* (i) suspended vcpus */ volatile cpuset_t halted_cpus; /* (x) cpus in a hard halt */ - struct mem_map mem_maps[VM_MAX_MEMMAPS]; /* (i) guest address space */ - struct mem_seg mem_segs[VM_MAX_MEMSEGS]; /* (o) guest memory regions */ struct vmspace *vmspace; /* (o) guest's address space */ + struct vm_mem mem; /* (i) guest memory */ char name[VM_MAX_NAMELEN]; /* (o) virtual machine name */ struct vcpu **vcpu; /* (i) guest vcpus */ struct vmm_mmio_region mmio_region[VM_MAX_MMIO_REGIONS]; @@ -156,7 +138,6 @@ struct vm { uint16_t cores; /* (o) num of cores/socket */ uint16_t threads; /* (o) num of threads/core */ uint16_t maxcpus; /* (o) max pluggable cpus */ - struct sx mem_segs_lock; /* (o) */ struct sx vcpus_init_lock; /* (o) */ }; @@ -234,10 +215,25 @@ u_int vm_maxcpu; SYSCTL_UINT(_hw_vmm, OID_AUTO, maxcpu, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &vm_maxcpu, 0, "Maximum number of vCPUs"); -static void vm_free_memmap(struct vm *vm, int ident); -static bool sysmem_mapping(struct vm *vm, struct mem_map *mm); static void vcpu_notify_event_locked(struct vcpu *vcpu); +/* global statistics */ +VMM_STAT(VMEXIT_COUNT, "total number of vm exits"); +VMM_STAT(VMEXIT_UNKNOWN, "number of vmexits for the unknown exception"); +VMM_STAT(VMEXIT_WFI, "number of times wfi was intercepted"); +VMM_STAT(VMEXIT_WFE, "number of times wfe was intercepted"); +VMM_STAT(VMEXIT_HVC, "number of times hvc was intercepted"); +VMM_STAT(VMEXIT_MSR, "number of times msr/mrs was intercepted"); +VMM_STAT(VMEXIT_DATA_ABORT, "number of vmexits for a data abort"); +VMM_STAT(VMEXIT_INSN_ABORT, "number of vmexits for an instruction abort"); +VMM_STAT(VMEXIT_UNHANDLED_SYNC, "number of vmexits for an unhandled synchronous exception"); +VMM_STAT(VMEXIT_IRQ, "number of vmexits for an irq"); +VMM_STAT(VMEXIT_FIQ, "number of vmexits for an interrupt"); +VMM_STAT(VMEXIT_BRK, "number of vmexits for a breakpoint exception"); +VMM_STAT(VMEXIT_SS, "number of vmexits for a single-step exception"); +VMM_STAT(VMEXIT_UNHANDLED_EL2, "number of vmexits for an unhandled EL2 exception"); +VMM_STAT(VMEXIT_UNHANDLED, "number of vmexits for an unhandled exception"); + /* * Upper limit on vm_maxcpu. We could increase this to 28 bits, but this * is a safe value for now. @@ -249,7 +245,8 @@ vmm_regs_init(struct vmm_regs *regs, const struct vmm_regs *masks) { #define _FETCH_KERN_REG(reg, field) do { \ regs->field = vmm_arch_regs_masks.field; \ - if (!get_kernel_reg_masked(reg, ®s->field, masks->field)) \ + if (!get_kernel_reg_iss_masked(reg ## _ISS, ®s->field, \ + masks->field)) \ regs->field = 0; \ } while (0) _FETCH_KERN_REG(ID_AA64AFR0_EL1, id_aa64afr0); @@ -315,6 +312,20 @@ vm_exitinfo(struct vcpu *vcpu) } static int +vmm_unsupported_quirk(void) +{ + /* + * Known to not load on Ampere eMAG + * https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=285051 + */ + if (CPU_MATCH(CPU_IMPL_MASK | CPU_PART_MASK, CPU_IMPL_APM, + CPU_PART_EMAG8180, 0, 0)) + return (ENXIO); + + return (0); +} + +static int vmm_init(void) { int error; @@ -343,19 +354,29 @@ vmm_handler(module_t mod, int what, void *arg) switch (what) { case MOD_LOAD: - /* TODO: if (vmm_is_hw_supported()) { */ - vmmdev_init(); + error = vmm_unsupported_quirk(); + if (error != 0) + break; + error = vmmdev_init(); + if (error != 0) + break; error = vmm_init(); if (error == 0) vmm_initialized = true; + else + (void)vmmdev_cleanup(); break; case MOD_UNLOAD: - /* TODO: if (vmm_is_hw_supported()) { */ error = vmmdev_cleanup(); if (error == 0 && vmm_initialized) { error = vmmops_modcleanup(); - if (error) + if (error) { + /* + * Something bad happened - prevent new + * VMs from being created + */ vmm_initialized = false; + } } break; default: @@ -376,8 +397,9 @@ static moduledata_t vmm_kmod = { * * - HYP initialization requires smp_rendezvous() and therefore must happen * after SMP is fully functional (after SI_SUB_SMP). + * - vmm device initialization requires an initialized devfs. */ -DECLARE_MODULE(vmm, vmm_kmod, SI_SUB_SMP + 1, SI_ORDER_ANY); +DECLARE_MODULE(vmm, vmm_kmod, MAX(SI_SUB_SMP, SI_SUB_DEVFS) + 1, SI_ORDER_ANY); MODULE_VERSION(vmm, 1); static void @@ -405,6 +427,14 @@ vm_init(struct vm *vm, bool create) } } +void +vm_disable_vcpu_creation(struct vm *vm) +{ + sx_xlock(&vm->vcpus_init_lock); + vm->dying = true; + sx_xunlock(&vm->vcpus_init_lock); +} + struct vcpu * vm_alloc_vcpu(struct vm *vm, int vcpuid) { @@ -417,13 +447,14 @@ vm_alloc_vcpu(struct vm *vm, int vcpuid) if (vcpuid >= vgic_max_cpu_count(vm->cookie)) return (NULL); - vcpu = atomic_load_ptr(&vm->vcpu[vcpuid]); + vcpu = (struct vcpu *) + atomic_load_acq_ptr((uintptr_t *)&vm->vcpu[vcpuid]); if (__predict_true(vcpu != NULL)) return (vcpu); sx_xlock(&vm->vcpus_init_lock); vcpu = vm->vcpu[vcpuid]; - if (vcpu == NULL/* && !vm->dying*/) { + if (vcpu == NULL && !vm->dying) { vcpu = vcpu_alloc(vm, vcpuid); vcpu_init(vcpu); @@ -473,7 +504,7 @@ vm_create(const char *name, struct vm **retvm) vm = malloc(sizeof(struct vm), M_VMM, M_WAITOK | M_ZERO); strcpy(vm->name, name); vm->vmspace = vmspace; - sx_init(&vm->mem_segs_lock, "vm mem_segs"); + vm_mem_init(&vm->mem); sx_init(&vm->vcpus_init_lock, "vm vcpus"); vm->sockets = 1; @@ -522,11 +553,11 @@ vm_set_topology(struct vm *vm, uint16_t sockets, uint16_t cores, static void vm_cleanup(struct vm *vm, bool destroy) { - struct mem_map *mm; pmap_t pmap __diagused; int i; if (destroy) { + vm_xlock_memsegs(vm); pmap = vmspace_pmap(vm->vmspace); sched_pin(); PCPU_SET(curvmpmap, NULL); @@ -534,7 +565,9 @@ vm_cleanup(struct vm *vm, bool destroy) CPU_FOREACH(i) { MPASS(cpuid_to_pcpu[i]->pc_curvmpmap != pmap); } - } + } else + vm_assert_memseg_xlocked(vm); + vgic_detach_from_vm(vm->cookie); @@ -545,25 +578,9 @@ vm_cleanup(struct vm *vm, bool destroy) vmmops_cleanup(vm->cookie); - /* - * System memory is removed from the guest address space only when - * the VM is destroyed. This is because the mapping remains the same - * across VM reset. - * - * Device memory can be relocated by the guest (e.g. using PCI BARs) - * so those mappings are removed on a VM reset. - */ - if (!destroy) { - for (i = 0; i < VM_MAX_MEMMAPS; i++) { - mm = &vm->mem_maps[i]; - if (destroy || !sysmem_mapping(vm, mm)) - vm_free_memmap(vm, i); - } - } - + vm_mem_cleanup(vm); if (destroy) { - for (i = 0; i < VM_MAX_MEMSEGS; i++) - vm_free_memseg(vm, i); + vm_mem_destroy(vm); vmmops_vmspace_free(vm->vmspace); vm->vmspace = NULL; @@ -572,7 +589,6 @@ vm_cleanup(struct vm *vm, bool destroy) free(vm->vcpu[i], M_VMM); free(vm->vcpu, M_VMM); sx_destroy(&vm->vcpus_init_lock); - sx_destroy(&vm->mem_segs_lock); } } @@ -608,290 +624,11 @@ vm_name(struct vm *vm) return (vm->name); } -void -vm_slock_memsegs(struct vm *vm) -{ - sx_slock(&vm->mem_segs_lock); -} - -void -vm_xlock_memsegs(struct vm *vm) -{ - sx_xlock(&vm->mem_segs_lock); -} - -void -vm_unlock_memsegs(struct vm *vm) -{ - sx_unlock(&vm->mem_segs_lock); -} - -/* - * Return 'true' if 'gpa' is allocated in the guest address space. - * - * This function is called in the context of a running vcpu which acts as - * an implicit lock on 'vm->mem_maps[]'. - */ -bool -vm_mem_allocated(struct vcpu *vcpu, vm_paddr_t gpa) -{ - struct vm *vm = vcpu->vm; - struct mem_map *mm; - int i; - -#ifdef INVARIANTS - int hostcpu, state; - state = vcpu_get_state(vcpu, &hostcpu); - KASSERT(state == VCPU_RUNNING && hostcpu == curcpu, - ("%s: invalid vcpu state %d/%d", __func__, state, hostcpu)); -#endif - - for (i = 0; i < VM_MAX_MEMMAPS; i++) { - mm = &vm->mem_maps[i]; - if (mm->len != 0 && gpa >= mm->gpa && gpa < mm->gpa + mm->len) - return (true); /* 'gpa' is sysmem or devmem */ - } - - return (false); -} - -int -vm_alloc_memseg(struct vm *vm, int ident, size_t len, bool sysmem) -{ - struct mem_seg *seg; - vm_object_t obj; - - sx_assert(&vm->mem_segs_lock, SX_XLOCKED); - - if (ident < 0 || ident >= VM_MAX_MEMSEGS) - return (EINVAL); - - if (len == 0 || (len & PAGE_MASK)) - return (EINVAL); - - seg = &vm->mem_segs[ident]; - if (seg->object != NULL) { - if (seg->len == len && seg->sysmem == sysmem) - return (EEXIST); - else - return (EINVAL); - } - - obj = vm_object_allocate(OBJT_DEFAULT, len >> PAGE_SHIFT); - if (obj == NULL) - return (ENOMEM); - - seg->len = len; - seg->object = obj; - seg->sysmem = sysmem; - return (0); -} - -int -vm_get_memseg(struct vm *vm, int ident, size_t *len, bool *sysmem, - vm_object_t *objptr) -{ - struct mem_seg *seg; - - sx_assert(&vm->mem_segs_lock, SX_LOCKED); - - if (ident < 0 || ident >= VM_MAX_MEMSEGS) - return (EINVAL); - - seg = &vm->mem_segs[ident]; - if (len) - *len = seg->len; - if (sysmem) - *sysmem = seg->sysmem; - if (objptr) - *objptr = seg->object; - return (0); -} - -void -vm_free_memseg(struct vm *vm, int ident) -{ - struct mem_seg *seg; - - KASSERT(ident >= 0 && ident < VM_MAX_MEMSEGS, - ("%s: invalid memseg ident %d", __func__, ident)); - - seg = &vm->mem_segs[ident]; - if (seg->object != NULL) { - vm_object_deallocate(seg->object); - bzero(seg, sizeof(struct mem_seg)); - } -} - -int -vm_mmap_memseg(struct vm *vm, vm_paddr_t gpa, int segid, vm_ooffset_t first, - size_t len, int prot, int flags) -{ - struct mem_seg *seg; - struct mem_map *m, *map; - vm_ooffset_t last; - int i, error; - - if (prot == 0 || (prot & ~(VM_PROT_ALL)) != 0) - return (EINVAL); - - if (flags & ~VM_MEMMAP_F_WIRED) - return (EINVAL); - - if (segid < 0 || segid >= VM_MAX_MEMSEGS) - return (EINVAL); - - seg = &vm->mem_segs[segid]; - if (seg->object == NULL) - return (EINVAL); - - last = first + len; - if (first < 0 || first >= last || last > seg->len) - return (EINVAL); - - if ((gpa | first | last) & PAGE_MASK) - return (EINVAL); - - map = NULL; - for (i = 0; i < VM_MAX_MEMMAPS; i++) { - m = &vm->mem_maps[i]; - if (m->len == 0) { - map = m; - break; - } - } - - if (map == NULL) - return (ENOSPC); - - error = vm_map_find(&vm->vmspace->vm_map, seg->object, first, &gpa, - len, 0, VMFS_NO_SPACE, prot, prot, 0); - if (error != KERN_SUCCESS) - return (EFAULT); - - vm_object_reference(seg->object); - - if (flags & VM_MEMMAP_F_WIRED) { - error = vm_map_wire(&vm->vmspace->vm_map, gpa, gpa + len, - VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); - if (error != KERN_SUCCESS) { - vm_map_remove(&vm->vmspace->vm_map, gpa, gpa + len); - return (error == KERN_RESOURCE_SHORTAGE ? ENOMEM : - EFAULT); - } - } - - map->gpa = gpa; - map->len = len; - map->segoff = first; - map->segid = segid; - map->prot = prot; - map->flags = flags; - return (0); -} - -int -vm_munmap_memseg(struct vm *vm, vm_paddr_t gpa, size_t len) -{ - struct mem_map *m; - int i; - - for (i = 0; i < VM_MAX_MEMMAPS; i++) { - m = &vm->mem_maps[i]; - if (m->gpa == gpa && m->len == len) { - vm_free_memmap(vm, i); - return (0); - } - } - - return (EINVAL); -} - -int -vm_mmap_getnext(struct vm *vm, vm_paddr_t *gpa, int *segid, - vm_ooffset_t *segoff, size_t *len, int *prot, int *flags) -{ - struct mem_map *mm, *mmnext; - int i; - - mmnext = NULL; - for (i = 0; i < VM_MAX_MEMMAPS; i++) { - mm = &vm->mem_maps[i]; - if (mm->len == 0 || mm->gpa < *gpa) - continue; - if (mmnext == NULL || mm->gpa < mmnext->gpa) - mmnext = mm; - } - - if (mmnext != NULL) { - *gpa = mmnext->gpa; - if (segid) - *segid = mmnext->segid; - if (segoff) - *segoff = mmnext->segoff; - if (len) - *len = mmnext->len; - if (prot) - *prot = mmnext->prot; - if (flags) - *flags = mmnext->flags; - return (0); - } else { - return (ENOENT); - } -} - -static void -vm_free_memmap(struct vm *vm, int ident) -{ - struct mem_map *mm; - int error __diagused; - - mm = &vm->mem_maps[ident]; - if (mm->len) { - error = vm_map_remove(&vm->vmspace->vm_map, mm->gpa, - mm->gpa + mm->len); - KASSERT(error == KERN_SUCCESS, ("%s: vm_map_remove error %d", - __func__, error)); - bzero(mm, sizeof(struct mem_map)); - } -} - -static __inline bool -sysmem_mapping(struct vm *vm, struct mem_map *mm) -{ - - if (mm->len != 0 && vm->mem_segs[mm->segid].sysmem) - return (true); - else - return (false); -} - -vm_paddr_t -vmm_sysmem_maxaddr(struct vm *vm) -{ - struct mem_map *mm; - vm_paddr_t maxaddr; - int i; - - maxaddr = 0; - for (i = 0; i < VM_MAX_MEMMAPS; i++) { - mm = &vm->mem_maps[i]; - if (sysmem_mapping(vm, mm)) { - if (maxaddr < mm->gpa + mm->len) - maxaddr = mm->gpa + mm->len; - } - } - return (maxaddr); -} - int vm_gla2gpa_nofault(struct vcpu *vcpu, struct vm_guest_paging *paging, uint64_t gla, int prot, uint64_t *gpa, int *is_fault) { - - vmmops_gla2gpa(vcpu->cookie, paging, gla, prot, gpa, is_fault); - return (0); + return (vmmops_gla2gpa(vcpu->cookie, paging, gla, prot, gpa, is_fault)); } static int @@ -1319,6 +1056,18 @@ vcpu_notify_event(struct vcpu *vcpu) vcpu_unlock(vcpu); } +struct vmspace * +vm_vmspace(struct vm *vm) +{ + return (vm->vmspace); +} + +struct vm_mem * +vm_mem(struct vm *vm) +{ + return (&vm->mem); +} + static void restore_guest_fpustate(struct vcpu *vcpu) { @@ -1506,70 +1255,6 @@ vcpu_get_state(struct vcpu *vcpu, int *hostcpu) return (state); } -static void * -_vm_gpa_hold(struct vm *vm, vm_paddr_t gpa, size_t len, int reqprot, - void **cookie) -{ - int i, count, pageoff; - struct mem_map *mm; - vm_page_t m; - - pageoff = gpa & PAGE_MASK; - if (len > PAGE_SIZE - pageoff) - panic("vm_gpa_hold: invalid gpa/len: 0x%016lx/%lu", gpa, len); - - count = 0; - for (i = 0; i < VM_MAX_MEMMAPS; i++) { - mm = &vm->mem_maps[i]; - if (sysmem_mapping(vm, mm) && gpa >= mm->gpa && - gpa < mm->gpa + mm->len) { - count = vm_fault_quick_hold_pages(&vm->vmspace->vm_map, - trunc_page(gpa), PAGE_SIZE, reqprot, &m, 1); - break; - } - } - - if (count == 1) { - *cookie = m; - return ((void *)(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)) + pageoff)); - } else { - *cookie = NULL; - return (NULL); - } -} - -void * -vm_gpa_hold(struct vcpu *vcpu, vm_paddr_t gpa, size_t len, int reqprot, - void **cookie) -{ -#ifdef INVARIANTS - /* - * The current vcpu should be frozen to ensure 'vm_memmap[]' - * stability. - */ - int state = vcpu_get_state(vcpu, NULL); - KASSERT(state == VCPU_FROZEN, ("%s: invalid vcpu state %d", - __func__, state)); -#endif - return (_vm_gpa_hold(vcpu->vm, gpa, len, reqprot, cookie)); -} - -void * -vm_gpa_hold_global(struct vm *vm, vm_paddr_t gpa, size_t len, int reqprot, - void **cookie) -{ - sx_assert(&vm->mem_segs_lock, SX_LOCKED); - return (_vm_gpa_hold(vm, gpa, len, reqprot, cookie)); -} - -void -vm_gpa_release(void *cookie) -{ - vm_page_t m = cookie; - - vm_page_unwire(m, PQ_ACTIVE); -} - int vm_get_register(struct vcpu *vcpu, int reg, uint64_t *retval) { diff --git a/sys/arm64/vmm/vmm_arm64.c b/sys/arm64/vmm/vmm_arm64.c index e71761f9ccef..de2425aae0a1 100644 --- a/sys/arm64/vmm/vmm_arm64.c +++ b/sys/arm64/vmm/vmm_arm64.c @@ -58,6 +58,8 @@ #include <machine/hypervisor.h> #include <machine/pmap.h> +#include <dev/vmm/vmm_mem.h> + #include "mmu.h" #include "arm64.h" #include "hyp.h" @@ -65,6 +67,7 @@ #include "io/vgic.h" #include "io/vgic_v3.h" #include "io/vtimer.h" +#include "vmm_handlers.h" #include "vmm_stat.h" #define HANDLED 1 @@ -101,9 +104,6 @@ static vm_offset_t stack_hyp_va[MAXCPU]; static vmem_t *el2_mem_alloc; static void arm_setup_vectors(void *arg); -static void vmm_pmap_clean_stage2_tlbi(void); -static void vmm_pmap_invalidate_range(uint64_t, vm_offset_t, vm_offset_t, bool); -static void vmm_pmap_invalidate_all(uint64_t); DPCPU_DEFINE_STATIC(struct hypctx *, vcpu); @@ -130,20 +130,6 @@ arm_setup_vectors(void *arg) el2_regs = arg; arm64_set_active_vcpu(NULL); - daif = intr_disable(); - - /* - * Install the temporary vectors which will be responsible for - * initializing the VMM when we next trap into EL2. - * - * x0: the exception vector table responsible for hypervisor - * initialization on the next call. - */ - vmm_call_hyp(vtophys(&vmm_hyp_code)); - - /* Create and map the hypervisor stack */ - stack_top = stack_hyp_va[PCPU_GET(cpuid)] + VMM_STACK_SIZE; - /* * Configure the system control register for EL2: * @@ -161,9 +147,27 @@ arm_setup_vectors(void *arg) sctlr_el2 |= SCTLR_EL2_WXN; sctlr_el2 &= ~SCTLR_EL2_EE; - /* Special call to initialize EL2 */ - vmm_call_hyp(vmmpmap_to_ttbr0(), stack_top, el2_regs->tcr_el2, - sctlr_el2, el2_regs->vtcr_el2); + daif = intr_disable(); + + if (in_vhe()) { + WRITE_SPECIALREG(vtcr_el2, el2_regs->vtcr_el2); + } else { + /* + * Install the temporary vectors which will be responsible for + * initializing the VMM when we next trap into EL2. + * + * x0: the exception vector table responsible for hypervisor + * initialization on the next call. + */ + vmm_call_hyp(vtophys(&vmm_hyp_code)); + + /* Create and map the hypervisor stack */ + stack_top = stack_hyp_va[PCPU_GET(cpuid)] + VMM_STACK_SIZE; + + /* Special call to initialize EL2 */ + vmm_call_hyp(vmmpmap_to_ttbr0(), stack_top, el2_regs->tcr_el2, + sctlr_el2, el2_regs->vtcr_el2); + } intr_restore(daif); } @@ -235,22 +239,15 @@ vmmops_modinit(int ipinum) vm_paddr_t vmm_base; uint64_t id_aa64mmfr0_el1, pa_range_bits, pa_range_field; uint64_t cnthctl_el2; - register_t daif; int cpu, i; bool rv __diagused; - if (!virt_enabled()) { + if (!has_hyp()) { printf( "vmm: Processor doesn't have support for virtualization\n"); return (ENXIO); } - /* TODO: Support VHE */ - if (in_vhe()) { - printf("vmm: VHE is unsupported\n"); - return (ENXIO); - } - if (!vgic_present()) { printf("vmm: No vgic found\n"); return (ENODEV); @@ -283,67 +280,72 @@ vmmops_modinit(int ipinum) } pa_range_bits = pa_range_field >> ID_AA64MMFR0_PARange_SHIFT; - /* Initialise the EL2 MMU */ - if (!vmmpmap_init()) { - printf("vmm: Failed to init the EL2 MMU\n"); - return (ENOMEM); + if (!in_vhe()) { + /* Initialise the EL2 MMU */ + if (!vmmpmap_init()) { + printf("vmm: Failed to init the EL2 MMU\n"); + return (ENOMEM); + } } /* Set up the stage 2 pmap callbacks */ MPASS(pmap_clean_stage2_tlbi == NULL); - pmap_clean_stage2_tlbi = vmm_pmap_clean_stage2_tlbi; - pmap_stage2_invalidate_range = vmm_pmap_invalidate_range; - pmap_stage2_invalidate_all = vmm_pmap_invalidate_all; - - /* - * Create an allocator for the virtual address space used by EL2. - * EL2 code is identity-mapped; the allocator is used to find space for - * VM structures. - */ - el2_mem_alloc = vmem_create("VMM EL2", 0, 0, PAGE_SIZE, 0, M_WAITOK); - - /* Create the mappings for the hypervisor translation table. */ - hyp_code_len = round_page(&vmm_hyp_code_end - &vmm_hyp_code); - - /* We need an physical identity mapping for when we activate the MMU */ - hyp_code_base = vmm_base = vtophys(&vmm_hyp_code); - rv = vmmpmap_enter(vmm_base, hyp_code_len, vmm_base, - VM_PROT_READ | VM_PROT_EXECUTE); - MPASS(rv); + pmap_clean_stage2_tlbi = vmm_clean_s2_tlbi; + pmap_stage2_invalidate_range = vmm_s2_tlbi_range; + pmap_stage2_invalidate_all = vmm_s2_tlbi_all; - next_hyp_va = roundup2(vmm_base + hyp_code_len, L2_SIZE); - - /* Create a per-CPU hypervisor stack */ - CPU_FOREACH(cpu) { - stack[cpu] = malloc(VMM_STACK_SIZE, M_HYP, M_WAITOK | M_ZERO); - stack_hyp_va[cpu] = next_hyp_va; - - for (i = 0; i < VMM_STACK_PAGES; i++) { - rv = vmmpmap_enter(stack_hyp_va[cpu] + ptoa(i), - PAGE_SIZE, vtophys(stack[cpu] + ptoa(i)), - VM_PROT_READ | VM_PROT_WRITE); - MPASS(rv); + if (!in_vhe()) { + /* + * Create an allocator for the virtual address space used by + * EL2. EL2 code is identity-mapped; the allocator is used to + * find space for VM structures. + */ + el2_mem_alloc = vmem_create("VMM EL2", 0, 0, PAGE_SIZE, 0, + M_WAITOK); + + /* Create the mappings for the hypervisor translation table. */ + hyp_code_len = round_page(&vmm_hyp_code_end - &vmm_hyp_code); + + /* We need an physical identity mapping for when we activate the MMU */ + hyp_code_base = vmm_base = vtophys(&vmm_hyp_code); + rv = vmmpmap_enter(vmm_base, hyp_code_len, vmm_base, + VM_PROT_READ | VM_PROT_EXECUTE); + MPASS(rv); + + next_hyp_va = roundup2(vmm_base + hyp_code_len, L2_SIZE); + + /* Create a per-CPU hypervisor stack */ + CPU_FOREACH(cpu) { + stack[cpu] = malloc(VMM_STACK_SIZE, M_HYP, M_WAITOK | M_ZERO); + stack_hyp_va[cpu] = next_hyp_va; + + for (i = 0; i < VMM_STACK_PAGES; i++) { + rv = vmmpmap_enter(stack_hyp_va[cpu] + ptoa(i), + PAGE_SIZE, vtophys(stack[cpu] + ptoa(i)), + VM_PROT_READ | VM_PROT_WRITE); + MPASS(rv); + } + next_hyp_va += L2_SIZE; } - next_hyp_va += L2_SIZE; - } - el2_regs.tcr_el2 = TCR_EL2_RES1; - el2_regs.tcr_el2 |= min(pa_range_bits << TCR_EL2_PS_SHIFT, - TCR_EL2_PS_52BITS); - el2_regs.tcr_el2 |= TCR_EL2_T0SZ(64 - EL2_VIRT_BITS); - el2_regs.tcr_el2 |= TCR_EL2_IRGN0_WBWA | TCR_EL2_ORGN0_WBWA; + el2_regs.tcr_el2 = TCR_EL2_RES1; + el2_regs.tcr_el2 |= min(pa_range_bits << TCR_EL2_PS_SHIFT, + TCR_EL2_PS_52BITS); + el2_regs.tcr_el2 |= TCR_EL2_T0SZ(64 - EL2_VIRT_BITS); + el2_regs.tcr_el2 |= TCR_EL2_IRGN0_WBWA | TCR_EL2_ORGN0_WBWA; #if PAGE_SIZE == PAGE_SIZE_4K - el2_regs.tcr_el2 |= TCR_EL2_TG0_4K; + el2_regs.tcr_el2 |= TCR_EL2_TG0_4K; #elif PAGE_SIZE == PAGE_SIZE_16K - el2_regs.tcr_el2 |= TCR_EL2_TG0_16K; + el2_regs.tcr_el2 |= TCR_EL2_TG0_16K; #else #error Unsupported page size #endif #ifdef SMP - el2_regs.tcr_el2 |= TCR_EL2_SH0_IS; + el2_regs.tcr_el2 |= TCR_EL2_SH0_IS; #endif + } - switch (el2_regs.tcr_el2 & TCR_EL2_PS_MASK) { + switch (pa_range_bits << TCR_EL2_PS_SHIFT) { case TCR_EL2_PS_32BITS: vmm_max_ipa_bits = 32; break; @@ -381,8 +383,6 @@ vmmops_modinit(int ipinum) * shareable */ el2_regs.vtcr_el2 = VTCR_EL2_RES1; - el2_regs.vtcr_el2 |= - min(pa_range_bits << VTCR_EL2_PS_SHIFT, VTCR_EL2_PS_48BIT); el2_regs.vtcr_el2 |= VTCR_EL2_IRGN0_WBWA | VTCR_EL2_ORGN0_WBWA; el2_regs.vtcr_el2 |= VTCR_EL2_T0SZ(64 - vmm_virt_bits); el2_regs.vtcr_el2 |= vmm_vtcr_el2_sl(vmm_pmap_levels); @@ -396,42 +396,55 @@ vmmops_modinit(int ipinum) #ifdef SMP el2_regs.vtcr_el2 |= VTCR_EL2_SH0_IS; #endif + /* + * If FEAT_LPA2 is enabled in the host then we need to enable it here + * so the page tables created by pmap.c are correct. The meaning of + * the shareability field changes to become address bits when this + * is set. + */ + if ((READ_SPECIALREG(tcr_el1) & TCR_DS) != 0) { + el2_regs.vtcr_el2 |= VTCR_EL2_DS; + el2_regs.vtcr_el2 |= + min(pa_range_bits << VTCR_EL2_PS_SHIFT, VTCR_EL2_PS_52BIT); + } else { + el2_regs.vtcr_el2 |= + min(pa_range_bits << VTCR_EL2_PS_SHIFT, VTCR_EL2_PS_48BIT); + } smp_rendezvous(NULL, arm_setup_vectors, NULL, &el2_regs); - /* Add memory to the vmem allocator (checking there is space) */ - if (vmm_base > (L2_SIZE + PAGE_SIZE)) { - /* - * Ensure there is an L2 block before the vmm code to check - * for buffer overflows on earlier data. Include the PAGE_SIZE - * of the minimum we can allocate. - */ - vmm_base -= L2_SIZE + PAGE_SIZE; - vmm_base = rounddown2(vmm_base, L2_SIZE); + if (!in_vhe()) { + /* Add memory to the vmem allocator (checking there is space) */ + if (vmm_base > (L2_SIZE + PAGE_SIZE)) { + /* + * Ensure there is an L2 block before the vmm code to check + * for buffer overflows on earlier data. Include the PAGE_SIZE + * of the minimum we can allocate. + */ + vmm_base -= L2_SIZE + PAGE_SIZE; + vmm_base = rounddown2(vmm_base, L2_SIZE); + + /* + * Check there is memory before the vmm code to add. + * + * Reserve the L2 block at address 0 so NULL dereference will + * raise an exception. + */ + if (vmm_base > L2_SIZE) + vmem_add(el2_mem_alloc, L2_SIZE, vmm_base - L2_SIZE, + M_WAITOK); + } /* - * Check there is memory before the vmm code to add. - * - * Reserve the L2 block at address 0 so NULL dereference will - * raise an exception. + * Add the memory after the stacks. There is most of an L2 block + * between the last stack and the first allocation so this should + * be safe without adding more padding. */ - if (vmm_base > L2_SIZE) - vmem_add(el2_mem_alloc, L2_SIZE, vmm_base - L2_SIZE, - M_WAITOK); + if (next_hyp_va < HYP_VM_MAX_ADDRESS - PAGE_SIZE) + vmem_add(el2_mem_alloc, next_hyp_va, + HYP_VM_MAX_ADDRESS - next_hyp_va, M_WAITOK); } - - /* - * Add the memory after the stacks. There is most of an L2 block - * between the last stack and the first allocation so this should - * be safe without adding more padding. - */ - if (next_hyp_va < HYP_VM_MAX_ADDRESS - PAGE_SIZE) - vmem_add(el2_mem_alloc, next_hyp_va, - HYP_VM_MAX_ADDRESS - next_hyp_va, M_WAITOK); - - daif = intr_disable(); - cnthctl_el2 = vmm_call_hyp(HYP_READ_REGISTER, HYP_REG_CNTHCTL); - intr_restore(daif); + cnthctl_el2 = vmm_read_reg(HYP_REG_CNTHCTL); vgic_init(); vtimer_init(cnthctl_el2); @@ -444,21 +457,25 @@ vmmops_modcleanup(void) { int cpu; - smp_rendezvous(NULL, arm_teardown_vectors, NULL, NULL); + if (!in_vhe()) { + smp_rendezvous(NULL, arm_teardown_vectors, NULL, NULL); - CPU_FOREACH(cpu) { - vmmpmap_remove(stack_hyp_va[cpu], VMM_STACK_PAGES * PAGE_SIZE, - false); - } + CPU_FOREACH(cpu) { + vmmpmap_remove(stack_hyp_va[cpu], + VMM_STACK_PAGES * PAGE_SIZE, false); + } - vmmpmap_remove(hyp_code_base, hyp_code_len, false); + vmmpmap_remove(hyp_code_base, hyp_code_len, false); + } vtimer_cleanup(); - vmmpmap_fini(); + if (!in_vhe()) { + vmmpmap_fini(); - CPU_FOREACH(cpu) - free(stack[cpu], M_HYP); + CPU_FOREACH(cpu) + free(stack[cpu], M_HYP); + } pmap_clean_stage2_tlbi = NULL; pmap_stage2_invalidate_range = NULL; @@ -510,8 +527,9 @@ vmmops_init(struct vm *vm, pmap_t pmap) vtimer_vminit(hyp); vgic_vminit(hyp); - hyp->el2_addr = el2_map_enter((vm_offset_t)hyp, size, - VM_PROT_READ | VM_PROT_WRITE); + if (!in_vhe()) + hyp->el2_addr = el2_map_enter((vm_offset_t)hyp, size, + VM_PROT_READ | VM_PROT_WRITE); return (hyp); } @@ -539,8 +557,9 @@ vmmops_vcpu_init(void *vmi, struct vcpu *vcpu1, int vcpuid) vtimer_cpuinit(hypctx); vgic_cpuinit(hypctx); - hypctx->el2_addr = el2_map_enter((vm_offset_t)hypctx, size, - VM_PROT_READ | VM_PROT_WRITE); + if (!in_vhe()) + hypctx->el2_addr = el2_map_enter((vm_offset_t)hypctx, size, + VM_PROT_READ | VM_PROT_WRITE); return (hypctx); } @@ -567,26 +586,6 @@ vmmops_vmspace_free(struct vmspace *vmspace) vmspace_free(vmspace); } -static void -vmm_pmap_clean_stage2_tlbi(void) -{ - vmm_call_hyp(HYP_CLEAN_S2_TLBI); -} - -static void -vmm_pmap_invalidate_range(uint64_t vttbr, vm_offset_t sva, vm_offset_t eva, - bool final_only) -{ - MPASS(eva > sva); - vmm_call_hyp(HYP_S2_TLBI_RANGE, vttbr, sva, eva, final_only); -} - -static void -vmm_pmap_invalidate_all(uint64_t vttbr) -{ - vmm_call_hyp(HYP_S2_TLBI_ALL, vttbr); -} - static inline void arm64_print_hyp_regs(struct vm_exit *vme) { @@ -700,7 +699,14 @@ handle_el1_sync_excp(struct hypctx *hypctx, struct vm_exit *vme_ret, arm64_gen_reg_emul_data(esr_iss, vme_ret); vme_ret->exitcode = VM_EXITCODE_REG_EMUL; break; - + case EXCP_BRK: + vmm_stat_incr(hypctx->vcpu, VMEXIT_BRK, 1); + vme_ret->exitcode = VM_EXITCODE_BRK; + break; + case EXCP_SOFTSTP_EL0: + vmm_stat_incr(hypctx->vcpu, VMEXIT_SS, 1); + vme_ret->exitcode = VM_EXITCODE_SS; + break; case EXCP_INSN_ABORT_L: case EXCP_DATA_ABORT_L: vmm_stat_incr(hypctx->vcpu, esr_ec == EXCP_DATA_ABORT_L ? @@ -1101,7 +1107,7 @@ vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo) * Update fields that may change on exeption entry * based on how sctlr_el1 is configured. */ - if ((hypctx->sctlr_el1 & SCTLR_SPAN) != 0) + if ((hypctx->sctlr_el1 & SCTLR_SPAN) == 0) hypctx->tf.tf_spsr |= PSR_PAN; if ((hypctx->sctlr_el1 & SCTLR_DSSBS) == 0) hypctx->tf.tf_spsr &= ~PSR_SSBS; @@ -1136,16 +1142,13 @@ vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo) vgic_flush_hwstate(hypctx); /* Call into EL2 to switch to the guest */ - excp_type = vmm_call_hyp(HYP_ENTER_GUEST, - hyp->el2_addr, hypctx->el2_addr); + excp_type = vmm_enter_guest(hyp, hypctx); vgic_sync_hwstate(hypctx); vtimer_sync_hwstate(hypctx); /* - * Deactivate the stage2 pmap. vmm_pmap_clean_stage2_tlbi - * depends on this meaning we activate the VM before entering - * the vm again + * Deactivate the stage2 pmap. */ PCPU_SET(curvmpmap, NULL); intr_restore(daif); @@ -1198,7 +1201,8 @@ vmmops_vcpu_cleanup(void *vcpui) vtimer_cpucleanup(hypctx); vgic_cpucleanup(hypctx); - vmmpmap_remove(hypctx->el2_addr, el2_hypctx_size(), true); + if (!in_vhe()) + vmmpmap_remove(hypctx->el2_addr, el2_hypctx_size(), true); free(hypctx, M_HYP); } @@ -1213,7 +1217,8 @@ vmmops_cleanup(void *vmi) smp_rendezvous(NULL, arm_pcpu_vmcleanup, NULL, hyp); - vmmpmap_remove(hyp->el2_addr, el2_hyp_size(hyp->vm), true); + if (!in_vhe()) + vmmpmap_remove(hyp->el2_addr, el2_hyp_size(hyp->vm), true); free(hyp, M_HYP); } @@ -1313,6 +1318,7 @@ vmmops_exception(void *vcpui, uint64_t esr, uint64_t far) int vmmops_getcap(void *vcpui, int num, int *retval) { + struct hypctx *hypctx = vcpui; int ret; ret = ENOENT; @@ -1322,6 +1328,11 @@ vmmops_getcap(void *vcpui, int num, int *retval) *retval = 1; ret = 0; break; + case VM_CAP_BRK_EXIT: + case VM_CAP_SS_EXIT: + case VM_CAP_MASK_HWINTR: + *retval = (hypctx->setcaps & (1ul << num)) != 0; + break; default: break; } @@ -1332,6 +1343,68 @@ vmmops_getcap(void *vcpui, int num, int *retval) int vmmops_setcap(void *vcpui, int num, int val) { + struct hypctx *hypctx = vcpui; + int ret; + + ret = 0; - return (ENOENT); + switch (num) { + case VM_CAP_BRK_EXIT: + if ((val != 0) == ((hypctx->setcaps & (1ul << num)) != 0)) + break; + if (val != 0) + hypctx->mdcr_el2 |= MDCR_EL2_TDE; + else + hypctx->mdcr_el2 &= ~MDCR_EL2_TDE; + break; + case VM_CAP_SS_EXIT: + if ((val != 0) == ((hypctx->setcaps & (1ul << num)) != 0)) + break; + + if (val != 0) { + hypctx->debug_spsr |= (hypctx->tf.tf_spsr & PSR_SS); + hypctx->debug_mdscr |= hypctx->mdscr_el1 & + (MDSCR_SS | MDSCR_KDE); + + hypctx->tf.tf_spsr |= PSR_SS; + hypctx->mdscr_el1 |= MDSCR_SS | MDSCR_KDE; + hypctx->mdcr_el2 |= MDCR_EL2_TDE; + } else { + hypctx->tf.tf_spsr &= ~PSR_SS; + hypctx->tf.tf_spsr |= hypctx->debug_spsr; + hypctx->debug_spsr &= ~PSR_SS; + hypctx->mdscr_el1 &= ~(MDSCR_SS | MDSCR_KDE); + hypctx->mdscr_el1 |= hypctx->debug_mdscr; + hypctx->debug_mdscr &= ~(MDSCR_SS | MDSCR_KDE); + hypctx->mdcr_el2 &= ~MDCR_EL2_TDE; + } + break; + case VM_CAP_MASK_HWINTR: + if ((val != 0) == ((hypctx->setcaps & (1ul << num)) != 0)) + break; + + if (val != 0) { + hypctx->debug_spsr |= (hypctx->tf.tf_spsr & + (PSR_I | PSR_F)); + hypctx->tf.tf_spsr |= PSR_I | PSR_F; + } else { + hypctx->tf.tf_spsr &= ~(PSR_I | PSR_F); + hypctx->tf.tf_spsr |= (hypctx->debug_spsr & + (PSR_I | PSR_F)); + hypctx->debug_spsr &= ~(PSR_I | PSR_F); + } + break; + default: + ret = ENOENT; + break; + } + + if (ret == 0) { + if (val == 0) + hypctx->setcaps &= ~(1ul << num); + else + hypctx->setcaps |= (1ul << num); + } + + return (ret); } diff --git a/sys/arm64/vmm/vmm_call.S b/sys/arm64/vmm/vmm_call.S index fc28e3f173eb..8caf0465f938 100644 --- a/sys/arm64/vmm/vmm_call.S +++ b/sys/arm64/vmm/vmm_call.S @@ -28,6 +28,7 @@ * SUCH DAMAGE. */ +#include <sys/elf_common.h> #include <machine/asm.h> @@ -37,3 +38,5 @@ ENTRY(vmm_call_hyp) hvc #0 ret END(vmm_call_hyp) + +GNU_PROPERTY_AARCH64_FEATURE_1_NOTE(GNU_PROPERTY_AARCH64_FEATURE_1_VAL) diff --git a/sys/arm64/vmm/vmm_dev.c b/sys/arm64/vmm/vmm_dev.c deleted file mode 100644 index 9f405384f2b3..000000000000 --- a/sys/arm64/vmm/vmm_dev.c +++ /dev/null @@ -1,1054 +0,0 @@ -/*- - * SPDX-License-Identifier: BSD-2-Clause - * - * Copyright (c) 2011 NetApp, Inc. - * Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com> - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include <sys/param.h> -#include <sys/kernel.h> -#include <sys/jail.h> -#include <sys/queue.h> -#include <sys/lock.h> -#include <sys/mutex.h> -#include <sys/malloc.h> -#include <sys/conf.h> -#include <sys/sysctl.h> -#include <sys/libkern.h> -#include <sys/ioccom.h> -#include <sys/mman.h> -#include <sys/uio.h> -#include <sys/proc.h> - -#include <vm/vm.h> -#include <vm/pmap.h> -#include <vm/vm_map.h> -#include <vm/vm_object.h> - -#include <machine/machdep.h> -#include <machine/vmparam.h> -#include <machine/vmm.h> -#include <machine/vmm_dev.h> - -#include "vmm_stat.h" - -#include "io/vgic.h" - -struct devmem_softc { - int segid; - char *name; - struct cdev *cdev; - struct vmmdev_softc *sc; - SLIST_ENTRY(devmem_softc) link; -}; - -struct vmmdev_softc { - struct vm *vm; /* vm instance cookie */ - struct cdev *cdev; - struct ucred *ucred; - SLIST_ENTRY(vmmdev_softc) link; - SLIST_HEAD(, devmem_softc) devmem; - int flags; -}; -#define VSC_LINKED 0x01 - -static SLIST_HEAD(, vmmdev_softc) head; - -static unsigned pr_allow_flag; -static struct mtx vmmdev_mtx; -MTX_SYSINIT(vmmdev_mtx, &vmmdev_mtx, "vmm device mutex", MTX_DEF); - -static MALLOC_DEFINE(M_VMMDEV, "vmmdev", "vmmdev"); - -SYSCTL_DECL(_hw_vmm); - -static int vmm_priv_check(struct ucred *ucred); -static int devmem_create_cdev(const char *vmname, int id, char *devmem); -static void devmem_destroy(void *arg); - -static int -vmm_priv_check(struct ucred *ucred) -{ - - if (jailed(ucred) && - !(ucred->cr_prison->pr_allow & pr_allow_flag)) - return (EPERM); - - return (0); -} - -static int -vcpu_lock_one(struct vcpu *vcpu) -{ - int error; - - error = vcpu_set_state(vcpu, VCPU_FROZEN, true); - return (error); -} - -static void -vcpu_unlock_one(struct vcpu *vcpu) -{ - enum vcpu_state state; - - state = vcpu_get_state(vcpu, NULL); - if (state != VCPU_FROZEN) { - panic("vcpu %s(%d) has invalid state %d", - vm_name(vcpu_vm(vcpu)), vcpu_vcpuid(vcpu), state); - } - - vcpu_set_state(vcpu, VCPU_IDLE, false); -} - -static int -vcpu_lock_all(struct vmmdev_softc *sc) -{ - struct vcpu *vcpu; - int error; - uint16_t i, j, maxcpus; - - error = 0; - vm_slock_vcpus(sc->vm); - maxcpus = vm_get_maxcpus(sc->vm); - for (i = 0; i < maxcpus; i++) { - vcpu = vm_vcpu(sc->vm, i); - if (vcpu == NULL) - continue; - error = vcpu_lock_one(vcpu); - if (error) - break; - } - - if (error) { - for (j = 0; j < i; j++) { - vcpu = vm_vcpu(sc->vm, j); - if (vcpu == NULL) - continue; - vcpu_unlock_one(vcpu); - } - vm_unlock_vcpus(sc->vm); - } - - return (error); -} - -static void -vcpu_unlock_all(struct vmmdev_softc *sc) -{ - struct vcpu *vcpu; - uint16_t i, maxcpus; - - maxcpus = vm_get_maxcpus(sc->vm); - for (i = 0; i < maxcpus; i++) { - vcpu = vm_vcpu(sc->vm, i); - if (vcpu == NULL) - continue; - vcpu_unlock_one(vcpu); - } - vm_unlock_vcpus(sc->vm); -} - -static struct vmmdev_softc * -vmmdev_lookup(const char *name) -{ - struct vmmdev_softc *sc; - -#ifdef notyet /* XXX kernel is not compiled with invariants */ - mtx_assert(&vmmdev_mtx, MA_OWNED); -#endif - - SLIST_FOREACH(sc, &head, link) { - if (strcmp(name, vm_name(sc->vm)) == 0) - break; - } - - if (sc == NULL) - return (NULL); - - if (cr_cansee(curthread->td_ucred, sc->ucred)) - return (NULL); - - return (sc); -} - -static struct vmmdev_softc * -vmmdev_lookup2(struct cdev *cdev) -{ - - return (cdev->si_drv1); -} - -static int -vmmdev_rw(struct cdev *cdev, struct uio *uio, int flags) -{ - int error, off, c, prot; - vm_paddr_t gpa, maxaddr; - void *hpa, *cookie; - struct vmmdev_softc *sc; - - error = vmm_priv_check(curthread->td_ucred); - if (error) - return (error); - - sc = vmmdev_lookup2(cdev); - if (sc == NULL) - return (ENXIO); - - /* - * Get a read lock on the guest memory map. - */ - vm_slock_memsegs(sc->vm); - - prot = (uio->uio_rw == UIO_WRITE ? VM_PROT_WRITE : VM_PROT_READ); - maxaddr = vmm_sysmem_maxaddr(sc->vm); - while (uio->uio_resid > 0 && error == 0) { - gpa = uio->uio_offset; - off = gpa & PAGE_MASK; - c = min(uio->uio_resid, PAGE_SIZE - off); - - /* - * The VM has a hole in its physical memory map. If we want to - * use 'dd' to inspect memory beyond the hole we need to - * provide bogus data for memory that lies in the hole. - * - * Since this device does not support lseek(2), dd(1) will - * read(2) blocks of data to simulate the lseek(2). - */ - hpa = vm_gpa_hold_global(sc->vm, gpa, c, prot, &cookie); - if (hpa == NULL) { - if (uio->uio_rw == UIO_READ && gpa < maxaddr) - error = uiomove(__DECONST(void *, zero_region), - c, uio); - else - error = EFAULT; - } else { - error = uiomove(hpa, c, uio); - vm_gpa_release(cookie); - } - } - vm_unlock_memsegs(sc->vm); - return (error); -} - -static int -get_memseg(struct vmmdev_softc *sc, struct vm_memseg *mseg) -{ - struct devmem_softc *dsc; - int error; - bool sysmem; - - error = vm_get_memseg(sc->vm, mseg->segid, &mseg->len, &sysmem, NULL); - if (error || mseg->len == 0) - return (error); - - if (!sysmem) { - SLIST_FOREACH(dsc, &sc->devmem, link) { - if (dsc->segid == mseg->segid) - break; - } - KASSERT(dsc != NULL, ("%s: devmem segment %d not found", - __func__, mseg->segid)); - error = copystr(dsc->name, mseg->name, sizeof(mseg->name), - NULL); - } else { - bzero(mseg->name, sizeof(mseg->name)); - } - - return (error); -} - -static int -alloc_memseg(struct vmmdev_softc *sc, struct vm_memseg *mseg) -{ - char *name; - int error; - bool sysmem; - - error = 0; - name = NULL; - sysmem = true; - - /* - * The allocation is lengthened by 1 to hold a terminating NUL. It'll - * by stripped off when devfs processes the full string. - */ - if (VM_MEMSEG_NAME(mseg)) { - sysmem = false; - name = malloc(sizeof(mseg->name), M_VMMDEV, M_WAITOK); - error = copystr(mseg->name, name, sizeof(mseg->name), NULL); - if (error) - goto done; - } - - error = vm_alloc_memseg(sc->vm, mseg->segid, mseg->len, sysmem); - if (error) - goto done; - - if (VM_MEMSEG_NAME(mseg)) { - error = devmem_create_cdev(vm_name(sc->vm), mseg->segid, name); - if (error) - vm_free_memseg(sc->vm, mseg->segid); - else - name = NULL; /* freed when 'cdev' is destroyed */ - } -done: - free(name, M_VMMDEV); - return (error); -} - -static int -vm_get_register_set(struct vcpu *vcpu, unsigned int count, int *regnum, - uint64_t *regval) -{ - int error, i; - - error = 0; - for (i = 0; i < count; i++) { - error = vm_get_register(vcpu, regnum[i], ®val[i]); - if (error) - break; - } - return (error); -} - -static int -vm_set_register_set(struct vcpu *vcpu, unsigned int count, int *regnum, - uint64_t *regval) -{ - int error, i; - - error = 0; - for (i = 0; i < count; i++) { - error = vm_set_register(vcpu, regnum[i], regval[i]); - if (error) - break; - } - return (error); -} - -static int -vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t data, int fflag, - struct thread *td) -{ - int error, vcpuid, size; - cpuset_t *cpuset; - struct vmmdev_softc *sc; - struct vcpu *vcpu; - struct vm_register *vmreg; - struct vm_register_set *vmregset; - struct vm_run *vmrun; - struct vm_vgic_version *vgv; - struct vm_vgic_descr *vgic; - struct vm_cpuset *vm_cpuset; - struct vm_irq *vi; - struct vm_capability *vmcap; - struct vm_stats *vmstats; - struct vm_stat_desc *statdesc; - struct vm_suspend *vmsuspend; - struct vm_exception *vmexc; - struct vm_gla2gpa *gg; - struct vm_memmap *mm; - struct vm_munmap *mu; - struct vm_msi *vmsi; - struct vm_cpu_topology *topology; - uint64_t *regvals; - int *regnums; - enum { NONE, SINGLE, ALL } vcpus_locked; - bool memsegs_locked; - - error = vmm_priv_check(curthread->td_ucred); - if (error) - return (error); - - sc = vmmdev_lookup2(cdev); - if (sc == NULL) - return (ENXIO); - - error = 0; - vcpuid = -1; - vcpu = NULL; - vcpus_locked = NONE; - memsegs_locked = false; - - /* - * Some VMM ioctls can operate only on vcpus that are not running. - */ - switch (cmd) { - case VM_RUN: - case VM_GET_REGISTER: - case VM_SET_REGISTER: - case VM_GET_REGISTER_SET: - case VM_SET_REGISTER_SET: - case VM_INJECT_EXCEPTION: - case VM_GET_CAPABILITY: - case VM_SET_CAPABILITY: - case VM_GLA2GPA_NOFAULT: - case VM_ACTIVATE_CPU: - /* - * ioctls that can operate only on vcpus that are not running. - */ - vcpuid = *(int *)data; - vcpu = vm_alloc_vcpu(sc->vm, vcpuid); - if (vcpu == NULL) { - error = EINVAL; - goto done; - } - error = vcpu_lock_one(vcpu); - if (error) - goto done; - vcpus_locked = SINGLE; - break; - - case VM_ALLOC_MEMSEG: - case VM_MMAP_MEMSEG: - case VM_MUNMAP_MEMSEG: - case VM_REINIT: - case VM_ATTACH_VGIC: - /* - * ioctls that modify the memory map must lock memory - * segments exclusively. - */ - vm_xlock_memsegs(sc->vm); - memsegs_locked = true; - - /* - * ioctls that operate on the entire virtual machine must - * prevent all vcpus from running. - */ - error = vcpu_lock_all(sc); - if (error) - goto done; - vcpus_locked = ALL; - break; - case VM_GET_MEMSEG: - case VM_MMAP_GETNEXT: - /* - * Lock the memory map while it is being inspected. - */ - vm_slock_memsegs(sc->vm); - memsegs_locked = true; - break; - - case VM_STATS: - /* - * These do not need the vCPU locked but do operate on - * a specific vCPU. - */ - vcpuid = *(int *)data; - vcpu = vm_alloc_vcpu(sc->vm, vcpuid); - if (vcpu == NULL) { - error = EINVAL; - goto done; - } - break; - - case VM_SUSPEND_CPU: - case VM_RESUME_CPU: - /* - * These can either operate on all CPUs via a vcpuid of - * -1 or on a specific vCPU. - */ - vcpuid = *(int *)data; - if (vcpuid == -1) - break; - vcpu = vm_alloc_vcpu(sc->vm, vcpuid); - if (vcpu == NULL) { - error = EINVAL; - goto done; - } - break; - - case VM_ASSERT_IRQ: - vi = (struct vm_irq *)data; - error = vm_assert_irq(sc->vm, vi->irq); - break; - case VM_DEASSERT_IRQ: - vi = (struct vm_irq *)data; - error = vm_deassert_irq(sc->vm, vi->irq); - break; - default: - break; - } - - switch (cmd) { - case VM_RUN: { - struct vm_exit *vme; - - vmrun = (struct vm_run *)data; - vme = vm_exitinfo(vcpu); - - error = vm_run(vcpu); - if (error != 0) - break; - - error = copyout(vme, vmrun->vm_exit, sizeof(*vme)); - if (error != 0) - break; - break; - } - case VM_SUSPEND: - vmsuspend = (struct vm_suspend *)data; - error = vm_suspend(sc->vm, vmsuspend->how); - break; - case VM_REINIT: - error = vm_reinit(sc->vm); - break; - case VM_STAT_DESC: { - statdesc = (struct vm_stat_desc *)data; - error = vmm_stat_desc_copy(statdesc->index, - statdesc->desc, sizeof(statdesc->desc)); - break; - } - case VM_STATS: { - CTASSERT(MAX_VM_STATS >= MAX_VMM_STAT_ELEMS); - vmstats = (struct vm_stats *)data; - getmicrotime(&vmstats->tv); - error = vmm_stat_copy(vcpu, vmstats->index, - nitems(vmstats->statbuf), - &vmstats->num_entries, vmstats->statbuf); - break; - } - case VM_MMAP_GETNEXT: - mm = (struct vm_memmap *)data; - error = vm_mmap_getnext(sc->vm, &mm->gpa, &mm->segid, - &mm->segoff, &mm->len, &mm->prot, &mm->flags); - break; - case VM_MMAP_MEMSEG: - mm = (struct vm_memmap *)data; - error = vm_mmap_memseg(sc->vm, mm->gpa, mm->segid, mm->segoff, - mm->len, mm->prot, mm->flags); - break; - case VM_MUNMAP_MEMSEG: - mu = (struct vm_munmap *)data; - error = vm_munmap_memseg(sc->vm, mu->gpa, mu->len); - break; - case VM_ALLOC_MEMSEG: - error = alloc_memseg(sc, (struct vm_memseg *)data); - break; - case VM_GET_MEMSEG: - error = get_memseg(sc, (struct vm_memseg *)data); - break; - case VM_GET_REGISTER: - vmreg = (struct vm_register *)data; - error = vm_get_register(vcpu, vmreg->regnum, &vmreg->regval); - break; - case VM_SET_REGISTER: - vmreg = (struct vm_register *)data; - error = vm_set_register(vcpu, vmreg->regnum, vmreg->regval); - break; - case VM_GET_REGISTER_SET: - vmregset = (struct vm_register_set *)data; - if (vmregset->count > VM_REG_LAST) { - error = EINVAL; - break; - } - regvals = malloc(sizeof(regvals[0]) * vmregset->count, M_VMMDEV, - M_WAITOK); - regnums = malloc(sizeof(regnums[0]) * vmregset->count, M_VMMDEV, - M_WAITOK); - error = copyin(vmregset->regnums, regnums, sizeof(regnums[0]) * - vmregset->count); - if (error == 0) - error = vm_get_register_set(vcpu, vmregset->count, - regnums, regvals); - if (error == 0) - error = copyout(regvals, vmregset->regvals, - sizeof(regvals[0]) * vmregset->count); - free(regvals, M_VMMDEV); - free(regnums, M_VMMDEV); - break; - case VM_SET_REGISTER_SET: - vmregset = (struct vm_register_set *)data; - if (vmregset->count > VM_REG_LAST) { - error = EINVAL; - break; - } - regvals = malloc(sizeof(regvals[0]) * vmregset->count, M_VMMDEV, - M_WAITOK); - regnums = malloc(sizeof(regnums[0]) * vmregset->count, M_VMMDEV, - M_WAITOK); - error = copyin(vmregset->regnums, regnums, sizeof(regnums[0]) * - vmregset->count); - if (error == 0) - error = copyin(vmregset->regvals, regvals, - sizeof(regvals[0]) * vmregset->count); - if (error == 0) - error = vm_set_register_set(vcpu, vmregset->count, - regnums, regvals); - free(regvals, M_VMMDEV); - free(regnums, M_VMMDEV); - break; - case VM_GET_CAPABILITY: - vmcap = (struct vm_capability *)data; - error = vm_get_capability(vcpu, - vmcap->captype, - &vmcap->capval); - break; - case VM_SET_CAPABILITY: - vmcap = (struct vm_capability *)data; - error = vm_set_capability(vcpu, - vmcap->captype, - vmcap->capval); - break; - case VM_INJECT_EXCEPTION: - vmexc = (struct vm_exception *)data; - error = vm_inject_exception(vcpu, vmexc->esr, vmexc->far); - break; - case VM_GLA2GPA_NOFAULT: - gg = (struct vm_gla2gpa *)data; - error = vm_gla2gpa_nofault(vcpu, &gg->paging, gg->gla, - gg->prot, &gg->gpa, &gg->fault); - KASSERT(error == 0 || error == EFAULT, - ("%s: vm_gla2gpa unknown error %d", __func__, error)); - break; - case VM_ACTIVATE_CPU: - error = vm_activate_cpu(vcpu); - break; - case VM_GET_CPUS: - error = 0; - vm_cpuset = (struct vm_cpuset *)data; - size = vm_cpuset->cpusetsize; - if (size < sizeof(cpuset_t) || size > CPU_MAXSIZE / NBBY) { - error = ERANGE; - break; - } - cpuset = malloc(size, M_TEMP, M_WAITOK | M_ZERO); - if (vm_cpuset->which == VM_ACTIVE_CPUS) - *cpuset = vm_active_cpus(sc->vm); - else if (vm_cpuset->which == VM_SUSPENDED_CPUS) - *cpuset = vm_suspended_cpus(sc->vm); - else if (vm_cpuset->which == VM_DEBUG_CPUS) - *cpuset = vm_debug_cpus(sc->vm); - else - error = EINVAL; - if (error == 0) - error = copyout(cpuset, vm_cpuset->cpus, size); - free(cpuset, M_TEMP); - break; - case VM_SUSPEND_CPU: - error = vm_suspend_cpu(sc->vm, vcpu); - break; - case VM_RESUME_CPU: - error = vm_resume_cpu(sc->vm, vcpu); - break; - case VM_GET_VGIC_VERSION: - vgv = (struct vm_vgic_version *)data; - /* TODO: Query the vgic driver for this */ - vgv->version = 3; - vgv->flags = 0; - error = 0; - break; - case VM_ATTACH_VGIC: - vgic = (struct vm_vgic_descr *)data; - error = vm_attach_vgic(sc->vm, vgic); - break; - case VM_RAISE_MSI: - vmsi = (struct vm_msi *)data; - error = vm_raise_msi(sc->vm, vmsi->msg, vmsi->addr, vmsi->bus, - vmsi->slot, vmsi->func); - break; - case VM_SET_TOPOLOGY: - topology = (struct vm_cpu_topology *)data; - error = vm_set_topology(sc->vm, topology->sockets, - topology->cores, topology->threads, topology->maxcpus); - break; - case VM_GET_TOPOLOGY: - topology = (struct vm_cpu_topology *)data; - vm_get_topology(sc->vm, &topology->sockets, &topology->cores, - &topology->threads, &topology->maxcpus); - error = 0; - break; - default: - error = ENOTTY; - break; - } - -done: - if (vcpus_locked == SINGLE) - vcpu_unlock_one(vcpu); - else if (vcpus_locked == ALL) - vcpu_unlock_all(sc); - if (memsegs_locked) - vm_unlock_memsegs(sc->vm); - - /* - * Make sure that no handler returns a kernel-internal - * error value to userspace. - */ - KASSERT(error == ERESTART || error >= 0, - ("vmmdev_ioctl: invalid error return %d", error)); - return (error); -} - -static int -vmmdev_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t mapsize, - struct vm_object **objp, int nprot) -{ - struct vmmdev_softc *sc; - vm_paddr_t gpa; - size_t len; - vm_ooffset_t segoff, first, last; - int error, found, segid; - bool sysmem; - - error = vmm_priv_check(curthread->td_ucred); - if (error) - return (error); - - first = *offset; - last = first + mapsize; - if ((nprot & PROT_EXEC) || first < 0 || first >= last) - return (EINVAL); - - sc = vmmdev_lookup2(cdev); - if (sc == NULL) { - /* virtual machine is in the process of being created */ - return (EINVAL); - } - - /* - * Get a read lock on the guest memory map. - */ - vm_slock_memsegs(sc->vm); - - gpa = 0; - found = 0; - while (!found) { - error = vm_mmap_getnext(sc->vm, &gpa, &segid, &segoff, &len, - NULL, NULL); - if (error) - break; - - if (first >= gpa && last <= gpa + len) - found = 1; - else - gpa += len; - } - - if (found) { - error = vm_get_memseg(sc->vm, segid, &len, &sysmem, objp); - KASSERT(error == 0 && *objp != NULL, - ("%s: invalid memory segment %d", __func__, segid)); - if (sysmem) { - vm_object_reference(*objp); - *offset = segoff + (first - gpa); - } else { - error = EINVAL; - } - } - vm_unlock_memsegs(sc->vm); - return (error); -} - -static void -vmmdev_destroy(void *arg) -{ - struct vmmdev_softc *sc = arg; - struct devmem_softc *dsc; - int error __diagused; - - error = vcpu_lock_all(sc); - KASSERT(error == 0, ("%s: error %d freezing vcpus", __func__, error)); - vm_unlock_vcpus(sc->vm); - - while ((dsc = SLIST_FIRST(&sc->devmem)) != NULL) { - KASSERT(dsc->cdev == NULL, ("%s: devmem not free", __func__)); - SLIST_REMOVE_HEAD(&sc->devmem, link); - free(dsc->name, M_VMMDEV); - free(dsc, M_VMMDEV); - } - - if (sc->cdev != NULL) - destroy_dev(sc->cdev); - - if (sc->vm != NULL) - vm_destroy(sc->vm); - - if (sc->ucred != NULL) - crfree(sc->ucred); - - if ((sc->flags & VSC_LINKED) != 0) { - mtx_lock(&vmmdev_mtx); - SLIST_REMOVE(&head, sc, vmmdev_softc, link); - mtx_unlock(&vmmdev_mtx); - } - - free(sc, M_VMMDEV); -} - -static int -sysctl_vmm_destroy(SYSCTL_HANDLER_ARGS) -{ - struct devmem_softc *dsc; - struct vmmdev_softc *sc; - struct cdev *cdev; - char *buf; - int error, buflen; - - error = vmm_priv_check(req->td->td_ucred); - if (error) - return (error); - - buflen = VM_MAX_NAMELEN + 1; - buf = malloc(buflen, M_VMMDEV, M_WAITOK | M_ZERO); - strlcpy(buf, "beavis", buflen); - error = sysctl_handle_string(oidp, buf, buflen, req); - if (error != 0 || req->newptr == NULL) - goto out; - - mtx_lock(&vmmdev_mtx); - sc = vmmdev_lookup(buf); - if (sc == NULL || sc->cdev == NULL) { - mtx_unlock(&vmmdev_mtx); - error = EINVAL; - goto out; - } - - /* - * Setting 'sc->cdev' to NULL is used to indicate that the VM - * is scheduled for destruction. - */ - cdev = sc->cdev; - sc->cdev = NULL; - mtx_unlock(&vmmdev_mtx); - - /* - * Destroy all cdevs: - * - * - any new operations on the 'cdev' will return an error (ENXIO). - * - * - the 'devmem' cdevs are destroyed before the virtual machine 'cdev' - */ - SLIST_FOREACH(dsc, &sc->devmem, link) { - KASSERT(dsc->cdev != NULL, ("devmem cdev already destroyed")); - destroy_dev(dsc->cdev); - devmem_destroy(dsc); - } - destroy_dev(cdev); - vmmdev_destroy(sc); - error = 0; - -out: - free(buf, M_VMMDEV); - return (error); -} -SYSCTL_PROC(_hw_vmm, OID_AUTO, destroy, - CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_MPSAFE, - NULL, 0, sysctl_vmm_destroy, "A", - NULL); - -static struct cdevsw vmmdevsw = { - .d_name = "vmmdev", - .d_version = D_VERSION, - .d_ioctl = vmmdev_ioctl, - .d_mmap_single = vmmdev_mmap_single, - .d_read = vmmdev_rw, - .d_write = vmmdev_rw, -}; - -static int -sysctl_vmm_create(SYSCTL_HANDLER_ARGS) -{ - struct vm *vm; - struct cdev *cdev; - struct vmmdev_softc *sc, *sc2; - char *buf; - int error, buflen; - - error = vmm_priv_check(req->td->td_ucred); - if (error) - return (error); - - buflen = VM_MAX_NAMELEN + 1; - buf = malloc(buflen, M_VMMDEV, M_WAITOK | M_ZERO); - strlcpy(buf, "beavis", buflen); - error = sysctl_handle_string(oidp, buf, buflen, req); - if (error != 0 || req->newptr == NULL) - goto out; - - mtx_lock(&vmmdev_mtx); - sc = vmmdev_lookup(buf); - mtx_unlock(&vmmdev_mtx); - if (sc != NULL) { - error = EEXIST; - goto out; - } - - error = vm_create(buf, &vm); - if (error != 0) - goto out; - - sc = malloc(sizeof(struct vmmdev_softc), M_VMMDEV, M_WAITOK | M_ZERO); - sc->ucred = crhold(curthread->td_ucred); - sc->vm = vm; - SLIST_INIT(&sc->devmem); - - /* - * Lookup the name again just in case somebody sneaked in when we - * dropped the lock. - */ - mtx_lock(&vmmdev_mtx); - sc2 = vmmdev_lookup(buf); - if (sc2 == NULL) { - SLIST_INSERT_HEAD(&head, sc, link); - sc->flags |= VSC_LINKED; - } - mtx_unlock(&vmmdev_mtx); - - if (sc2 != NULL) { - vmmdev_destroy(sc); - error = EEXIST; - goto out; - } - - error = make_dev_p(MAKEDEV_CHECKNAME, &cdev, &vmmdevsw, sc->ucred, - UID_ROOT, GID_WHEEL, 0600, "vmm/%s", buf); - if (error != 0) { - vmmdev_destroy(sc); - goto out; - } - - mtx_lock(&vmmdev_mtx); - sc->cdev = cdev; - sc->cdev->si_drv1 = sc; - mtx_unlock(&vmmdev_mtx); - -out: - free(buf, M_VMMDEV); - return (error); -} -SYSCTL_PROC(_hw_vmm, OID_AUTO, create, - CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_MPSAFE, - NULL, 0, sysctl_vmm_create, "A", - NULL); - -void -vmmdev_init(void) -{ - pr_allow_flag = prison_add_allow(NULL, "vmm", NULL, - "Allow use of vmm in a jail."); -} - -int -vmmdev_cleanup(void) -{ - int error; - - if (SLIST_EMPTY(&head)) - error = 0; - else - error = EBUSY; - - return (error); -} - -static int -devmem_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t len, - struct vm_object **objp, int nprot) -{ - struct devmem_softc *dsc; - vm_ooffset_t first, last; - size_t seglen; - int error; - bool sysmem; - - dsc = cdev->si_drv1; - if (dsc == NULL) { - /* 'cdev' has been created but is not ready for use */ - return (ENXIO); - } - - first = *offset; - last = *offset + len; - if ((nprot & PROT_EXEC) || first < 0 || first >= last) - return (EINVAL); - - vm_slock_memsegs(dsc->sc->vm); - - error = vm_get_memseg(dsc->sc->vm, dsc->segid, &seglen, &sysmem, objp); - KASSERT(error == 0 && !sysmem && *objp != NULL, - ("%s: invalid devmem segment %d", __func__, dsc->segid)); - - if (seglen >= last) - vm_object_reference(*objp); - else - error = 0; - vm_unlock_memsegs(dsc->sc->vm); - return (error); -} - -static struct cdevsw devmemsw = { - .d_name = "devmem", - .d_version = D_VERSION, - .d_mmap_single = devmem_mmap_single, -}; - -static int -devmem_create_cdev(const char *vmname, int segid, char *devname) -{ - struct devmem_softc *dsc; - struct vmmdev_softc *sc; - struct cdev *cdev; - int error; - - error = make_dev_p(MAKEDEV_CHECKNAME, &cdev, &devmemsw, NULL, - UID_ROOT, GID_WHEEL, 0600, "vmm.io/%s.%s", vmname, devname); - if (error) - return (error); - - dsc = malloc(sizeof(struct devmem_softc), M_VMMDEV, M_WAITOK | M_ZERO); - - mtx_lock(&vmmdev_mtx); - sc = vmmdev_lookup(vmname); - KASSERT(sc != NULL, ("%s: vm %s softc not found", __func__, vmname)); - if (sc->cdev == NULL) { - /* virtual machine is being created or destroyed */ - mtx_unlock(&vmmdev_mtx); - free(dsc, M_VMMDEV); - destroy_dev_sched_cb(cdev, NULL, 0); - return (ENODEV); - } - - dsc->segid = segid; - dsc->name = devname; - dsc->cdev = cdev; - dsc->sc = sc; - SLIST_INSERT_HEAD(&sc->devmem, dsc, link); - mtx_unlock(&vmmdev_mtx); - - /* The 'cdev' is ready for use after 'si_drv1' is initialized */ - cdev->si_drv1 = dsc; - return (0); -} - -static void -devmem_destroy(void *arg) -{ - struct devmem_softc *dsc = arg; - - KASSERT(dsc->cdev, ("%s: devmem cdev already destroyed", __func__)); - dsc->cdev = NULL; - dsc->sc = NULL; -} diff --git a/sys/arm64/vmm/vmm_dev_machdep.c b/sys/arm64/vmm/vmm_dev_machdep.c new file mode 100644 index 000000000000..926a74fa528b --- /dev/null +++ b/sys/arm64/vmm/vmm_dev_machdep.c @@ -0,0 +1,138 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2011 NetApp, Inc. + * Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/param.h> +#include <sys/kernel.h> +#include <sys/conf.h> +#include <sys/libkern.h> +#include <sys/ioccom.h> +#include <sys/mman.h> +#include <sys/uio.h> +#include <sys/proc.h> + +#include <vm/vm.h> +#include <vm/pmap.h> +#include <vm/vm_map.h> + +#include <machine/machdep.h> +#include <machine/vmparam.h> +#include <machine/vmm.h> + +#include <dev/vmm/vmm_dev.h> +#include <dev/vmm/vmm_mem.h> + +#include "io/vgic.h" + +const struct vmmdev_ioctl vmmdev_machdep_ioctls[] = { + VMMDEV_IOCTL(VM_RUN, VMMDEV_IOCTL_LOCK_ONE_VCPU), + VMMDEV_IOCTL(VM_INJECT_EXCEPTION, VMMDEV_IOCTL_LOCK_ONE_VCPU), + VMMDEV_IOCTL(VM_GLA2GPA_NOFAULT, VMMDEV_IOCTL_LOCK_ONE_VCPU), + + VMMDEV_IOCTL(VM_ATTACH_VGIC, + VMMDEV_IOCTL_XLOCK_MEMSEGS | VMMDEV_IOCTL_LOCK_ALL_VCPUS), + + VMMDEV_IOCTL(VM_GET_VGIC_VERSION, 0), + VMMDEV_IOCTL(VM_RAISE_MSI, 0), + VMMDEV_IOCTL(VM_ASSERT_IRQ, 0), + VMMDEV_IOCTL(VM_DEASSERT_IRQ, 0), +}; +const size_t vmmdev_machdep_ioctl_count = nitems(vmmdev_machdep_ioctls); + +int +vmmdev_machdep_ioctl(struct vm *vm, struct vcpu *vcpu, u_long cmd, caddr_t data, + int fflag, struct thread *td) +{ + struct vm_run *vmrun; + struct vm_vgic_version *vgv; + struct vm_vgic_descr *vgic; + struct vm_irq *vi; + struct vm_exception *vmexc; + struct vm_gla2gpa *gg; + struct vm_msi *vmsi; + int error; + + error = 0; + switch (cmd) { + case VM_RUN: { + struct vm_exit *vme; + + vmrun = (struct vm_run *)data; + vme = vm_exitinfo(vcpu); + + error = vm_run(vcpu); + if (error != 0) + break; + + error = copyout(vme, vmrun->vm_exit, sizeof(*vme)); + if (error != 0) + break; + break; + } + case VM_INJECT_EXCEPTION: + vmexc = (struct vm_exception *)data; + error = vm_inject_exception(vcpu, vmexc->esr, vmexc->far); + break; + case VM_GLA2GPA_NOFAULT: + gg = (struct vm_gla2gpa *)data; + error = vm_gla2gpa_nofault(vcpu, &gg->paging, gg->gla, + gg->prot, &gg->gpa, &gg->fault); + KASSERT(error == 0 || error == EFAULT, + ("%s: vm_gla2gpa unknown error %d", __func__, error)); + break; + case VM_GET_VGIC_VERSION: + vgv = (struct vm_vgic_version *)data; + /* TODO: Query the vgic driver for this */ + vgv->version = 3; + vgv->flags = 0; + error = 0; + break; + case VM_ATTACH_VGIC: + vgic = (struct vm_vgic_descr *)data; + error = vm_attach_vgic(vm, vgic); + break; + case VM_RAISE_MSI: + vmsi = (struct vm_msi *)data; + error = vm_raise_msi(vm, vmsi->msg, vmsi->addr, vmsi->bus, + vmsi->slot, vmsi->func); + break; + case VM_ASSERT_IRQ: + vi = (struct vm_irq *)data; + error = vm_assert_irq(vm, vi->irq); + break; + case VM_DEASSERT_IRQ: + vi = (struct vm_irq *)data; + error = vm_deassert_irq(vm, vi->irq); + break; + default: + error = ENOTTY; + break; + } + + return (error); +} diff --git a/sys/arm64/vmm/vmm_handlers.c b/sys/arm64/vmm/vmm_handlers.c new file mode 100644 index 000000000000..c567b585eb06 --- /dev/null +++ b/sys/arm64/vmm/vmm_handlers.c @@ -0,0 +1,113 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Arm Ltd + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include <sys/types.h> +#include <sys/systm.h> + +#include <machine/ifunc.h> + +#include "arm64.h" +#include "vmm_handlers.h" + +/* Read an EL2 register */ +static uint64_t +vmm_nvhe_read_reg(uint64_t reg) +{ + return (vmm_call_hyp(HYP_READ_REGISTER, reg)); +} + +DEFINE_IFUNC(, uint64_t, vmm_read_reg, (uint64_t reg)) +{ + if (in_vhe()) + return (vmm_vhe_read_reg); + return (vmm_nvhe_read_reg); +} + +/* Enter the guest */ +static uint64_t +vmm_nvhe_enter_guest(struct hyp *hyp, struct hypctx *hypctx) +{ + return (vmm_call_hyp(HYP_ENTER_GUEST, hyp->el2_addr, hypctx->el2_addr)); +} + +DEFINE_IFUNC(, uint64_t, vmm_enter_guest, + (struct hyp *hyp, struct hypctx *hypctx)) +{ + if (in_vhe()) + return (vmm_vhe_enter_guest); + return (vmm_nvhe_enter_guest); +} + +/* Clean the TLB for all guests */ +static void +vmm_nvhe_clean_s2_tlbi(void) +{ + vmm_call_hyp(HYP_CLEAN_S2_TLBI); +} + +DEFINE_IFUNC(, void, vmm_clean_s2_tlbi, (void)) +{ + if (in_vhe()) + return (vmm_vhe_clean_s2_tlbi); + return (vmm_nvhe_clean_s2_tlbi); +} + +/* + * Switch to a guest vttbr and clean the TLB for a range of guest + * virtual address space. + */ +static void +vmm_nvhe_s2_tlbi_range(uint64_t vttbr, vm_offset_t sva, vm_offset_t eva, + bool final_only) +{ + vmm_call_hyp(HYP_S2_TLBI_RANGE, vttbr, sva, eva, final_only); +} + +DEFINE_IFUNC(, void, vmm_s2_tlbi_range, + (uint64_t vttbr, vm_offset_t sva, vm_offset_t eva, bool final_only)) +{ + if (in_vhe()) + return (vmm_vhe_s2_tlbi_range); + return (vmm_nvhe_s2_tlbi_range); +} + +/* + * Switch to a guest vttbr and clean the TLB for all the guest + * virtual address space. + */ +static void +vmm_nvhe_s2_tlbi_all(uint64_t vttbr) +{ + vmm_call_hyp(HYP_S2_TLBI_ALL, vttbr); +} + +DEFINE_IFUNC(, void, vmm_s2_tlbi_all, (uint64_t vttbr)) +{ + if (in_vhe()) + return (vmm_vhe_s2_tlbi_all); + return (vmm_nvhe_s2_tlbi_all); +} diff --git a/sys/arm64/vmm/vmm_handlers.h b/sys/arm64/vmm/vmm_handlers.h new file mode 100644 index 000000000000..f651fce6f32d --- /dev/null +++ b/sys/arm64/vmm/vmm_handlers.h @@ -0,0 +1,48 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Arm Ltd + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _VMM_VMM_HANDLERS_H_ +#define _VMM_VMM_HANDLERS_H_ + +#include <sys/types.h> + +struct hyp; +struct hypctx; + +void vmm_clean_s2_tlbi(void); +uint64_t vmm_enter_guest(struct hyp *, struct hypctx *); +uint64_t vmm_read_reg(uint64_t); +void vmm_s2_tlbi_range(uint64_t, vm_offset_t, vm_offset_t, bool); +void vmm_s2_tlbi_all(uint64_t); + +void vmm_vhe_clean_s2_tlbi(void); +uint64_t vmm_vhe_enter_guest(struct hyp *, struct hypctx *); +uint64_t vmm_vhe_read_reg(uint64_t); +void vmm_vhe_s2_tlbi_range(uint64_t, vm_offset_t, vm_offset_t, bool); +void vmm_vhe_s2_tlbi_all(uint64_t); + +#endif /* _VMM_VMM_HANDLERS_H_ */ diff --git a/sys/arm64/vmm/vmm_hyp.c b/sys/arm64/vmm/vmm_hyp.c index 9ff250e798e7..d61885c15871 100644 --- a/sys/arm64/vmm/vmm_hyp.c +++ b/sys/arm64/vmm/vmm_hyp.c @@ -39,9 +39,7 @@ struct hypctx; -uint64_t vmm_hyp_enter(uint64_t, uint64_t, uint64_t, uint64_t, uint64_t, - uint64_t, uint64_t, uint64_t); -uint64_t vmm_enter_guest(struct hypctx *); +uint64_t VMM_HYP_FUNC(do_call_guest)(struct hypctx *); static void vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest) @@ -51,11 +49,12 @@ vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest) /* Store the guest VFP registers */ if (guest) { /* Store the timer registers */ - hypctx->vtimer_cpu.cntkctl_el1 = READ_SPECIALREG(cntkctl_el1); + hypctx->vtimer_cpu.cntkctl_el1 = + READ_SPECIALREG(EL1_REG(CNTKCTL)); hypctx->vtimer_cpu.virt_timer.cntx_cval_el0 = - READ_SPECIALREG(cntv_cval_el0); + READ_SPECIALREG(EL0_REG(CNTV_CVAL)); hypctx->vtimer_cpu.virt_timer.cntx_ctl_el0 = - READ_SPECIALREG(cntv_ctl_el0); + READ_SPECIALREG(EL0_REG(CNTV_CTL)); /* Store the GICv3 registers */ hypctx->vgic_v3_regs.ich_eisr_el2 = @@ -221,41 +220,53 @@ vmm_hyp_reg_store(struct hypctx *hypctx, struct hyp *hyp, bool guest) hypctx->tf.tf_spsr = READ_SPECIALREG(spsr_el2); if (guest) { hypctx->tf.tf_esr = READ_SPECIALREG(esr_el2); + hypctx->par_el1 = READ_SPECIALREG(par_el1); } /* Store the guest special registers */ - hypctx->elr_el1 = READ_SPECIALREG(elr_el1); hypctx->sp_el0 = READ_SPECIALREG(sp_el0); hypctx->tpidr_el0 = READ_SPECIALREG(tpidr_el0); hypctx->tpidrro_el0 = READ_SPECIALREG(tpidrro_el0); hypctx->tpidr_el1 = READ_SPECIALREG(tpidr_el1); - hypctx->vbar_el1 = READ_SPECIALREG(vbar_el1); hypctx->actlr_el1 = READ_SPECIALREG(actlr_el1); - hypctx->afsr0_el1 = READ_SPECIALREG(afsr0_el1); - hypctx->afsr1_el1 = READ_SPECIALREG(afsr1_el1); - hypctx->amair_el1 = READ_SPECIALREG(amair_el1); - hypctx->contextidr_el1 = READ_SPECIALREG(contextidr_el1); - hypctx->cpacr_el1 = READ_SPECIALREG(cpacr_el1); hypctx->csselr_el1 = READ_SPECIALREG(csselr_el1); - hypctx->esr_el1 = READ_SPECIALREG(esr_el1); - hypctx->far_el1 = READ_SPECIALREG(far_el1); - hypctx->mair_el1 = READ_SPECIALREG(mair_el1); hypctx->mdccint_el1 = READ_SPECIALREG(mdccint_el1); hypctx->mdscr_el1 = READ_SPECIALREG(mdscr_el1); - hypctx->par_el1 = READ_SPECIALREG(par_el1); - hypctx->sctlr_el1 = READ_SPECIALREG(sctlr_el1); - hypctx->spsr_el1 = READ_SPECIALREG(spsr_el1); - hypctx->tcr_el1 = READ_SPECIALREG(tcr_el1); - /* TODO: Support when this is not res0 */ - hypctx->tcr2_el1 = 0; - hypctx->ttbr0_el1 = READ_SPECIALREG(ttbr0_el1); - hypctx->ttbr1_el1 = READ_SPECIALREG(ttbr1_el1); + + if (guest_or_nonvhe(guest)) { + hypctx->elr_el1 = READ_SPECIALREG(EL1_REG(ELR)); + hypctx->vbar_el1 = READ_SPECIALREG(EL1_REG(VBAR)); + + hypctx->afsr0_el1 = READ_SPECIALREG(EL1_REG(AFSR0)); + hypctx->afsr1_el1 = READ_SPECIALREG(EL1_REG(AFSR1)); + hypctx->amair_el1 = READ_SPECIALREG(EL1_REG(AMAIR)); + hypctx->contextidr_el1 = READ_SPECIALREG(EL1_REG(CONTEXTIDR)); + hypctx->cpacr_el1 = READ_SPECIALREG(EL1_REG(CPACR)); + hypctx->esr_el1 = READ_SPECIALREG(EL1_REG(ESR)); + hypctx->far_el1 = READ_SPECIALREG(EL1_REG(FAR)); + hypctx->mair_el1 = READ_SPECIALREG(EL1_REG(MAIR)); + hypctx->sctlr_el1 = READ_SPECIALREG(EL1_REG(SCTLR)); + hypctx->spsr_el1 = READ_SPECIALREG(EL1_REG(SPSR)); + hypctx->tcr_el1 = READ_SPECIALREG(EL1_REG(TCR)); + /* TODO: Support when this is not res0 */ + hypctx->tcr2_el1 = 0; + hypctx->ttbr0_el1 = READ_SPECIALREG(EL1_REG(TTBR0)); + hypctx->ttbr1_el1 = READ_SPECIALREG(EL1_REG(TTBR1)); + } hypctx->cptr_el2 = READ_SPECIALREG(cptr_el2); hypctx->hcr_el2 = READ_SPECIALREG(hcr_el2); hypctx->vpidr_el2 = READ_SPECIALREG(vpidr_el2); hypctx->vmpidr_el2 = READ_SPECIALREG(vmpidr_el2); + +#ifndef VMM_VHE + /* hcrx_el2 depends on feat_hcx */ + uint64_t mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); + if (ID_AA64MMFR1_HCX_VAL(mmfr1) >> ID_AA64MMFR1_HCX_SHIFT) { + hypctx->hcrx_el2 = READ_SPECIALREG(MRS_REG_ALT_NAME(HCRX_EL2)); + } +#endif } static void @@ -264,35 +275,52 @@ vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest) uint64_t dfr0; /* Restore the special registers */ - WRITE_SPECIALREG(elr_el1, hypctx->elr_el1); + WRITE_SPECIALREG(hcr_el2, hypctx->hcr_el2); + + if (guest_or_nonvhe(guest)) { + uint64_t mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); + if (ID_AA64MMFR1_HCX_VAL(mmfr1) >> ID_AA64MMFR1_HCX_SHIFT) { + WRITE_SPECIALREG(MRS_REG_ALT_NAME(HCRX_EL2), hypctx->hcrx_el2); + } + } + isb(); + WRITE_SPECIALREG(sp_el0, hypctx->sp_el0); WRITE_SPECIALREG(tpidr_el0, hypctx->tpidr_el0); WRITE_SPECIALREG(tpidrro_el0, hypctx->tpidrro_el0); WRITE_SPECIALREG(tpidr_el1, hypctx->tpidr_el1); - WRITE_SPECIALREG(vbar_el1, hypctx->vbar_el1); WRITE_SPECIALREG(actlr_el1, hypctx->actlr_el1); - WRITE_SPECIALREG(afsr0_el1, hypctx->afsr0_el1); - WRITE_SPECIALREG(afsr1_el1, hypctx->afsr1_el1); - WRITE_SPECIALREG(amair_el1, hypctx->amair_el1); - WRITE_SPECIALREG(contextidr_el1, hypctx->contextidr_el1); - WRITE_SPECIALREG(cpacr_el1, hypctx->cpacr_el1); WRITE_SPECIALREG(csselr_el1, hypctx->csselr_el1); - WRITE_SPECIALREG(esr_el1, hypctx->esr_el1); - WRITE_SPECIALREG(far_el1, hypctx->far_el1); WRITE_SPECIALREG(mdccint_el1, hypctx->mdccint_el1); WRITE_SPECIALREG(mdscr_el1, hypctx->mdscr_el1); - WRITE_SPECIALREG(mair_el1, hypctx->mair_el1); - WRITE_SPECIALREG(par_el1, hypctx->par_el1); - WRITE_SPECIALREG(sctlr_el1, hypctx->sctlr_el1); - WRITE_SPECIALREG(tcr_el1, hypctx->tcr_el1); - /* TODO: tcr2_el1 */ - WRITE_SPECIALREG(ttbr0_el1, hypctx->ttbr0_el1); - WRITE_SPECIALREG(ttbr1_el1, hypctx->ttbr1_el1); - WRITE_SPECIALREG(spsr_el1, hypctx->spsr_el1); + + if (guest_or_nonvhe(guest)) { + WRITE_SPECIALREG(EL1_REG(ELR), hypctx->elr_el1); + WRITE_SPECIALREG(EL1_REG(VBAR), hypctx->vbar_el1); + + WRITE_SPECIALREG(EL1_REG(AFSR0), hypctx->afsr0_el1); + WRITE_SPECIALREG(EL1_REG(AFSR1), hypctx->afsr1_el1); + WRITE_SPECIALREG(EL1_REG(AMAIR), hypctx->amair_el1); + WRITE_SPECIALREG(EL1_REG(CONTEXTIDR), hypctx->contextidr_el1); + WRITE_SPECIALREG(EL1_REG(CPACR), hypctx->cpacr_el1); + WRITE_SPECIALREG(EL1_REG(ESR), hypctx->esr_el1); + WRITE_SPECIALREG(EL1_REG(FAR), hypctx->far_el1); + WRITE_SPECIALREG(EL1_REG(MAIR), hypctx->mair_el1); // + + WRITE_SPECIALREG(EL1_REG(SCTLR), hypctx->sctlr_el1); + WRITE_SPECIALREG(EL1_REG(SPSR), hypctx->spsr_el1); + WRITE_SPECIALREG(EL1_REG(TCR), hypctx->tcr_el1); + /* TODO: tcr2_el1 */ + WRITE_SPECIALREG(EL1_REG(TTBR0), hypctx->ttbr0_el1); + WRITE_SPECIALREG(EL1_REG(TTBR1), hypctx->ttbr1_el1); + } + + if (guest) { + WRITE_SPECIALREG(par_el1, hypctx->par_el1); + } WRITE_SPECIALREG(cptr_el2, hypctx->cptr_el2); - WRITE_SPECIALREG(hcr_el2, hypctx->hcr_el2); WRITE_SPECIALREG(vpidr_el2, hypctx->vpidr_el2); WRITE_SPECIALREG(vmpidr_el2, hypctx->vmpidr_el2); @@ -413,10 +441,11 @@ vmm_hyp_reg_restore(struct hypctx *hypctx, struct hyp *hyp, bool guest) if (guest) { /* Load the timer registers */ - WRITE_SPECIALREG(cntkctl_el1, hypctx->vtimer_cpu.cntkctl_el1); - WRITE_SPECIALREG(cntv_cval_el0, + WRITE_SPECIALREG(EL1_REG(CNTKCTL), + hypctx->vtimer_cpu.cntkctl_el1); + WRITE_SPECIALREG(EL0_REG(CNTV_CVAL), hypctx->vtimer_cpu.virt_timer.cntx_cval_el0); - WRITE_SPECIALREG(cntv_ctl_el0, + WRITE_SPECIALREG(EL0_REG(CNTV_CTL), hypctx->vtimer_cpu.virt_timer.cntx_ctl_el0); WRITE_SPECIALREG(cnthctl_el2, hyp->vtimer.cnthctl_el2); WRITE_SPECIALREG(cntvoff_el2, hyp->vtimer.cntvoff_el2); @@ -496,7 +525,7 @@ vmm_hyp_call_guest(struct hyp *hyp, struct hypctx *hypctx) WRITE_SPECIALREG(mdcr_el2, hypctx->mdcr_el2); /* Call into the guest */ - ret = vmm_enter_guest(hypctx); + ret = VMM_HYP_FUNC(do_call_guest)(hypctx); WRITE_SPECIALREG(mdcr_el2, host_hypctx.mdcr_el2); isb(); @@ -566,8 +595,20 @@ vmm_hyp_call_guest(struct hyp *hyp, struct hypctx *hypctx) return (ret); } -static uint64_t -vmm_hyp_read_reg(uint64_t reg) +VMM_STATIC uint64_t +VMM_HYP_FUNC(enter_guest)(struct hyp *hyp, struct hypctx *hypctx) +{ + uint64_t ret; + + do { + ret = vmm_hyp_call_guest(hyp, hypctx); + } while (ret == EXCP_TYPE_REENTER); + + return (ret); +} + +VMM_STATIC uint64_t +VMM_HYP_FUNC(read_reg)(uint64_t reg) { switch (reg) { case HYP_REG_ICH_VTR: @@ -579,22 +620,27 @@ vmm_hyp_read_reg(uint64_t reg) return (0); } -static int -vmm_clean_s2_tlbi(void) +VMM_STATIC void +VMM_HYP_FUNC(clean_s2_tlbi)(void) { dsb(ishst); __asm __volatile("tlbi alle1is"); dsb(ish); - - return (0); } -static int -vm_s2_tlbi_range(uint64_t vttbr, vm_offset_t sva, vm_size_t eva, +VMM_STATIC void +VMM_HYP_FUNC(s2_tlbi_range)(uint64_t vttbr, vm_offset_t sva, vm_offset_t eva, bool final_only) { uint64_t end, r, start; uint64_t host_vttbr; +#ifdef VMM_VHE + uint64_t host_tcr; +#endif + +#ifdef VMM_VHE + dsb(ishst); +#endif #define TLBI_VA_SHIFT 12 #define TLBI_VA_MASK ((1ul << 44) - 1) @@ -607,6 +653,12 @@ vm_s2_tlbi_range(uint64_t vttbr, vm_offset_t sva, vm_size_t eva, WRITE_SPECIALREG(vttbr_el2, vttbr); isb(); +#ifdef VMM_VHE + host_tcr = READ_SPECIALREG(tcr_el2); + WRITE_SPECIALREG(tcr_el2, host_tcr & ~HCR_TGE); + isb(); +#endif + /* * The CPU can cache the stage 1 + 2 combination so we need to ensure * the stage 2 is invalidated first, then when this has completed we @@ -631,18 +683,25 @@ vm_s2_tlbi_range(uint64_t vttbr, vm_offset_t sva, vm_size_t eva, dsb(ish); isb(); - /* Switch back t othe host vttbr */ - WRITE_SPECIALREG(vttbr_el2, host_vttbr); +#ifdef VMM_VHE + WRITE_SPECIALREG(tcr_el2, host_tcr); isb(); +#endif - return (0); + /* Switch back to the host vttbr */ + WRITE_SPECIALREG(vttbr_el2, host_vttbr); + isb(); } -static int -vm_s2_tlbi_all(uint64_t vttbr) +VMM_STATIC void +VMM_HYP_FUNC(s2_tlbi_all)(uint64_t vttbr) { uint64_t host_vttbr; +#ifdef VMM_VHE + dsb(ishst); +#endif + /* Switch to the guest vttbr */ /* TODO: Handle Cortex-A57/A72 erratum 131936 */ host_vttbr = READ_SPECIALREG(vttbr_el2); @@ -656,80 +715,4 @@ vm_s2_tlbi_all(uint64_t vttbr) /* Switch back t othe host vttbr */ WRITE_SPECIALREG(vttbr_el2, host_vttbr); isb(); - - return (0); -} - -static int -vmm_dc_civac(uint64_t start, uint64_t len) -{ - size_t line_size, end; - uint64_t ctr; - - ctr = READ_SPECIALREG(ctr_el0); - line_size = sizeof(int) << CTR_DLINE_SIZE(ctr); - end = start + len; - dsb(ishst); - /* Clean and Invalidate the D-cache */ - for (; start < end; start += line_size) - __asm __volatile("dc civac, %0" :: "r" (start) : "memory"); - dsb(ish); - return (0); -} - -static int -vmm_el2_tlbi(uint64_t type, uint64_t start, uint64_t len) -{ - uint64_t end, r; - - dsb(ishst); - switch (type) { - default: - case HYP_EL2_TLBI_ALL: - __asm __volatile("tlbi alle2" ::: "memory"); - break; - case HYP_EL2_TLBI_VA: - end = TLBI_VA(start + len); - start = TLBI_VA(start); - for (r = start; r < end; r += TLBI_VA_L3_INCR) { - __asm __volatile("tlbi vae2is, %0" :: "r"(r)); - } - break; - } - dsb(ish); - - return (0); -} - -uint64_t -vmm_hyp_enter(uint64_t handle, uint64_t x1, uint64_t x2, uint64_t x3, - uint64_t x4, uint64_t x5, uint64_t x6, uint64_t x7) -{ - uint64_t ret; - - switch (handle) { - case HYP_ENTER_GUEST: - do { - ret = vmm_hyp_call_guest((struct hyp *)x1, - (struct hypctx *)x2); - } while (ret == EXCP_TYPE_REENTER); - return (ret); - case HYP_READ_REGISTER: - return (vmm_hyp_read_reg(x1)); - case HYP_CLEAN_S2_TLBI: - return (vmm_clean_s2_tlbi()); - case HYP_DC_CIVAC: - return (vmm_dc_civac(x1, x2)); - case HYP_EL2_TLBI: - return (vmm_el2_tlbi(x1, x2, x3)); - case HYP_S2_TLBI_RANGE: - return (vm_s2_tlbi_range(x1, x2, x3, x4)); - case HYP_S2_TLBI_ALL: - return (vm_s2_tlbi_all(x1)); - case HYP_CLEANUP: /* Handled in vmm_hyp_exception.S */ - default: - break; - } - - return (0); } diff --git a/sys/arm64/vmm/vmm_hyp_el2.S b/sys/arm64/vmm/vmm_hyp_el2.S index 7b49d3144dff..0ba040ee7bad 100644 --- a/sys/arm64/vmm/vmm_hyp_el2.S +++ b/sys/arm64/vmm/vmm_hyp_el2.S @@ -28,12 +28,17 @@ * SUCH DAMAGE. */ +#include <sys/elf_common.h> + +#include <machine/asm.h> #include <machine/param.h> - .rodata + .section .rodata .align PAGE_SHIFT .globl vmm_hyp_code vmm_hyp_code: .incbin "vmm_hyp_blob.bin" .globl vmm_hyp_code_end vmm_hyp_code_end: + +GNU_PROPERTY_AARCH64_FEATURE_1_NOTE(GNU_PROPERTY_AARCH64_FEATURE_1_VAL) diff --git a/sys/arm64/vmm/vmm_hyp_exception.S b/sys/arm64/vmm/vmm_hyp_exception.S index 0e8b31ae8b12..50c2490f37bf 100644 --- a/sys/arm64/vmm/vmm_hyp_exception.S +++ b/sys/arm64/vmm/vmm_hyp_exception.S @@ -30,6 +30,7 @@ */ +#include <sys/elf_common.h> #include <machine/asm.h> #include <machine/hypervisor.h> @@ -145,29 +146,6 @@ b handle_\name .endm - .section ".vmm_vectors","ax" - .align 11 -hyp_init_vectors: - vempty /* Synchronous EL2t */ - vempty /* IRQ EL2t */ - vempty /* FIQ EL2t */ - vempty /* Error EL2t */ - - vempty /* Synchronous EL2h */ - vempty /* IRQ EL2h */ - vempty /* FIQ EL2h */ - vempty /* Error EL2h */ - - vector hyp_init /* Synchronous 64-bit EL1 */ - vempty /* IRQ 64-bit EL1 */ - vempty /* FIQ 64-bit EL1 */ - vempty /* Error 64-bit EL1 */ - - vempty /* Synchronous 32-bit EL1 */ - vempty /* IRQ 32-bit EL1 */ - vempty /* FIQ 32-bit EL1 */ - vempty /* Error 32-bit EL1 */ - .text .align 11 hyp_vectors: @@ -191,50 +169,6 @@ hyp_vectors: vempty /* FIQ 32-bit EL1 */ vempty /* Error 32-bit EL1 */ -/* - * Initialize the hypervisor mode with a new exception vector table, translation - * table and stack. - * - * Expecting: - * x0 - translation tables physical address - * x1 - stack top virtual address - * x2 - TCR_EL2 value - * x3 - SCTLR_EL2 value - * x4 - VTCR_EL2 value - */ -LENTRY(handle_hyp_init) - /* Install the new exception vectors */ - adrp x6, hyp_vectors - add x6, x6, :lo12:hyp_vectors - msr vbar_el2, x6 - /* Set the stack top address */ - mov sp, x1 - /* Use the host VTTBR_EL2 to tell the host and the guests apart */ - mov x9, #VTTBR_HOST - msr vttbr_el2, x9 - /* Load the base address for the translation tables */ - msr ttbr0_el2, x0 - /* Invalidate the TLB */ - dsb ish - tlbi alle2 - dsb ishst - isb - /* Use the same memory attributes as EL1 */ - mrs x9, mair_el1 - msr mair_el2, x9 - /* Configure address translation */ - msr tcr_el2, x2 - isb - /* Set the system control register for EL2 */ - msr sctlr_el2, x3 - /* Set the Stage 2 translation control register */ - msr vtcr_el2, x4 - /* Return success */ - mov x0, #0 - /* MMU is up and running */ - ERET -LEND(handle_hyp_init) - .macro do_world_switch_to_host save_guest_registers restore_host_registers @@ -242,10 +176,15 @@ LEND(handle_hyp_init) /* Restore host VTTBR */ mov x9, #VTTBR_HOST msr vttbr_el2, x9 + +#ifdef VMM_VHE + msr vbar_el1, x1 +#endif .endm .macro handle_el2_excp type +#ifndef VMM_VHE /* Save registers before modifying so we can restore them */ str x9, [sp, #-16]! @@ -256,15 +195,18 @@ LEND(handle_hyp_init) /* We got the exception while the guest was running */ ldr x9, [sp], #16 +#endif /* !VMM_VHE */ do_world_switch_to_host mov x0, \type ret +#ifndef VMM_VHE 1: /* We got the exception while the host was running */ ldr x9, [sp], #16 mov x0, \type ERET +#endif /* !VMM_VHE */ .endm @@ -286,6 +228,7 @@ LEND(handle_el2_el2h_error) LENTRY(handle_el2_el1_sync64) +#ifndef VMM_VHE /* Save registers before modifying so we can restore them */ str x9, [sp, #-16]! @@ -308,7 +251,9 @@ LENTRY(handle_el2_el1_sync64) ldr lr, [sp], #16 ERET -1: /* Guest exception taken to EL2 */ +1: +#endif + /* Guest exception taken to EL2 */ do_world_switch_to_host mov x0, #EXCP_TYPE_EL1_SYNC ret @@ -332,7 +277,7 @@ LENTRY(handle_el2_el1_irq64) 2: ldr x9, [sp], #16 ret -LEND(handle_el2_el1_irq) +LEND(handle_el2_el1_irq64) LENTRY(handle_el2_el1_fiq64) do_world_switch_to_host @@ -349,12 +294,20 @@ LEND(handle_el2_el1_error64) /* * Usage: - * uint64_t vmm_enter_guest(struct hypctx *hypctx) + * uint64_t vmm_do_call_guest(struct hypctx *hypctx) * * Expecting: * x0 - hypctx address */ -ENTRY(vmm_enter_guest) +ENTRY(VMM_HYP_FUNC(do_call_guest)) +#ifdef VMM_VHE + mrs x1, vbar_el1 + adrp x2, hyp_vectors + add x2, x2, :lo12:hyp_vectors + msr vbar_el1, x2 + isb +#endif + /* Save hypctx address */ msr tpidr_el2, x0 @@ -363,25 +316,6 @@ ENTRY(vmm_enter_guest) /* Enter guest */ ERET -END(vmm_enter_guest) - -/* - * Usage: - * void vmm_cleanup(uint64_t handle, void *hyp_stub_vectors) - * - * Expecting: - * x1 - physical address of hyp_stub_vectors - */ -LENTRY(vmm_cleanup) - /* Restore the stub vectors */ - msr vbar_el2, x1 - - /* Disable the MMU */ - dsb sy - mrs x2, sctlr_el2 - bic x2, x2, #SCTLR_EL2_M - msr sctlr_el2, x2 - isb +END(VMM_HYP_FUNC(do_call_guest)) - ERET -LEND(vmm_cleanup) +GNU_PROPERTY_AARCH64_FEATURE_1_NOTE(GNU_PROPERTY_AARCH64_FEATURE_1_VAL) diff --git a/sys/arm64/vmm/vmm_ktr.h b/sys/arm64/vmm/vmm_ktr.h deleted file mode 100644 index 965f440ae874..000000000000 --- a/sys/arm64/vmm/vmm_ktr.h +++ /dev/null @@ -1,69 +0,0 @@ -/*- - * SPDX-License-Identifier: BSD-2-Clause-FreeBSD - * - * Copyright (c) 2011 NetApp, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#ifndef _VMM_KTR_H_ -#define _VMM_KTR_H_ - -#include <sys/ktr.h> -#include <sys/pcpu.h> - -#ifndef KTR_VMM -#define KTR_VMM KTR_GEN -#endif - -#define VCPU_CTR0(vm, vcpuid, format) \ -CTR2(KTR_VMM, "vm %s[%d]: " format, vm_name((vm)), (vcpuid)) - -#define VCPU_CTR1(vm, vcpuid, format, p1) \ -CTR3(KTR_VMM, "vm %s[%d]: " format, vm_name((vm)), (vcpuid), (p1)) - -#define VCPU_CTR2(vm, vcpuid, format, p1, p2) \ -CTR4(KTR_VMM, "vm %s[%d]: " format, vm_name((vm)), (vcpuid), (p1), (p2)) - -#define VCPU_CTR3(vm, vcpuid, format, p1, p2, p3) \ -CTR5(KTR_VMM, "vm %s[%d]: " format, vm_name((vm)), (vcpuid), (p1), (p2), (p3)) - -#define VCPU_CTR4(vm, vcpuid, format, p1, p2, p3, p4) \ -CTR6(KTR_VMM, "vm %s[%d]: " format, vm_name((vm)), (vcpuid), \ - (p1), (p2), (p3), (p4)) - -#define VM_CTR0(vm, format) \ -CTR1(KTR_VMM, "vm %s: " format, vm_name((vm))) - -#define VM_CTR1(vm, format, p1) \ -CTR2(KTR_VMM, "vm %s: " format, vm_name((vm)), (p1)) - -#define VM_CTR2(vm, format, p1, p2) \ -CTR3(KTR_VMM, "vm %s: " format, vm_name((vm)), (p1), (p2)) - -#define VM_CTR3(vm, format, p1, p2, p3) \ -CTR4(KTR_VMM, "vm %s: " format, vm_name((vm)), (p1), (p2), (p3)) - -#define VM_CTR4(vm, format, p1, p2, p3, p4) \ -CTR5(KTR_VMM, "vm %s: " format, vm_name((vm)), (p1), (p2), (p3), (p4)) -#endif diff --git a/sys/arm64/vmm/vmm_mmu.c b/sys/arm64/vmm/vmm_mmu.c index 1f2d248a743b..42537254e27b 100644 --- a/sys/arm64/vmm/vmm_mmu.c +++ b/sys/arm64/vmm/vmm_mmu.c @@ -294,7 +294,7 @@ vmmpmap_enter(vm_offset_t va, vm_size_t size, vm_paddr_t pa, vm_prot_t prot) KASSERT((size & PAGE_MASK) == 0, ("%s: Mapping is not page-sized", __func__)); - l3e = ATTR_DEFAULT | L3_PAGE; + l3e = ATTR_AF | ATTR_SH(ATTR_SH_IS) | L3_PAGE; /* This bit is res1 at EL2 */ l3e |= ATTR_S1_AP(ATTR_S1_AP_USER); /* Only normal memory is used at EL2 */ diff --git a/sys/arm64/vmm/vmm_nvhe.c b/sys/arm64/vmm/vmm_nvhe.c new file mode 100644 index 000000000000..025b1308ce68 --- /dev/null +++ b/sys/arm64/vmm/vmm_nvhe.c @@ -0,0 +1,118 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2021 Andrew Turner + * Copyright (c) 2024 Arm Ltd + * + * This work was supported by Innovate UK project 105694, "Digital Security + * by Design (DSbD) Technology Platform Prototype". + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#define VMM_STATIC static +#define VMM_HYP_FUNC(func) vmm_nvhe_ ## func + +#define guest_or_nonvhe(guest) (true) +#define EL1_REG(reg) MRS_REG_ALT_NAME(reg ## _EL1) +#define EL0_REG(reg) MRS_REG_ALT_NAME(reg ## _EL0) + +#include "vmm_hyp.c" + +uint64_t vmm_hyp_enter(uint64_t, uint64_t, uint64_t, uint64_t, uint64_t, + uint64_t, uint64_t, uint64_t); + +/* + * Handlers for EL2 addres space. Only needed by non-VHE code as in VHE the + * kernel is in EL2 so pmap will manage the address space. + */ +static int +vmm_dc_civac(uint64_t start, uint64_t len) +{ + size_t line_size, end; + uint64_t ctr; + + ctr = READ_SPECIALREG(ctr_el0); + line_size = sizeof(int) << CTR_DLINE_SIZE(ctr); + end = start + len; + dsb(ishst); + /* Clean and Invalidate the D-cache */ + for (; start < end; start += line_size) + __asm __volatile("dc civac, %0" :: "r" (start) : "memory"); + dsb(ish); + return (0); +} + +static int +vmm_el2_tlbi(uint64_t type, uint64_t start, uint64_t len) +{ + uint64_t end, r; + + dsb(ishst); + switch (type) { + default: + case HYP_EL2_TLBI_ALL: + __asm __volatile("tlbi alle2" ::: "memory"); + break; + case HYP_EL2_TLBI_VA: + end = TLBI_VA(start + len); + start = TLBI_VA(start); + for (r = start; r < end; r += TLBI_VA_L3_INCR) { + __asm __volatile("tlbi vae2is, %0" :: "r"(r)); + } + break; + } + dsb(ish); + + return (0); +} + +uint64_t +vmm_hyp_enter(uint64_t handle, uint64_t x1, uint64_t x2, uint64_t x3, + uint64_t x4, uint64_t x5, uint64_t x6, uint64_t x7) +{ + switch (handle) { + case HYP_ENTER_GUEST: + return (VMM_HYP_FUNC(enter_guest)((struct hyp *)x1, + (struct hypctx *)x2)); + case HYP_READ_REGISTER: + return (VMM_HYP_FUNC(read_reg)(x1)); + case HYP_CLEAN_S2_TLBI: + VMM_HYP_FUNC(clean_s2_tlbi()); + return (0); + case HYP_DC_CIVAC: + return (vmm_dc_civac(x1, x2)); + case HYP_EL2_TLBI: + return (vmm_el2_tlbi(x1, x2, x3)); + case HYP_S2_TLBI_RANGE: + VMM_HYP_FUNC(s2_tlbi_range)(x1, x2, x3, x4); + return (0); + case HYP_S2_TLBI_ALL: + VMM_HYP_FUNC(s2_tlbi_all)(x1); + return (0); + case HYP_CLEANUP: /* Handled in vmm_hyp_exception.S */ + default: + break; + } + + return (0); +} diff --git a/sys/arm64/vmm/vmm_nvhe_exception.S b/sys/arm64/vmm/vmm_nvhe_exception.S new file mode 100644 index 000000000000..17bc4cb70366 --- /dev/null +++ b/sys/arm64/vmm/vmm_nvhe_exception.S @@ -0,0 +1,120 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Arm Ltd + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#define VMM_HYP_FUNC(func) vmm_nvhe_ ## func + +#include "vmm_hyp_exception.S" + + .section ".vmm_vectors","ax" + .align 11 +hyp_init_vectors: + vempty /* Synchronous EL2t */ + vempty /* IRQ EL2t */ + vempty /* FIQ EL2t */ + vempty /* Error EL2t */ + + vempty /* Synchronous EL2h */ + vempty /* IRQ EL2h */ + vempty /* FIQ EL2h */ + vempty /* Error EL2h */ + + vector hyp_init /* Synchronous 64-bit EL1 */ + vempty /* IRQ 64-bit EL1 */ + vempty /* FIQ 64-bit EL1 */ + vempty /* Error 64-bit EL1 */ + + vempty /* Synchronous 32-bit EL1 */ + vempty /* IRQ 32-bit EL1 */ + vempty /* FIQ 32-bit EL1 */ + vempty /* Error 32-bit EL1 */ + + .text + +/* + * Initialize the hypervisor mode with a new exception vector table, translation + * table and stack. + * + * Expecting: + * x0 - translation tables physical address + * x1 - stack top virtual address + * x2 - TCR_EL2 value + * x3 - SCTLR_EL2 value + * x4 - VTCR_EL2 value + */ +LENTRY(handle_hyp_init) + /* Install the new exception vectors */ + adrp x6, hyp_vectors + add x6, x6, :lo12:hyp_vectors + msr vbar_el2, x6 + /* Set the stack top address */ + mov sp, x1 + /* Use the host VTTBR_EL2 to tell the host and the guests apart */ + mov x9, #VTTBR_HOST + msr vttbr_el2, x9 + /* Load the base address for the translation tables */ + msr ttbr0_el2, x0 + /* Invalidate the TLB */ + dsb ish + tlbi alle2 + dsb ishst + isb + /* Use the same memory attributes as EL1 */ + mrs x9, mair_el1 + msr mair_el2, x9 + /* Configure address translation */ + msr tcr_el2, x2 + isb + /* Set the system control register for EL2 */ + msr sctlr_el2, x3 + /* Set the Stage 2 translation control register */ + msr vtcr_el2, x4 + /* Return success */ + mov x0, #0 + /* MMU is up and running */ + ERET +LEND(handle_hyp_init) + +/* + * Usage: + * void vmm_cleanup(uint64_t handle, void *hyp_stub_vectors) + * + * Expecting: + * x1 - physical address of hyp_stub_vectors + */ +LENTRY(vmm_cleanup) + /* Restore the stub vectors */ + msr vbar_el2, x1 + + /* Disable the MMU */ + dsb sy + mrs x2, sctlr_el2 + bic x2, x2, #SCTLR_EL2_M + msr sctlr_el2, x2 + isb + + ERET +LEND(vmm_cleanup) diff --git a/sys/arm64/vmm/vmm_reset.c b/sys/arm64/vmm/vmm_reset.c index a929a60c9474..79d022cf33e8 100644 --- a/sys/arm64/vmm/vmm_reset.c +++ b/sys/arm64/vmm/vmm_reset.c @@ -136,7 +136,12 @@ reset_vm_el2_regs(void *vcpu) */ el2ctx->hcr_el2 = HCR_RW | HCR_TID3 | HCR_TWI | HCR_BSU_IS | HCR_FB | HCR_AMO | HCR_IMO | HCR_FMO | HCR_SWIO | HCR_VM; + if (in_vhe()) { + el2ctx->hcr_el2 |= HCR_E2H; + } + /* Set the Extended Hypervisor Configuration Register */ + el2ctx->hcrx_el2 = 0; /* TODO: Trap all extensions we don't support */ el2ctx->mdcr_el2 = 0; /* PMCR_EL0.N is read from MDCR_EL2.HPMN */ @@ -166,7 +171,11 @@ reset_vm_el2_regs(void *vcpu) * Don't trap accesses to CPACR_EL1, trace, SVE, Advanced SIMD * and floating point functionality to EL2. */ - el2ctx->cptr_el2 = CPTR_RES1; + if (in_vhe()) + el2ctx->cptr_el2 = CPTR_E2H_TRAP_ALL | CPTR_E2H_FPEN; + else + el2ctx->cptr_el2 = CPTR_TRAP_ALL & ~CPTR_TFP; + el2ctx->cptr_el2 &= ~CPTR_TCPAC; /* * Disable interrupts in the guest. The guest OS will re-enable * them. diff --git a/sys/arm64/vmm/vmm_stat.c b/sys/arm64/vmm/vmm_stat.c deleted file mode 100644 index 858ce980843a..000000000000 --- a/sys/arm64/vmm/vmm_stat.c +++ /dev/null @@ -1,165 +0,0 @@ -/*- - * SPDX-License-Identifier: BSD-2-Clause - * - * Copyright (c) 2011 NetApp, Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include <sys/cdefs.h> - -#include <sys/param.h> -#include <sys/kernel.h> -#include <sys/systm.h> -#include <sys/malloc.h> - -#include <machine/machdep.h> -#include <machine/vmm.h> -#include "vmm_stat.h" - -/* - * 'vst_num_elems' is the total number of addressable statistic elements - * 'vst_num_types' is the number of unique statistic types - * - * It is always true that 'vst_num_elems' is greater than or equal to - * 'vst_num_types'. This is because a stat type may represent more than - * one element (for e.g. VMM_STAT_ARRAY). - */ -static int vst_num_elems, vst_num_types; -static struct vmm_stat_type *vsttab[MAX_VMM_STAT_ELEMS]; - -static MALLOC_DEFINE(M_VMM_STAT, "vmm stat", "vmm stat"); - -#define vst_size ((size_t)vst_num_elems * sizeof(uint64_t)) - -void -vmm_stat_register(void *arg) -{ - struct vmm_stat_type *vst = arg; - - /* We require all stats to identify themselves with a description */ - if (vst->desc == NULL) - return; - - if (vst_num_elems + vst->nelems >= MAX_VMM_STAT_ELEMS) { - printf("Cannot accommodate vmm stat type \"%s\"!\n", vst->desc); - return; - } - - vst->index = vst_num_elems; - vst_num_elems += vst->nelems; - - vsttab[vst_num_types++] = vst; -} - -int -vmm_stat_copy(struct vcpu *vcpu, int index, int count, int *num_stats, - uint64_t *buf) -{ - struct vmm_stat_type *vst; - uint64_t *stats; - int i, tocopy; - - if (index < 0 || count < 0) - return (EINVAL); - - if (index > vst_num_elems) - return (ENOENT); - - if (index == vst_num_elems) { - *num_stats = 0; - return (0); - } - - tocopy = min(vst_num_elems - index, count); - - /* Let stats functions update their counters */ - for (i = 0; i < vst_num_types; i++) { - vst = vsttab[i]; - if (vst->func != NULL) - (*vst->func)(vcpu, vst); - } - - /* Copy over the stats */ - stats = vcpu_stats(vcpu); - memcpy(buf, stats + index, tocopy * sizeof(stats[0])); - *num_stats = tocopy; - return (0); -} - -void * -vmm_stat_alloc(void) -{ - - return (malloc(vst_size, M_VMM_STAT, M_WAITOK)); -} - -void -vmm_stat_init(void *vp) -{ - - bzero(vp, vst_size); -} - -void -vmm_stat_free(void *vp) -{ - free(vp, M_VMM_STAT); -} - -int -vmm_stat_desc_copy(int index, char *buf, int bufsize) -{ - int i; - struct vmm_stat_type *vst; - - for (i = 0; i < vst_num_types; i++) { - vst = vsttab[i]; - if (index >= vst->index && index < vst->index + vst->nelems) { - if (vst->nelems > 1) { - snprintf(buf, bufsize, "%s[%d]", - vst->desc, index - vst->index); - } else { - strlcpy(buf, vst->desc, bufsize); - } - return (0); /* found it */ - } - } - - return (EINVAL); -} - -/* global statistics */ -VMM_STAT(VMEXIT_COUNT, "total number of vm exits"); -VMM_STAT(VMEXIT_UNKNOWN, "number of vmexits for the unknown exception"); -VMM_STAT(VMEXIT_WFI, "number of times wfi was intercepted"); -VMM_STAT(VMEXIT_WFE, "number of times wfe was intercepted"); -VMM_STAT(VMEXIT_HVC, "number of times hvc was intercepted"); -VMM_STAT(VMEXIT_MSR, "number of times msr/mrs was intercepted"); -VMM_STAT(VMEXIT_DATA_ABORT, "number of vmexits for a data abort"); -VMM_STAT(VMEXIT_INSN_ABORT, "number of vmexits for an instruction abort"); -VMM_STAT(VMEXIT_UNHANDLED_SYNC, "number of vmexits for an unhandled synchronous exception"); -VMM_STAT(VMEXIT_IRQ, "number of vmexits for an irq"); -VMM_STAT(VMEXIT_FIQ, "number of vmexits for an interrupt"); -VMM_STAT(VMEXIT_UNHANDLED_EL2, "number of vmexits for an unhandled EL2 exception"); -VMM_STAT(VMEXIT_UNHANDLED, "number of vmexits for an unhandled exception"); diff --git a/sys/arm64/vmm/vmm_stat.h b/sys/arm64/vmm/vmm_stat.h index b0a06ef79253..0dc3eeced603 100644 --- a/sys/arm64/vmm/vmm_stat.h +++ b/sys/arm64/vmm/vmm_stat.h @@ -32,102 +32,7 @@ #ifndef _VMM_STAT_H_ #define _VMM_STAT_H_ -struct vm; - -#define MAX_VMM_STAT_ELEMS 64 /* arbitrary */ - -enum vmm_stat_scope { - VMM_STAT_SCOPE_ANY, -}; - -struct vmm_stat_type; -typedef void (*vmm_stat_func_t)(struct vcpu *vcpu, - struct vmm_stat_type *stat); - -struct vmm_stat_type { - int index; /* position in the stats buffer */ - int nelems; /* standalone or array */ - const char *desc; /* description of statistic */ - vmm_stat_func_t func; - enum vmm_stat_scope scope; -}; - -void vmm_stat_register(void *arg); - -#define VMM_STAT_FDEFINE(type, nelems, desc, func, scope) \ - struct vmm_stat_type type[1] = { \ - { -1, nelems, desc, func, scope } \ - }; \ - SYSINIT(type##_stat, SI_SUB_KLD, SI_ORDER_ANY, vmm_stat_register, type) - -#define VMM_STAT_DEFINE(type, nelems, desc, scope) \ - VMM_STAT_FDEFINE(type, nelems, desc, NULL, scope) - -#define VMM_STAT_DECLARE(type) \ - extern struct vmm_stat_type type[1] - -#define VMM_STAT(type, desc) \ - VMM_STAT_DEFINE(type, 1, desc, VMM_STAT_SCOPE_ANY) - -#define VMM_STAT_FUNC(type, desc, func) \ - VMM_STAT_FDEFINE(type, 1, desc, func, VMM_STAT_SCOPE_ANY) - -#define VMM_STAT_ARRAY(type, nelems, desc) \ - VMM_STAT_DEFINE(type, nelems, desc, VMM_STAT_SCOPE_ANY) - -void *vmm_stat_alloc(void); -void vmm_stat_init(void *vp); -void vmm_stat_free(void *vp); - -int vmm_stat_copy(struct vcpu *vcpu, int index, int count, - int *num_stats, uint64_t *buf); -int vmm_stat_desc_copy(int index, char *buf, int buflen); - -static void __inline -vmm_stat_array_incr(struct vcpu *vcpu, struct vmm_stat_type *vst, int statidx, - uint64_t x) -{ -#ifdef VMM_KEEP_STATS - uint64_t *stats; - - stats = vcpu_stats(vcpu); - - if (vst->index >= 0 && statidx < vst->nelems) - stats[vst->index + statidx] += x; -#endif -} - -static void __inline -vmm_stat_array_set(struct vcpu *vcpu, struct vmm_stat_type *vst, int statidx, - uint64_t val) -{ -#ifdef VMM_KEEP_STATS - uint64_t *stats; - - stats = vcpu_stats(vcpu); - - if (vst->index >= 0 && statidx < vst->nelems) - stats[vst->index + statidx] = val; -#endif -} - -static void __inline -vmm_stat_incr(struct vcpu *vcpu, struct vmm_stat_type *vst, uint64_t x) -{ - -#ifdef VMM_KEEP_STATS - vmm_stat_array_incr(vcpu, vst, 0, x); -#endif -} - -static void __inline -vmm_stat_set(struct vcpu *vcpu, struct vmm_stat_type *vst, uint64_t val) -{ - -#ifdef VMM_KEEP_STATS - vmm_stat_array_set(vcpu, vst, 0, val); -#endif -} +#include <dev/vmm/vmm_stat.h> VMM_STAT_DECLARE(VMEXIT_COUNT); VMM_STAT_DECLARE(VMEXIT_UNKNOWN); @@ -140,6 +45,9 @@ VMM_STAT_DECLARE(VMEXIT_INSN_ABORT); VMM_STAT_DECLARE(VMEXIT_UNHANDLED_SYNC); VMM_STAT_DECLARE(VMEXIT_IRQ); VMM_STAT_DECLARE(VMEXIT_FIQ); +VMM_STAT_DECLARE(VMEXIT_BRK); +VMM_STAT_DECLARE(VMEXIT_SS); VMM_STAT_DECLARE(VMEXIT_UNHANDLED_EL2); VMM_STAT_DECLARE(VMEXIT_UNHANDLED); + #endif diff --git a/sys/arm64/vmm/vmm_vhe.c b/sys/arm64/vmm/vmm_vhe.c new file mode 100644 index 000000000000..8a12852e2a7a --- /dev/null +++ b/sys/arm64/vmm/vmm_vhe.c @@ -0,0 +1,39 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Arm Ltd + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "vmm_handlers.h" + +#define VMM_VHE + +#define VMM_STATIC +#define VMM_HYP_FUNC(func) vmm_vhe_ ## func + +#define guest_or_nonvhe(guest) (guest) +#define EL1_REG(reg) MRS_REG_ALT_NAME(reg ## _EL12) +#define EL0_REG(reg) MRS_REG_ALT_NAME(reg ## _EL02) + +#include "vmm_hyp.c" diff --git a/sys/arm64/vmm/vmm_vhe_exception.S b/sys/arm64/vmm/vmm_vhe_exception.S new file mode 100644 index 000000000000..286f5df03707 --- /dev/null +++ b/sys/arm64/vmm/vmm_vhe_exception.S @@ -0,0 +1,31 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2024 Arm Ltd + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#define VMM_VHE +#define VMM_HYP_FUNC(func) vmm_vhe_ ## func + +#include "vmm_hyp_exception.S" |