aboutsummaryrefslogtreecommitdiff
path: root/sys/arm64
diff options
context:
space:
mode:
Diffstat (limited to 'sys/arm64')
-rw-r--r--sys/arm64/arm64/exception.S36
-rw-r--r--sys/arm64/arm64/exec_machdep.c1
-rw-r--r--sys/arm64/arm64/genassym.c5
-rw-r--r--sys/arm64/arm64/locore.S3
-rw-r--r--sys/arm64/arm64/machdep.c5
-rw-r--r--sys/arm64/arm64/mte.c191
-rw-r--r--sys/arm64/arm64/pmap.c79
-rw-r--r--sys/arm64/arm64/swtch.S10
-rw-r--r--sys/arm64/arm64/vm_machdep.c37
-rw-r--r--sys/arm64/include/cpu.h10
-rw-r--r--sys/arm64/include/elf.h6
-rw-r--r--sys/arm64/include/pcpu.h1
-rw-r--r--sys/arm64/include/pmap.h7
-rw-r--r--sys/arm64/include/proc.h6
-rw-r--r--sys/arm64/include/stack.h7
-rw-r--r--sys/arm64/rockchip/rk3328_codec.c14
-rw-r--r--sys/arm64/rockchip/rk_gpio.c106
17 files changed, 458 insertions, 66 deletions
diff --git a/sys/arm64/arm64/exception.S b/sys/arm64/arm64/exception.S
index 5a4181348a54..5efbc4b36710 100644
--- a/sys/arm64/arm64/exception.S
+++ b/sys/arm64/arm64/exception.S
@@ -92,10 +92,34 @@
blr x1
1:
- ldr x0, [x18, #PC_CURTHREAD]
+ ldr x19, [x18, #PC_CURTHREAD]
+
+ ldr x1, [x19, #TD_MD_SCTLR]
+ /*
+ * If the upper bit in SCTLR_EL1.TCF0 is set we are either in async
+ * or asym modes. Either of which could set TFSRE0_EL1.
+ */
+ tbz x1, #(SCTLR_TCF0_SHIFT + 1), 2f
+ /* Check for a tag fault */
+ mrs x1, TFSRE0_EL1_REG
+ tbz x1, #TFSRE0_TF0_SHIFT, 2f
+
+ /*
+ * A fault has happened, set MD_FLAG_MTE_ASYNC_FAULT. As FEAT_LSE
+ * is a required feature where FEAT_MTE_ASYNC could be implemented
+ * we can depend on it being present to set the flag.
+ */
+ ldr w1, =MD_FLAG_MTE_ASYNC_FAULT
+ add x2, x19, #TD_MD_FLAGS
+.arch_extension lse
+ stset w1, [x2]
+.arch_extension nolse
+
+2:
+ mov x0, x19
bl ptrauth_exit_el0
- ldr x0, [x18, #(PC_CURTHREAD)]
+ mov x0, x19
bl dbg_monitor_enter
/* Unmask debug and SError exceptions */
@@ -118,6 +142,14 @@
msr daifset, #(DAIF_ALL)
.if \el == 0
ldr x0, [x18, #PC_CURTHREAD]
+
+ ldr x1, [x0, #TD_MD_SCTLR]
+ /* See above for why we check this field */
+ tbz x1, #(SCTLR_TCF0_SHIFT + 1), 1f
+ dsb ish
+ msr TFSRE0_EL1_REG, xzr
+1:
+
mov x1, sp
bl dbg_monitor_exit
diff --git a/sys/arm64/arm64/exec_machdep.c b/sys/arm64/arm64/exec_machdep.c
index a2e1e42249b4..d0a7302e2f7d 100644
--- a/sys/arm64/arm64/exec_machdep.c
+++ b/sys/arm64/arm64/exec_machdep.c
@@ -471,6 +471,7 @@ exec_setregs(struct thread *td, struct image_params *imgp, uintptr_t stack)
/* Generate new pointer authentication keys */
ptrauth_exec(td);
+ mte_exec(td);
}
/* Sanity check these are the same size, they will be memcpy'd to and from */
diff --git a/sys/arm64/arm64/genassym.c b/sys/arm64/arm64/genassym.c
index 22696796e69d..6c86f190282d 100644
--- a/sys/arm64/arm64/genassym.c
+++ b/sys/arm64/arm64/genassym.c
@@ -53,8 +53,6 @@ ASSYM(PC_CURPCB, offsetof(struct pcpu, pc_curpcb));
ASSYM(PC_CURTHREAD, offsetof(struct pcpu, pc_curthread));
ASSYM(PC_SSBD, offsetof(struct pcpu, pc_ssbd));
-/* Size of pcb, rounded to keep stack alignment */
-ASSYM(PCB_SIZE, roundup2(sizeof(struct pcb), STACKALIGNBYTES + 1));
ASSYM(PCB_SINGLE_STEP_SHIFT, PCB_SINGLE_STEP_SHIFT);
ASSYM(PCB_REGS, offsetof(struct pcb, pcb_x));
ASSYM(PCB_X19, PCB_X19);
@@ -76,6 +74,9 @@ ASSYM(TD_FRAME, offsetof(struct thread, td_frame));
ASSYM(TD_LOCK, offsetof(struct thread, td_lock));
ASSYM(TD_MD_CANARY, offsetof(struct thread, td_md.md_canary));
ASSYM(TD_MD_EFIRT_TMP, offsetof(struct thread, td_md.md_efirt_tmp));
+ASSYM(TD_MD_FLAGS, offsetof(struct thread, td_md.md_flags));
+ASSYM(MD_FLAG_MTE_ASYNC_FAULT, MD_FLAG_MTE_ASYNC_FAULT);
+ASSYM(TD_MD_SCTLR, offsetof(struct thread, td_md.md_sctlr));
ASSYM(TF_SIZE, sizeof(struct trapframe));
ASSYM(TF_SP, offsetof(struct trapframe, tf_sp));
diff --git a/sys/arm64/arm64/locore.S b/sys/arm64/arm64/locore.S
index b200aa93c281..bd61b485edf7 100644
--- a/sys/arm64/arm64/locore.S
+++ b/sys/arm64/arm64/locore.S
@@ -128,8 +128,7 @@ virtdone:
/* Set up the stack */
adrp x25, initstack_end
- add x25, x25, :lo12:initstack_end
- sub sp, x25, #PCB_SIZE
+ add sp, x25, :lo12:initstack_end
/* Zero the BSS */
ldr x15, .Lbss
diff --git a/sys/arm64/arm64/machdep.c b/sys/arm64/arm64/machdep.c
index d219c737c215..f35ec7ab2e2e 100644
--- a/sys/arm64/arm64/machdep.c
+++ b/sys/arm64/arm64/machdep.c
@@ -131,6 +131,7 @@ uintptr_t boot_canary = 0x49a2d892bc05a0b1ul;
#endif
static struct trapframe proc0_tf;
+static struct pcb pcb0;
int early_boot = 1;
int cold = 1;
@@ -443,14 +444,14 @@ init_proc0(void *kstack)
#if defined(PERTHREAD_SSP)
thread0.td_md.md_canary = boot_canary;
#endif
- thread0.td_pcb = (struct pcb *)(thread0.td_kstack +
- thread0.td_kstack_pages * PAGE_SIZE) - 1;
+ thread0.td_pcb = &pcb0;
thread0.td_pcb->pcb_flags = 0;
thread0.td_pcb->pcb_fpflags = 0;
thread0.td_pcb->pcb_fpusaved = &thread0.td_pcb->pcb_fpustate;
thread0.td_pcb->pcb_vfpcpu = UINT_MAX;
thread0.td_frame = &proc0_tf;
ptrauth_thread0(&thread0);
+ mte_thread0(&thread0);
pcpup->pc_curpcb = thread0.td_pcb;
/*
diff --git a/sys/arm64/arm64/mte.c b/sys/arm64/arm64/mte.c
new file mode 100644
index 000000000000..6e902858a8b9
--- /dev/null
+++ b/sys/arm64/arm64/mte.c
@@ -0,0 +1,191 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2024-2026 Arm Ltd
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/libkern.h>
+#include <sys/proc.h>
+
+#include <machine/cpu_feat.h>
+#include <machine/pcb.h>
+#include <machine/pte.h>
+#include <machine/sysarch.h>
+#include <vm/vm.h>
+#include <vm/vm_page.h>
+
+/* Version of MTE implemented. 0 == unimplemented */
+static u_int __read_mostly mte_version = 0;
+
+/*
+ * FEAT_MTE (mte_version == 1) has userspace instructions, but no tag
+ * checking. May of the registers/fields need FEAT_MTE2 to be implemented
+ * before we can access them.
+ */
+#define MTE_HAS_TAG_CHECK (mte_version >= 2)
+
+struct thread *mte_switch(struct thread *);
+
+#define load_tags(addr) ({ \
+ uint64_t __val; \
+ asm volatile( \
+ ".arch_extension memtag \n" \
+ "ldgm %0, [%1] \n" \
+ ".arch_extension nomemtag" : "=r" (__val) : "r" (addr)); \
+ __val; \
+})
+
+#define set_tags(tags, addr) do { \
+ asm volatile( \
+ ".arch_extension memtag \n" \
+ "stgm %0, [%1] \n" \
+ ".arch_extension nomemtag" : "=r" (tags) : "r" (addr)); \
+} while (0)
+
+/* Fetch the block size used by tag load and store instructions */
+static inline size_t
+mte_block_size(void)
+{
+ return (sizeof(int) << GMID_BS_SIZE(READ_SPECIALREG(GMID_EL1_REG)));
+}
+
+static void
+mte_update_sctlr(struct thread *td, uint64_t sctlr)
+{
+ MPASS((sctlr & ~(SCTLR_ATA0 | SCTLR_TCF0_MASK)) == 0);
+ td->td_md.md_sctlr &= ~(SCTLR_ATA0 | SCTLR_TCF0_MASK);
+ td->td_md.md_sctlr |= sctlr;
+}
+
+/**
+ * Clear/sync the allocation tags for a given page. This should be done on
+ * allocation of a page to ensure a tag check fault does not occur immediately
+ * after accessing newly tagged memory.
+ */
+void
+mte_sync_tags(vm_page_t page)
+{
+ char *addr;
+ size_t block_size;
+
+ if (!MTE_HAS_TAG_CHECK)
+ return;
+
+ /* don't clear the tags on a page that's already setup for mte */
+ if ((page->md.pv_flags & PV_MTE_TAGGED) != 0)
+ return;
+
+ block_size = mte_block_size();
+ addr = PHYS_TO_DMAP(page->phys_addr);
+
+ for (size_t count = 0; count < PAGE_SIZE;
+ count += block_size, addr += block_size)
+ asm volatile(
+ ".arch_extension memtag \n"
+ "stgm xzr, [%0] \n"
+ ".arch_extension nomemtag" : : "r" (addr));
+
+ page->md.pv_flags |= PV_MTE_TAGGED;
+}
+
+/**
+ * Copy the allocation tags from given target to destination page. This is called
+ * on a copy-on-write and anything that causes a pmap_copy_page call.
+ */
+void
+mte_copy_tags(vm_page_t srcpage, vm_page_t dstpage, char *src, char *dst)
+{
+ size_t block_size;
+ uint64_t tags;
+
+ MPASS((srcpage->md.pv_flags & PV_MTE_TAGGED) != 0);
+
+ /*
+ * Copy the tags from the source page to the destination page,
+ * incrementing by the block count read from GMID_EL1
+ */
+ block_size = mte_block_size();
+ for (size_t count = 0; count < PAGE_SIZE;
+ count += block_size, src += block_size, dst += block_size) {
+ tags = load_tags(src);
+ set_tags(tags, dst);
+ }
+ dstpage->md.pv_flags |= PV_MTE_TAGGED;
+}
+
+void
+mte_fork(struct thread *new_td, struct thread *orig_td)
+{
+ if (!MTE_HAS_TAG_CHECK)
+ return;
+
+ mte_update_sctlr(new_td,
+ orig_td->td_md.md_sctlr & SCTLR_TCF0_MASK);
+ new_td->td_md.md_gcr = orig_td->td_md.md_gcr;
+}
+
+void
+mte_exec(struct thread *td)
+{
+ if (!MTE_HAS_TAG_CHECK)
+ return;
+
+ mte_update_sctlr(td, SCTLR_TCF0_NONE);
+ td->td_md.md_gcr = GCR_RRND;
+}
+
+void
+mte_copy_thread(struct thread *new_td, struct thread *orig_td)
+{
+ if (!MTE_HAS_TAG_CHECK)
+ return;
+
+ mte_update_sctlr(new_td,
+ orig_td->td_md.md_sctlr & SCTLR_TCF0_MASK);
+ new_td->td_md.md_gcr = orig_td->td_md.md_gcr;
+}
+
+/* Only for kernel threads */
+void
+mte_thread_alloc(struct thread *td)
+{
+}
+
+/* Only for a kernel thread */
+void
+mte_thread0(struct thread *td)
+{
+}
+
+
+struct thread *
+mte_switch(struct thread *td)
+{
+ if (MTE_HAS_TAG_CHECK) {
+ WRITE_SPECIALREG(GCR_EL1_REG, td->td_md.md_gcr);
+ }
+ return (td);
+}
diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
index adc583812e5b..1fb9ac2011aa 100644
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -146,6 +146,7 @@
#include <vm/uma.h>
#include <machine/asan.h>
+#include <machine/cpu.h>
#include <machine/cpu_feat.h>
#include <machine/elf.h>
#include <machine/ifunc.h>
@@ -358,6 +359,7 @@ struct pv_chunks_list __exclusive_cache_line pv_chunks[PMAP_MEMDOM];
vm_paddr_t dmap_phys_base; /* The start of the dmap region */
vm_paddr_t dmap_phys_max; /* The limit of the dmap region */
vm_offset_t dmap_max_addr; /* The virtual address limit of the dmap */
+static int dmap_attr = VM_MEMATTR_WRITE_BACK;
extern pt_entry_t pagetable_l0_ttbr1[];
@@ -483,7 +485,7 @@ static void pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte);
static bool pmap_activate_int(struct thread *td, pmap_t pmap);
static void pmap_alloc_asid(pmap_t pmap);
static int pmap_change_props_locked(void *addr, vm_size_t size,
- vm_prot_t prot, int mode, bool skip_unmapped);
+ vm_prot_t prot, int mode, int old_mode, bool skip_unmapped);
static bool pmap_copy_l3c(pmap_t pmap, pt_entry_t *l3p, vm_offset_t va,
pt_entry_t l3e, vm_page_t ml3, struct rwlock **lockp);
static pt_entry_t *pmap_demote_l1(pmap_t pmap, pt_entry_t *l1, vm_offset_t va);
@@ -1586,6 +1588,7 @@ pmap_page_init(vm_page_t m)
TAILQ_INIT(&m->md.pv_list);
m->md.pv_memattr = VM_MEMATTR_WRITE_BACK;
+ m->md.pv_flags = 0;
}
static void
@@ -6920,6 +6923,7 @@ pmap_zero_page(vm_page_t m)
void *va = VM_PAGE_TO_DMAP(m);
pagezero(va);
+ m->md.pv_flags &= ~PV_MTE_TAGGED;
}
/*
@@ -6951,6 +6955,15 @@ pmap_copy_page(vm_page_t msrc, vm_page_t mdst)
void *src = VM_PAGE_TO_DMAP(msrc);
void *dst = VM_PAGE_TO_DMAP(mdst);
+ /*
+ * On a page copy, check whether the src page is tagged. If it is,
+ * we must copy the tags before copying the contents of the page.
+ */
+ if ((msrc->md.pv_flags & PV_MTE_TAGGED) != 0)
+ mte_copy_tags(msrc, mdst, src, dst);
+ else
+ mdst->md.pv_flags &= ~PV_MTE_TAGGED;
+
pagecopy(src, dst);
}
@@ -6967,6 +6980,9 @@ pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[],
int cnt;
while (xfersize > 0) {
+ KASSERT(ADDR_IS_CANONICAL(a_offset),
+ ("%s: Address not in canonical form: %lx", __func__, a_offset));
+
a_pg_offset = a_offset & PAGE_MASK;
m_a = ma[a_offset >> PAGE_SHIFT];
p_a = m_a->phys_addr;
@@ -8161,7 +8177,7 @@ pmap_unmapbios(void *p, vm_size_t size)
/* Ensure the attributes are as expected for the DMAP region */
PMAP_LOCK(kernel_pmap);
error = pmap_change_props_locked(va, size,
- PROT_READ | PROT_WRITE, VM_MEMATTR_DEFAULT, false);
+ PROT_READ | PROT_WRITE, VM_MEMATTR_DEFAULT, -1, false);
PMAP_UNLOCK(kernel_pmap);
KASSERT(error == 0, ("%s: Failed to reset DMAP attributes: %d",
__func__, error));
@@ -8267,7 +8283,25 @@ pmap_change_attr(void *va, vm_size_t size, int mode)
int error;
PMAP_LOCK(kernel_pmap);
- error = pmap_change_props_locked(va, size, PROT_NONE, mode, false);
+ error = pmap_change_props_locked(va, size, PROT_NONE, mode, -1, false);
+ PMAP_UNLOCK(kernel_pmap);
+ return (error);
+}
+
+int
+pmap_change_dmap_attr(int mode)
+{
+ int error;
+
+ KASSERT(mode == VM_MEMATTR_WRITE_BACK ||
+ mode == VM_MEMATTR_TAGGED,
+ ("%s: mode %d must be compatible with write-back", __func__, mode));
+
+ PMAP_LOCK(kernel_pmap);
+ error = pmap_change_props_locked((void *)DMAP_MIN_ADDRESS,
+ dmap_max_addr - DMAP_MIN_ADDRESS, PROT_NONE, mode, dmap_attr, true);
+ if (error == 0)
+ dmap_attr = mode;
PMAP_UNLOCK(kernel_pmap);
return (error);
}
@@ -8289,20 +8323,20 @@ pmap_change_prot(void *va, vm_size_t size, vm_prot_t prot)
return (EINVAL);
PMAP_LOCK(kernel_pmap);
- error = pmap_change_props_locked(va, size, prot, -1, false);
+ error = pmap_change_props_locked(va, size, prot, -1, -1, false);
PMAP_UNLOCK(kernel_pmap);
return (error);
}
static int
pmap_change_props_locked(void *addr, vm_size_t size, vm_prot_t prot,
- int mode, bool skip_unmapped)
+ int mode, int old_mode, bool skip_unmapped)
{
vm_offset_t base, offset, tmpva, va;
vm_size_t pte_size;
vm_paddr_t pa;
pt_entry_t pte, *ptep, *newpte;
- pt_entry_t bits, mask;
+ pt_entry_t bits, mask, old_mode_bits, old_mode_mask;
char *tmpptep;
int lvl, rv;
@@ -8316,8 +8350,8 @@ pmap_change_props_locked(void *addr, vm_size_t size, vm_prot_t prot,
!(base >= VM_MIN_KERNEL_ADDRESS && base < VM_MAX_KERNEL_ADDRESS))
return (EINVAL);
- bits = 0;
- mask = 0;
+ bits = old_mode_bits = 0;
+ mask = old_mode_mask = 0;
if (mode != -1) {
bits = ATTR_S1_IDX(mode);
mask = ATTR_S1_IDX_MASK;
@@ -8326,6 +8360,10 @@ pmap_change_props_locked(void *addr, vm_size_t size, vm_prot_t prot,
bits |= ATTR_S1_XN;
}
}
+ if (old_mode != -1) {
+ old_mode_bits = ATTR_S1_IDX(old_mode);
+ old_mode_mask = ATTR_S1_IDX_MASK;
+ }
if (prot != VM_PROT_NONE) {
/* Don't mark the DMAP as executable. It never is on arm64. */
if (VIRT_IN_DMAP(base)) {
@@ -8353,11 +8391,14 @@ pmap_change_props_locked(void *addr, vm_size_t size, vm_prot_t prot,
if (ptep == NULL && !skip_unmapped) {
return (EINVAL);
} else if ((ptep == NULL && skip_unmapped) ||
- (pmap_load(ptep) & mask) == bits) {
+ (pmap_load(ptep) & mask) == bits ||
+ (pmap_load(ptep) & old_mode_mask) != old_mode_bits) {
/*
- * We already have the correct attribute or there
- * is no memory mapped at this address and we are
- * skipping unmapped memory.
+ * We already have one of the following meaning
+ * we can skip this memory region::
+ * - No memory mapped at this address
+ * - The new attributes are already set
+ * - The expected attributes are incorrect
*/
switch (lvl) {
default:
@@ -8487,12 +8528,24 @@ pmap_change_props_locked(void *addr, vm_size_t size, vm_prot_t prot,
pa = PTE_TO_PHYS(pte);
if (!VIRT_IN_DMAP(tmpva) && PHYS_IN_DMAP(pa)) {
+ int dmap_mode;
+
+ /*
+ * When booting on HW with MTE enabled we may
+ * need to swap to a tagged type for the DMAP
+ * to allow tags to be set through it.
+ */
+ if (mode == VM_MEMATTR_WRITE_BACK)
+ dmap_mode = dmap_attr;
+ else
+ dmap_mode = mode;
+
/*
* Keep the DMAP memory in sync.
*/
rv = pmap_change_props_locked(
PHYS_TO_DMAP(pa), pte_size,
- prot, mode, true);
+ prot, dmap_mode, old_mode, true);
if (rv != 0)
return (rv);
}
diff --git a/sys/arm64/arm64/swtch.S b/sys/arm64/arm64/swtch.S
index b3bf88135e57..b349072c06f4 100644
--- a/sys/arm64/arm64/swtch.S
+++ b/sys/arm64/arm64/swtch.S
@@ -75,7 +75,7 @@
* void cpu_throw(struct thread *old, struct thread *new)
*/
ENTRY(cpu_throw)
- /* Of old == NULL skip disabling stepping */
+ /* If old == NULL skip disabling stepping */
cbz x0, 1f
/* If we were single stepping, disable it */
@@ -96,8 +96,9 @@ ENTRY(cpu_throw)
mov x0, x1
#endif
- /* This returns the thread pointer so no need to save it */
+ /* These return the thread pointer so no need to save it */
bl ptrauth_switch
+ bl mte_switch
#ifdef PERTHREAD_SSP
mov x19, x0
#endif
@@ -176,8 +177,9 @@ ENTRY(cpu_switch)
mov x0, x1
#endif
- /* This returns the thread pointer so no need to save it */
+ /* These return the thread pointer so no need to save it */
bl ptrauth_switch
+ bl mte_switch
/* This returns the thread pcb */
bl pmap_switch
/* Move the new pcb out of the way */
@@ -276,6 +278,8 @@ ENTRY(fork_trampoline)
ldp x26, x27, [sp, #TF_X + 26 * 8]
ldp x28, x29, [sp, #TF_X + 28 * 8]
+ add sp, sp, #(TF_SIZE)
+
/*
* No need for interrupts reenabling since PSR
* will be set to the desired value anyway.
diff --git a/sys/arm64/arm64/vm_machdep.c b/sys/arm64/arm64/vm_machdep.c
index 4cb87ca9856e..635bdcef7025 100644
--- a/sys/arm64/arm64/vm_machdep.c
+++ b/sys/arm64/arm64/vm_machdep.c
@@ -27,8 +27,8 @@
#include "opt_platform.h"
-#include <sys/param.h>
#include <sys/systm.h>
+#include <sys/kernel.h>
#include <sys/limits.h>
#include <sys/proc.h>
#include <sys/sf_buf.h>
@@ -61,6 +61,8 @@
*/
cpu_reset_hook_t cpu_reset_hook = psci_reset;
+static uma_zone_t pcb_zone;
+
/*
* Finish a fork operation, with process p2 nearly set up.
* Copy and update the pcb, set up the stack so that the child
@@ -89,25 +91,21 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
#endif
}
- pcb2 = (struct pcb *)(td2->td_kstack +
- td2->td_kstack_pages * PAGE_SIZE) - 1;
-
- td2->td_pcb = pcb2;
+ pcb2 = td2->td_pcb;
bcopy(td1->td_pcb, pcb2, sizeof(*pcb2));
/* Clear the debug register state. */
bzero(&pcb2->pcb_dbg_regs, sizeof(pcb2->pcb_dbg_regs));
ptrauth_fork(td2, td1);
+ mte_fork(td2, td1);
- tf = STACKALIGN((struct trapframe *)pcb2 - 1);
+ tf = td2->td_frame;
bcopy(td1->td_frame, tf, sizeof(*tf));
tf->tf_x[0] = 0;
tf->tf_x[1] = 0;
tf->tf_spsr = td1->td_frame->tf_spsr & (PSR_M_32 | PSR_DAIF);
- td2->td_frame = tf;
-
/* Set the return value registers for fork() */
td2->td_pcb->pcb_x[PCB_X19] = (uintptr_t)fork_return;
td2->td_pcb->pcb_x[PCB_X20] = (uintptr_t)td2;
@@ -203,6 +201,7 @@ cpu_copy_thread(struct thread *td, struct thread *td0)
/* Generate new pointer authentication keys. */
ptrauth_copy_thread(td, td0);
+ mte_copy_thread(td, td0);
}
/*
@@ -265,17 +264,21 @@ cpu_thread_exit(struct thread *td)
void
cpu_thread_alloc(struct thread *td)
{
-
- td->td_pcb = (struct pcb *)(td->td_kstack +
- td->td_kstack_pages * PAGE_SIZE) - 1;
- td->td_frame = (struct trapframe *)STACKALIGN(
- (struct trapframe *)td->td_pcb - 1);
+ td->td_pcb = uma_zalloc(pcb_zone, M_WAITOK);
ptrauth_thread_alloc(td);
+ mte_thread_alloc(td);
+}
+
+void
+cpu_thread_new_kstack(struct thread *td)
+{
+ td->td_frame = (struct trapframe *)td_kstack_top(td) - 1;
}
void
cpu_thread_free(struct thread *td)
{
+ uma_zfree(pcb_zone, td->td_pcb);
}
void
@@ -335,3 +338,11 @@ cpu_sync_core(void)
* return from ELx is a context synchronization event.
*/
}
+
+static void
+pcbinit(void *dummy __unused)
+{
+ pcb_zone = uma_zcreate("pcb", sizeof(struct pcb), NULL, NULL, NULL,
+ NULL, UMA_ALIGNOF(struct pcb), 0);
+}
+SYSINIT(pcbinit, SI_SUB_INTRINSIC, SI_ORDER_ANY, pcbinit, NULL);
diff --git a/sys/arm64/include/cpu.h b/sys/arm64/include/cpu.h
index 05844ad63036..bdbc601edd26 100644
--- a/sys/arm64/include/cpu.h
+++ b/sys/arm64/include/cpu.h
@@ -277,6 +277,16 @@ void ptrauth_thread0(struct thread *);
void ptrauth_mp_start(uint64_t);
#endif
+/* Memory Tagging Extension (MTE) support */
+void mte_fork(struct thread *, struct thread *);
+void mte_exec(struct thread *);
+void mte_copy_thread(struct thread *, struct thread *);
+void mte_thread_alloc(struct thread *);
+void mte_thread0(struct thread *);
+
+void mte_sync_tags(vm_page_t page);
+void mte_copy_tags(vm_page_t, vm_page_t, char *, char *);
+
/* Functions to read the sanitised view of the special registers */
void update_special_regs(u_int);
void update_special_reg_iss(u_int, uint64_t, uint64_t);
diff --git a/sys/arm64/include/elf.h b/sys/arm64/include/elf.h
index a5a90f8c7712..7940bb259256 100644
--- a/sys/arm64/include/elf.h
+++ b/sys/arm64/include/elf.h
@@ -96,6 +96,12 @@ __ElfType(Auxinfo);
/* First __FreeBSD_version that supports Top Byte Ignore (TBI) */
#define TBI_VERSION 1500058
+/*
+ * The HWCAP values must be identical to Linux. Many userspace programs
+ * will define missing HWCAP values to the Linux version. To keep these
+ * working when we add the HWCAP it must be the same.
+ */
+
/* HWCAP */
#define HWCAP_FP (1 << 0)
#define HWCAP_ASIMD (1 << 1)
diff --git a/sys/arm64/include/pcpu.h b/sys/arm64/include/pcpu.h
index 286a40e7de3d..d04f975350d8 100644
--- a/sys/arm64/include/pcpu.h
+++ b/sys/arm64/include/pcpu.h
@@ -55,7 +55,6 @@ struct debug_monitor_state;
#ifdef _KERNEL
-struct pcb;
struct pcpu;
static inline struct pcpu *
diff --git a/sys/arm64/include/pmap.h b/sys/arm64/include/pmap.h
index 00b54a874e12..cf20827fa666 100644
--- a/sys/arm64/include/pmap.h
+++ b/sys/arm64/include/pmap.h
@@ -70,9 +70,13 @@ struct md_page {
TAILQ_HEAD(,pv_entry) pv_list;
int pv_gen;
vm_memattr_t pv_memattr;
- uint8_t pv_reserve[3];
+ uint8_t pv_flags;
+ uint8_t pv_reserve[2];
};
+/* machine page flags */
+#define PV_MTE_TAGGED 0x01 /* page is tagged with MTE */
+
enum pmap_stage {
PM_INVALID,
PM_STAGE1,
@@ -148,6 +152,7 @@ void pmap_activate_vm(pmap_t);
void pmap_bootstrap_dmap(vm_size_t);
void pmap_bootstrap(void);
int pmap_change_attr(void *va, vm_size_t size, int mode);
+int pmap_change_dmap_attr(int);
int pmap_change_prot(void *va, vm_size_t size, vm_prot_t prot);
void pmap_kenter(vm_offset_t sva, vm_size_t size, vm_paddr_t pa, int mode);
void pmap_kenter_device(vm_offset_t, vm_size_t, vm_paddr_t);
diff --git a/sys/arm64/include/proc.h b/sys/arm64/include/proc.h
index d5879a794269..22ceb614413d 100644
--- a/sys/arm64/include/proc.h
+++ b/sys/arm64/include/proc.h
@@ -69,9 +69,11 @@ struct mdthread {
uint64_t md_efirt_tmp;
int md_efirt_dis_pf;
- int md_reserved0;
+ u_int md_flags;
+#define MD_FLAG_MTE_ASYNC_FAULT_SHIFT 0
+#define MD_FLAG_MTE_ASYNC_FAULT (1u << 0)
uint64_t md_sctlr;
- uint64_t md_reserved1;
+ uint64_t md_gcr; /* FEAT_MTE: Tag Control Register */
};
struct mdproc {
diff --git a/sys/arm64/include/stack.h b/sys/arm64/include/stack.h
index 19e9e837e3ee..23e7a5af27de 100644
--- a/sys/arm64/include/stack.h
+++ b/sys/arm64/include/stack.h
@@ -39,11 +39,9 @@ bool unwind_frame(struct thread *, struct unwind_state *);
#ifdef _SYS_PROC_H_
-#include <machine/pcb.h>
-
#define GET_STACK_USAGE(total, used) do { \
struct thread *td = curthread; \
- (total) = td->td_kstack_pages * PAGE_SIZE - sizeof(struct pcb); \
+ (total) = ptoa(td->td_kstack_pages); \
(used) = td->td_kstack + (total) - (char *)&td; \
} while (0)
@@ -51,8 +49,7 @@ static __inline bool
kstack_contains(struct thread *td, vm_offset_t va, size_t len)
{
return (va >= (vm_offset_t)td->td_kstack && va + len >= va &&
- va + len <= (vm_offset_t)td->td_kstack + td->td_kstack_pages *
- PAGE_SIZE - sizeof(struct pcb));
+ va + len <= (vm_offset_t)td_kstack_top(td));
}
#endif /* _SYS_PROC_H_ */
diff --git a/sys/arm64/rockchip/rk3328_codec.c b/sys/arm64/rockchip/rk3328_codec.c
index 22e3cde9093e..a019cab27cc9 100644
--- a/sys/arm64/rockchip/rk3328_codec.c
+++ b/sys/arm64/rockchip/rk3328_codec.c
@@ -416,18 +416,8 @@ static int
rkcodec_mixer_set(struct snd_mixer *m, unsigned dev, unsigned left, unsigned right)
{
struct rkcodec_softc *sc;
- struct mtx *mixer_lock;
- uint8_t do_unlock;
sc = device_get_softc(mix_getdevinfo(m));
- mixer_lock = mixer_get_lock(m);
-
- if (mtx_owned(mixer_lock)) {
- do_unlock = 0;
- } else {
- do_unlock = 1;
- mtx_lock(mixer_lock);
- }
right = left;
@@ -443,10 +433,6 @@ rkcodec_mixer_set(struct snd_mixer *m, unsigned dev, unsigned left, unsigned rig
}
RKCODEC_UNLOCK(sc);
- if (do_unlock) {
- mtx_unlock(mixer_lock);
- }
-
return (left | (right << 8));
}
diff --git a/sys/arm64/rockchip/rk_gpio.c b/sys/arm64/rockchip/rk_gpio.c
index 8da37d516802..7c2071d2d178 100644
--- a/sys/arm64/rockchip/rk_gpio.c
+++ b/sys/arm64/rockchip/rk_gpio.c
@@ -227,8 +227,22 @@ rk_gpio_intr(void *arg)
status &= ~(1 << pin);
if (intr_isrc_dispatch(RK_GPIO_ISRC(sc, pin), tf)) {
- device_printf(sc->sc_dev, "Interrupt pin=%d unhandled\n",
- pin);
+ /*
+ * Pin asserted but no consumer is registered for it
+ * yet (or anymore). Level-triggered sources keep
+ * firing on every interrupt cycle, so a single stuck
+ * pin floods the console with thousands of these
+ * messages per second. Mask the pin's IRQ at the
+ * controller and disable further dispatches; if a
+ * consumer attaches later it will re-enable through
+ * pic_enable_intr / rk_gpio_pic_enable_intr.
+ */
+ RK_GPIO_LOCK(sc);
+ rk_gpio_write_bit(sc, RK_GPIO_INTMASK, pin, 1);
+ rk_gpio_write_bit(sc, RK_GPIO_INTEN, pin, 0);
+ RK_GPIO_UNLOCK(sc);
+ device_printf(sc->sc_dev,
+ "Interrupt pin=%d unhandled — masked\n", pin);
continue;
}
@@ -818,10 +832,14 @@ rk_pic_setup_intr(device_t dev, struct intr_irqsrc *isrc,
return (EINVAL);
}
rk_gpio_write_bit(sc, RK_GPIO_DEBOUNCE, pin, 1);
- rk_gpio_write_bit(sc, RK_GPIO_INTMASK, pin, 0);
- rk_gpio_write_bit(sc, RK_GPIO_INTEN, pin, 1);
RK_GPIO_UNLOCK(sc);
+ /*
+ * Leave the interrupt masked + disabled here. INTRNG will call
+ * pic_enable_intr() next to make it live. That keeps the
+ * masking responsibility cleanly in enable/disable rather than
+ * split between setup and disable.
+ */
return (0);
}
@@ -837,14 +855,86 @@ rk_pic_teardown_intr(device_t dev, struct intr_irqsrc *isrc,
if (isrc->isrc_handlers == 0) {
irqsrc->mode = GPIO_INTR_CONFORM;
RK_GPIO_LOCK(sc);
- rk_gpio_write_bit(sc, RK_GPIO_INTEN, irqsrc->irq, 0);
- rk_gpio_write_bit(sc, RK_GPIO_INTMASK, irqsrc->irq, 0);
+ /*
+ * INTEN/INTMASK are already cleared by pic_disable_intr,
+ * which INTRNG calls before teardown of the last handler.
+ * We only need to undo what setup_intr configured -- here,
+ * the debounce filter.
+ */
rk_gpio_write_bit(sc, RK_GPIO_DEBOUNCE, irqsrc->irq, 0);
RK_GPIO_UNLOCK(sc);
}
return (0);
}
+/*
+ * INTRNG calls pic_disable_intr() during teardown of the final handler
+ * for a source, OR when a consumer explicitly wants the source off.
+ * Clear INTEN so the controller will not raise this pin at all.
+ *
+ * The in-flight masking between FILTER_SCHEDULE_THREAD and ithread
+ * completion is handled by pic_pre_ithread() / pic_post_ithread()
+ * below, NOT by this method.
+ */
+static void
+rk_pic_disable_intr(device_t dev, struct intr_irqsrc *isrc)
+{
+ struct rk_gpio_softc *sc = device_get_softc(dev);
+ struct rk_pin_irqsrc *rkisrc = (struct rk_pin_irqsrc *)isrc;
+
+ RK_GPIO_LOCK(sc);
+ rk_gpio_write_bit(sc, RK_GPIO_INTMASK, rkisrc->irq, 1);
+ rk_gpio_write_bit(sc, RK_GPIO_INTEN, rkisrc->irq, 0);
+ RK_GPIO_UNLOCK(sc);
+}
+
+/*
+ * INTRNG calls pic_enable_intr() to make a source live for the first
+ * time (after setup_intr), or to re-enable after a prior
+ * pic_disable_intr(). Set INTEN and unmask so the controller starts
+ * delivering this pin.
+ */
+static void
+rk_pic_enable_intr(device_t dev, struct intr_irqsrc *isrc)
+{
+ struct rk_gpio_softc *sc = device_get_softc(dev);
+ struct rk_pin_irqsrc *rkisrc = (struct rk_pin_irqsrc *)isrc;
+
+ RK_GPIO_LOCK(sc);
+ rk_gpio_write_bit(sc, RK_GPIO_INTEN, rkisrc->irq, 1);
+ rk_gpio_write_bit(sc, RK_GPIO_INTMASK, rkisrc->irq, 0);
+ RK_GPIO_UNLOCK(sc);
+}
+
+/*
+ * Called by INTRNG before delivering to the ithread. Mask the source
+ * so it cannot re-fire during the ithread window -- without this,
+ * level-low IRQs (e.g. FUSB302 INT_N) re-trigger continuously and
+ * starve the ithread (~210 kHz storm observed via dtrace).
+ * Re-unmasked in pic_post_ithread() once the ithread acks the source.
+ */
+static void
+rk_pic_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
+{
+ struct rk_gpio_softc *sc = device_get_softc(dev);
+ struct rk_pin_irqsrc *rkisrc = (struct rk_pin_irqsrc *)isrc;
+
+ RK_GPIO_LOCK(sc);
+ rk_gpio_write_bit(sc, RK_GPIO_INTMASK, rkisrc->irq, 1);
+ RK_GPIO_UNLOCK(sc);
+}
+
+static void
+rk_pic_post_ithread(device_t dev, struct intr_irqsrc *isrc)
+{
+ struct rk_gpio_softc *sc = device_get_softc(dev);
+ struct rk_pin_irqsrc *rkisrc = (struct rk_pin_irqsrc *)isrc;
+
+ RK_GPIO_LOCK(sc);
+ rk_gpio_write_bit(sc, RK_GPIO_INTMASK, rkisrc->irq, 0);
+ RK_GPIO_UNLOCK(sc);
+}
+
static device_method_t rk_gpio_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, rk_gpio_probe),
@@ -873,6 +963,10 @@ static device_method_t rk_gpio_methods[] = {
DEVMETHOD(pic_map_intr, rk_pic_map_intr),
DEVMETHOD(pic_setup_intr, rk_pic_setup_intr),
DEVMETHOD(pic_teardown_intr, rk_pic_teardown_intr),
+ DEVMETHOD(pic_disable_intr, rk_pic_disable_intr),
+ DEVMETHOD(pic_enable_intr, rk_pic_enable_intr),
+ DEVMETHOD(pic_pre_ithread, rk_pic_pre_ithread),
+ DEVMETHOD(pic_post_ithread, rk_pic_post_ithread),
/* ofw_bus interface */
DEVMETHOD(ofw_bus_get_node, rk_gpio_get_node),
generated by cgit v1.3 (git 2.53.0) at 2026-05-30 23:46:12 +0000