diff options
Diffstat (limited to 'sys/arm64/arm64/locore.S')
| -rw-r--r-- | sys/arm64/arm64/locore.S | 117 |
1 files changed, 89 insertions, 28 deletions
diff --git a/sys/arm64/arm64/locore.S b/sys/arm64/arm64/locore.S index fd77938edae9..fffebe8f2b02 100644 --- a/sys/arm64/arm64/locore.S +++ b/sys/arm64/arm64/locore.S @@ -29,6 +29,7 @@ #include <sys/syscall.h> #include <machine/asm.h> #include <machine/armreg.h> +#include <machine/cpu.h> #include <machine/hypervisor.h> #include <machine/param.h> #include <machine/pte.h> @@ -192,12 +193,50 @@ END(_start) #ifdef SMP /* - * mpentry(unsigned long) + * void + * mpentry_psci(unsigned long) * - * Called by a core when it is being brought online. + * Called by a core when it is being brought online with psci. * The data in x0 is passed straight to init_secondary. */ -ENTRY(mpentry) +ENTRY(mpentry_psci) + mov x26, xzr + b mpentry_common +END(mpentry_psci) + +/* + * void + * mpentry_spintable(void) + * + * Called by a core when it is being brought online with a spin-table. + * Reads the new CPU ID and passes this to init_secondary. + */ +ENTRY(mpentry_spintable) + ldr x26, =spintable_wait + b mpentry_common +END(mpentry_spintable) + +/* Wait for the current CPU to be released */ +LENTRY(spintable_wait) + /* Read the affinity bits from mpidr_el1 */ + mrs x1, mpidr_el1 + ldr x2, =CPU_AFF_MASK + and x1, x1, x2 + + adrp x2, ap_cpuid +1: + ldr x0, [x2, :lo12:ap_cpuid] + cmp x0, x1 + b.ne 1b + + str xzr, [x2, :lo12:ap_cpuid] + dsb sy + sev + + ret +LEND(mpentry_spintable) + +LENTRY(mpentry_common) /* Disable interrupts */ msr daifset, #DAIF_INTR @@ -228,6 +267,14 @@ ENTRY(mpentry) mp_virtdone: BTI_J + /* + * Allow this CPU to wait until the kernel is ready for it, + * e.g. with spin-table but each CPU uses the same release address + */ + cbz x26, 1f + blr x26 +1: + /* Start using the AP boot stack */ adrp x4, bootstack ldr x4, [x4, :lo12:bootstack] @@ -258,7 +305,7 @@ mp_virtdone: msr tpidr_el1, x18 b init_secondary -END(mpentry) +LEND(mpentry_common) #endif /* @@ -382,14 +429,16 @@ LEND(get_load_phys_addr) * All the memory must not cross a 1GiB boundaty * x28 contains the physical address we were loaded from * - * TODO: This is out of date. - * There are at least 5 pages before that address for the page tables + * There are 7 or 8 pages before that address for the page tables * The pages used are: + * - The Kernel L3 tables (only for 16k kernel) * - The Kernel L2 table * - The Kernel L1 table * - The Kernel L0 table (TTBR1) + * - The identity (PA = VA) L2 table * - The identity (PA = VA) L1 table - * - The identity (PA = VA) L0 table (TTBR0) + * - The identity (PA = VA) L0 table (Early TTBR0) + * - The Kernel empty L0 table (Late TTBR0) */ LENTRY(create_pagetables) /* Save the Link register */ @@ -467,23 +516,32 @@ booti_no_fdt: common: #if PAGE_SIZE != PAGE_SIZE_4K /* - * Create L3 pages. The kernel will be loaded at a 2M aligned - * address, however L2 blocks are too large when the page size is - * not 4k to map the kernel with such an aligned address. However, - * when the page size is larger than 4k, L2 blocks are too large to - * map the kernel with such an alignment. + * Create L3 and L3C pages. The kernel will be loaded at a 2M aligned + * address, enabling the creation of L3C pages. However, when the page + * size is larger than 4k, L2 blocks are too large to map the kernel + * with 2M alignment. */ +#define PTE_SHIFT L3_SHIFT +#define BUILD_PTE_FUNC build_l3_page_pagetable +#else +#define PTE_SHIFT L2_SHIFT +#define BUILD_PTE_FUNC build_l2_block_pagetable +#endif - /* Get the number of l3 pages to allocate, rounded down */ - lsr x10, x8, #(L3_SHIFT) + /* Get the number of blocks/pages to allocate, rounded down */ + lsr x10, x8, #(PTE_SHIFT) - /* Create the kernel space L2 table */ + /* Create the kernel space PTE table */ mov x6, x26 mov x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) mov x8, #(KERNBASE) mov x9, x28 - bl build_l3_page_pagetable + bl BUILD_PTE_FUNC + +#undef PTE_SHIFT +#undef BUILD_PTE_FUNC +#if PAGE_SIZE != PAGE_SIZE_4K /* Move to the l2 table */ ldr x9, =(PAGE_SIZE * L3_PAGE_COUNT) add x26, x26, x9 @@ -492,16 +550,6 @@ common: mov x9, x6 mov x6, x26 bl link_l2_pagetable -#else - /* Get the number of l2 pages to allocate, rounded down */ - lsr x10, x8, #(L2_SHIFT) - - /* Create the kernel space L2 table */ - mov x6, x26 - mov x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) - mov x8, #(KERNBASE) - mov x9, x28 - bl build_l2_block_pagetable #endif /* Move to the l1 table */ @@ -735,13 +783,17 @@ LENTRY(link_l2_pagetable) LEND(link_l2_pagetable) /* - * Builds count level 3 page table entries + * Builds count level 3 page table entries. Uses ATTR_CONTIGUOUS to create + * large page (L3C) mappings when the current VA and remaining count allow + * it. * x6 = L3 table * x7 = Block attributes * x8 = VA start * x9 = PA start (trashed) * x10 = Entry count (trashed) * x11, x12 and x13 are trashed + * + * VA start (x8) modulo L3C_SIZE must equal PA start (x9) modulo L3C_SIZE. */ LENTRY(build_l3_page_pagetable) /* @@ -762,8 +814,17 @@ LENTRY(build_l3_page_pagetable) /* Only use the output address bits */ lsr x9, x9, #L3_SHIFT + /* Check if an ATTR_CONTIGUOUS mapping is possible */ +1: tst x11, #(L3C_ENTRIES - 1) + b.ne 2f + cmp x10, #L3C_ENTRIES + b.lo 3f + orr x12, x12, #(ATTR_CONTIGUOUS) + b 2f +3: and x12, x12, #(~ATTR_CONTIGUOUS) + /* Set the physical address for this virtual address */ -1: orr x13, x12, x9, lsl #L3_SHIFT +2: orr x13, x12, x9, lsl #L3_SHIFT /* Store the entry */ str x13, [x6, x11, lsl #3] |