Skip to content

Commit 97a6f43

Browse files
ardbiesheuvelctmarinas
ardbiesheuvel
authored and
ctmarinas
committed
arm64: head: Move early kernel mapping routines into C code
The asm version of the kernel mapping code works fine for creating a coarse grained identity map, but for mapping the kernel down to its exact boundaries with the right attributes, it is not suitable. This is why we create a preliminary RWX kernel mapping first, and then rebuild it from scratch later on. So let's reimplement this in C, in a way that will make it unnecessary to create the kernel page tables yet another time in paging_init(). Signed-off-by: Ard Biesheuvel <[email protected]> Link: https://lore.kernel.org/r/[email protected] Signed-off-by: Catalin Marinas <[email protected]>
1 parent 82ca151 commit 97a6f43

File tree

15 files changed

+315
-121
lines changed

15 files changed

+315
-121
lines changed

‎arch/arm64/include/asm/archrandom.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,4 @@ static inline bool __init __early_cpu_has_rndr(void)
129129
return (ftr >> ID_AA64ISAR0_EL1_RNDR_SHIFT) & 0xf;
130130
}
131131

132-
u64 kaslr_early_init(void *fdt);
133-
134132
#endif /* _ASM_ARCHRANDOM_H */

‎arch/arm64/include/asm/scs.h

Lines changed: 3 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -33,37 +33,11 @@
3333
#include <asm/cpufeature.h>
3434

3535
#ifdef CONFIG_UNWIND_PATCH_PAC_INTO_SCS
36-
static inline bool should_patch_pac_into_scs(void)
37-
{
38-
u64 reg;
39-
40-
/*
41-
* We only enable the shadow call stack dynamically if we are running
42-
* on a system that does not implement PAC or BTI. PAC and SCS provide
43-
* roughly the same level of protection, and BTI relies on the PACIASP
44-
* instructions serving as landing pads, preventing us from patching
45-
* those instructions into something else.
46-
*/
47-
reg = read_sysreg_s(SYS_ID_AA64ISAR1_EL1);
48-
if (SYS_FIELD_GET(ID_AA64ISAR1_EL1, APA, reg) |
49-
SYS_FIELD_GET(ID_AA64ISAR1_EL1, API, reg))
50-
return false;
51-
52-
reg = read_sysreg_s(SYS_ID_AA64ISAR2_EL1);
53-
if (SYS_FIELD_GET(ID_AA64ISAR2_EL1, APA3, reg))
54-
return false;
55-
56-
if (IS_ENABLED(CONFIG_ARM64_BTI_KERNEL)) {
57-
reg = read_sysreg_s(SYS_ID_AA64PFR1_EL1);
58-
if (reg & (0xf << ID_AA64PFR1_EL1_BT_SHIFT))
59-
return false;
60-
}
61-
return true;
62-
}
63-
6436
static inline void dynamic_scs_init(void)
6537
{
66-
if (should_patch_pac_into_scs()) {
38+
extern bool __pi_dynamic_scs_is_enabled;
39+
40+
if (__pi_dynamic_scs_is_enabled) {
6741
pr_info("Enabling dynamic shadow call stack\n");
6842
static_branch_enable(&dynamic_scs_enabled);
6943
}

‎arch/arm64/kernel/head.S

Lines changed: 1 addition & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,6 @@
8181
* x20 primary_entry() .. __primary_switch() CPU boot mode
8282
* x21 primary_entry() .. start_kernel() FDT pointer passed at boot in x0
8383
* x22 create_idmap() .. start_kernel() ID map VA of the DT blob
84-
* x23 __primary_switch() physical misalignment/KASLR offset
8584
* x25 primary_entry() .. start_kernel() supported VA size
8685
* x28 create_idmap() callee preserved temp register
8786
*/
@@ -408,24 +407,6 @@ SYM_FUNC_START_LOCAL(create_idmap)
408407
0: ret x28
409408
SYM_FUNC_END(create_idmap)
410409

411-
SYM_FUNC_START_LOCAL(create_kernel_mapping)
412-
adrp x0, init_pg_dir
413-
mov_q x5, KIMAGE_VADDR // compile time __va(_text)
414-
#ifdef CONFIG_RELOCATABLE
415-
add x5, x5, x23 // add KASLR displacement
416-
#endif
417-
adrp x6, _end // runtime __pa(_end)
418-
adrp x3, _text // runtime __pa(_text)
419-
sub x6, x6, x3 // _end - _text
420-
add x6, x6, x5 // runtime __va(_end)
421-
mov_q x7, SWAPPER_RW_MMUFLAGS
422-
423-
map_memory x0, x1, x5, x6, x7, x3, (VA_BITS - PGDIR_SHIFT), x10, x11, x12, x13, x14
424-
425-
dsb ishst // sync with page table walker
426-
ret
427-
SYM_FUNC_END(create_kernel_mapping)
428-
429410
/*
430411
* Initialize CPU registers with task-specific and cpu-specific context.
431412
*
@@ -752,44 +733,13 @@ SYM_FUNC_START_LOCAL(__primary_switch)
752733
adrp x2, init_idmap_pg_dir
753734
bl __enable_mmu
754735

755-
// Clear BSS
756-
adrp x0, __bss_start
757-
mov x1, xzr
758-
adrp x2, init_pg_end
759-
sub x2, x2, x0
760-
bl __pi_memset
761-
dsb ishst // Make zero page visible to PTW
762-
763736
adrp x1, early_init_stack
764737
mov sp, x1
765738
mov x29, xzr
766739
mov x0, x20 // pass the full boot status
767740
mov x1, x22 // pass the low FDT mapping
768-
bl __pi_init_feature_override // Parse cpu feature overrides
769-
770-
#ifdef CONFIG_RELOCATABLE
771-
adrp x23, KERNEL_START
772-
and x23, x23, MIN_KIMG_ALIGN - 1
773-
#ifdef CONFIG_RANDOMIZE_BASE
774-
mov x0, x22
775-
bl __pi_kaslr_early_init
776-
bic x0, x0, #SZ_2M - 1
777-
orr x23, x23, x0 // record kernel offset
778-
#endif
779-
#endif
780-
bl create_kernel_mapping
741+
bl __pi_early_map_kernel // Map and relocate the kernel
781742

782-
adrp x1, init_pg_dir
783-
load_ttbr1 x1, x1, x2
784-
#ifdef CONFIG_RELOCATABLE
785-
mov x0, x23
786-
bl __pi_relocate_kernel
787-
#endif
788-
#ifdef CONFIG_UNWIND_PATCH_PAC_INTO_SCS
789-
ldr x0, =__eh_frame_start
790-
ldr x1, =__eh_frame_end
791-
bl __pi_scs_patch_vmlinux
792-
#endif
793743
ldr x8, =__primary_switched
794744
adrp x0, KERNEL_START // __pa(KERNEL_START)
795745
br x8

‎arch/arm64/kernel/image-vars.h

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,28 @@ PROVIDE(__pi_id_aa64pfr1_override = id_aa64pfr1_override);
4545
PROVIDE(__pi_id_aa64smfr0_override = id_aa64smfr0_override);
4646
PROVIDE(__pi_id_aa64zfr0_override = id_aa64zfr0_override);
4747
PROVIDE(__pi_arm64_sw_feature_override = arm64_sw_feature_override);
48+
PROVIDE(__pi_arm64_use_ng_mappings = arm64_use_ng_mappings);
49+
#ifdef CONFIG_CAVIUM_ERRATUM_27456
50+
PROVIDE(__pi_cavium_erratum_27456_cpus = cavium_erratum_27456_cpus);
51+
#endif
4852
PROVIDE(__pi__ctype = _ctype);
4953
PROVIDE(__pi_memstart_offset_seed = memstart_offset_seed);
5054

55+
PROVIDE(__pi_init_pg_dir = init_pg_dir);
56+
PROVIDE(__pi_init_pg_end = init_pg_end);
57+
58+
PROVIDE(__pi__text = _text);
59+
PROVIDE(__pi__stext = _stext);
60+
PROVIDE(__pi__etext = _etext);
61+
PROVIDE(__pi___start_rodata = __start_rodata);
62+
PROVIDE(__pi___inittext_begin = __inittext_begin);
63+
PROVIDE(__pi___inittext_end = __inittext_end);
64+
PROVIDE(__pi___initdata_begin = __initdata_begin);
65+
PROVIDE(__pi___initdata_end = __initdata_end);
66+
PROVIDE(__pi__data = _data);
67+
PROVIDE(__pi___bss_start = __bss_start);
68+
PROVIDE(__pi__end = _end);
69+
5170
#ifdef CONFIG_KVM
5271

5372
/*

‎arch/arm64/kernel/pi/Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ $(obj)/lib-%.o: $(srctree)/lib/%.c FORCE
3939
$(call if_changed_rule,cc_o_c)
4040

4141
obj-y := idreg-override.pi.o \
42+
map_kernel.pi.o map_range.pi.o \
4243
lib-fdt.pi.o lib-fdt_ro.pi.o
4344
obj-$(CONFIG_RELOCATABLE) += relocate.pi.o
4445
obj-$(CONFIG_RANDOMIZE_BASE) += kaslr_early.pi.o

‎arch/arm64/kernel/pi/idreg-override.c

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -308,37 +308,35 @@ static __init void __parse_cmdline(const char *cmdline, bool parse_aliases)
308308
} while (1);
309309
}
310310

311-
static __init const u8 *get_bootargs_cmdline(const void *fdt)
311+
static __init const u8 *get_bootargs_cmdline(const void *fdt, int node)
312312
{
313+
static char const bootargs[] __initconst = "bootargs";
313314
const u8 *prop;
314-
int node;
315315

316-
node = fdt_path_offset(fdt, "/chosen");
317316
if (node < 0)
318317
return NULL;
319318

320-
prop = fdt_getprop(fdt, node, "bootargs", NULL);
319+
prop = fdt_getprop(fdt, node, bootargs, NULL);
321320
if (!prop)
322321
return NULL;
323322

324323
return strlen(prop) ? prop : NULL;
325324
}
326325

327-
static __init void parse_cmdline(const void *fdt)
326+
static __init void parse_cmdline(const void *fdt, int chosen)
328327
{
329-
const u8 *prop = get_bootargs_cmdline(fdt);
328+
static char const cmdline[] __initconst = CONFIG_CMDLINE;
329+
const u8 *prop = get_bootargs_cmdline(fdt, chosen);
330330

331331
if (IS_ENABLED(CONFIG_CMDLINE_FORCE) || !prop)
332-
__parse_cmdline(CONFIG_CMDLINE, true);
332+
__parse_cmdline(cmdline, true);
333333

334334
if (!IS_ENABLED(CONFIG_CMDLINE_FORCE) && prop)
335335
__parse_cmdline(prop, true);
336336
}
337337

338-
/* Keep checkers quiet */
339-
void init_feature_override(u64 boot_status, const void *fdt);
340-
341-
asmlinkage void __init init_feature_override(u64 boot_status, const void *fdt)
338+
void __init init_feature_override(u64 boot_status, const void *fdt,
339+
int chosen)
342340
{
343341
struct arm64_ftr_override *override;
344342
const struct ftr_set_desc *reg;
@@ -354,7 +352,7 @@ asmlinkage void __init init_feature_override(u64 boot_status, const void *fdt)
354352

355353
__boot_status = boot_status;
356354

357-
parse_cmdline(fdt);
355+
parse_cmdline(fdt, chosen);
358356

359357
for (i = 0; i < ARRAY_SIZE(regs); i++) {
360358
reg = prel64_pointer(regs[i].reg);

‎arch/arm64/kernel/pi/kaslr_early.c

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,17 +16,17 @@
1616
#include <asm/memory.h>
1717
#include <asm/pgtable.h>
1818

19+
#include "pi.h"
20+
1921
extern u16 memstart_offset_seed;
2022

21-
static u64 __init get_kaslr_seed(void *fdt)
23+
static u64 __init get_kaslr_seed(void *fdt, int node)
2224
{
23-
static char const chosen_str[] __initconst = "chosen";
2425
static char const seed_str[] __initconst = "kaslr-seed";
25-
int node, len;
2626
fdt64_t *prop;
2727
u64 ret;
28+
int len;
2829

29-
node = fdt_path_offset(fdt, chosen_str);
3030
if (node < 0)
3131
return 0;
3232

@@ -39,14 +39,14 @@ static u64 __init get_kaslr_seed(void *fdt)
3939
return ret;
4040
}
4141

42-
asmlinkage u64 __init kaslr_early_init(void *fdt)
42+
u64 __init kaslr_early_init(void *fdt, int chosen)
4343
{
4444
u64 seed, range;
4545

4646
if (kaslr_disabled_cmdline())
4747
return 0;
4848

49-
seed = get_kaslr_seed(fdt);
49+
seed = get_kaslr_seed(fdt, chosen);
5050
if (!seed) {
5151
if (!__early_cpu_has_rndr() ||
5252
!__arm64_rndr((unsigned long *)&seed))

0 commit comments

Comments
 (0)