Annotation of src/sys/arch/aarch64/aarch64/locore.S, Revision 1.63
1.63 ! ryo 1: /* $NetBSD: locore.S,v 1.62 2020/05/26 09:16:33 ryo Exp $ */
1.1 matt 2:
1.5 ryo 3: /*
4: * Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org>
1.1 matt 5: * All rights reserved.
6: *
7: * Redistribution and use in source and binary forms, with or without
8: * modification, are permitted provided that the following conditions
9: * are met:
10: * 1. Redistributions of source code must retain the above copyright
11: * notice, this list of conditions and the following disclaimer.
12: * 2. Redistributions in binary form must reproduce the above copyright
13: * notice, this list of conditions and the following disclaimer in the
14: * documentation and/or other materials provided with the distribution.
15: *
1.5 ryo 16: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18: * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19: * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
20: * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21: * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22: * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
24: * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
25: * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
1.1 matt 26: * POSSIBILITY OF SUCH DAMAGE.
27: */
28:
1.28 ryo 29: #include "opt_arm_debug.h"
30: #include "opt_console.h"
1.5 ryo 31: #include "opt_cpuoptions.h"
1.28 ryo 32: #include "opt_ddb.h"
33: #include "opt_fdt.h"
1.31 maxv 34: #include "opt_kasan.h"
1.5 ryo 35: #include "opt_multiprocessor.h"
36:
1.1 matt 37: #include <aarch64/asm.h>
1.5 ryo 38: #include <aarch64/hypervisor.h>
1.1 matt 39: #include "assym.h"
40:
1.63 ! ryo 41: RCSID("$NetBSD: locore.S,v 1.62 2020/05/26 09:16:33 ryo Exp $")
1.28 ryo 42:
1.49 jmcneill 43: #ifdef AARCH64_DEVICE_MEM_STRONGLY_ORDERED
1.48 jmcneill 44: #define MAIR_DEVICE_MEM MAIR_DEVICE_nGnRnE
45: #else
46: #define MAIR_DEVICE_MEM MAIR_DEVICE_nGnRE
47: #endif
1.49 jmcneill 48: #define MAIR_DEVICE_MEM_SO MAIR_DEVICE_nGnRnE
1.5 ryo 49:
1.57 tnn 50: /*#define DEBUG_LOCORE // debug print */
51: /*#define DEBUG_LOCORE_PRINT_LOCK // avoid mixing AP's output */
52: /*#define DEBUG_MMU // dump MMU table */
1.28 ryo 53:
54: #define LOCORE_EL2
55:
56: #define BOOT_AP_STACKSIZE 256 /* size of temporally stack for APs */
57: #define BOOTPAGE_ALLOC_MAX (1024 * 1024) /* reserved size from _end[] */
1.1 matt 58:
1.8 ryo 59: #if (defined(VERBOSE_INIT_ARM) || defined(DEBUG_LOCORE)) && defined(EARLYCONS)
1.5 ryo 60: #define VERBOSE_LOCORE
61: #endif
1.2 nisimura 62:
1.28 ryo 63: #ifdef VERBOSE_LOCORE
64: #define VPRINT(string) PRINT(string)
65: #else
66: #define VPRINT(string)
67: #endif
1.1 matt 68:
1.28 ryo 69: /* DPRINTREG macro use x19 internally. x0-x15 may be broken */
70: #if (defined(DEBUG_LOCORE) && defined(EARLYCONS))
71: #define DPRINT(string) PRINT(string)
72: #define DPRINTREG(str, reg) mov x19,reg; PRINT(str); mov x0,x19; bl print_x0
73: #define DPRINTSREG(str, reg) mrs x19,reg; PRINT(str); mov x0,x19; bl print_x0
1.1 matt 74: #else
1.28 ryo 75: #define DPRINT(string)
76: #define DPRINTREG(str, reg)
77: #define DPRINTSREG(str, reg)
1.5 ryo 78: #endif
79:
1.28 ryo 80: #define PRINT(string) bl xprint; .asciz string; .align 2
81:
82:
83: .text
84: .align 3
85: ASENTRY_NP(aarch64_start)
86: /* keep lr & sp for return to bootloader if possible */
87: mov x27, lr
88: mov x28, sp
1.5 ryo 89:
1.8 ryo 90: /* set stack pointer for boot */
1.53 skrll 91: adrl x0, bootstk
1.8 ryo 92: mov sp, x0
1.5 ryo 93:
1.28 ryo 94: bl clear_bss
1.24 skrll 95:
1.28 ryo 96: PRINT("boot NetBSD/aarch64\n")
1.5 ryo 97:
98: bl 1f
1.28 ryo 99: 1: DPRINTREG("PC = ", lr)
100: DPRINTREG("SP = ", sp)
101: mrs x20, CurrentEL
102: lsr x20, x20, #2
103: DPRINTREG("CurrentEL = ", x20)
104: cmp x20, #2
105: bcc 1f
1.5 ryo 106: /* EL2 registers can be accessed in EL2 or higher */
1.28 ryo 107: DPRINTSREG("SCTLR_EL2 = ", sctlr_el2)
108: DPRINTSREG("HCR_EL2 = ", hcr_el2)
1.5 ryo 109: 1:
1.28 ryo 110: DPRINTSREG("SPSR_EL1 = ", spsr_el1)
111: DPRINTSREG("CNTFREQ_EL0 = ", cntfrq_el0)
112: DPRINTSREG("SCTLR_EL1 = ", sctlr_el1)
113: DPRINTSREG("MIDR_EL1 = ", midr_el1)
114: DPRINTSREG("MPIDR_EL1 = ", mpidr_el1)
115: DPRINTSREG("ID_AA64MPFR0_EL1 = ", id_aa64pfr0_el1)
116: DPRINTSREG("ID_AA64MPFR1_EL1 = ", id_aa64pfr1_el1)
117: DPRINTSREG("ID_AA64ISAR0_EL1 = ", id_aa64isar0_el1)
118: DPRINTSREG("ID_AA64ISAR1_EL1 = ", id_aa64isar1_el1)
119: DPRINTSREG("ID_AA64MMFR0_EL1 = ", id_aa64mmfr0_el1)
120: DPRINTSREG("ID_AA64MMFR1_EL1 = ", id_aa64mmfr1_el1)
1.1 matt 121:
1.5 ryo 122: #ifdef LOCORE_EL2
1.28 ryo 123: VPRINT("Drop to EL1...")
1.5 ryo 124: # include <aarch64/aarch64/locore_el2.S>
1.28 ryo 125: VPRINT("OK\n")
126: mrs x20, CurrentEL
127: lsr x20, x20, #2
128: DPRINTREG("CurrentEL = ", x20)
1.5 ryo 129: #endif /* LOCORE_EL2 */
130:
131:
132: bl mmu_disable
133: bl init_sysregs
1.28 ryo 134: bl init_mmutable
135: cbnz x0, aarch64_fatal
136: bl save_ttbrs
1.5 ryo 137:
1.28 ryo 138: VPRINT("MMU Enable...")
1.5 ryo 139: bl mmu_enable
1.28 ryo 140: VPRINT("OK\n")
1.5 ryo 141:
1.28 ryo 142: ldr x20, =vstart /* virtual address of vstart */
143: DPRINTSREG("SPSR_EL1 = ", spsr_el1)
144: DPRINTSREG("DAIF = ", daif)
145: DPRINTREG("vstart = ", x20)
146: br x20 /* jump to the kernel virtual address */
147:
148: aarch64_fatal:
149: PRINT("fatal error occured while booting\n")
150: /* return to bootloader. if switched from EL2 to EL1, It might fail */
151: mov lr, x27
152: mov sp, x28
153: ret
1.5 ryo 154:
155: /*
156: * vstart is in kernel virtual address
157: */
158: vstart:
1.28 ryo 159: DPRINTREG("PC = ", x20)
160:
161: /* set exception vector */
1.53 skrll 162: adrl x0, _C_LABEL(el1_vectors)
1.28 ryo 163: msr vbar_el1, x0
164:
165: /* set lwp0 stack */
1.53 skrll 166: adrl x0, lwp0uspace
1.5 ryo 167: add x0, x0, #(UPAGES * PAGE_SIZE)
168: sub x0, x0, #TF_SIZE /* lwp0space + USPACE - TF_SIZE */
169: mov sp, x0 /* define lwp0 ksp bottom */
1.28 ryo 170: DPRINTREG("SP(lwp0,kvm) = ", sp)
1.5 ryo 171:
1.28 ryo 172: /* lwp-private = NULL */
173: msr tpidr_el0, xzr
1.29 ryo 174: msr tpidrro_el0, xzr
1.1 matt 175:
1.28 ryo 176: /* set curcpu() */
1.53 skrll 177: adrl x0, cpu_info_store /* cpu_info_store is cpu_info[0] */
1.8 ryo 178: msr tpidr_el1, x0 /* curcpu is cpu_info[0] */
1.28 ryo 179: DPRINTREG("curcpu = ", x0);
180:
1.59 maxv 181: /* init PAC if supported */
182: mov x0, #1
183: bl aarch64_pac_init
1.60 ryo 184: cbnz w0, 1f /* if (aarch64_pac_init() == 0) */
185: mrs x0, sctlr_el1
186: orr x0, x0, #SCTLR_EnIA /* enable PAC */
187: msr sctlr_el1, x0
188: 1:
1.59 maxv 189:
1.55 skrll 190: /* set topology information */
1.45 mlelstv 191: mrs x0, tpidr_el1 /* curcpu */
192: mrs x1, mpidr_el1
1.51 mrg 193: mov x2, #0
1.52 mrg 194: bl arm_cpu_topology_set
1.55 skrll 195:
196: /* get cache configuration */
1.45 mlelstv 197: mov x0, xzr
198: bl aarch64_getcacheinfo
199:
1.31 maxv 200: #ifdef KASAN
1.53 skrll 201: adrl x0, lwp0uspace
1.31 maxv 202: bl _C_LABEL(kasan_early_init)
203: #endif
1.5 ryo 204:
205: mov fp, #0 /* trace back starts here */
1.28 ryo 206: PRINT("initarm\n")
1.5 ryo 207: bl _C_LABEL(initarm) /* Off we go */
208:
1.28 ryo 209: PRINT("main\n")
1.5 ryo 210: bl _C_LABEL(main) /* call main() */
211:
212: adr x0, .Lmainreturned
213: b _C_LABEL(panic)
214: /* NOTREACHED */
1.28 ryo 215: ASEND(aarch64_start)
1.5 ryo 216:
217: .Lmainreturned:
218: .asciz "main() returned"
1.28 ryo 219: .align 2
220:
221:
222: ASENTRY_NP(clear_bss)
223: /* Zero the BSS. The size must be aligned 16, usually it should be. */
1.53 skrll 224: adrl x14, __bss_start__
225: adrl x15, __bss_end__
1.28 ryo 226: b 2f
227: 1: stp xzr, xzr, [x14], #16
228: 2: cmp x14, x15
229: b.lo 1b
230: ret
231: ASEND(clear_bss)
232:
233:
234: init_sysregs:
235: stp x0, lr, [sp, #-16]!
236:
1.32 ryo 237: /* init debug event */
238: ldr x0, mdscr_setting
239: msr mdscr_el1, x0
240: msr oslar_el1, xzr
1.28 ryo 241:
242: /* Clear context id register */
243: msr contextidr_el1, xzr
244:
245: /* No trap system register access, and Trap FP/SIMD access */
246: msr cpacr_el1, xzr
247:
248: /* allow to read CNTVCT_EL0 and CNTFRQ_EL0 from EL0 */
249: mrs x0, cntkctl_el1
250: orr x0, x0, #CNTKCTL_EL0VCTEN
251: msr cntkctl_el1, x0
252:
253: /* any exception not masked */
254: msr daif, xzr
255:
256: ldp x0, lr, [sp], #16
257: ret
1.5 ryo 258:
1.8 ryo 259:
260: #ifdef MULTIPROCESSOR
261:
1.28 ryo 262: #ifdef DEBUG_LOCORE
1.8 ryo 263: /*
1.28 ryo 264: * atomic_ops doesn't work before MMU enabled, so using Peterson's algorithm.
265: * this is only used to serialize debug print and avoid mixing output.
266: * Not absolutely necessary.
267: *
268: * x27 for cpuindex.
1.8 ryo 269: */
1.28 ryo 270: locore_lock_enter:
271: #ifdef DEBUG_LOCORE_PRINT_LOCK
272: mov x3, xzr /* x3 = level */
273: levelloop:
274: /* lock_level[] and lock_turn[] are always accessed via PA(devmap) */
1.53 skrll 275: adrl x0, kern_vtopdiff
1.28 ryo 276: ldr x0, [x0]
277: ldr x4, =lock_level
278: sub x4, x4, x0
279: ldr x5, =lock_turn
280: sub x5, x5, x0
281:
282: strh w3, [x4, x27, lsl #1] /* lock_level[i] = level */
283: dsb sy
284: strh w27, [x5, x3, lsl #1] /* lock_turn[level] = i */
285: dsb sy
286: waitloop:
287: dmb sy
288: ldrh w0, [x5, x3, lsl #1] /* lock_turn[level] == i ? */
289: cmp x27, x0
290: bne nextlevel
291:
292: mov x2, xzr /* k = 0 */
293: levelcheck:
294: cmp x2, x27
295: beq levelcheck_next
296:
297: dmb sy
298: ldrsh w0, [x4, x2, lsl #1] /* lock_level[k] >= level */
299: cmp w0, w3
300: bge waitloop
301: levelcheck_next:
302: add x2, x2, #1 /* k++ */
303: cmp x2, #MAXCPUS
304: bne levelcheck
305: nextlevel:
306: add x3, x3, #1
307: cmp x3, #(MAXCPUS - 1)
308: bne levelloop
309: #endif /* DEBUG_LOCORE_PRINT_LOCK */
310: ret
311:
312:
313: locore_lock_exit:
314: #ifdef DEBUG_LOCORE_PRINT_LOCK
315: /* lock_level[] and lock_turn[] are always accessed via PA(devmap) */
1.53 skrll 316: adrl x0, kern_vtopdiff
1.28 ryo 317: ldr x0, [x0]
318: ldr x1, =lock_level
319: sub x1, x1, x0
320: mvn x0, xzr
321: strh w0, [x1, x27, lsl #1] /* lock_level[i] = -1 */
322: dsb sy
323: #endif /* DEBUG_LOCORE_PRINT_LOCK */
324: ret
325:
326:
327: /* print "[CPU$x27] " (x27 for cpuindex) */
328: printcpu:
1.8 ryo 329: stp x0, lr, [sp, #-16]!
1.28 ryo 330: PRINT("[CPU"); \
331: mov x0, x27; \
332: bl _printdec_x0; \
333: PRINT("] "); \
1.8 ryo 334: ldp x0, lr, [sp], #16
335: ret
336:
1.28 ryo 337: #define CPU_DPRINT(str) \
338: bl locore_lock_enter; \
339: bl printcpu; \
340: DPRINT(str); \
341: bl locore_lock_exit
342:
343: /*
344: * CPU_DPRINTREG macro use x19 internally. x0-x15 may be broken.
345: * x27 for cpuindex.
346: */
347: #define CPU_DPRINTREG(str,reg) \
348: mov x19, reg; \
349: bl locore_lock_enter; \
350: bl printcpu; \
351: PRINT(str); \
352: mov x0, x19; \
353: bl print_x0; \
354: bl locore_lock_exit
355:
356: #define CPU_DPRINTSREG(str, reg) \
357: mrs x19, reg; \
358: CPU_DPRINTREG(str, x19)
359:
360: #else /* DEBUG_LOCORE */
361:
362: #define CPU_DPRINT(str)
363: #define CPU_DPRINTREG(str,reg)
364: #define CPU_DPRINTSREG(str, reg)
365:
366: #endif /* DEBUG_LOCORE */
1.8 ryo 367:
1.28 ryo 368: ENTRY_NP(cpu_mpstart)
1.25 ryo 369: mrs x3, mpidr_el1
370: ldr x0, =(MPIDR_AFF0|MPIDR_AFF1|MPIDR_AFF2|MPIDR_AFF3)
371: and x3, x3, x0
372:
1.28 ryo 373: /*
374: * resolve own cpuindex. my mpidr is stored in
375: * extern uint64_t cpu_mpidr[MAXCPUS]
376: */
1.53 skrll 377: adrl x0, _C_LABEL(cpu_mpidr)
1.25 ryo 378: mov x1, xzr
379: 1:
380: add x1, x1, #1
1.44 jmcneill 381: cmp x1, #MAXCPUS /* cpuindex >= MAXCPUS ? */
1.25 ryo 382: bge toomanycpus
1.35 skrll 383: ldr x2, [x0, x1, lsl #3] /* cpu_mpidr[cpuindex] */
1.25 ryo 384: cmp x2, x3 /* == mpidr_el1 & MPIDR_AFF ? */
385: bne 1b
386:
387: mov x27, x1 /* x27 = cpuindex */
1.28 ryo 388:
389: /*
390: * x27 = cpuindex
391: */
1.8 ryo 392:
393: /* set stack pointer for boot */
1.28 ryo 394: mov x1, #BOOT_AP_STACKSIZE
1.8 ryo 395: mul x1, x1, x27
1.53 skrll 396: adrl x0, bootstk
1.43 skrll 397: add sp, x0, x1 /* sp = bootstk + (BOOT_AP_STACKSIZE * cpuindex) */
1.8 ryo 398:
399: bl 1f
1.28 ryo 400: 1: CPU_DPRINTREG("PC = ", lr)
401: CPU_DPRINTREG("SP = ", sp)
402: mrs x20, CurrentEL
403: lsr x20, x20, #2
404: CPU_DPRINTREG("CurrentEL = ", x20)
405: cmp x20, #2
406: bcc 1f
407: /* EL2 registers can be accessed in EL2 or higher */
408: CPU_DPRINTSREG("SCTLR_EL2 = ", sctlr_el2)
409: CPU_DPRINTSREG("HCR_EL2 = ", hcr_el2)
410: 1:
411: CPU_DPRINTSREG("SPSR_EL1 = ", spsr_el1)
412: CPU_DPRINTSREG("SCTLR_EL1 = ", sctlr_el1)
413: CPU_DPRINTSREG("MIDR_EL1 = ", midr_el1)
414: CPU_DPRINTSREG("MPIDR_EL1 = ", mpidr_el1)
1.8 ryo 415:
416: #ifdef LOCORE_EL2
1.28 ryo 417: CPU_DPRINT("Drop to EL1...\n")
1.8 ryo 418: bl drop_to_el1
1.28 ryo 419: CPU_DPRINT("Drop to EL1 OK\n")
420: mrs x20, CurrentEL
421: lsr x20, x20, #2
422: CPU_DPRINTREG("CurrentEL = ", x20)
1.8 ryo 423: #endif /* LOCORE_EL2 */
424:
1.28 ryo 425:
1.8 ryo 426: bl mmu_disable
427: bl init_sysregs
428:
1.28 ryo 429: CPU_DPRINT("MMU Enable...\n")
430: bl load_ttbrs
1.8 ryo 431: bl mmu_enable
1.28 ryo 432: CPU_DPRINT("MMU Enable OK\n")
1.8 ryo 433:
434: /* jump to virtual address */
1.28 ryo 435: ldr x20, =mp_vstart
436: br x20
1.8 ryo 437:
438: mp_vstart:
1.61 ryo 439: hint 0x24 /* bti j */
1.28 ryo 440: CPU_DPRINTREG("PC = ", x20)
441:
1.8 ryo 442: /* set exception vector */
1.53 skrll 443: adrl x0, _C_LABEL(el1_vectors)
1.8 ryo 444: msr vbar_el1, x0
445:
1.28 ryo 446: /* lwp-private = NULL */
447: msr tpidr_el0, xzr
1.29 ryo 448: msr tpidrro_el0, xzr
1.20 ryo 449:
450: /* set curcpu(), and fill curcpu()->ci_{midr,mpidr} */
451: mov x0, #CPU_INFO_SIZE
452: mul x0, x27, x0
1.53 skrll 453: adrl x1, _C_LABEL(cpu_info_store)
1.20 ryo 454: add x0, x0, x1 /* x0 = &cpu_info_store[cpuindex] */
455: msr tpidr_el1, x0 /* tpidr_el1 = curcpu() = x0 */
456:
1.8 ryo 457: mrs x1, midr_el1
1.20 ryo 458: str x1, [x0, #CI_MIDR] /* curcpu()->ci_cpuid = midr_el1 */
1.8 ryo 459: mrs x1, mpidr_el1
1.20 ryo 460: str x1, [x0, #CI_MPIDR] /* curcpu()->ci_mpidr = mpidr_el1 */
1.8 ryo 461:
1.56 skrll 462: /* set topology information */
463: mov x2, #0
464: bl arm_cpu_topology_set
465:
466: /* x28 = &arm_cpu_hatched[cpuindex / (sizeof(u_long) * NBBY)] */
467: adrl x0, _C_LABEL(arm_cpu_hatched)
1.58 skrll 468: // Appease clang - mov x1, x27, lsr #6
469: orr x1, xzr, x27, lsr #6
1.56 skrll 470: add x28, x0, x1, lsl #3
471:
472: /* x29 = __BIT(cpuindex % (sizeof(u_long) * NBBY)) */
1.44 jmcneill 473: mov x0, #1
1.56 skrll 474: and x2, x27, #63
475: lsl x29, x0, x2
1.44 jmcneill 476:
1.8 ryo 477: /*
1.56 skrll 478: * atomic_or_ulong(&arm_cpu_hatched[cpuindex / (sizeof(u_long) * NBBY)],
479: * _BIT(cpuindex % ((sizeof(u_long) * NBBY)
480: * to inform the boot processor.
1.8 ryo 481: */
1.42 jmcneill 482: mov x0, x28
1.44 jmcneill 483: mov x1, x29
1.56 skrll 484: bl _C_LABEL(atomic_or_ulong) /* hatched! */
1.42 jmcneill 485: dsb sy
1.8 ryo 486: sev
487:
1.56 skrll 488: /* x28 = &arm_cpu_mbox[cpuindex / (sizeof(u_long) * NBBY)] */
489: adrl x0, _C_LABEL(arm_cpu_mbox)
1.58 skrll 490: // Appease clang - mov x1, x27, lsr #6
491: orr x1, xzr, x27, lsr #6
1.56 skrll 492: add x28, x0, x1, lsl #3
1.44 jmcneill 493:
1.42 jmcneill 494: /* wait for the mailbox start bit to become true */
1.8 ryo 495: 1:
496: dmb sy
1.56 skrll 497: ldr x20, [x28]
498: tst x20, x29
1.8 ryo 499: bne 9f
500: wfe
501: b 1b
502: 9:
503:
504: /* fill my cpu_info */
1.20 ryo 505: mrs x0, tpidr_el1 /* curcpu() */
1.8 ryo 506:
507: ldr x1, [x0, #CI_IDLELWP] /* x1 = curcpu()->ci_data.cpu_idlelwp */
508: str x1, [x0, #CI_CURLWP] /* curlwp is idlelwp */
509:
1.28 ryo 510: /* get my stack from lwp */
1.8 ryo 511: ldr x2, [x1, #L_PCB] /* x2 = lwp_getpcb(idlelwp) */
512: add x2, x2, #(UPAGES * PAGE_SIZE)
513: sub sp, x2, #TF_SIZE /* sp = pcb + USPACE - TF_SIZE */
514:
1.59 maxv 515: /* init PAC if supported */
516: mov x0, #0
517: bl aarch64_pac_init
1.60 ryo 518: cbnz w0, 1f /* if (aarch64_pac_init() == 0) */
519: mrs x0, sctlr_el1
520: orr x0, x0, #SCTLR_EnIA /* enable PAC */
521: msr sctlr_el1, x0
522: 1:
1.8 ryo 523:
524: mov fp, xzr /* trace back starts here */
1.59 maxv 525: mrs x0, tpidr_el1 /* curcpu() */
1.8 ryo 526: bl _C_LABEL(cpu_hatch)
527: mov x0, xzr
528: b _C_LABEL(idle_loop) /* never to return */
1.30 skrll 529: END(cpu_mpstart)
1.8 ryo 530:
1.20 ryo 531: toomanycpus:
1.28 ryo 532: CPU_DPRINT("too many cpus, or MPIDR not exists in cpu_mpidr[]\n")
1.20 ryo 533: 1: wfi
534: b 1b
535:
536:
1.8 ryo 537: #else /* MULTIPROCESSOR */
538:
1.30 skrll 539: ENTRY_NP(cpu_mpstart)
1.8 ryo 540: 1: wfi
541: b 1b
1.30 skrll 542: END(cpu_mpstart)
1.8 ryo 543:
544: #endif /* MULTIPROCESSOR */
545:
1.28 ryo 546:
1.5 ryo 547: /*
548: * xprint - print strings pointed by $PC(LR)
549: * and return to the end of string.
1.28 ryo 550: * "\n" will be replaced "\r\n"
1.5 ryo 551: * e.g.)
1.28 ryo 552: * bl xprint <- call
553: * .ascii "Hello\n\0" <- wouldn't return here
1.5 ryo 554: * .align 2
1.28 ryo 555: * nop <- return to here
1.22 skrll 556: *
1.5 ryo 557: */
1.28 ryo 558: xprint:
559: mov x0, lr
560: bl _C_LABEL(uartputs)
561: add x0, x0, #3
562: bic lr, x0, #3
563: ret
1.22 skrll 564:
1.28 ryo 565: /*
566: * uartputs(str) - print strings with replacing "\n" to "\r\n".
567: * returns the address after the end of the string. (x0 = next of '\0')
568: */
569: ENTRY_NP(uartputs)
570: stp x19, lr, [sp, #-16]!
571: mov x19, x0
1.22 skrll 572: ldrb w0, [x19], #1
1.28 ryo 573: cbz w0, 9f
1.5 ryo 574: 1:
1.28 ryo 575: cmp x0, #'\n'
576: bne 2f
1.62 ryo 577: mov x0, #0x0d /* '\r' */
1.28 ryo 578: bl uartputc
579: mov x0, #'\n'
580: 2:
1.5 ryo 581: bl uartputc
1.22 skrll 582: ldrb w0, [x19], #1
1.5 ryo 583: cbnz w0, 1b
584: 9:
1.28 ryo 585: mov x0, x19
586: ldp x19, lr, [sp], #16
1.5 ryo 587: ret
1.21 maxv 588: END(uartputs)
1.5 ryo 589:
1.28 ryo 590: /*
591: * print x0 in 16 widths hexadecimal.
592: *
593: * x0 is preserved despite being caller saved.
594: * other caller saved registers will be broken.
595: */
596: _print_x0:
1.5 ryo 597: stp x0, lr, [sp, #-16]!
1.22 skrll 598: stp x20, x21, [sp, #-16]!
1.5 ryo 599:
1.22 skrll 600: mov x21, x0 /* number to display */
601: mov x20, #60 /* num of shift */
1.5 ryo 602: 1:
1.22 skrll 603: ror x0, x21, x20
604: and x0, x0, #0xf
1.5 ryo 605: cmp x0, #10
606: blt 2f
607: add x0, x0, #('a' - 10 - '0')
608: 2: add x0, x0, #'0'
609: bl uartputc
1.22 skrll 610: subs x20, x20, #4
1.5 ryo 611: bge 1b
612:
1.22 skrll 613: ldp x20, x21, [sp], #16
1.5 ryo 614: ldp x0, lr, [sp], #16
615: ret
616:
1.28 ryo 617: /*
618: * print x0 in decimal.
619: *
620: * x0 is preserved despite being caller saved.
621: * other caller saved registers will be broken.
622: */
623: _printdec_x0:
624: stp x0, lr, [sp, #-(16+32)]!
625: add x8, sp, #(16+32)
1.5 ryo 626:
1.28 ryo 627: strb wzr, [x8, #-1]!
628: 1:
629: mov x10, #10
630: udiv x1, x0, x10 /* x1 = x0 / 10 */
631: msub x3, x1, x10, x0 /* x3 = x0 % 10 */
1.5 ryo 632: mov x0, x1
1.1 matt 633:
1.28 ryo 634: add x3, x3, #'0'
635: strb w3, [x8, #-1]!
636: cbnz x0, 1b
1.1 matt 637:
1.28 ryo 638: mov x0, x8
639: bl uartputs
1.5 ryo 640:
1.28 ryo 641: ldp x0, lr, [sp], #(16+32)
1.5 ryo 642: ret
643:
644: /*
1.28 ryo 645: * print x0 in 16 widths hexadecimal with crlf.
646: *
647: * x0 is preserved despite being caller saved.
648: * other caller saved registers will be broken.
1.5 ryo 649: */
1.28 ryo 650: print_x0:
1.5 ryo 651: stp x0, lr, [sp, #-16]!
1.28 ryo 652: bl _print_x0
653: PRINT("\n")
1.5 ryo 654: ldp x0, lr, [sp], #16
1.1 matt 655: ret
656:
1.28 ryo 657: #ifdef DEBUG_MMU
1.1 matt 658: /*
1.28 ryo 659: * tinyprintf() supports only maximum 7 '%x', '%d' and '%s' formats.
660: * width and any modifiers are ignored. '\n' will be replaced to '\r\n'.
661: *
662: * '%x' will be always expanded 16 widths hexadicimal.
663: * e.g., tinyprintf("Hello %s %x\n", "World", 0x12345)
664: * outputs "Hello World 0000000000012345\r\n"
1.2 nisimura 665: */
1.28 ryo 666: tinyprintf:
1.5 ryo 667: stp x0, lr, [sp, #-16]!
1.22 skrll 668: stp x19, x20, [sp, #-16]!
1.28 ryo 669: stp x7, x8, [sp, #-16]!
670: stp x5, x6, [sp, #-16]!
671: stp x3, x4, [sp, #-16]!
672: stp x1, x2, [sp, #-16]!
1.22 skrll 673:
1.28 ryo 674: mov x20, xzr
675: mov x19, x0
676: ldrb w0, [x19], #1
677: cbz w0, tinyprintf_done
1.5 ryo 678:
1.28 ryo 679: tinyprintf_loop:
680: cmp x0, #'\n'
681: bne 1f
682: /* '\n' -> '\r', '\n' */
1.62 ryo 683: mov x0, #0x0d /* '\r' */
1.28 ryo 684: bl uartputc
685: mov x0, #'\n'
1.5 ryo 686: 1:
1.22 skrll 687:
1.28 ryo 688: cmp x0, #'%'
689: bne tinyprintf_putc
690: cmp x20, #8
691: bcs tinyprintf_putc
692:
693: tinyprintf_fetch_fmt:
694: ldrb w9, [x19], #1
695: cbz w9, tinyprintf_done
696:
697: /* width and modifier are ignored */
698: cmp x9, #'h'
699: beq tinyprintf_fetch_fmt
700: cmp x9, #'l'
701: beq tinyprintf_fetch_fmt
702: cmp x9, #'j'
703: beq tinyprintf_fetch_fmt
704: cmp x9, #'t'
705: beq tinyprintf_fetch_fmt
706: cmp x9, #'z'
707: beq tinyprintf_fetch_fmt
708: cmp x9, #'0'
709: bcc 1f
710: cmp x9, #'9'
711: bls tinyprintf_fetch_fmt
712: 1:
713: ldr x0, [sp, x20, lsl #3] /* get Nth argument */
1.22 skrll 714: add x20, x20, #1
1.5 ryo 715:
1.28 ryo 716: cmp x9, #'x'
717: bne 5f
718: /* "%x" format */
719: bl _print_x0
720: b tinyprintf_next
721: 5:
722: cmp x9, #'d'
723: bne 5f
724: /* "%d" format */
725: bl _printdec_x0
726: b tinyprintf_next
727: 5:
728: cmp x9, #'s'
729: bne 5f
730: /* "%s" format */
731: bl _C_LABEL(uartputs)
732: b tinyprintf_next
733: 5:
734:
735: tinyprintf_putc:
736: bl uartputc
737: tinyprintf_next:
738: ldrb w0, [x19], #1
739: cbnz w0, tinyprintf_loop
740:
741: tinyprintf_done:
742: mov x0, x19
743:
744: ldp x1, x2, [sp], #16
745: ldp x3, x4, [sp], #16
746: ldp x5, x6, [sp], #16
747: ldp x7, x8, [sp], #16
1.22 skrll 748: ldp x19, x20, [sp], #16
1.5 ryo 749: ldp x0, lr, [sp], #16
1.2 nisimura 750: ret
1.28 ryo 751: #endif /* defined(DEBUG_LOCORE) || defined(DEBUG_MMU) */
1.2 nisimura 752:
1.5 ryo 753:
1.28 ryo 754: save_ttbrs:
755: /* save ttbr[01]_el1 for AP */
756: mrs x0, ttbr0_el1
757: mrs x1, ttbr1_el1
1.53 skrll 758: adrl x2, ttbr_save
1.28 ryo 759: stp x0, x1, [x2]
760: ret
1.5 ryo 761:
1.28 ryo 762: load_ttbrs:
763: /* load ttbr[01]_el1 */
1.53 skrll 764: adrl x2, ttbr_save
1.28 ryo 765: ldp x0, x1, [x2]
766: msr ttbr0_el1, x0
767: msr ttbr1_el1, x1
1.3 nisimura 768: ret
1.2 nisimura 769:
1.5 ryo 770:
1.28 ryo 771: init_mmutable:
772: stp x26, lr, [sp, #-16]!
1.22 skrll 773:
1.28 ryo 774: /* first allocated page must be kernel l0pt = ARM_BOOTSTRAP_LxPT */
775: bl bootpage_alloc
776: cbz x0, init_mmutable_error
777: msr ttbr1_el1, x0
778:
779: bl bootpage_alloc
780: cbz x0, init_mmutable_error
781: msr ttbr0_el1, x0
1.22 skrll 782:
1.5 ryo 783: #ifdef DEBUG_MMU
1.28 ryo 784: adr x26, tinyprintf
785: #else
786: mov x26, xzr
1.5 ryo 787: #endif
788:
1.28 ryo 789: /*
790: * int
791: * pmapboot_enter(
792: * x0: vaddr_t va,
793: * x1: paddr_t pa,
794: * x2: psize_t size,
795: * x3: psize_t blocksize, // L[123]_SIZE
796: * x4: pt_entry_t attr, // pte attributes. LX_BLKPAG_*
797: * x5: flags,
798: * x6: pd_entry_t *(*physpage_allocator)(void),
799: * x7: void (*pr)(const char *, ...)
800: * );
801: */
1.5 ryo 802:
1.28 ryo 803: #ifdef CONSADDR
804: VPRINT("Creating VA=PA tables for CONSADDR\n")
805: mov x7, x26 /* pr func */
806: adr x6, bootpage_alloc /* allocator */
807: mov x5, xzr /* flags = 0 */
808: mov x4, #LX_BLKPAG_ATTR_DEVICE_MEM|LX_BLKPAG_AP_RW /* attr */
1.40 jmcneill 809: orr x4, x4, #LX_BLKPAG_UXN|LX_BLKPAG_PXN
1.28 ryo 810: mov x3, #L2_SIZE /* blocksize */
811: mov x2, #L2_SIZE /* size */
812: ldr x1, =CONSADDR /* pa */
813: mov x0, x1 /* va */
814: bl pmapboot_enter
815: cbnz x0, init_mmutable_error
816: #elif defined(EARLYCONS)
817: /* CONSADDR is unknown, but need to map UART */
818: VPRINT("Creating VA=PA tables (0x00000000-0xffffffff)\n")
819: mov x7, x26 /* pr func */
820: adr x6, bootpage_alloc /* allocator */
821: mov x5, xzr /* flags = 0 */
822: mov x4, #LX_BLKPAG_ATTR_DEVICE_MEM|LX_BLKPAG_AP_RW /* attr */
1.40 jmcneill 823: orr x4, x4, #LX_BLKPAG_UXN|LX_BLKPAG_PXN
1.28 ryo 824: mov x3, #L2_SIZE /* blocksize */
825: mov x2, #(1024*1024*1024*4) /* size */
826: mov x1, xzr /* pa */
827: mov x0, xzr /* va */
828: bl pmapboot_enter
829: cbnz x0, init_mmutable_error
830: #endif
831:
832: /* identity mapping for kernel image */
833: VPRINT("Creating VA=PA tables for kernel image\n")
834: mov x7, x26 /* pr func */
835: adr x6, bootpage_alloc /* allocator */
836: mov x5, xzr /* flags = 0 */
837: mov x4, #LX_BLKPAG_ATTR_NORMAL_NC|LX_BLKPAG_AP_RW /* attr */
838: mov x3, #L2_SIZE /* blocksize */
839: adr x0, start /* va = start */
1.53 skrll 840: adrl x2, _end
1.28 ryo 841: sub x2, x2, x0 /* size = _end - start */
1.41 skrll 842: add x2, x2, #BOOTPAGE_ALLOC_MAX /* for bootpage_alloc() */
1.28 ryo 843: mov x1, x0 /* pa */
844: bl pmapboot_enter
845: cbnz x0, init_mmutable_error
846:
847: #ifdef FDT
1.63 ! ryo 848: VPRINT("Creating VA=PA tables for FDT\n")
1.53 skrll 849: adrl x8, _C_LABEL(fdt_addr_r)
1.28 ryo 850: ldr x8, [x8]
851:
852: mov x7, x26 /* pr func */
853: adr x6, bootpage_alloc /* allocator */
854: mov x5, xzr /* flags = 0 */
855: mov x4, #LX_BLKPAG_ATTR_NORMAL_NC|LX_BLKPAG_AP_RW /* attr */
1.46 skrll 856: orr x4, x4, #LX_BLKPAG_UXN|LX_BLKPAG_PXN
1.28 ryo 857: mov x3, #L2_SIZE /* blocksize */
858: mov x2, #L2_SIZE /* size */
859: mov x1, x8 /* pa */
860: mov x0, x8 /* va */
861: bl pmapboot_enter
862: cbnz x0, init_mmutable_error
863: #endif
864:
865: VPRINT("Creating KVA=PA tables\n")
866: mov x7, x26 /* pr func */
867: adr x6, bootpage_alloc /* allocator */
868: mov x5, xzr /* flags = 0 */
869: mov x4, #LX_BLKPAG_ATTR_NORMAL_WB|LX_BLKPAG_AP_RW /* attr */
870: orr x4, x4, #LX_BLKPAG_UXN
871: mov x3, #L2_SIZE /* blocksize */
1.38 skrll 872: adr x1, start /* pa = start */
1.53 skrll 873: adrl x2, _end
1.28 ryo 874: sub x2, x2, x1 /* size = _end - start */
1.37 skrll 875: ldr x0, =start /* va */
1.28 ryo 876: bl pmapboot_enter
877: cbnz x0, init_mmutable_error
1.5 ryo 878:
1.28 ryo 879: VPRINT("OK\n");
880: mov x0, xzr
881: b init_mmutable_done
882: init_mmutable_error:
883: mvn x0, xzr
884: init_mmutable_done:
885: ldp x26, lr, [sp], #16
886: ret
1.5 ryo 887:
1.28 ryo 888: /* return PA of allocated page */
889: ENTRY_NP(bootpage_alloc)
890: /* x2 = kernend_extra */
1.53 skrll 891: adrl x3, kernend_extra
1.28 ryo 892: ldr x2, [x3]
893: /* if (kernend_extra < 0) return NULL */
894: mov x0, xzr
895: cmp x2, xzr
896: bmi bootpage_alloc_done
1.5 ryo 897:
1.28 ryo 898: /* x0 = PA of _end[] */
1.53 skrll 899: adrl x1, kern_vtopdiff
1.28 ryo 900: ldr x1, [x1]
901: ldr x0, =ARM_BOOTSTRAP_LxPT
902: sub x0, x0, x1
903:
904: /* x0 = ARM_BOOTSTRAP_LxPT + kernend_extra */
905: add x0, x0, x2
906:
907: /* kernend_extra += PAGE_SIZE; */
908: add x2, x2, #PAGE_SIZE
909: str x2, [x3]
910:
911: /* clear allocated page */
912: mov x1, x0
913: add x2, x1, #PAGE_SIZE
914: 1: stp xzr, xzr, [x1], #16
915: cmp x1, x2
916: bcc 1b
917: bootpage_alloc_done:
918: ret
919: END(bootpage_alloc)
1.5 ryo 920:
921:
1.28 ryo 922: mmu_disable:
1.5 ryo 923: dsb sy
924: mrs x0, sctlr_el1
925: bic x0, x0, SCTLR_M /* clear MMU enable bit */
926: msr sctlr_el1, x0
927: isb
928: ret
929:
1.28 ryo 930: mmu_enable:
1.5 ryo 931: dsb sy
932:
933: /* Invalidate all TLB */
934: dsb ishst
935: #ifdef MULTIPROCESSOR
936: tlbi vmalle1is
937: #else
938: tlbi vmalle1
939: #endif
940: dsb ish
941: isb
942:
943: ldr x0, mair_setting
944: msr mair_el1, x0
945:
946:
947: /* TCR_EL1:IPS[34:32] = AA64MMFR0:PARange[3:0] */
948: ldr x0, tcr_setting
949: mrs x1, id_aa64mmfr0_el1
950: bfi x0, x1, #32, #3
951: msr tcr_el1, x0
1.1 matt 952:
953: /*
1.5 ryo 954: * configure SCTLR
1.1 matt 955: */
1.5 ryo 956: mrs x0, sctlr_el1
957: ldr x1, sctlr_clear
958: bic x0, x0, x1
959: ldr x1, sctlr_set
960: orr x0, x0, x1
961:
962: ldr x1, sctlr_ee
963: #ifdef __AARCH64EB__
964: orr x0, x0, x1 /* set: BigEndian */
965: #else
966: bic x0, x0, x1 /* clear: LittleEndian */
967: #endif
968: msr sctlr_el1, x0 /* enabling MMU! */
969: isb
1.1 matt 970:
971: ret
1.28 ryo 972:
1.1 matt 973:
1.5 ryo 974: .align 3
975: mair_setting:
976: .quad ( \
977: __SHIFTIN(MAIR_NORMAL_WB, MAIR_ATTR0) | \
978: __SHIFTIN(MAIR_NORMAL_NC, MAIR_ATTR1) | \
979: __SHIFTIN(MAIR_NORMAL_WT, MAIR_ATTR2) | \
1.49 jmcneill 980: __SHIFTIN(MAIR_DEVICE_MEM, MAIR_ATTR3) | \
981: __SHIFTIN(MAIR_DEVICE_MEM_SO, MAIR_ATTR4))
1.5 ryo 982:
983: #define VIRT_BIT 48
1.28 ryo 984:
985: #ifdef MULTIPROCESSOR
986: #define TCR_SHAREABLE (TCR_SH0_INNER | TCR_SH1_INNER)
987: #else
988: #define TCR_SHAREABLE (TCR_SH0_NONE | TCR_SH1_NONE)
989: #endif
990:
1.5 ryo 991: tcr_setting:
992: .quad ( \
993: __SHIFTIN(64 - VIRT_BIT, TCR_T1SZ) | \
994: __SHIFTIN(64 - VIRT_BIT, TCR_T0SZ) | \
995: TCR_AS64K | \
996: TCR_TG1_4KB | TCR_TG0_4KB | \
997: TCR_ORGN0_WB_WA | \
998: TCR_IRGN0_WB_WA | \
999: TCR_ORGN1_WB_WA | \
1.28 ryo 1000: TCR_IRGN1_WB_WA) | TCR_SHAREABLE
1.5 ryo 1001:
1002:
1003: #ifdef AARCH64_ALIGNMENT_CHECK
1004: #define SCTLR_A_CONFIG SCTLR_A
1005: #else
1006: #define SCTLR_A_CONFIG 0
1007: #endif
1008:
1009: #ifdef AARCH64_EL0_STACK_ALIGNMENT_CHECK
1010: #define SCTLR_SA0_CONFIG SCTLR_SA0
1011: #else
1012: #define SCTLR_SA0_CONFIG 0
1013: #endif
1014:
1015: #ifdef AARCH64_EL1_STACK_ALIGNMENT_CHECK
1016: #define SCTLR_SA_CONFIG SCTLR_SA
1017: #else
1018: #define SCTLR_SA_CONFIG 0
1.3 nisimura 1019: #endif
1020:
1.5 ryo 1021:
1022: sctlr_ee:
1.39 skrll 1023: .quad (SCTLR_EE | SCTLR_EOE) /* Endianness of Exception and EL0 */
1.5 ryo 1024: sctlr_set:
1025: .quad ( \
1026: SCTLR_LSMAOE | /* Load/Store Multiple Atomicity and Ordering */ \
1027: SCTLR_nTLSMD | /* no Trap Load/Store Multiple to Device */ \
1.6 ryo 1028: SCTLR_UCI | /* Enables EL0 DC {CVAU,CIVAC,CVAC}, IC IVAU */ \
1.5 ryo 1029: SCTLR_SPAN | /* This field resets to 1 */ \
1030: SCTLR_UCT | /* Enables EL0 access to the CTR_EL0 */ \
1.9 ryo 1031: SCTLR_nTWE | /* EL0 WFE non-trapping */ \
1032: SCTLR_nTWI | /* EL0 WFI non-trapping */ \
1.5 ryo 1033: SCTLR_DZE | /* Enables access to the DC ZVA instruction */ \
1034: SCTLR_I | /* Instruction cache enable */ \
1035: SCTLR_SED | /* SETEND instruction disable */ \
1036: SCTLR_C | /* Cache enable */ \
1037: SCTLR_M | /* MMU Enable */ \
1038: SCTLR_SA0_CONFIG | \
1039: SCTLR_SA_CONFIG | \
1040: SCTLR_A_CONFIG | \
1041: 0)
1042: sctlr_clear:
1043: .quad ( \
1044: SCTLR_IESB | /* Enable Implicit ErrorSynchronizationBarrier */ \
1045: SCTLR_WXN | /* Write permission implies Execute Never (W^X) */ \
1046: SCTLR_UMA | /* EL0 Controls access to interrupt masks */ \
1047: SCTLR_ITD | /* IT instruction disable */ \
1.54 maxv 1048: SCTLR_nAA | /* ? */ \
1.5 ryo 1049: SCTLR_CP15BEN | /* CP15 barrier enable */ \
1050: SCTLR_SA0 | /* Enable EL0 stack alignment check */ \
1051: SCTLR_SA | /* Enable SP alignment check */ \
1052: SCTLR_A | /* Alignment check enable */ \
1053: 0)
1054:
1.32 ryo 1055: mdscr_setting:
1056: .quad ( \
1057: MDSCR_TDCC | /* Trap Debug Communications Channel access */ \
1058: 0)
1059:
1.28 ryo 1060: .L_devmap_addr:
1061: .quad VM_KERNEL_IO_ADDRESS
1062:
1063: .data
1064:
1065: #ifdef DEBUG_LOCORE_PRINT_LOCK
1066: .align 2
1067: lock_level:
1068: .fill MAXCPUS, 2, -1
1069: lock_turn:
1070: .fill (MAXCPUS - 1), 2, -1
1071: #endif /* DEBUG_LOCORE_PRINT_LOCK */
1072:
1073: .align 3
1074: ttbr_save:
1075: .space 8 * 2
1.5 ryo 1076:
1.8 ryo 1077: .bss
1078:
1.5 ryo 1079: .align PGSHIFT
1080: .global _C_LABEL(lwp0uspace)
1081: _C_LABEL(lwp0uspace):
1082: .space UPAGES * PAGE_SIZE
1.8 ryo 1083: bootstk:
1084:
1085: #ifdef MULTIPROCESSOR
1.28 ryo 1086: .space BOOT_AP_STACKSIZE * (MAXCPUS - 1)
1.8 ryo 1087: #endif
1088:
1.28 ryo 1089: .section ".init_pagetable", "aw", %nobits
1.5 ryo 1090: .align PGSHIFT
1.28 ryo 1091: .global ARM_BOOTSTRAP_LxPT
1092: ARM_BOOTSTRAP_LxPT:
1093: l0pt_kern:
1.30 skrll 1094:
1095: .section "_init_memory", "aw", %nobits
1096: .align PGSHIFT
1097:
1098: /* None currently */
CVSweb <webmaster@jp.NetBSD.org>