[BACK]Return to locore.S CVS log [TXT][DIR] Up to [cvs.NetBSD.org] / src / sys / arch / aarch64 / aarch64

Annotation of src/sys/arch/aarch64/aarch64/locore.S, Revision 1.63

1.63    ! ryo         1: /*     $NetBSD: locore.S,v 1.62 2020/05/26 09:16:33 ryo Exp $  */
1.1       matt        2:
1.5       ryo         3: /*
                      4:  * Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org>
1.1       matt        5:  * All rights reserved.
                      6:  *
                      7:  * Redistribution and use in source and binary forms, with or without
                      8:  * modification, are permitted provided that the following conditions
                      9:  * are met:
                     10:  * 1. Redistributions of source code must retain the above copyright
                     11:  *    notice, this list of conditions and the following disclaimer.
                     12:  * 2. Redistributions in binary form must reproduce the above copyright
                     13:  *    notice, this list of conditions and the following disclaimer in the
                     14:  *    documentation and/or other materials provided with the distribution.
                     15:  *
1.5       ryo        16:  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
                     17:  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
                     18:  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
                     19:  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
                     20:  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
                     21:  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
                     22:  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
                     23:  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
                     24:  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
                     25:  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
1.1       matt       26:  * POSSIBILITY OF SUCH DAMAGE.
                     27:  */
                     28:
1.28      ryo        29: #include "opt_arm_debug.h"
                     30: #include "opt_console.h"
1.5       ryo        31: #include "opt_cpuoptions.h"
1.28      ryo        32: #include "opt_ddb.h"
                     33: #include "opt_fdt.h"
1.31      maxv       34: #include "opt_kasan.h"
1.5       ryo        35: #include "opt_multiprocessor.h"
                     36:
1.1       matt       37: #include <aarch64/asm.h>
1.5       ryo        38: #include <aarch64/hypervisor.h>
1.1       matt       39: #include "assym.h"
                     40:
1.63    ! ryo        41: RCSID("$NetBSD: locore.S,v 1.62 2020/05/26 09:16:33 ryo Exp $")
1.28      ryo        42:
1.49      jmcneill   43: #ifdef AARCH64_DEVICE_MEM_STRONGLY_ORDERED
1.48      jmcneill   44: #define        MAIR_DEVICE_MEM         MAIR_DEVICE_nGnRnE
                     45: #else
                     46: #define        MAIR_DEVICE_MEM         MAIR_DEVICE_nGnRE
                     47: #endif
1.49      jmcneill   48: #define        MAIR_DEVICE_MEM_SO      MAIR_DEVICE_nGnRnE
1.5       ryo        49:
1.57      tnn        50: /*#define DEBUG_LOCORE                 // debug print */
                     51: /*#define DEBUG_LOCORE_PRINT_LOCK      // avoid mixing AP's output */
                     52: /*#define DEBUG_MMU                    // dump MMU table */
1.28      ryo        53:
                     54: #define LOCORE_EL2
                     55:
                     56: #define BOOT_AP_STACKSIZE      256     /* size of temporally stack for APs */
                     57: #define BOOTPAGE_ALLOC_MAX     (1024 * 1024)   /* reserved size from _end[] */
1.1       matt       58:
1.8       ryo        59: #if (defined(VERBOSE_INIT_ARM) || defined(DEBUG_LOCORE)) && defined(EARLYCONS)
1.5       ryo        60: #define VERBOSE_LOCORE
                     61: #endif
1.2       nisimura   62:
1.28      ryo        63: #ifdef VERBOSE_LOCORE
                     64: #define VPRINT(string)         PRINT(string)
                     65: #else
                     66: #define VPRINT(string)
                     67: #endif
1.1       matt       68:
1.28      ryo        69: /* DPRINTREG macro use x19 internally. x0-x15 may be broken */
                     70: #if (defined(DEBUG_LOCORE) && defined(EARLYCONS))
                     71: #define DPRINT(string)         PRINT(string)
                     72: #define DPRINTREG(str, reg)    mov x19,reg; PRINT(str); mov x0,x19; bl print_x0
                     73: #define DPRINTSREG(str, reg)   mrs x19,reg; PRINT(str); mov x0,x19; bl print_x0
1.1       matt       74: #else
1.28      ryo        75: #define DPRINT(string)
                     76: #define DPRINTREG(str, reg)
                     77: #define DPRINTSREG(str, reg)
1.5       ryo        78: #endif
                     79:
1.28      ryo        80: #define PRINT(string)  bl xprint; .asciz string; .align 2
                     81:
                     82:
                     83:        .text
                     84:        .align  3
                     85: ASENTRY_NP(aarch64_start)
                     86:        /* keep lr & sp for return to bootloader if possible */
                     87:        mov     x27, lr
                     88:        mov     x28, sp
1.5       ryo        89:
1.8       ryo        90:        /* set stack pointer for boot */
1.53      skrll      91:        adrl    x0, bootstk
1.8       ryo        92:        mov     sp, x0
1.5       ryo        93:
1.28      ryo        94:        bl      clear_bss
1.24      skrll      95:
1.28      ryo        96:        PRINT("boot NetBSD/aarch64\n")
1.5       ryo        97:
                     98:        bl      1f
1.28      ryo        99: 1:     DPRINTREG("PC               = ", lr)
                    100:        DPRINTREG("SP               = ", sp)
                    101:        mrs     x20, CurrentEL
                    102:        lsr     x20, x20, #2
                    103:        DPRINTREG("CurrentEL        = ", x20)
                    104:        cmp     x20, #2
                    105:        bcc     1f
1.5       ryo       106:        /* EL2 registers can be accessed in EL2 or higher */
1.28      ryo       107:        DPRINTSREG("SCTLR_EL2        = ", sctlr_el2)
                    108:        DPRINTSREG("HCR_EL2          = ", hcr_el2)
1.5       ryo       109: 1:
1.28      ryo       110:        DPRINTSREG("SPSR_EL1         = ", spsr_el1)
                    111:        DPRINTSREG("CNTFREQ_EL0      = ", cntfrq_el0)
                    112:        DPRINTSREG("SCTLR_EL1        = ", sctlr_el1)
                    113:        DPRINTSREG("MIDR_EL1         = ", midr_el1)
                    114:        DPRINTSREG("MPIDR_EL1        = ", mpidr_el1)
                    115:        DPRINTSREG("ID_AA64MPFR0_EL1 = ", id_aa64pfr0_el1)
                    116:        DPRINTSREG("ID_AA64MPFR1_EL1 = ", id_aa64pfr1_el1)
                    117:        DPRINTSREG("ID_AA64ISAR0_EL1 = ", id_aa64isar0_el1)
                    118:        DPRINTSREG("ID_AA64ISAR1_EL1 = ", id_aa64isar1_el1)
                    119:        DPRINTSREG("ID_AA64MMFR0_EL1 = ", id_aa64mmfr0_el1)
                    120:        DPRINTSREG("ID_AA64MMFR1_EL1 = ", id_aa64mmfr1_el1)
1.1       matt      121:
1.5       ryo       122: #ifdef LOCORE_EL2
1.28      ryo       123:        VPRINT("Drop to EL1...")
1.5       ryo       124: # include <aarch64/aarch64/locore_el2.S>
1.28      ryo       125:        VPRINT("OK\n")
                    126:        mrs     x20, CurrentEL
                    127:        lsr     x20, x20, #2
                    128:        DPRINTREG("CurrentEL        = ", x20)
1.5       ryo       129: #endif /* LOCORE_EL2 */
                    130:
                    131:
                    132:        bl      mmu_disable
                    133:        bl      init_sysregs
1.28      ryo       134:        bl      init_mmutable
                    135:        cbnz    x0, aarch64_fatal
                    136:        bl      save_ttbrs
1.5       ryo       137:
1.28      ryo       138:        VPRINT("MMU Enable...")
1.5       ryo       139:        bl      mmu_enable
1.28      ryo       140:        VPRINT("OK\n")
1.5       ryo       141:
1.28      ryo       142:        ldr     x20, =vstart    /* virtual address of vstart */
                    143:        DPRINTSREG("SPSR_EL1         = ", spsr_el1)
                    144:        DPRINTSREG("DAIF             = ", daif)
                    145:        DPRINTREG("vstart           = ", x20)
                    146:        br      x20             /* jump to the kernel virtual address */
                    147:
                    148: aarch64_fatal:
                    149:        PRINT("fatal error occured while booting\n")
                    150:        /* return to bootloader. if switched from EL2 to EL1, It might fail */
                    151:        mov     lr, x27
                    152:        mov     sp, x28
                    153:        ret
1.5       ryo       154:
                    155: /*
                    156:  * vstart is in kernel virtual address
                    157:  */
                    158: vstart:
1.28      ryo       159:        DPRINTREG("PC               = ", x20)
                    160:
                    161:        /* set exception vector */
1.53      skrll     162:        adrl    x0, _C_LABEL(el1_vectors)
1.28      ryo       163:        msr     vbar_el1, x0
                    164:
                    165:        /* set lwp0 stack */
1.53      skrll     166:        adrl    x0, lwp0uspace
1.5       ryo       167:        add     x0, x0, #(UPAGES * PAGE_SIZE)
                    168:        sub     x0, x0, #TF_SIZE        /* lwp0space + USPACE - TF_SIZE */
                    169:        mov     sp, x0                  /* define lwp0 ksp bottom */
1.28      ryo       170:        DPRINTREG("SP(lwp0,kvm)     = ", sp)
1.5       ryo       171:
1.28      ryo       172:        /* lwp-private = NULL */
                    173:        msr     tpidr_el0, xzr
1.29      ryo       174:        msr     tpidrro_el0, xzr
1.1       matt      175:
1.28      ryo       176:        /* set curcpu() */
1.53      skrll     177:        adrl    x0, cpu_info_store      /* cpu_info_store is cpu_info[0] */
1.8       ryo       178:        msr     tpidr_el1, x0           /* curcpu is cpu_info[0] */
1.28      ryo       179:        DPRINTREG("curcpu           = ", x0);
                    180:
1.59      maxv      181:        /* init PAC if supported */
                    182:        mov     x0, #1
                    183:        bl      aarch64_pac_init
1.60      ryo       184:        cbnz    w0, 1f                  /* if (aarch64_pac_init() == 0) */
                    185:        mrs     x0, sctlr_el1
                    186:        orr     x0, x0, #SCTLR_EnIA     /*  enable PAC */
                    187:        msr     sctlr_el1, x0
                    188: 1:
1.59      maxv      189:
1.55      skrll     190:        /* set topology information */
1.45      mlelstv   191:        mrs     x0, tpidr_el1           /* curcpu */
                    192:        mrs     x1, mpidr_el1
1.51      mrg       193:        mov     x2, #0
1.52      mrg       194:        bl      arm_cpu_topology_set
1.55      skrll     195:
                    196:        /* get cache configuration */
1.45      mlelstv   197:        mov     x0, xzr
                    198:        bl      aarch64_getcacheinfo
                    199:
1.31      maxv      200: #ifdef KASAN
1.53      skrll     201:        adrl    x0, lwp0uspace
1.31      maxv      202:        bl      _C_LABEL(kasan_early_init)
                    203: #endif
1.5       ryo       204:
                    205:        mov     fp, #0                  /* trace back starts here */
1.28      ryo       206:        PRINT("initarm\n")
1.5       ryo       207:        bl      _C_LABEL(initarm)       /* Off we go */
                    208:
1.28      ryo       209:        PRINT("main\n")
1.5       ryo       210:        bl      _C_LABEL(main)          /* call main() */
                    211:
                    212:        adr     x0, .Lmainreturned
                    213:        b       _C_LABEL(panic)
                    214:        /* NOTREACHED */
1.28      ryo       215: ASEND(aarch64_start)
1.5       ryo       216:
                    217: .Lmainreturned:
                    218:        .asciz  "main() returned"
1.28      ryo       219:        .align 2
                    220:
                    221:
                    222: ASENTRY_NP(clear_bss)
                    223:        /* Zero the BSS. The size must be aligned 16, usually it should be. */
1.53      skrll     224:        adrl    x14, __bss_start__
                    225:        adrl    x15, __bss_end__
1.28      ryo       226:        b       2f
                    227: 1:     stp     xzr, xzr, [x14], #16
                    228: 2:     cmp     x14, x15
                    229:        b.lo    1b
                    230:        ret
                    231: ASEND(clear_bss)
                    232:
                    233:
                    234: init_sysregs:
                    235:        stp     x0, lr, [sp, #-16]!
                    236:
1.32      ryo       237:        /* init debug event */
                    238:        ldr     x0, mdscr_setting
                    239:        msr     mdscr_el1, x0
                    240:        msr     oslar_el1, xzr
1.28      ryo       241:
                    242:        /* Clear context id register */
                    243:        msr     contextidr_el1, xzr
                    244:
                    245:        /* No trap system register access, and Trap FP/SIMD access */
                    246:        msr     cpacr_el1, xzr
                    247:
                    248:        /* allow to read CNTVCT_EL0 and CNTFRQ_EL0 from EL0 */
                    249:        mrs     x0, cntkctl_el1
                    250:        orr     x0, x0, #CNTKCTL_EL0VCTEN
                    251:        msr     cntkctl_el1, x0
                    252:
                    253:        /* any exception not masked */
                    254:        msr     daif, xzr
                    255:
                    256:        ldp     x0, lr, [sp], #16
                    257:        ret
1.5       ryo       258:
1.8       ryo       259:
                    260: #ifdef MULTIPROCESSOR
                    261:
1.28      ryo       262: #ifdef DEBUG_LOCORE
1.8       ryo       263: /*
1.28      ryo       264:  * atomic_ops doesn't work before MMU enabled, so using Peterson's algorithm.
                    265:  * this is only used to serialize debug print and avoid mixing output.
                    266:  * Not absolutely necessary.
                    267:  *
                    268:  * x27 for cpuindex.
1.8       ryo       269:  */
1.28      ryo       270: locore_lock_enter:
                    271: #ifdef DEBUG_LOCORE_PRINT_LOCK
                    272:        mov     x3, xzr                 /* x3 = level */
                    273: levelloop:
                    274:        /* lock_level[] and lock_turn[] are always accessed via PA(devmap) */
1.53      skrll     275:        adrl    x0, kern_vtopdiff
1.28      ryo       276:        ldr     x0, [x0]
                    277:        ldr     x4, =lock_level
                    278:        sub     x4, x4, x0
                    279:        ldr     x5, =lock_turn
                    280:        sub     x5, x5, x0
                    281:
                    282:        strh    w3, [x4, x27, lsl #1]   /* lock_level[i] = level */
                    283:        dsb     sy
                    284:        strh    w27, [x5, x3, lsl #1]   /* lock_turn[level] = i */
                    285:        dsb     sy
                    286: waitloop:
                    287:        dmb     sy
                    288:        ldrh    w0, [x5, x3, lsl #1]    /* lock_turn[level] == i ? */
                    289:        cmp     x27, x0
                    290:        bne     nextlevel
                    291:
                    292:        mov     x2, xzr                 /* k = 0 */
                    293: levelcheck:
                    294:        cmp     x2, x27
                    295:        beq     levelcheck_next
                    296:
                    297:        dmb     sy
                    298:        ldrsh   w0, [x4, x2, lsl #1]    /* lock_level[k] >= level */
                    299:        cmp     w0, w3
                    300:        bge     waitloop
                    301: levelcheck_next:
                    302:        add     x2, x2, #1              /* k++ */
                    303:        cmp     x2, #MAXCPUS
                    304:        bne     levelcheck
                    305: nextlevel:
                    306:        add     x3, x3, #1
                    307:        cmp     x3, #(MAXCPUS - 1)
                    308:        bne     levelloop
                    309: #endif /* DEBUG_LOCORE_PRINT_LOCK */
                    310:        ret
                    311:
                    312:
                    313: locore_lock_exit:
                    314: #ifdef DEBUG_LOCORE_PRINT_LOCK
                    315:        /* lock_level[] and lock_turn[] are always accessed via PA(devmap) */
1.53      skrll     316:        adrl    x0, kern_vtopdiff
1.28      ryo       317:        ldr     x0, [x0]
                    318:        ldr     x1, =lock_level
                    319:        sub     x1, x1, x0
                    320:        mvn     x0, xzr
                    321:        strh    w0, [x1, x27, lsl #1]   /* lock_level[i] = -1 */
                    322:        dsb     sy
                    323: #endif /* DEBUG_LOCORE_PRINT_LOCK */
                    324:        ret
                    325:
                    326:
                    327: /* print "[CPU$x27] " (x27 for cpuindex) */
                    328: printcpu:
1.8       ryo       329:        stp     x0, lr, [sp, #-16]!
1.28      ryo       330:        PRINT("[CPU");                  \
                    331:        mov     x0, x27;                \
                    332:        bl      _printdec_x0;           \
                    333:        PRINT("] ");                    \
1.8       ryo       334:        ldp     x0, lr, [sp], #16
                    335:        ret
                    336:
1.28      ryo       337: #define CPU_DPRINT(str)                        \
                    338:        bl      locore_lock_enter;      \
                    339:        bl      printcpu;               \
                    340:        DPRINT(str);                    \
                    341:        bl      locore_lock_exit
                    342:
                    343: /*
                    344:  * CPU_DPRINTREG macro use x19 internally. x0-x15 may be broken.
                    345:  * x27 for cpuindex.
                    346:  */
                    347: #define CPU_DPRINTREG(str,reg)         \
                    348:        mov     x19, reg;               \
                    349:        bl      locore_lock_enter;      \
                    350:        bl      printcpu;               \
                    351:        PRINT(str);                     \
                    352:        mov     x0, x19;                \
                    353:        bl      print_x0;               \
                    354:        bl      locore_lock_exit
                    355:
                    356: #define CPU_DPRINTSREG(str, reg)       \
                    357:        mrs     x19, reg;               \
                    358:        CPU_DPRINTREG(str, x19)
                    359:
                    360: #else /* DEBUG_LOCORE */
                    361:
                    362: #define CPU_DPRINT(str)
                    363: #define CPU_DPRINTREG(str,reg)
                    364: #define CPU_DPRINTSREG(str, reg)
                    365:
                    366: #endif /* DEBUG_LOCORE */
1.8       ryo       367:
1.28      ryo       368: ENTRY_NP(cpu_mpstart)
1.25      ryo       369:        mrs     x3, mpidr_el1
                    370:        ldr     x0, =(MPIDR_AFF0|MPIDR_AFF1|MPIDR_AFF2|MPIDR_AFF3)
                    371:        and     x3, x3, x0
                    372:
1.28      ryo       373:        /*
                    374:         * resolve own cpuindex. my mpidr is stored in
                    375:         * extern uint64_t cpu_mpidr[MAXCPUS]
                    376:         */
1.53      skrll     377:        adrl    x0, _C_LABEL(cpu_mpidr)
1.25      ryo       378:        mov     x1, xzr
                    379: 1:
                    380:        add     x1, x1, #1
1.44      jmcneill  381:        cmp     x1, #MAXCPUS            /* cpuindex >= MAXCPUS ? */
1.25      ryo       382:        bge     toomanycpus
1.35      skrll     383:        ldr     x2, [x0, x1, lsl #3]    /* cpu_mpidr[cpuindex] */
1.25      ryo       384:        cmp     x2, x3                  /* == mpidr_el1 & MPIDR_AFF ? */
                    385:        bne     1b
                    386:
                    387:        mov     x27, x1                 /* x27 = cpuindex */
1.28      ryo       388:
                    389:        /*
                    390:         * x27 = cpuindex
                    391:         */
1.8       ryo       392:
                    393:        /* set stack pointer for boot */
1.28      ryo       394:        mov     x1, #BOOT_AP_STACKSIZE
1.8       ryo       395:        mul     x1, x1, x27
1.53      skrll     396:        adrl    x0, bootstk
1.43      skrll     397:        add     sp, x0, x1  /* sp = bootstk + (BOOT_AP_STACKSIZE * cpuindex) */
1.8       ryo       398:
                    399:        bl      1f
1.28      ryo       400: 1:     CPU_DPRINTREG("PC               = ", lr)
                    401:        CPU_DPRINTREG("SP               = ", sp)
                    402:        mrs     x20, CurrentEL
                    403:        lsr     x20, x20, #2
                    404:        CPU_DPRINTREG("CurrentEL        = ", x20)
                    405:        cmp     x20, #2
                    406:        bcc     1f
                    407:        /* EL2 registers can be accessed in EL2 or higher */
                    408:        CPU_DPRINTSREG("SCTLR_EL2        = ", sctlr_el2)
                    409:        CPU_DPRINTSREG("HCR_EL2          = ", hcr_el2)
                    410: 1:
                    411:        CPU_DPRINTSREG("SPSR_EL1         = ", spsr_el1)
                    412:        CPU_DPRINTSREG("SCTLR_EL1        = ", sctlr_el1)
                    413:        CPU_DPRINTSREG("MIDR_EL1         = ", midr_el1)
                    414:        CPU_DPRINTSREG("MPIDR_EL1        = ", mpidr_el1)
1.8       ryo       415:
                    416: #ifdef LOCORE_EL2
1.28      ryo       417:        CPU_DPRINT("Drop to EL1...\n")
1.8       ryo       418:        bl      drop_to_el1
1.28      ryo       419:        CPU_DPRINT("Drop to EL1 OK\n")
                    420:        mrs     x20, CurrentEL
                    421:        lsr     x20, x20, #2
                    422:        CPU_DPRINTREG("CurrentEL        = ", x20)
1.8       ryo       423: #endif /* LOCORE_EL2 */
                    424:
1.28      ryo       425:
1.8       ryo       426:        bl      mmu_disable
                    427:        bl      init_sysregs
                    428:
1.28      ryo       429:        CPU_DPRINT("MMU Enable...\n")
                    430:        bl      load_ttbrs
1.8       ryo       431:        bl      mmu_enable
1.28      ryo       432:        CPU_DPRINT("MMU Enable OK\n")
1.8       ryo       433:
                    434:        /* jump to virtual address */
1.28      ryo       435:        ldr     x20, =mp_vstart
                    436:        br      x20
1.8       ryo       437:
                    438: mp_vstart:
1.61      ryo       439:        hint    0x24            /* bti j */
1.28      ryo       440:        CPU_DPRINTREG("PC               = ", x20)
                    441:
1.8       ryo       442:        /* set exception vector */
1.53      skrll     443:        adrl    x0, _C_LABEL(el1_vectors)
1.8       ryo       444:        msr     vbar_el1, x0
                    445:
1.28      ryo       446:        /* lwp-private = NULL */
                    447:        msr     tpidr_el0, xzr
1.29      ryo       448:        msr     tpidrro_el0, xzr
1.20      ryo       449:
                    450:        /* set curcpu(), and fill curcpu()->ci_{midr,mpidr} */
                    451:        mov     x0, #CPU_INFO_SIZE
                    452:        mul     x0, x27, x0
1.53      skrll     453:        adrl    x1, _C_LABEL(cpu_info_store)
1.20      ryo       454:        add     x0, x0, x1              /* x0 = &cpu_info_store[cpuindex] */
                    455:        msr     tpidr_el1, x0           /* tpidr_el1 = curcpu() = x0 */
                    456:
1.8       ryo       457:        mrs     x1, midr_el1
1.20      ryo       458:        str     x1, [x0, #CI_MIDR]      /* curcpu()->ci_cpuid = midr_el1 */
1.8       ryo       459:        mrs     x1, mpidr_el1
1.20      ryo       460:        str     x1, [x0, #CI_MPIDR]     /* curcpu()->ci_mpidr = mpidr_el1 */
1.8       ryo       461:
1.56      skrll     462:        /* set topology information */
                    463:        mov     x2, #0
                    464:        bl      arm_cpu_topology_set
                    465:
                    466:        /* x28 = &arm_cpu_hatched[cpuindex / (sizeof(u_long) * NBBY)] */
                    467:        adrl    x0, _C_LABEL(arm_cpu_hatched)
1.58      skrll     468:        // Appease clang - mov  x1, x27, lsr #6
                    469:        orr     x1, xzr, x27, lsr #6
1.56      skrll     470:        add     x28, x0, x1, lsl #3
                    471:
                    472:        /* x29 = __BIT(cpuindex % (sizeof(u_long) * NBBY)) */
1.44      jmcneill  473:        mov     x0, #1
1.56      skrll     474:        and     x2, x27, #63
                    475:        lsl     x29, x0, x2
1.44      jmcneill  476:
1.8       ryo       477:        /*
1.56      skrll     478:         * atomic_or_ulong(&arm_cpu_hatched[cpuindex / (sizeof(u_long)  * NBBY)],
                    479:         *   _BIT(cpuindex % ((sizeof(u_long) * NBBY)
                    480:         * to inform the boot processor.
1.8       ryo       481:         */
1.42      jmcneill  482:        mov     x0, x28
1.44      jmcneill  483:        mov     x1, x29
1.56      skrll     484:        bl      _C_LABEL(atomic_or_ulong)       /* hatched! */
1.42      jmcneill  485:        dsb     sy
1.8       ryo       486:        sev
                    487:
1.56      skrll     488:        /* x28 = &arm_cpu_mbox[cpuindex / (sizeof(u_long) * NBBY)] */
                    489:        adrl    x0, _C_LABEL(arm_cpu_mbox)
1.58      skrll     490:        // Appease clang - mov  x1, x27, lsr #6
                    491:        orr     x1, xzr, x27, lsr #6
1.56      skrll     492:        add     x28, x0, x1, lsl #3
1.44      jmcneill  493:
1.42      jmcneill  494:        /* wait for the mailbox start bit to become true */
1.8       ryo       495: 1:
                    496:        dmb     sy
1.56      skrll     497:        ldr     x20, [x28]
                    498:        tst     x20, x29
1.8       ryo       499:        bne     9f
                    500:        wfe
                    501:        b       1b
                    502: 9:
                    503:
                    504:        /* fill my cpu_info */
1.20      ryo       505:        mrs     x0, tpidr_el1           /* curcpu() */
1.8       ryo       506:
                    507:        ldr     x1, [x0, #CI_IDLELWP]   /* x1 = curcpu()->ci_data.cpu_idlelwp */
                    508:        str     x1, [x0, #CI_CURLWP]    /* curlwp is idlelwp */
                    509:
1.28      ryo       510:        /* get my stack from lwp */
1.8       ryo       511:        ldr     x2, [x1, #L_PCB]        /* x2 = lwp_getpcb(idlelwp) */
                    512:        add     x2, x2, #(UPAGES * PAGE_SIZE)
                    513:        sub     sp, x2, #TF_SIZE        /* sp = pcb + USPACE - TF_SIZE */
                    514:
1.59      maxv      515:        /* init PAC if supported */
                    516:        mov     x0, #0
                    517:        bl      aarch64_pac_init
1.60      ryo       518:        cbnz    w0, 1f                  /* if (aarch64_pac_init() == 0) */
                    519:        mrs     x0, sctlr_el1
                    520:        orr     x0, x0, #SCTLR_EnIA     /*  enable PAC */
                    521:        msr     sctlr_el1, x0
                    522: 1:
1.8       ryo       523:
                    524:        mov     fp, xzr                 /* trace back starts here */
1.59      maxv      525:        mrs     x0, tpidr_el1           /* curcpu() */
1.8       ryo       526:        bl      _C_LABEL(cpu_hatch)
                    527:        mov     x0, xzr
                    528:        b       _C_LABEL(idle_loop)     /* never to return */
1.30      skrll     529: END(cpu_mpstart)
1.8       ryo       530:
1.20      ryo       531: toomanycpus:
1.28      ryo       532:        CPU_DPRINT("too many cpus, or MPIDR not exists in cpu_mpidr[]\n")
1.20      ryo       533: 1:     wfi
                    534:        b       1b
                    535:
                    536:
1.8       ryo       537: #else /* MULTIPROCESSOR */
                    538:
1.30      skrll     539: ENTRY_NP(cpu_mpstart)
1.8       ryo       540: 1:     wfi
                    541:        b       1b
1.30      skrll     542: END(cpu_mpstart)
1.8       ryo       543:
                    544: #endif /* MULTIPROCESSOR */
                    545:
1.28      ryo       546:
1.5       ryo       547: /*
                    548:  * xprint - print strings pointed by $PC(LR)
                    549:  *          and return to the end of string.
1.28      ryo       550:  *          "\n" will be replaced "\r\n"
1.5       ryo       551:  * e.g.)
1.28      ryo       552:  *    bl        xprint      <- call
                    553:  *    .ascii    "Hello\n\0" <- wouldn't return here
1.5       ryo       554:  *    .align    2
1.28      ryo       555:  *    nop                   <- return to here
1.22      skrll     556:  *
1.5       ryo       557:  */
1.28      ryo       558: xprint:
                    559:        mov     x0, lr
                    560:        bl      _C_LABEL(uartputs)
                    561:        add     x0, x0, #3
                    562:        bic     lr, x0, #3
                    563:        ret
1.22      skrll     564:
1.28      ryo       565: /*
                    566:  * uartputs(str) - print strings with replacing "\n" to "\r\n".
                    567:  * returns the address after the end of the string. (x0 = next of '\0')
                    568:  */
                    569: ENTRY_NP(uartputs)
                    570:        stp     x19, lr, [sp, #-16]!
                    571:        mov     x19, x0
1.22      skrll     572:        ldrb    w0, [x19], #1
1.28      ryo       573:        cbz     w0, 9f
1.5       ryo       574: 1:
1.28      ryo       575:        cmp     x0, #'\n'
                    576:        bne     2f
1.62      ryo       577:        mov     x0, #0x0d       /* '\r' */
1.28      ryo       578:        bl      uartputc
                    579:        mov     x0, #'\n'
                    580: 2:
1.5       ryo       581:        bl      uartputc
1.22      skrll     582:        ldrb    w0, [x19], #1
1.5       ryo       583:        cbnz    w0, 1b
                    584: 9:
1.28      ryo       585:        mov     x0, x19
                    586:        ldp     x19, lr, [sp], #16
1.5       ryo       587:        ret
1.21      maxv      588: END(uartputs)
1.5       ryo       589:
1.28      ryo       590: /*
                    591:  * print x0 in 16 widths hexadecimal.
                    592:  *
                    593:  * x0 is preserved despite being caller saved.
                    594:  * other caller saved registers will be broken.
                    595:  */
                    596: _print_x0:
1.5       ryo       597:        stp     x0, lr, [sp, #-16]!
1.22      skrll     598:        stp     x20, x21, [sp, #-16]!
1.5       ryo       599:
1.22      skrll     600:        mov     x21, x0         /* number to display */
                    601:        mov     x20, #60        /* num of shift */
1.5       ryo       602: 1:
1.22      skrll     603:        ror     x0, x21, x20
                    604:        and     x0, x0, #0xf
1.5       ryo       605:        cmp     x0, #10
                    606:        blt     2f
                    607:        add     x0, x0, #('a' - 10 - '0')
                    608: 2:     add     x0, x0, #'0'
                    609:        bl      uartputc
1.22      skrll     610:        subs    x20, x20, #4
1.5       ryo       611:        bge     1b
                    612:
1.22      skrll     613:        ldp     x20, x21, [sp], #16
1.5       ryo       614:        ldp     x0, lr, [sp], #16
                    615:        ret
                    616:
1.28      ryo       617: /*
                    618:  * print x0 in decimal.
                    619:  *
                    620:  * x0 is preserved despite being caller saved.
                    621:  * other caller saved registers will be broken.
                    622:  */
                    623: _printdec_x0:
                    624:        stp     x0, lr, [sp, #-(16+32)]!
                    625:        add     x8, sp, #(16+32)
1.5       ryo       626:
1.28      ryo       627:        strb    wzr, [x8, #-1]!
                    628: 1:
                    629:        mov     x10, #10
                    630:        udiv    x1, x0, x10     /* x1 = x0 / 10 */
                    631:        msub    x3, x1, x10, x0 /* x3 = x0 % 10 */
1.5       ryo       632:        mov     x0, x1
1.1       matt      633:
1.28      ryo       634:        add     x3, x3, #'0'
                    635:        strb    w3, [x8, #-1]!
                    636:        cbnz    x0, 1b
1.1       matt      637:
1.28      ryo       638:        mov     x0, x8
                    639:        bl      uartputs
1.5       ryo       640:
1.28      ryo       641:        ldp     x0, lr, [sp], #(16+32)
1.5       ryo       642:        ret
                    643:
                    644: /*
1.28      ryo       645:  * print x0 in 16 widths hexadecimal with crlf.
                    646:  *
                    647:  * x0 is preserved despite being caller saved.
                    648:  * other caller saved registers will be broken.
1.5       ryo       649:  */
1.28      ryo       650: print_x0:
1.5       ryo       651:        stp     x0, lr, [sp, #-16]!
1.28      ryo       652:        bl      _print_x0
                    653:        PRINT("\n")
1.5       ryo       654:        ldp     x0, lr, [sp], #16
1.1       matt      655:        ret
                    656:
1.28      ryo       657: #ifdef DEBUG_MMU
1.1       matt      658: /*
1.28      ryo       659:  * tinyprintf() supports only maximum 7 '%x', '%d' and '%s' formats.
                    660:  * width and any modifiers are ignored. '\n' will be replaced to '\r\n'.
                    661:  *
                    662:  * '%x' will be always expanded 16 widths hexadicimal.
                    663:  * e.g., tinyprintf("Hello %s %x\n", "World", 0x12345)
                    664:  * outputs "Hello World 0000000000012345\r\n"
1.2       nisimura  665:  */
1.28      ryo       666: tinyprintf:
1.5       ryo       667:        stp     x0, lr, [sp, #-16]!
1.22      skrll     668:        stp     x19, x20, [sp, #-16]!
1.28      ryo       669:        stp     x7, x8, [sp, #-16]!
                    670:        stp     x5, x6, [sp, #-16]!
                    671:        stp     x3, x4, [sp, #-16]!
                    672:        stp     x1, x2, [sp, #-16]!
1.22      skrll     673:
1.28      ryo       674:        mov     x20, xzr
                    675:        mov     x19, x0
                    676:        ldrb    w0, [x19], #1
                    677:        cbz     w0, tinyprintf_done
1.5       ryo       678:
1.28      ryo       679: tinyprintf_loop:
                    680:        cmp     x0, #'\n'
                    681:        bne     1f
                    682:        /* '\n' -> '\r', '\n' */
1.62      ryo       683:        mov     x0, #0x0d       /* '\r' */
1.28      ryo       684:        bl      uartputc
                    685:        mov     x0, #'\n'
1.5       ryo       686: 1:
1.22      skrll     687:
1.28      ryo       688:        cmp     x0, #'%'
                    689:        bne     tinyprintf_putc
                    690:        cmp     x20, #8
                    691:        bcs     tinyprintf_putc
                    692:
                    693: tinyprintf_fetch_fmt:
                    694:        ldrb    w9, [x19], #1
                    695:        cbz     w9, tinyprintf_done
                    696:
                    697:        /* width and modifier are ignored */
                    698:        cmp     x9, #'h'
                    699:        beq     tinyprintf_fetch_fmt
                    700:        cmp     x9, #'l'
                    701:        beq     tinyprintf_fetch_fmt
                    702:        cmp     x9, #'j'
                    703:        beq     tinyprintf_fetch_fmt
                    704:        cmp     x9, #'t'
                    705:        beq     tinyprintf_fetch_fmt
                    706:        cmp     x9, #'z'
                    707:        beq     tinyprintf_fetch_fmt
                    708:        cmp     x9, #'0'
                    709:        bcc     1f
                    710:        cmp     x9, #'9'
                    711:        bls     tinyprintf_fetch_fmt
                    712: 1:
                    713:        ldr     x0, [sp, x20, lsl #3]   /* get Nth argument */
1.22      skrll     714:        add     x20, x20, #1
1.5       ryo       715:
1.28      ryo       716:        cmp     x9, #'x'
                    717:        bne     5f
                    718:        /* "%x" format */
                    719:        bl      _print_x0
                    720:        b       tinyprintf_next
                    721: 5:
                    722:        cmp     x9, #'d'
                    723:        bne     5f
                    724:        /* "%d" format */
                    725:        bl      _printdec_x0
                    726:        b       tinyprintf_next
                    727: 5:
                    728:        cmp     x9, #'s'
                    729:        bne     5f
                    730:        /* "%s" format */
                    731:        bl      _C_LABEL(uartputs)
                    732:        b       tinyprintf_next
                    733: 5:
                    734:
                    735: tinyprintf_putc:
                    736:        bl      uartputc
                    737: tinyprintf_next:
                    738:        ldrb    w0, [x19], #1
                    739:        cbnz    w0, tinyprintf_loop
                    740:
                    741: tinyprintf_done:
                    742:        mov     x0, x19
                    743:
                    744:        ldp     x1, x2, [sp], #16
                    745:        ldp     x3, x4, [sp], #16
                    746:        ldp     x5, x6, [sp], #16
                    747:        ldp     x7, x8, [sp], #16
1.22      skrll     748:        ldp     x19, x20, [sp], #16
1.5       ryo       749:        ldp     x0, lr, [sp], #16
1.2       nisimura  750:        ret
1.28      ryo       751: #endif /* defined(DEBUG_LOCORE) || defined(DEBUG_MMU) */
1.2       nisimura  752:
1.5       ryo       753:
1.28      ryo       754: save_ttbrs:
                    755:        /* save ttbr[01]_el1 for AP */
                    756:        mrs     x0, ttbr0_el1
                    757:        mrs     x1, ttbr1_el1
1.53      skrll     758:        adrl    x2, ttbr_save
1.28      ryo       759:        stp     x0, x1, [x2]
                    760:        ret
1.5       ryo       761:
1.28      ryo       762: load_ttbrs:
                    763:        /* load ttbr[01]_el1 */
1.53      skrll     764:        adrl    x2, ttbr_save
1.28      ryo       765:        ldp     x0, x1, [x2]
                    766:        msr     ttbr0_el1, x0
                    767:        msr     ttbr1_el1, x1
1.3       nisimura  768:        ret
1.2       nisimura  769:
1.5       ryo       770:
1.28      ryo       771: init_mmutable:
                    772:        stp     x26, lr, [sp, #-16]!
1.22      skrll     773:
1.28      ryo       774:        /* first allocated page must be kernel l0pt = ARM_BOOTSTRAP_LxPT */
                    775:        bl      bootpage_alloc
                    776:        cbz     x0, init_mmutable_error
                    777:        msr     ttbr1_el1, x0
                    778:
                    779:        bl      bootpage_alloc
                    780:        cbz     x0, init_mmutable_error
                    781:        msr     ttbr0_el1, x0
1.22      skrll     782:
1.5       ryo       783: #ifdef DEBUG_MMU
1.28      ryo       784:        adr     x26, tinyprintf
                    785: #else
                    786:        mov     x26, xzr
1.5       ryo       787: #endif
                    788:
1.28      ryo       789:        /*
                    790:         * int
                    791:         * pmapboot_enter(
                    792:         *     x0: vaddr_t va,
                    793:         *     x1: paddr_t pa,
                    794:         *     x2: psize_t size,
                    795:         *     x3: psize_t blocksize,  // L[123]_SIZE
                    796:         *     x4: pt_entry_t attr,    // pte attributes. LX_BLKPAG_*
                    797:         *     x5: flags,
                    798:         *     x6: pd_entry_t *(*physpage_allocator)(void),
                    799:         *     x7: void (*pr)(const char *, ...)
                    800:         *  );
                    801:         */
1.5       ryo       802:
1.28      ryo       803: #ifdef CONSADDR
                    804:        VPRINT("Creating VA=PA tables for CONSADDR\n")
                    805:        mov     x7, x26                         /* pr func */
                    806:        adr     x6, bootpage_alloc              /* allocator */
                    807:        mov     x5, xzr                         /* flags = 0 */
                    808:        mov     x4, #LX_BLKPAG_ATTR_DEVICE_MEM|LX_BLKPAG_AP_RW  /* attr */
1.40      jmcneill  809:        orr     x4, x4, #LX_BLKPAG_UXN|LX_BLKPAG_PXN
1.28      ryo       810:        mov     x3, #L2_SIZE                    /* blocksize */
                    811:        mov     x2, #L2_SIZE                    /* size */
                    812:        ldr     x1, =CONSADDR                   /* pa */
                    813:        mov     x0, x1                          /* va */
                    814:        bl      pmapboot_enter
                    815:        cbnz    x0, init_mmutable_error
                    816: #elif defined(EARLYCONS)
                    817:        /* CONSADDR is unknown, but need to map UART */
                    818:        VPRINT("Creating VA=PA tables (0x00000000-0xffffffff)\n")
                    819:        mov     x7, x26                         /* pr func */
                    820:        adr     x6, bootpage_alloc              /* allocator */
                    821:        mov     x5, xzr                         /* flags = 0 */
                    822:        mov     x4, #LX_BLKPAG_ATTR_DEVICE_MEM|LX_BLKPAG_AP_RW  /* attr */
1.40      jmcneill  823:        orr     x4, x4, #LX_BLKPAG_UXN|LX_BLKPAG_PXN
1.28      ryo       824:        mov     x3, #L2_SIZE                    /* blocksize */
                    825:        mov     x2, #(1024*1024*1024*4)         /* size */
                    826:        mov     x1, xzr                         /* pa */
                    827:        mov     x0, xzr                         /* va */
                    828:        bl      pmapboot_enter
                    829:        cbnz    x0, init_mmutable_error
                    830: #endif
                    831:
                    832:        /* identity mapping for kernel image */
                    833:        VPRINT("Creating VA=PA tables for kernel image\n")
                    834:        mov     x7, x26                         /* pr func */
                    835:        adr     x6, bootpage_alloc              /* allocator */
                    836:        mov     x5, xzr                         /* flags = 0 */
                    837:        mov     x4, #LX_BLKPAG_ATTR_NORMAL_NC|LX_BLKPAG_AP_RW   /* attr */
                    838:        mov     x3, #L2_SIZE                    /* blocksize */
                    839:        adr     x0, start                       /* va = start */
1.53      skrll     840:        adrl    x2, _end
1.28      ryo       841:        sub     x2, x2, x0                      /* size = _end - start */
1.41      skrll     842:        add     x2, x2, #BOOTPAGE_ALLOC_MAX     /* for bootpage_alloc() */
1.28      ryo       843:        mov     x1, x0                          /* pa */
                    844:        bl      pmapboot_enter
                    845:        cbnz    x0, init_mmutable_error
                    846:
                    847: #ifdef FDT
1.63    ! ryo       848:        VPRINT("Creating VA=PA tables for FDT\n")
1.53      skrll     849:        adrl    x8, _C_LABEL(fdt_addr_r)
1.28      ryo       850:        ldr     x8, [x8]
                    851:
                    852:        mov     x7, x26                         /* pr func */
                    853:        adr     x6, bootpage_alloc              /* allocator */
                    854:        mov     x5, xzr                         /* flags = 0 */
                    855:        mov     x4, #LX_BLKPAG_ATTR_NORMAL_NC|LX_BLKPAG_AP_RW   /* attr */
1.46      skrll     856:        orr     x4, x4, #LX_BLKPAG_UXN|LX_BLKPAG_PXN
1.28      ryo       857:        mov     x3, #L2_SIZE                    /* blocksize */
                    858:        mov     x2, #L2_SIZE                    /* size */
                    859:        mov     x1, x8                          /* pa */
                    860:        mov     x0, x8                          /* va */
                    861:        bl      pmapboot_enter
                    862:        cbnz    x0, init_mmutable_error
                    863: #endif
                    864:
                    865:        VPRINT("Creating KVA=PA tables\n")
                    866:        mov     x7, x26                         /* pr func */
                    867:        adr     x6, bootpage_alloc              /* allocator */
                    868:        mov     x5, xzr                         /* flags = 0 */
                    869:        mov     x4, #LX_BLKPAG_ATTR_NORMAL_WB|LX_BLKPAG_AP_RW   /* attr */
                    870:        orr     x4, x4, #LX_BLKPAG_UXN
                    871:        mov     x3, #L2_SIZE                    /* blocksize */
1.38      skrll     872:        adr     x1, start                       /* pa = start */
1.53      skrll     873:        adrl    x2, _end
1.28      ryo       874:        sub     x2, x2, x1                      /* size = _end - start */
1.37      skrll     875:        ldr     x0, =start                      /* va */
1.28      ryo       876:        bl      pmapboot_enter
                    877:        cbnz    x0, init_mmutable_error
1.5       ryo       878:
1.28      ryo       879:        VPRINT("OK\n");
                    880:        mov     x0, xzr
                    881:        b       init_mmutable_done
                    882: init_mmutable_error:
                    883:        mvn     x0, xzr
                    884: init_mmutable_done:
                    885:        ldp     x26, lr, [sp], #16
                    886:        ret
1.5       ryo       887:
1.28      ryo       888: /* return PA of allocated page */
                    889: ENTRY_NP(bootpage_alloc)
                    890:        /* x2 = kernend_extra */
1.53      skrll     891:        adrl    x3, kernend_extra
1.28      ryo       892:        ldr     x2, [x3]
                    893:        /* if (kernend_extra < 0) return NULL */
                    894:        mov     x0, xzr
                    895:        cmp     x2, xzr
                    896:        bmi     bootpage_alloc_done
1.5       ryo       897:
1.28      ryo       898:        /* x0 = PA of _end[] */
1.53      skrll     899:        adrl    x1, kern_vtopdiff
1.28      ryo       900:        ldr     x1, [x1]
                    901:        ldr     x0, =ARM_BOOTSTRAP_LxPT
                    902:        sub     x0, x0, x1
                    903:
                    904:        /* x0 = ARM_BOOTSTRAP_LxPT + kernend_extra */
                    905:        add     x0, x0, x2
                    906:
                    907:        /* kernend_extra += PAGE_SIZE; */
                    908:        add     x2, x2, #PAGE_SIZE
                    909:        str     x2, [x3]
                    910:
                    911:        /* clear allocated page */
                    912:        mov     x1, x0
                    913:        add     x2, x1, #PAGE_SIZE
                    914: 1:     stp     xzr, xzr, [x1], #16
                    915:        cmp     x1, x2
                    916:        bcc     1b
                    917: bootpage_alloc_done:
                    918:        ret
                    919: END(bootpage_alloc)
1.5       ryo       920:
                    921:
1.28      ryo       922: mmu_disable:
1.5       ryo       923:        dsb     sy
                    924:        mrs     x0, sctlr_el1
                    925:        bic     x0, x0, SCTLR_M         /* clear MMU enable bit */
                    926:        msr     sctlr_el1, x0
                    927:        isb
                    928:        ret
                    929:
1.28      ryo       930: mmu_enable:
1.5       ryo       931:        dsb     sy
                    932:
                    933:        /* Invalidate all TLB */
                    934:        dsb     ishst
                    935: #ifdef MULTIPROCESSOR
                    936:        tlbi    vmalle1is
                    937: #else
                    938:        tlbi    vmalle1
                    939: #endif
                    940:        dsb     ish
                    941:        isb
                    942:
                    943:        ldr     x0, mair_setting
                    944:        msr     mair_el1, x0
                    945:
                    946:
                    947:        /* TCR_EL1:IPS[34:32] = AA64MMFR0:PARange[3:0] */
                    948:        ldr     x0, tcr_setting
                    949:        mrs     x1, id_aa64mmfr0_el1
                    950:        bfi     x0, x1, #32, #3
                    951:        msr     tcr_el1, x0
1.1       matt      952:
                    953:        /*
1.5       ryo       954:         * configure SCTLR
1.1       matt      955:         */
1.5       ryo       956:        mrs     x0, sctlr_el1
                    957:        ldr     x1, sctlr_clear
                    958:        bic     x0, x0, x1
                    959:        ldr     x1, sctlr_set
                    960:        orr     x0, x0, x1
                    961:
                    962:        ldr     x1, sctlr_ee
                    963: #ifdef __AARCH64EB__
                    964:        orr     x0, x0, x1      /* set: BigEndian */
                    965: #else
                    966:        bic     x0, x0, x1      /* clear: LittleEndian */
                    967: #endif
                    968:        msr     sctlr_el1, x0   /* enabling MMU! */
                    969:        isb
1.1       matt      970:
                    971:        ret
1.28      ryo       972:
1.1       matt      973:
1.5       ryo       974:        .align 3
                    975: mair_setting:
                    976:        .quad (                                         \
                    977:            __SHIFTIN(MAIR_NORMAL_WB, MAIR_ATTR0) |     \
                    978:            __SHIFTIN(MAIR_NORMAL_NC, MAIR_ATTR1) |     \
                    979:            __SHIFTIN(MAIR_NORMAL_WT, MAIR_ATTR2) |     \
1.49      jmcneill  980:            __SHIFTIN(MAIR_DEVICE_MEM, MAIR_ATTR3) |    \
                    981:            __SHIFTIN(MAIR_DEVICE_MEM_SO, MAIR_ATTR4))
1.5       ryo       982:
                    983: #define VIRT_BIT       48
1.28      ryo       984:
                    985: #ifdef MULTIPROCESSOR
                    986: #define TCR_SHAREABLE  (TCR_SH0_INNER | TCR_SH1_INNER)
                    987: #else
                    988: #define TCR_SHAREABLE  (TCR_SH0_NONE | TCR_SH1_NONE)
                    989: #endif
                    990:
1.5       ryo       991: tcr_setting:
                    992:        .quad (                                         \
                    993:            __SHIFTIN(64 - VIRT_BIT, TCR_T1SZ) |        \
                    994:            __SHIFTIN(64 - VIRT_BIT, TCR_T0SZ) |        \
                    995:            TCR_AS64K |                                 \
                    996:            TCR_TG1_4KB | TCR_TG0_4KB |                 \
                    997:            TCR_ORGN0_WB_WA |                           \
                    998:            TCR_IRGN0_WB_WA |                           \
                    999:            TCR_ORGN1_WB_WA |                           \
1.28      ryo      1000:            TCR_IRGN1_WB_WA) | TCR_SHAREABLE
1.5       ryo      1001:
                   1002:
                   1003: #ifdef AARCH64_ALIGNMENT_CHECK
                   1004: #define SCTLR_A_CONFIG         SCTLR_A
                   1005: #else
                   1006: #define SCTLR_A_CONFIG         0
                   1007: #endif
                   1008:
                   1009: #ifdef AARCH64_EL0_STACK_ALIGNMENT_CHECK
                   1010: #define SCTLR_SA0_CONFIG       SCTLR_SA0
                   1011: #else
                   1012: #define SCTLR_SA0_CONFIG       0
                   1013: #endif
                   1014:
                   1015: #ifdef AARCH64_EL1_STACK_ALIGNMENT_CHECK
                   1016: #define SCTLR_SA_CONFIG                SCTLR_SA
                   1017: #else
                   1018: #define SCTLR_SA_CONFIG                0
1.3       nisimura 1019: #endif
                   1020:
1.5       ryo      1021:
                   1022: sctlr_ee:
1.39      skrll    1023:        .quad (SCTLR_EE | SCTLR_EOE)    /* Endianness of Exception and EL0 */
1.5       ryo      1024: sctlr_set:
                   1025:        .quad ( \
                   1026:            SCTLR_LSMAOE |  /* Load/Store Multiple Atomicity and Ordering */ \
                   1027:            SCTLR_nTLSMD |  /* no Trap Load/Store Multiple to Device */ \
1.6       ryo      1028:            SCTLR_UCI |     /* Enables EL0 DC {CVAU,CIVAC,CVAC}, IC IVAU */ \
1.5       ryo      1029:            SCTLR_SPAN |    /* This field resets to 1 */ \
                   1030:            SCTLR_UCT |     /* Enables EL0 access to the CTR_EL0 */ \
1.9       ryo      1031:            SCTLR_nTWE |    /* EL0 WFE non-trapping */ \
                   1032:            SCTLR_nTWI |    /* EL0 WFI non-trapping */ \
1.5       ryo      1033:            SCTLR_DZE |     /* Enables access to the DC ZVA instruction */ \
                   1034:            SCTLR_I |       /* Instruction cache enable */ \
                   1035:            SCTLR_SED |     /* SETEND instruction disable */ \
                   1036:            SCTLR_C |       /* Cache enable */ \
                   1037:            SCTLR_M |       /* MMU Enable */ \
                   1038:            SCTLR_SA0_CONFIG | \
                   1039:            SCTLR_SA_CONFIG | \
                   1040:            SCTLR_A_CONFIG | \
                   1041:            0)
                   1042: sctlr_clear:
                   1043:        .quad ( \
                   1044:            SCTLR_IESB |    /* Enable Implicit ErrorSynchronizationBarrier */ \
                   1045:            SCTLR_WXN |     /* Write permission implies Execute Never (W^X) */ \
                   1046:            SCTLR_UMA |     /* EL0 Controls access to interrupt masks */ \
                   1047:            SCTLR_ITD |     /* IT instruction disable */ \
1.54      maxv     1048:            SCTLR_nAA |     /* ? */ \
1.5       ryo      1049:            SCTLR_CP15BEN | /* CP15 barrier enable */ \
                   1050:            SCTLR_SA0 |     /* Enable EL0 stack alignment check */ \
                   1051:            SCTLR_SA |      /* Enable SP alignment check */ \
                   1052:            SCTLR_A |       /* Alignment check enable */ \
                   1053:            0)
                   1054:
1.32      ryo      1055: mdscr_setting:
                   1056:        .quad ( \
                   1057:            MDSCR_TDCC |    /* Trap Debug Communications Channel access */ \
                   1058:            0)
                   1059:
1.28      ryo      1060: .L_devmap_addr:
                   1061:        .quad   VM_KERNEL_IO_ADDRESS
                   1062:
                   1063:        .data
                   1064:
                   1065: #ifdef DEBUG_LOCORE_PRINT_LOCK
                   1066:        .align 2
                   1067: lock_level:
                   1068:        .fill   MAXCPUS, 2, -1
                   1069: lock_turn:
                   1070:        .fill   (MAXCPUS - 1), 2, -1
                   1071: #endif /* DEBUG_LOCORE_PRINT_LOCK */
                   1072:
                   1073:        .align 3
                   1074: ttbr_save:
                   1075:        .space  8 * 2
1.5       ryo      1076:
1.8       ryo      1077:        .bss
                   1078:
1.5       ryo      1079:        .align PGSHIFT
                   1080:        .global _C_LABEL(lwp0uspace)
                   1081: _C_LABEL(lwp0uspace):
                   1082:        .space  UPAGES * PAGE_SIZE
1.8       ryo      1083: bootstk:
                   1084:
                   1085: #ifdef MULTIPROCESSOR
1.28      ryo      1086:        .space  BOOT_AP_STACKSIZE * (MAXCPUS - 1)
1.8       ryo      1087: #endif
                   1088:
1.28      ryo      1089:        .section ".init_pagetable", "aw", %nobits
1.5       ryo      1090:        .align PGSHIFT
1.28      ryo      1091:        .global ARM_BOOTSTRAP_LxPT
                   1092: ARM_BOOTSTRAP_LxPT:
                   1093: l0pt_kern:
1.30      skrll    1094:
                   1095:        .section "_init_memory", "aw", %nobits
                   1096:        .align PGSHIFT
                   1097:
                   1098:        /* None currently */

CVSweb <webmaster@jp.NetBSD.org>