Annotation of src/sys/arch/aarch64/aarch64/cpu.c, Revision 1.56.2.1
1.56.2.1! thorpej 1: /* $NetBSD: cpu.c,v 1.57 2020/12/11 18:03:33 skrll Exp $ */
1.1 ryo 2:
3: /*
4: * Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org>
5: * All rights reserved.
6: *
7: * Redistribution and use in source and binary forms, with or without
8: * modification, are permitted provided that the following conditions
9: * are met:
10: * 1. Redistributions of source code must retain the above copyright
11: * notice, this list of conditions and the following disclaimer.
12: * 2. Redistributions in binary form must reproduce the above copyright
13: * notice, this list of conditions and the following disclaimer in the
14: * documentation and/or other materials provided with the distribution.
15: *
16: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18: * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19: * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
20: * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21: * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22: * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
24: * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
25: * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26: * POSSIBILITY OF SUCH DAMAGE.
27: */
28:
29: #include <sys/cdefs.h>
1.56.2.1! thorpej 30: __KERNEL_RCSID(1, "$NetBSD: cpu.c,v 1.57 2020/12/11 18:03:33 skrll Exp $");
1.1 ryo 31:
32: #include "locators.h"
1.2 ryo 33: #include "opt_arm_debug.h"
34: #include "opt_fdt.h"
1.1 ryo 35: #include "opt_multiprocessor.h"
36:
37: #include <sys/param.h>
1.2 ryo 38: #include <sys/atomic.h>
1.35 skrll 39: #include <sys/cpu.h>
1.1 ryo 40: #include <sys/device.h>
41: #include <sys/kmem.h>
1.14 ryo 42: #include <sys/reboot.h>
1.45 riastrad 43: #include <sys/rndsource.h>
1.13 mrg 44: #include <sys/sysctl.h>
1.35 skrll 45: #include <sys/systm.h>
1.1 ryo 46:
1.53 riastrad 47: #include <crypto/aes/aes_impl.h>
1.49 riastrad 48: #include <crypto/aes/arch/arm/aes_armv8.h>
1.50 riastrad 49: #include <crypto/aes/arch/arm/aes_neon.h>
1.54 riastrad 50: #include <crypto/chacha/chacha_impl.h>
51: #include <crypto/chacha/arch/arm/chacha_neon.h>
1.49 riastrad 52:
1.1 ryo 53: #include <aarch64/armreg.h>
54: #include <aarch64/cpu.h>
1.42 jmcneill 55: #include <aarch64/cpu_counter.h>
1.1 ryo 56: #include <aarch64/machdep.h>
57:
1.56.2.1! thorpej 58: #include <arm/cpufunc.h>
1.34 mrg 59: #include <arm/cpu_topology.h>
1.2 ryo 60: #ifdef FDT
61: #include <arm/fdt/arm_fdtvar.h>
62: #endif
63:
1.4 skrll 64: #ifdef VERBOSE_INIT_ARM
65: #define VPRINTF(...) printf(__VA_ARGS__)
66: #else
1.12 skrll 67: #define VPRINTF(...) __nothing
1.4 skrll 68: #endif
69:
1.1 ryo 70: void cpu_attach(device_t, cpuid_t);
1.2 ryo 71: static void identify_aarch64_model(uint32_t, char *, size_t);
1.6 ryo 72: static void cpu_identify(device_t self, struct cpu_info *);
1.2 ryo 73: static void cpu_identify1(device_t self, struct cpu_info *);
1.1 ryo 74: static void cpu_identify2(device_t self, struct cpu_info *);
1.42 jmcneill 75: static void cpu_init_counter(struct cpu_info *);
1.13 mrg 76: static void cpu_setup_id(struct cpu_info *);
77: static void cpu_setup_sysctl(device_t, struct cpu_info *);
1.45 riastrad 78: static void cpu_setup_rng(device_t, struct cpu_info *);
1.49 riastrad 79: static void cpu_setup_aes(device_t, struct cpu_info *);
1.54 riastrad 80: static void cpu_setup_chacha(device_t, struct cpu_info *);
1.1 ryo 81:
1.2 ryo 82: #ifdef MULTIPROCESSOR
83: #define NCPUINFO MAXCPUS
84: #else
85: #define NCPUINFO 1
86: #endif /* MULTIPROCESSOR */
87:
1.6 ryo 88: /*
89: * Our exported CPU info;
90: * these will be refered from secondary cpus in the middle of hatching.
91: */
92: struct cpu_info cpu_info_store[NCPUINFO] = {
93: [0] = {
94: .ci_cpl = IPL_HIGH,
95: .ci_curlwp = &lwp0
96: }
97: };
98:
1.1 ryo 99: void
100: cpu_attach(device_t dv, cpuid_t id)
101: {
102: struct cpu_info *ci;
1.6 ryo 103: const int unit = device_unit(dv);
1.1 ryo 104:
1.6 ryo 105: if (unit == 0) {
1.1 ryo 106: ci = curcpu();
1.6 ryo 107: ci->ci_cpuid = id;
1.13 mrg 108: cpu_setup_id(ci);
1.1 ryo 109: } else {
110: #ifdef MULTIPROCESSOR
1.14 ryo 111: if ((boothowto & RB_MD1) != 0) {
112: aprint_naive("\n");
113: aprint_normal(": multiprocessor boot disabled\n");
114: return;
115: }
116:
1.6 ryo 117: KASSERT(unit < MAXCPUS);
118: ci = &cpu_info_store[unit];
119:
1.1 ryo 120: ci->ci_cpl = IPL_HIGH;
121: ci->ci_cpuid = id;
1.13 mrg 122: /* ci_id is stored by own cpus when hatching */
1.2 ryo 123:
1.6 ryo 124: cpu_info[ncpu] = ci;
1.23 jmcneill 125: if (cpu_hatched_p(unit) == 0) {
1.1 ryo 126: ci->ci_dev = dv;
127: dv->dv_private = ci;
1.6 ryo 128: ci->ci_index = -1;
1.1 ryo 129:
130: aprint_naive(": disabled\n");
1.2 ryo 131: aprint_normal(": disabled (unresponsive)\n");
1.1 ryo 132: return;
133: }
1.2 ryo 134: #else /* MULTIPROCESSOR */
1.1 ryo 135: aprint_naive(": disabled\n");
136: aprint_normal(": disabled (uniprocessor kernel)\n");
137: return;
1.2 ryo 138: #endif /* MULTIPROCESSOR */
139: }
140:
1.1 ryo 141: ci->ci_dev = dv;
142: dv->dv_private = ci;
143:
1.48 riastrad 144: ci->ci_kfpu_spl = -1;
145:
1.34 mrg 146: arm_cpu_do_topology(ci);
1.33 mrg 147: cpu_identify(ci->ci_dev, ci);
1.26 mlelstv 148:
1.1 ryo 149: #ifdef MULTIPROCESSOR
1.6 ryo 150: if (unit != 0) {
1.1 ryo 151: mi_cpu_attach(ci);
1.2 ryo 152: return;
1.1 ryo 153: }
1.2 ryo 154: #endif /* MULTIPROCESSOR */
1.1 ryo 155:
1.15 ryo 156: set_cpufuncs();
1.1 ryo 157: fpu_attach(ci);
158:
1.2 ryo 159: cpu_identify1(dv, ci);
1.40 skrll 160:
161: /* aarch64_getcacheinfo(0) was called by locore.S */
1.6 ryo 162: aarch64_printcacheinfo(dv);
1.1 ryo 163: cpu_identify2(dv, ci);
1.13 mrg 164:
1.42 jmcneill 165: cpu_init_counter(ci);
166:
1.13 mrg 167: cpu_setup_sysctl(dv, ci);
1.45 riastrad 168: cpu_setup_rng(dv, ci);
1.49 riastrad 169: cpu_setup_aes(dv, ci);
1.54 riastrad 170: cpu_setup_chacha(dv, ci);
1.1 ryo 171: }
172:
173: struct cpuidtab {
174: uint32_t cpu_partnum;
175: const char *cpu_name;
1.43 jmcneill 176: const char *cpu_vendor;
1.1 ryo 177: const char *cpu_architecture;
178: };
179:
180: #define CPU_PARTMASK (CPU_ID_IMPLEMENTOR_MASK | CPU_ID_PARTNO_MASK)
181:
182: const struct cpuidtab cpuids[] = {
1.43 jmcneill 183: { CPU_ID_CORTEXA35R0 & CPU_PARTMASK, "Cortex-A35", "Arm", "v8-A" },
184: { CPU_ID_CORTEXA53R0 & CPU_PARTMASK, "Cortex-A53", "Arm", "v8-A" },
185: { CPU_ID_CORTEXA57R0 & CPU_PARTMASK, "Cortex-A57", "Arm", "v8-A" },
186: { CPU_ID_CORTEXA55R1 & CPU_PARTMASK, "Cortex-A55", "Arm", "v8.2-A+" },
187: { CPU_ID_CORTEXA65R0 & CPU_PARTMASK, "Cortex-A65", "Arm", "v8.2-A+" },
188: { CPU_ID_CORTEXA72R0 & CPU_PARTMASK, "Cortex-A72", "Arm", "v8-A" },
189: { CPU_ID_CORTEXA73R0 & CPU_PARTMASK, "Cortex-A73", "Arm", "v8-A" },
190: { CPU_ID_CORTEXA75R2 & CPU_PARTMASK, "Cortex-A75", "Arm", "v8.2-A+" },
191: { CPU_ID_CORTEXA76R3 & CPU_PARTMASK, "Cortex-A76", "Arm", "v8.2-A+" },
192: { CPU_ID_CORTEXA76AER1 & CPU_PARTMASK, "Cortex-A76AE", "Arm", "v8.2-A+" },
193: { CPU_ID_CORTEXA77R0 & CPU_PARTMASK, "Cortex-A77", "Arm", "v8.2-A+" },
194: { CPU_ID_NVIDIADENVER2 & CPU_PARTMASK, "Denver2", "NVIDIA", "v8-A" },
195: { CPU_ID_EMAG8180 & CPU_PARTMASK, "eMAG", "Ampere", "v8-A" },
196: { CPU_ID_NEOVERSEE1R1 & CPU_PARTMASK, "Neoverse E1", "Arm", "v8.2-A+" },
197: { CPU_ID_NEOVERSEN1R3 & CPU_PARTMASK, "Neoverse N1", "Arm", "v8.2-A+" },
198: { CPU_ID_THUNDERXRX, "ThunderX", "Cavium", "v8-A" },
199: { CPU_ID_THUNDERX81XXRX, "ThunderX CN81XX", "Cavium", "v8-A" },
200: { CPU_ID_THUNDERX83XXRX, "ThunderX CN83XX", "Cavium", "v8-A" },
201: { CPU_ID_THUNDERX2RX, "ThunderX2", "Marvell", "v8.1-A" },
1.1 ryo 202: };
203:
204: static void
205: identify_aarch64_model(uint32_t cpuid, char *buf, size_t len)
206: {
207: int i;
208: uint32_t cpupart, variant, revision;
209:
210: cpupart = cpuid & CPU_PARTMASK;
211: variant = __SHIFTOUT(cpuid, CPU_ID_VARIANT_MASK);
212: revision = __SHIFTOUT(cpuid, CPU_ID_REVISION_MASK);
213:
214: for (i = 0; i < __arraycount(cpuids); i++) {
215: if (cpupart == cpuids[i].cpu_partnum) {
1.43 jmcneill 216: snprintf(buf, len, "%s %s r%dp%d (%s)",
217: cpuids[i].cpu_vendor, cpuids[i].cpu_name,
218: variant, revision,
1.1 ryo 219: cpuids[i].cpu_architecture);
220: return;
221: }
222: }
223:
224: snprintf(buf, len, "unknown CPU (ID = 0x%08x)", cpuid);
225: }
226:
227: static void
1.6 ryo 228: cpu_identify(device_t self, struct cpu_info *ci)
1.1 ryo 229: {
230: char model[128];
1.32 martin 231: const char *m;
1.1 ryo 232:
1.13 mrg 233: identify_aarch64_model(ci->ci_id.ac_midr, model, sizeof(model));
1.36 skrll 234: if (ci->ci_index == 0) {
1.32 martin 235: m = cpu_getmodel();
236: if (m == NULL || *m == 0)
237: cpu_setmodel("%s", model);
238: }
1.1 ryo 239:
240: aprint_naive("\n");
1.43 jmcneill 241: aprint_normal(": %s, id 0x%lx\n", model, ci->ci_cpuid);
1.2 ryo 242: }
1.1 ryo 243:
1.2 ryo 244: static void
245: cpu_identify1(device_t self, struct cpu_info *ci)
246: {
1.51 ryo 247: uint64_t ctr, clidr, sctlr; /* for cache */
1.1 ryo 248:
249: /* SCTLR - System Control Register */
250: sctlr = reg_sctlr_el1_read();
251: if (sctlr & SCTLR_I)
1.43 jmcneill 252: aprint_verbose_dev(self, "IC enabled");
1.1 ryo 253: else
1.43 jmcneill 254: aprint_verbose_dev(self, "IC disabled");
1.1 ryo 255:
256: if (sctlr & SCTLR_C)
1.43 jmcneill 257: aprint_verbose(", DC enabled");
1.1 ryo 258: else
1.43 jmcneill 259: aprint_verbose(", DC disabled");
1.1 ryo 260:
261: if (sctlr & SCTLR_A)
1.43 jmcneill 262: aprint_verbose(", Alignment check enabled\n");
1.1 ryo 263: else {
264: switch (sctlr & (SCTLR_SA | SCTLR_SA0)) {
265: case SCTLR_SA | SCTLR_SA0:
1.43 jmcneill 266: aprint_verbose(
1.1 ryo 267: ", EL0/EL1 stack Alignment check enabled\n");
268: break;
269: case SCTLR_SA:
1.43 jmcneill 270: aprint_verbose(", EL1 stack Alignment check enabled\n");
1.1 ryo 271: break;
272: case SCTLR_SA0:
1.43 jmcneill 273: aprint_verbose(", EL0 stack Alignment check enabled\n");
1.1 ryo 274: break;
275: case 0:
1.43 jmcneill 276: aprint_verbose(", Alignment check disabled\n");
1.1 ryo 277: break;
278: }
279: }
280:
281: /*
282: * CTR - Cache Type Register
283: */
284: ctr = reg_ctr_el0_read();
1.51 ryo 285: clidr = reg_clidr_el1_read();
1.43 jmcneill 286: aprint_verbose_dev(self, "Cache Writeback Granule %" PRIu64 "B,"
1.3 christos 287: " Exclusives Reservation Granule %" PRIu64 "B\n",
1.1 ryo 288: __SHIFTOUT(ctr, CTR_EL0_CWG_LINE) * 4,
289: __SHIFTOUT(ctr, CTR_EL0_ERG_LINE) * 4);
290:
1.51 ryo 291: aprint_verbose_dev(self, "Dcache line %ld, Icache line %ld"
292: ", DIC=%lu, IDC=%lu, LoUU=%lu, LoC=%lu, LoUIS=%lu\n",
1.1 ryo 293: sizeof(int) << __SHIFTOUT(ctr, CTR_EL0_DMIN_LINE),
1.51 ryo 294: sizeof(int) << __SHIFTOUT(ctr, CTR_EL0_IMIN_LINE),
295: __SHIFTOUT(ctr, CTR_EL0_DIC),
296: __SHIFTOUT(ctr, CTR_EL0_IDC),
297: __SHIFTOUT(clidr, CLIDR_LOUU),
298: __SHIFTOUT(clidr, CLIDR_LOC),
299: __SHIFTOUT(clidr, CLIDR_LOUIS));
1.1 ryo 300: }
301:
302:
303: /*
304: * identify vfp, etc.
305: */
306: static void
307: cpu_identify2(device_t self, struct cpu_info *ci)
308: {
1.13 mrg 309: struct aarch64_sysctl_cpu_id *id = &ci->ci_id;
310: uint64_t dfr0;
1.1 ryo 311:
1.13 mrg 312: if (!CPU_IS_PRIMARY(ci)) {
313: cpu_setup_id(ci);
314: cpu_setup_sysctl(self, ci);
315: }
1.1 ryo 316:
317: dfr0 = reg_id_aa64dfr0_el1_read();
318:
1.33 mrg 319: aprint_debug_dev(self, "midr=0x%" PRIx32 " mpidr=0x%" PRIx32 "\n",
320: (uint32_t)ci->ci_id.ac_midr, (uint32_t)ci->ci_id.ac_mpidr);
1.43 jmcneill 321: aprint_verbose_dev(self, "revID=0x%" PRIx64, id->ac_revidr);
1.1 ryo 322:
323: /* ID_AA64DFR0_EL1 */
324: switch (__SHIFTOUT(dfr0, ID_AA64DFR0_EL1_PMUVER)) {
325: case ID_AA64DFR0_EL1_PMUVER_V3:
1.43 jmcneill 326: aprint_verbose(", PMCv3");
1.1 ryo 327: break;
328: case ID_AA64DFR0_EL1_PMUVER_NOV3:
1.43 jmcneill 329: aprint_verbose(", PMC");
1.1 ryo 330: break;
331: }
332:
333: /* ID_AA64MMFR0_EL1 */
1.13 mrg 334: switch (__SHIFTOUT(id->ac_aa64mmfr0, ID_AA64MMFR0_EL1_TGRAN4)) {
1.1 ryo 335: case ID_AA64MMFR0_EL1_TGRAN4_4KB:
1.43 jmcneill 336: aprint_verbose(", 4k table");
1.1 ryo 337: break;
338: }
1.13 mrg 339: switch (__SHIFTOUT(id->ac_aa64mmfr0, ID_AA64MMFR0_EL1_TGRAN16)) {
1.1 ryo 340: case ID_AA64MMFR0_EL1_TGRAN16_16KB:
1.43 jmcneill 341: aprint_verbose(", 16k table");
1.1 ryo 342: break;
343: }
1.13 mrg 344: switch (__SHIFTOUT(id->ac_aa64mmfr0, ID_AA64MMFR0_EL1_TGRAN64)) {
1.1 ryo 345: case ID_AA64MMFR0_EL1_TGRAN64_64KB:
1.43 jmcneill 346: aprint_verbose(", 64k table");
1.1 ryo 347: break;
348: }
349:
1.13 mrg 350: switch (__SHIFTOUT(id->ac_aa64mmfr0, ID_AA64MMFR0_EL1_ASIDBITS)) {
1.1 ryo 351: case ID_AA64MMFR0_EL1_ASIDBITS_8BIT:
1.43 jmcneill 352: aprint_verbose(", 8bit ASID");
1.1 ryo 353: break;
354: case ID_AA64MMFR0_EL1_ASIDBITS_16BIT:
1.43 jmcneill 355: aprint_verbose(", 16bit ASID");
1.1 ryo 356: break;
357: }
1.43 jmcneill 358: aprint_verbose("\n");
1.1 ryo 359:
360:
361:
1.43 jmcneill 362: aprint_verbose_dev(self, "auxID=0x%" PRIx64, ci->ci_id.ac_aa64isar0);
1.1 ryo 363:
364: /* PFR0 */
1.47 riastrad 365: switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_CSV3)) {
366: case ID_AA64PFR0_EL1_CSV3_IMPL:
367: aprint_verbose(", CSV3");
368: break;
369: }
370: switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_CSV2)) {
371: case ID_AA64PFR0_EL1_CSV2_IMPL:
372: aprint_verbose(", CSV2");
373: break;
374: }
1.13 mrg 375: switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_GIC)) {
1.1 ryo 376: case ID_AA64PFR0_EL1_GIC_CPUIF_EN:
1.43 jmcneill 377: aprint_verbose(", GICv3");
1.1 ryo 378: break;
379: }
1.13 mrg 380: switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_FP)) {
1.56 jmcneill 381: case ID_AA64PFR0_EL1_FP_NONE:
382: break;
383: default:
1.43 jmcneill 384: aprint_verbose(", FP");
1.1 ryo 385: break;
386: }
387:
388: /* ISAR0 */
1.13 mrg 389: switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_CRC32)) {
1.1 ryo 390: case ID_AA64ISAR0_EL1_CRC32_CRC32X:
1.43 jmcneill 391: aprint_verbose(", CRC32");
1.1 ryo 392: break;
393: }
1.13 mrg 394: switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_SHA1)) {
1.1 ryo 395: case ID_AA64ISAR0_EL1_SHA1_SHA1CPMHSU:
1.43 jmcneill 396: aprint_verbose(", SHA1");
1.1 ryo 397: break;
398: }
1.13 mrg 399: switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_SHA2)) {
1.1 ryo 400: case ID_AA64ISAR0_EL1_SHA2_SHA256HSU:
1.43 jmcneill 401: aprint_verbose(", SHA256");
1.1 ryo 402: break;
403: }
1.13 mrg 404: switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_AES)) {
1.1 ryo 405: case ID_AA64ISAR0_EL1_AES_AES:
1.43 jmcneill 406: aprint_verbose(", AES");
1.1 ryo 407: break;
408: case ID_AA64ISAR0_EL1_AES_PMUL:
1.43 jmcneill 409: aprint_verbose(", AES+PMULL");
1.1 ryo 410: break;
411: }
1.44 riastrad 412: switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_RNDR)) {
413: case ID_AA64ISAR0_EL1_RNDR_RNDRRS:
414: aprint_verbose(", RNDRRS");
415: break;
416: }
1.1 ryo 417:
1.47 riastrad 418: /* PFR0:DIT -- data-independent timing support */
419: switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_DIT)) {
420: case ID_AA64PFR0_EL1_DIT_IMPL:
421: aprint_verbose(", DIT");
422: break;
423: }
1.1 ryo 424:
425: /* PFR0:AdvSIMD */
1.13 mrg 426: switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_ADVSIMD)) {
1.56 jmcneill 427: case ID_AA64PFR0_EL1_ADV_SIMD_NONE:
428: break;
429: default:
1.43 jmcneill 430: aprint_verbose(", NEON");
1.1 ryo 431: break;
432: }
433:
434: /* MVFR0/MVFR1 */
1.13 mrg 435: switch (__SHIFTOUT(id->ac_mvfr0, MVFR0_FPROUND)) {
1.1 ryo 436: case MVFR0_FPROUND_ALL:
1.43 jmcneill 437: aprint_verbose(", rounding");
1.1 ryo 438: break;
439: }
1.13 mrg 440: switch (__SHIFTOUT(id->ac_mvfr0, MVFR0_FPTRAP)) {
1.1 ryo 441: case MVFR0_FPTRAP_TRAP:
1.43 jmcneill 442: aprint_verbose(", exceptions");
1.1 ryo 443: break;
444: }
1.13 mrg 445: switch (__SHIFTOUT(id->ac_mvfr1, MVFR1_FPDNAN)) {
1.1 ryo 446: case MVFR1_FPDNAN_NAN:
1.43 jmcneill 447: aprint_verbose(", NaN propagation");
1.1 ryo 448: break;
449: }
1.13 mrg 450: switch (__SHIFTOUT(id->ac_mvfr1, MVFR1_FPFTZ)) {
1.1 ryo 451: case MVFR1_FPFTZ_DENORMAL:
1.43 jmcneill 452: aprint_verbose(", denormals");
1.1 ryo 453: break;
454: }
1.13 mrg 455: switch (__SHIFTOUT(id->ac_mvfr0, MVFR0_SIMDREG)) {
1.1 ryo 456: case MVFR0_SIMDREG_16x64:
1.43 jmcneill 457: aprint_verbose(", 16x64bitRegs");
1.1 ryo 458: break;
459: case MVFR0_SIMDREG_32x64:
1.43 jmcneill 460: aprint_verbose(", 32x64bitRegs");
1.1 ryo 461: break;
462: }
1.13 mrg 463: switch (__SHIFTOUT(id->ac_mvfr1, MVFR1_SIMDFMAC)) {
1.1 ryo 464: case MVFR1_SIMDFMAC_FMAC:
1.43 jmcneill 465: aprint_verbose(", Fused Multiply-Add");
1.1 ryo 466: break;
467: }
468:
1.43 jmcneill 469: aprint_verbose("\n");
1.1 ryo 470: }
1.2 ryo 471:
1.13 mrg 472: /*
1.42 jmcneill 473: * Enable the performance counter, then estimate frequency for
474: * the current PE and store the result in cpu_cc_freq.
475: */
476: static void
477: cpu_init_counter(struct cpu_info *ci)
478: {
1.55 jmcneill 479: const uint64_t dfr0 = reg_id_aa64dfr0_el1_read();
480: const u_int pmuver = __SHIFTOUT(dfr0, ID_AA64DFR0_EL1_PMUVER);
481: if (pmuver == ID_AA64DFR0_EL1_PMUVER_NONE) {
482: /* Performance Monitors Extension not implemented. */
483: return;
484: }
485:
1.42 jmcneill 486: reg_pmcr_el0_write(PMCR_E | PMCR_C);
487: reg_pmcntenset_el0_write(PMCNTEN_C);
488:
489: const uint32_t prev = cpu_counter32();
490: delay(100000);
491: ci->ci_data.cpu_cc_freq = (cpu_counter32() - prev) * 10;
492: }
493:
494: /*
1.13 mrg 495: * Fill in this CPUs id data. Must be called from hatched cpus.
496: */
497: static void
498: cpu_setup_id(struct cpu_info *ci)
499: {
500: struct aarch64_sysctl_cpu_id *id = &ci->ci_id;
501:
502: memset(id, 0, sizeof *id);
503:
504: id->ac_midr = reg_midr_el1_read();
505: id->ac_revidr = reg_revidr_el1_read();
506: id->ac_mpidr = reg_mpidr_el1_read();
507:
508: id->ac_aa64dfr0 = reg_id_aa64dfr0_el1_read();
509: id->ac_aa64dfr1 = reg_id_aa64dfr1_el1_read();
510:
511: id->ac_aa64isar0 = reg_id_aa64isar0_el1_read();
512: id->ac_aa64isar1 = reg_id_aa64isar1_el1_read();
513:
514: id->ac_aa64mmfr0 = reg_id_aa64mmfr0_el1_read();
515: id->ac_aa64mmfr1 = reg_id_aa64mmfr1_el1_read();
1.39 maxv 516: id->ac_aa64mmfr2 = reg_id_aa64mmfr2_el1_read();
1.13 mrg 517:
518: id->ac_mvfr0 = reg_mvfr0_el1_read();
519: id->ac_mvfr1 = reg_mvfr1_el1_read();
520: id->ac_mvfr2 = reg_mvfr2_el1_read();
521:
1.52 ryo 522: id->ac_clidr = reg_clidr_el1_read();
523: id->ac_ctr = reg_ctr_el0_read();
524:
1.13 mrg 525: /* Only in ARMv8.2. */
526: id->ac_aa64zfr0 = 0 /* reg_id_aa64zfr0_el1_read() */;
527:
528: id->ac_aa64pfr0 = reg_id_aa64pfr0_el1_read();
529: id->ac_aa64pfr1 = reg_id_aa64pfr1_el1_read();
530: }
531:
532: /*
533: * setup the per-cpu sysctl tree.
534: */
535: static void
536: cpu_setup_sysctl(device_t dv, struct cpu_info *ci)
537: {
538: const struct sysctlnode *cpunode = NULL;
539:
540: sysctl_createv(NULL, 0, NULL, &cpunode,
541: CTLFLAG_PERMANENT,
542: CTLTYPE_NODE, device_xname(dv), NULL,
543: NULL, 0, NULL, 0,
544: CTL_MACHDEP,
545: CTL_CREATE, CTL_EOL);
546:
547: if (cpunode == NULL)
548: return;
549:
550: sysctl_createv(NULL, 0, &cpunode, NULL,
551: CTLFLAG_PERMANENT,
552: CTLTYPE_STRUCT, "cpu_id", NULL,
553: NULL, 0, &ci->ci_id, sizeof(ci->ci_id),
554: CTL_CREATE, CTL_EOL);
555: }
556:
1.45 riastrad 557: static struct krndsource rndrrs_source;
558:
559: static void
560: rndrrs_get(size_t nbytes, void *cookie)
561: {
562: /* Entropy bits per data byte, wild-arse guess. */
563: const unsigned bpb = 4;
564: size_t nbits = nbytes*NBBY;
565: uint64_t x;
566: int error;
567:
568: while (nbits) {
569: /*
570: * x := random 64-bit sample
571: * error := Z bit, set to 1 if sample is bad
572: *
573: * XXX This should be done by marking the function
574: * __attribute__((target("arch=armv8.5-a+rng"))) and
575: * using `mrs %0, rndrrs', but:
576: *
577: * (a) the version of gcc we use doesn't support that,
578: * and
579: * (b) clang doesn't seem to like `rndrrs' itself.
580: *
581: * So we use the numeric encoding for now.
582: */
583: __asm __volatile(""
584: "mrs %0, s3_3_c2_c4_1\n"
585: "cset %w1, eq"
586: : "=r"(x), "=r"(error));
587: if (error)
588: break;
589: rnd_add_data_sync(&rndrrs_source, &x, sizeof(x),
590: bpb*sizeof(x));
591: nbits -= MIN(nbits, bpb*sizeof(x));
592: }
593:
594: explicit_memset(&x, 0, sizeof x);
595: }
596:
597: /*
598: * setup the RNDRRS entropy source
599: */
600: static void
601: cpu_setup_rng(device_t dv, struct cpu_info *ci)
602: {
603: struct aarch64_sysctl_cpu_id *id = &ci->ci_id;
604:
605: /* Probably shared between cores. */
606: if (!CPU_IS_PRIMARY(ci))
607: return;
608:
609: /* Verify that it is supported. */
610: switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_RNDR)) {
611: case ID_AA64ISAR0_EL1_RNDR_RNDRRS:
612: break;
613: default:
614: return;
615: }
616:
617: /* Attach it. */
618: rndsource_setcb(&rndrrs_source, rndrrs_get, NULL);
619: rnd_attach_source(&rndrrs_source, "rndrrs", RND_TYPE_RNG,
620: RND_FLAG_DEFAULT|RND_FLAG_HASCB);
621: }
622:
1.49 riastrad 623: /*
624: * setup the AES implementation
625: */
626: static void
627: cpu_setup_aes(device_t dv, struct cpu_info *ci)
628: {
629: struct aarch64_sysctl_cpu_id *id = &ci->ci_id;
630:
1.50 riastrad 631: /* Check for ARMv8.0-AES support. */
1.49 riastrad 632: switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_AES)) {
633: case ID_AA64ISAR0_EL1_AES_AES:
634: case ID_AA64ISAR0_EL1_AES_PMUL:
1.50 riastrad 635: aes_md_init(&aes_armv8_impl);
636: return;
637: default:
1.49 riastrad 638: break;
1.50 riastrad 639: }
640:
641: /* Failing that, check for SIMD support. */
642: switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_ADVSIMD)) {
643: case ID_AA64PFR0_EL1_ADV_SIMD_IMPL:
644: aes_md_init(&aes_neon_impl);
645: return;
1.49 riastrad 646: default:
1.50 riastrad 647: break;
1.49 riastrad 648: }
649: }
650:
1.54 riastrad 651: /*
652: * setup the ChaCha implementation
653: */
654: static void
655: cpu_setup_chacha(device_t dv, struct cpu_info *ci)
656: {
657: struct aarch64_sysctl_cpu_id *id = &ci->ci_id;
658:
659: /* Check for SIMD support. */
660: switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_ADVSIMD)) {
661: case ID_AA64PFR0_EL1_ADV_SIMD_IMPL:
662: chacha_md_init(&chacha_neon_impl);
663: return;
664: default:
665: break;
666: }
667: }
668:
1.2 ryo 669: #ifdef MULTIPROCESSOR
670: void
671: cpu_hatch(struct cpu_info *ci)
672: {
673: KASSERT(curcpu() == ci);
674:
675: mutex_enter(&cpu_hatch_lock);
676:
1.15 ryo 677: set_cpufuncs();
1.2 ryo 678: fpu_attach(ci);
679:
680: cpu_identify1(ci->ci_dev, ci);
1.26 mlelstv 681: aarch64_getcacheinfo(device_unit(ci->ci_dev));
1.6 ryo 682: aarch64_printcacheinfo(ci->ci_dev);
1.2 ryo 683: cpu_identify2(ci->ci_dev, ci);
684:
685: mutex_exit(&cpu_hatch_lock);
686:
1.42 jmcneill 687: cpu_init_counter(ci);
688:
1.2 ryo 689: intr_cpu_init(ci);
690:
691: #ifdef FDT
692: arm_fdt_cpu_hatch(ci);
693: #endif
694: #ifdef MD_CPU_HATCH
695: MD_CPU_HATCH(ci); /* for non-fdt arch? */
696: #endif
697:
1.6 ryo 698: /*
1.41 skrll 699: * clear my bit of arm_cpu_mbox to tell cpu_boot_secondary_processors().
1.6 ryo 700: * there are cpu0,1,2,3, and if cpu2 is unresponsive,
701: * ci_index are each cpu0=0, cpu1=1, cpu2=undef, cpu3=2.
702: * therefore we have to use device_unit instead of ci_index for mbox.
703: */
1.23 jmcneill 704:
1.41 skrll 705: cpu_clr_mbox(device_unit(ci->ci_dev));
1.23 jmcneill 706: }
1.2 ryo 707: #endif /* MULTIPROCESSOR */
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to cpu.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / arch / aarch64 / aarch64