Annotation of src/sys/arch/x86/x86/cpu.c, Revision 1.130.2.7
1.130.2.7! martin 1: /* $NetBSD: cpu.c,v 1.130.2.6 2018/04/14 10:11:49 martin Exp $ */
1.2 ad 2:
3: /*-
1.98 rmind 4: * Copyright (c) 2000-2012 NetBSD Foundation, Inc.
1.2 ad 5: * All rights reserved.
6: *
7: * This code is derived from software contributed to The NetBSD Foundation
1.11 ad 8: * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran.
1.2 ad 9: *
10: * Redistribution and use in source and binary forms, with or without
11: * modification, are permitted provided that the following conditions
12: * are met:
13: * 1. Redistributions of source code must retain the above copyright
14: * notice, this list of conditions and the following disclaimer.
15: * 2. Redistributions in binary form must reproduce the above copyright
16: * notice, this list of conditions and the following disclaimer in the
17: * documentation and/or other materials provided with the distribution.
18: *
19: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29: * POSSIBILITY OF SUCH DAMAGE.
30: */
31:
32: /*
33: * Copyright (c) 1999 Stefan Grefen
34: *
35: * Redistribution and use in source and binary forms, with or without
36: * modification, are permitted provided that the following conditions
37: * are met:
38: * 1. Redistributions of source code must retain the above copyright
39: * notice, this list of conditions and the following disclaimer.
40: * 2. Redistributions in binary form must reproduce the above copyright
41: * notice, this list of conditions and the following disclaimer in the
42: * documentation and/or other materials provided with the distribution.
43: * 3. All advertising materials mentioning features or use of this software
44: * must display the following acknowledgement:
45: * This product includes software developed by the NetBSD
46: * Foundation, Inc. and its contributors.
47: * 4. Neither the name of The NetBSD Foundation nor the names of its
48: * contributors may be used to endorse or promote products derived
49: * from this software without specific prior written permission.
50: *
51: * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
52: * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
53: * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
54: * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
55: * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
56: * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
57: * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
58: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
59: * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
60: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
61: * SUCH DAMAGE.
62: */
63:
64: #include <sys/cdefs.h>
1.130.2.7! martin 65: __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.130.2.6 2018/04/14 10:11:49 martin Exp $");
1.2 ad 66:
67: #include "opt_ddb.h"
68: #include "opt_mpbios.h" /* for MPDEBUG */
69: #include "opt_mtrr.h"
1.101 kiyohara 70: #include "opt_multiprocessor.h"
1.130.2.5 martin 71: #include "opt_svs.h"
1.2 ad 72:
73: #include "lapic.h"
74: #include "ioapic.h"
75:
76: #include <sys/param.h>
77: #include <sys/proc.h>
78: #include <sys/systm.h>
79: #include <sys/device.h>
1.61 cegger 80: #include <sys/kmem.h>
1.9 ad 81: #include <sys/cpu.h>
1.93 jruoho 82: #include <sys/cpufreq.h>
1.98 rmind 83: #include <sys/idle.h>
1.9 ad 84: #include <sys/atomic.h>
1.35 ad 85: #include <sys/reboot.h>
1.2 ad 86:
1.78 uebayasi 87: #include <uvm/uvm.h>
1.2 ad 88:
1.102 pgoyette 89: #include "acpica.h" /* for NACPICA, for mp_verbose */
90:
1.2 ad 91: #include <machine/cpufunc.h>
92: #include <machine/cpuvar.h>
93: #include <machine/pmap.h>
94: #include <machine/vmparam.h>
1.102 pgoyette 95: #if defined(MULTIPROCESSOR)
1.2 ad 96: #include <machine/mpbiosvar.h>
1.101 kiyohara 97: #endif
1.102 pgoyette 98: #include <machine/mpconfig.h> /* for mp_verbose */
1.2 ad 99: #include <machine/pcb.h>
100: #include <machine/specialreg.h>
101: #include <machine/segments.h>
102: #include <machine/gdt.h>
103: #include <machine/mtrr.h>
104: #include <machine/pio.h>
1.38 ad 105: #include <machine/cpu_counter.h>
1.2 ad 106:
1.109 dsl 107: #include <x86/fpu.h>
108:
1.2 ad 109: #ifdef i386
110: #include <machine/tlog.h>
111: #endif
112:
1.101 kiyohara 113: #if NLAPIC > 0
1.2 ad 114: #include <machine/apicvar.h>
115: #include <machine/i82489reg.h>
116: #include <machine/i82489var.h>
1.101 kiyohara 117: #endif
1.2 ad 118:
119: #include <dev/ic/mc146818reg.h>
120: #include <i386/isa/nvram.h>
121: #include <dev/isa/isareg.h>
122:
1.38 ad 123: #include "tsc.h"
124:
1.87 jruoho 125: static int cpu_match(device_t, cfdata_t, void *);
126: static void cpu_attach(device_t, device_t, void *);
127: static void cpu_defer(device_t);
128: static int cpu_rescan(device_t, const char *, const int *);
129: static void cpu_childdetached(device_t, device_t);
1.96 jruoho 130: static bool cpu_stop(device_t);
1.69 dyoung 131: static bool cpu_suspend(device_t, const pmf_qual_t *);
132: static bool cpu_resume(device_t, const pmf_qual_t *);
1.79 jruoho 133: static bool cpu_shutdown(device_t, int);
1.12 jmcneill 134:
1.2 ad 135: struct cpu_softc {
1.23 cube 136: device_t sc_dev; /* device tree glue */
1.2 ad 137: struct cpu_info *sc_info; /* pointer to CPU info */
1.20 jmcneill 138: bool sc_wasonline;
1.2 ad 139: };
140:
1.101 kiyohara 141: #ifdef MULTIPROCESSOR
1.120 msaitoh 142: int mp_cpu_start(struct cpu_info *, paddr_t);
1.2 ad 143: void mp_cpu_start_cleanup(struct cpu_info *);
144: const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL,
145: mp_cpu_start_cleanup };
1.101 kiyohara 146: #endif
1.2 ad 147:
148:
1.81 jmcneill 149: CFATTACH_DECL2_NEW(cpu, sizeof(struct cpu_softc),
150: cpu_match, cpu_attach, NULL, NULL, cpu_rescan, cpu_childdetached);
1.2 ad 151:
152: /*
153: * Statically-allocated CPU info for the primary CPU (or the only
154: * CPU, on uniprocessors). The CPU info list is initialized to
155: * point at it.
156: */
157: #ifdef TRAPLOG
158: struct tlog tlog_primary;
159: #endif
1.21 ad 160: struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = {
1.2 ad 161: .ci_dev = 0,
162: .ci_self = &cpu_info_primary,
163: .ci_idepth = -1,
164: .ci_curlwp = &lwp0,
1.43 ad 165: .ci_curldt = -1,
1.2 ad 166: #ifdef TRAPLOG
167: .ci_tlog_base = &tlog_primary,
1.130.2.3 martin 168: #endif
1.2 ad 169: };
170:
171: struct cpu_info *cpu_info_list = &cpu_info_primary;
172:
173: #ifdef i386
1.130.2.3 martin 174: void cpu_set_tss_gates(struct cpu_info *);
1.2 ad 175: #endif
176:
1.12 jmcneill 177: static void cpu_init_idle_lwp(struct cpu_info *);
178:
1.122 maxv 179: uint32_t cpu_feature[7] __read_mostly; /* X86 CPUID feature bits */
1.117 maxv 180: /* [0] basic features cpuid.1:%edx
181: * [1] basic features cpuid.1:%ecx (CPUID2_xxx bits)
182: * [2] extended features cpuid:80000001:%edx
183: * [3] extended features cpuid:80000001:%ecx
184: * [4] VIA padlock features
185: * [5] structured extended features cpuid.7:%ebx
186: * [6] structured extended features cpuid.7:%ecx
187: */
1.70 jym 188:
1.101 kiyohara 189: #ifdef MULTIPROCESSOR
1.12 jmcneill 190: bool x86_mp_online;
191: paddr_t mp_trampoline_paddr = MP_TRAMPOLINE;
1.101 kiyohara 192: #endif
193: #if NLAPIC > 0
1.14 joerg 194: static vaddr_t cmos_data_mapping;
1.101 kiyohara 195: #endif
1.45 ad 196: struct cpu_info *cpu_starting;
1.2 ad 197:
1.101 kiyohara 198: #ifdef MULTIPROCESSOR
1.2 ad 199: void cpu_hatch(void *);
200: static void cpu_boot_secondary(struct cpu_info *ci);
201: static void cpu_start_secondary(struct cpu_info *ci);
1.101 kiyohara 202: #endif
203: #if NLAPIC > 0
1.2 ad 204: static void cpu_copy_trampoline(void);
1.101 kiyohara 205: #endif
1.2 ad 206:
207: /*
208: * Runs once per boot once multiprocessor goo has been detected and
209: * the local APIC on the boot processor has been mapped.
210: *
211: * Called from lapic_boot_init() (from mpbios_scan()).
212: */
1.101 kiyohara 213: #if NLAPIC > 0
1.2 ad 214: void
1.9 ad 215: cpu_init_first(void)
1.2 ad 216: {
217:
1.45 ad 218: cpu_info_primary.ci_cpuid = lapic_cpu_number();
1.2 ad 219: cpu_copy_trampoline();
1.14 joerg 220:
221: cmos_data_mapping = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY);
222: if (cmos_data_mapping == 0)
223: panic("No KVA for page 0");
1.64 cegger 224: pmap_kenter_pa(cmos_data_mapping, 0, VM_PROT_READ|VM_PROT_WRITE, 0);
1.14 joerg 225: pmap_update(pmap_kernel());
1.2 ad 226: }
1.101 kiyohara 227: #endif
1.2 ad 228:
1.87 jruoho 229: static int
1.23 cube 230: cpu_match(device_t parent, cfdata_t match, void *aux)
1.2 ad 231: {
232:
233: return 1;
234: }
235:
1.130.2.4 martin 236: #ifdef __HAVE_PCPU_AREA
237: void
238: cpu_pcpuarea_init(struct cpu_info *ci)
239: {
240: struct vm_page *pg;
241: size_t i, npages;
242: vaddr_t base, va;
243: paddr_t pa;
244:
245: CTASSERT(sizeof(struct pcpu_entry) % PAGE_SIZE == 0);
246:
247: npages = sizeof(struct pcpu_entry) / PAGE_SIZE;
248: base = (vaddr_t)&pcpuarea->ent[cpu_index(ci)];
249:
250: for (i = 0; i < npages; i++) {
251: pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO);
252: if (pg == NULL) {
253: panic("failed to allocate pcpu PA");
254: }
255:
256: va = base + i * PAGE_SIZE;
257: pa = VM_PAGE_TO_PHYS(pg);
258:
259: pmap_kenter_pa(va, pa, VM_PROT_READ|VM_PROT_WRITE, 0);
260: }
261:
262: pmap_update(pmap_kernel());
263: }
264: #endif
265:
1.2 ad 266: static void
267: cpu_vm_init(struct cpu_info *ci)
268: {
269: int ncolors = 2, i;
270:
271: for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) {
272: struct x86_cache_info *cai;
273: int tcolors;
274:
275: cai = &ci->ci_cinfo[i];
276:
277: tcolors = atop(cai->cai_totalsize);
278: switch(cai->cai_associativity) {
279: case 0xff:
280: tcolors = 1; /* fully associative */
281: break;
282: case 0:
283: case 1:
284: break;
285: default:
286: tcolors /= cai->cai_associativity;
287: }
288: ncolors = max(ncolors, tcolors);
1.32 tls 289: /*
290: * If the desired number of colors is not a power of
291: * two, it won't be good. Find the greatest power of
292: * two which is an even divisor of the number of colors,
293: * to preserve even coloring of pages.
294: */
295: if (ncolors & (ncolors - 1) ) {
296: int try, picked = 1;
297: for (try = 1; try < ncolors; try *= 2) {
298: if (ncolors % try == 0) picked = try;
299: }
300: if (picked == 1) {
301: panic("desired number of cache colors %d is "
302: " > 1, but not even!", ncolors);
303: }
304: ncolors = picked;
305: }
1.2 ad 306: }
307:
308: /*
1.94 mrg 309: * Knowing the size of the largest cache on this CPU, potentially
310: * re-color our pages.
1.2 ad 311: */
1.52 ad 312: aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors);
1.2 ad 313: uvm_page_recolor(ncolors);
1.98 rmind 314:
315: pmap_tlb_cpu_init(ci);
1.123 maxv 316: #ifndef __HAVE_DIRECT_MAP
317: pmap_vpage_cpu_init(ci);
318: #endif
1.2 ad 319: }
320:
1.87 jruoho 321: static void
1.23 cube 322: cpu_attach(device_t parent, device_t self, void *aux)
1.2 ad 323: {
1.23 cube 324: struct cpu_softc *sc = device_private(self);
1.2 ad 325: struct cpu_attach_args *caa = aux;
326: struct cpu_info *ci;
1.21 ad 327: uintptr_t ptr;
1.101 kiyohara 328: #if NLAPIC > 0
1.2 ad 329: int cpunum = caa->cpu_number;
1.101 kiyohara 330: #endif
1.51 ad 331: static bool again;
1.2 ad 332:
1.23 cube 333: sc->sc_dev = self;
334:
1.98 rmind 335: if (ncpu == maxcpus) {
336: #ifndef _LP64
337: aprint_error(": too many CPUs, please use NetBSD/amd64\n");
338: #else
339: aprint_error(": too many CPUs\n");
340: #endif
1.48 ad 341: return;
342: }
343:
1.2 ad 344: /*
345: * If we're an Application Processor, allocate a cpu_info
346: * structure, otherwise use the primary's.
347: */
348: if (caa->cpu_role == CPU_ROLE_AP) {
1.36 ad 349: if ((boothowto & RB_MD1) != 0) {
1.35 ad 350: aprint_error(": multiprocessor boot disabled\n");
1.56 jmcneill 351: if (!pmf_device_register(self, NULL, NULL))
352: aprint_error_dev(self,
353: "couldn't establish power handler\n");
1.35 ad 354: return;
355: }
1.2 ad 356: aprint_naive(": Application Processor\n");
1.72 rmind 357: ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,
1.61 cegger 358: KM_SLEEP);
1.67 jym 359: ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);
1.43 ad 360: ci->ci_curldt = -1;
1.2 ad 361: #ifdef TRAPLOG
1.61 cegger 362: ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP);
1.2 ad 363: #endif
364: } else {
365: aprint_naive(": %s Processor\n",
366: caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
367: ci = &cpu_info_primary;
1.101 kiyohara 368: #if NLAPIC > 0
1.2 ad 369: if (cpunum != lapic_cpu_number()) {
1.51 ad 370: /* XXX should be done earlier. */
1.39 ad 371: uint32_t reg;
372: aprint_verbose("\n");
1.47 ad 373: aprint_verbose_dev(self, "running CPU at apic %d"
374: " instead of at expected %d", lapic_cpu_number(),
1.23 cube 375: cpunum);
1.125 nonaka 376: reg = lapic_readreg(LAPIC_ID);
377: lapic_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) |
1.39 ad 378: (cpunum << LAPIC_ID_SHIFT));
1.2 ad 379: }
1.47 ad 380: if (cpunum != lapic_cpu_number()) {
381: aprint_error_dev(self, "unable to reset apic id\n");
382: }
1.101 kiyohara 383: #endif
1.2 ad 384: }
385:
386: ci->ci_self = ci;
387: sc->sc_info = ci;
388: ci->ci_dev = self;
1.74 jruoho 389: ci->ci_acpiid = caa->cpu_id;
1.42 ad 390: ci->ci_cpuid = caa->cpu_number;
1.2 ad 391: ci->ci_func = caa->cpu_func;
1.112 msaitoh 392: aprint_normal("\n");
1.2 ad 393:
1.55 ad 394: /* Must be before mi_cpu_attach(). */
395: cpu_vm_init(ci);
396:
1.2 ad 397: if (caa->cpu_role == CPU_ROLE_AP) {
398: int error;
399:
400: error = mi_cpu_attach(ci);
401: if (error != 0) {
1.47 ad 402: aprint_error_dev(self,
1.30 cegger 403: "mi_cpu_attach failed with %d\n", error);
1.2 ad 404: return;
405: }
1.130.2.4 martin 406: #ifdef __HAVE_PCPU_AREA
407: cpu_pcpuarea_init(ci);
408: #endif
1.15 yamt 409: cpu_init_tss(ci);
1.2 ad 410: } else {
411: KASSERT(ci->ci_data.cpu_idlelwp != NULL);
412: }
413:
1.130.2.5 martin 414: #ifdef SVS
415: cpu_svs_init(ci);
416: #endif
417:
1.2 ad 418: pmap_reference(pmap_kernel());
419: ci->ci_pmap = pmap_kernel();
420: ci->ci_tlbstate = TLBSTATE_STALE;
421:
1.51 ad 422: /*
423: * Boot processor may not be attached first, but the below
424: * must be done to allow booting other processors.
425: */
426: if (!again) {
427: atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);
428: /* Basic init. */
1.2 ad 429: cpu_intr_init(ci);
1.40 ad 430: cpu_get_tsc_freq(ci);
1.2 ad 431: cpu_init(ci);
1.130.2.3 martin 432: #ifdef i386
1.2 ad 433: cpu_set_tss_gates(ci);
1.130.2.3 martin 434: #endif
1.2 ad 435: pmap_cpu_init_late(ci);
1.101 kiyohara 436: #if NLAPIC > 0
1.51 ad 437: if (caa->cpu_role != CPU_ROLE_SP) {
438: /* Enable lapic. */
439: lapic_enable();
440: lapic_set_lvt();
441: lapic_calibrate_timer(ci);
442: }
1.101 kiyohara 443: #endif
1.51 ad 444: /* Make sure DELAY() is initialized. */
445: DELAY(1);
446: again = true;
447: }
448:
449: /* further PCB init done later. */
450:
451: switch (caa->cpu_role) {
452: case CPU_ROLE_SP:
453: atomic_or_32(&ci->ci_flags, CPUF_SP);
454: cpu_identify(ci);
1.53 ad 455: x86_errata();
1.37 joerg 456: x86_cpu_idle_init();
1.2 ad 457: break;
458:
459: case CPU_ROLE_BP:
1.51 ad 460: atomic_or_32(&ci->ci_flags, CPUF_BSP);
1.40 ad 461: cpu_identify(ci);
1.53 ad 462: x86_errata();
1.37 joerg 463: x86_cpu_idle_init();
1.2 ad 464: break;
465:
1.101 kiyohara 466: #ifdef MULTIPROCESSOR
1.2 ad 467: case CPU_ROLE_AP:
468: /*
469: * report on an AP
470: */
471: cpu_intr_init(ci);
472: gdt_alloc_cpu(ci);
1.130.2.3 martin 473: #ifdef i386
1.2 ad 474: cpu_set_tss_gates(ci);
1.130.2.3 martin 475: #endif
1.2 ad 476: pmap_cpu_init_late(ci);
477: cpu_start_secondary(ci);
478: if (ci->ci_flags & CPUF_PRESENT) {
1.59 cegger 479: struct cpu_info *tmp;
480:
1.40 ad 481: cpu_identify(ci);
1.59 cegger 482: tmp = cpu_info_list;
483: while (tmp->ci_next)
484: tmp = tmp->ci_next;
485:
486: tmp->ci_next = ci;
1.2 ad 487: }
488: break;
1.101 kiyohara 489: #endif
1.2 ad 490:
491: default:
492: panic("unknown processor type??\n");
493: }
1.51 ad 494:
1.71 cegger 495: pat_init(ci);
1.2 ad 496:
1.79 jruoho 497: if (!pmf_device_register1(self, cpu_suspend, cpu_resume, cpu_shutdown))
1.12 jmcneill 498: aprint_error_dev(self, "couldn't establish power handler\n");
499:
1.101 kiyohara 500: #ifdef MULTIPROCESSOR
1.2 ad 501: if (mp_verbose) {
502: struct lwp *l = ci->ci_data.cpu_idlelwp;
1.65 rmind 503: struct pcb *pcb = lwp_getpcb(l);
1.2 ad 504:
1.47 ad 505: aprint_verbose_dev(self,
1.28 cegger 506: "idle lwp at %p, idle sp at %p\n",
507: l,
1.2 ad 508: #ifdef i386
1.65 rmind 509: (void *)pcb->pcb_esp
1.2 ad 510: #else
1.65 rmind 511: (void *)pcb->pcb_rsp
1.2 ad 512: #endif
513: );
514: }
1.101 kiyohara 515: #endif
1.81 jmcneill 516:
1.89 jruoho 517: /*
518: * Postpone the "cpufeaturebus" scan.
519: * It is safe to scan the pseudo-bus
520: * only after all CPUs have attached.
521: */
1.87 jruoho 522: (void)config_defer(self, cpu_defer);
523: }
524:
525: static void
526: cpu_defer(device_t self)
527: {
1.81 jmcneill 528: cpu_rescan(self, NULL, NULL);
529: }
530:
1.87 jruoho 531: static int
1.81 jmcneill 532: cpu_rescan(device_t self, const char *ifattr, const int *locators)
533: {
1.83 jruoho 534: struct cpu_softc *sc = device_private(self);
1.81 jmcneill 535: struct cpufeature_attach_args cfaa;
536: struct cpu_info *ci = sc->sc_info;
537:
538: memset(&cfaa, 0, sizeof(cfaa));
539: cfaa.ci = ci;
540:
541: if (ifattr_match(ifattr, "cpufeaturebus")) {
1.83 jruoho 542: if (ci->ci_frequency == NULL) {
1.86 jruoho 543: cfaa.name = "frequency";
1.84 jruoho 544: ci->ci_frequency = config_found_ia(self,
545: "cpufeaturebus", &cfaa, NULL);
546: }
547:
1.81 jmcneill 548: if (ci->ci_padlock == NULL) {
549: cfaa.name = "padlock";
550: ci->ci_padlock = config_found_ia(self,
551: "cpufeaturebus", &cfaa, NULL);
552: }
1.82 jruoho 553:
1.86 jruoho 554: if (ci->ci_temperature == NULL) {
555: cfaa.name = "temperature";
556: ci->ci_temperature = config_found_ia(self,
1.85 jruoho 557: "cpufeaturebus", &cfaa, NULL);
558: }
1.95 jmcneill 559:
560: if (ci->ci_vm == NULL) {
561: cfaa.name = "vm";
562: ci->ci_vm = config_found_ia(self,
563: "cpufeaturebus", &cfaa, NULL);
564: }
1.81 jmcneill 565: }
566:
567: return 0;
568: }
569:
1.87 jruoho 570: static void
1.81 jmcneill 571: cpu_childdetached(device_t self, device_t child)
572: {
573: struct cpu_softc *sc = device_private(self);
574: struct cpu_info *ci = sc->sc_info;
575:
1.83 jruoho 576: if (ci->ci_frequency == child)
577: ci->ci_frequency = NULL;
1.82 jruoho 578:
1.81 jmcneill 579: if (ci->ci_padlock == child)
580: ci->ci_padlock = NULL;
1.83 jruoho 581:
1.86 jruoho 582: if (ci->ci_temperature == child)
583: ci->ci_temperature = NULL;
1.95 jmcneill 584:
585: if (ci->ci_vm == child)
586: ci->ci_vm = NULL;
1.2 ad 587: }
588:
589: /*
590: * Initialize the processor appropriately.
591: */
592:
593: void
1.9 ad 594: cpu_init(struct cpu_info *ci)
1.2 ad 595: {
1.113 christos 596: uint32_t cr4 = 0;
1.2 ad 597:
598: lcr0(rcr0() | CR0_WP);
599:
600: /*
601: * On a P6 or above, enable global TLB caching if the
602: * hardware supports it.
603: */
1.70 jym 604: if (cpu_feature[0] & CPUID_PGE)
1.130.2.5 martin 605: #ifdef SVS
606: if (!svs_enabled)
607: #endif
1.110 dsl 608: cr4 |= CR4_PGE; /* enable global TLB caching */
1.2 ad 609:
610: /*
611: * If we have FXSAVE/FXRESTOR, use them.
612: */
1.70 jym 613: if (cpu_feature[0] & CPUID_FXSR) {
1.110 dsl 614: cr4 |= CR4_OSFXSR;
1.2 ad 615:
616: /*
617: * If we have SSE/SSE2, enable XMM exceptions.
618: */
1.70 jym 619: if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2))
1.110 dsl 620: cr4 |= CR4_OSXMMEXCPT;
1.2 ad 621: }
622:
1.110 dsl 623: /* If xsave is supported, enable it */
624: if (cpu_feature[1] & CPUID2_XSAVE)
625: cr4 |= CR4_OSXSAVE;
626:
1.118 maxv 627: /* If SMEP is supported, enable it */
628: if (cpu_feature[5] & CPUID_SEF_SMEP)
629: cr4 |= CR4_SMEP;
630:
1.130.2.6 martin 631: #ifdef amd64
632: /* If SMAP is supported, enable it */
633: if (cpu_feature[5] & CPUID_SEF_SMAP)
634: cr4 |= CR4_SMAP;
635: #endif
636:
1.113 christos 637: if (cr4) {
638: cr4 |= rcr4();
639: lcr4(cr4);
640: }
1.110 dsl 641:
642: /* If xsave is enabled, enable all fpu features */
643: if (cr4 & CR4_OSXSAVE)
644: wrxcr(0, x86_xsave_features & XCR0_FPU);
645:
1.2 ad 646: #ifdef MTRR
647: /*
648: * On a P6 or above, initialize MTRR's if the hardware supports them.
649: */
1.70 jym 650: if (cpu_feature[0] & CPUID_MTRR) {
1.2 ad 651: if ((ci->ci_flags & CPUF_AP) == 0)
652: i686_mtrr_init_first();
653: mtrr_init_cpu(ci);
654: }
655:
656: #ifdef i386
657: if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) {
658: /*
659: * Must be a K6-2 Step >= 7 or a K6-III.
660: */
1.106 msaitoh 661: if (CPUID_TO_FAMILY(ci->ci_signature) == 5) {
662: if (CPUID_TO_MODEL(ci->ci_signature) > 8 ||
663: (CPUID_TO_MODEL(ci->ci_signature) == 8 &&
664: CPUID_TO_STEPPING(ci->ci_signature) >= 7)) {
1.2 ad 665: mtrr_funcs = &k6_mtrr_funcs;
666: k6_mtrr_init_first();
667: mtrr_init_cpu(ci);
668: }
669: }
670: }
671: #endif /* i386 */
672: #endif /* MTRR */
673:
1.38 ad 674: if (ci != &cpu_info_primary) {
675: /* Synchronize TSC again, and check for drift. */
676: wbinvd();
677: atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
678: tsc_sync_ap(ci);
679: } else {
680: atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
681: }
1.2 ad 682: }
683:
1.101 kiyohara 684: #ifdef MULTIPROCESSOR
1.2 ad 685: void
1.12 jmcneill 686: cpu_boot_secondary_processors(void)
1.2 ad 687: {
688: struct cpu_info *ci;
1.100 chs 689: kcpuset_t *cpus;
1.2 ad 690: u_long i;
691:
1.5 ad 692: /* Now that we know the number of CPUs, patch the text segment. */
1.60 ad 693: x86_patch(false);
1.5 ad 694:
1.100 chs 695: kcpuset_create(&cpus, true);
696: kcpuset_set(cpus, cpu_index(curcpu()));
697: for (i = 0; i < maxcpus; i++) {
1.57 ad 698: ci = cpu_lookup(i);
1.2 ad 699: if (ci == NULL)
700: continue;
701: if (ci->ci_data.cpu_idlelwp == NULL)
702: continue;
703: if ((ci->ci_flags & CPUF_PRESENT) == 0)
704: continue;
705: if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))
706: continue;
707: cpu_boot_secondary(ci);
1.100 chs 708: kcpuset_set(cpus, cpu_index(ci));
1.2 ad 709: }
1.100 chs 710: while (!kcpuset_match(cpus, kcpuset_running))
711: ;
712: kcpuset_destroy(cpus);
1.2 ad 713:
714: x86_mp_online = true;
1.38 ad 715:
716: /* Now that we know about the TSC, attach the timecounter. */
717: tsc_tc_init();
1.55 ad 718:
719: /* Enable zeroing of pages in the idle loop if we have SSE2. */
1.70 jym 720: vm_page_zero_enable = ((cpu_feature[0] & CPUID_SSE2) != 0);
1.2 ad 721: }
1.101 kiyohara 722: #endif
1.2 ad 723:
724: static void
725: cpu_init_idle_lwp(struct cpu_info *ci)
726: {
727: struct lwp *l = ci->ci_data.cpu_idlelwp;
1.65 rmind 728: struct pcb *pcb = lwp_getpcb(l);
1.2 ad 729:
730: pcb->pcb_cr0 = rcr0();
731: }
732:
733: void
1.12 jmcneill 734: cpu_init_idle_lwps(void)
1.2 ad 735: {
736: struct cpu_info *ci;
737: u_long i;
738:
1.54 ad 739: for (i = 0; i < maxcpus; i++) {
1.57 ad 740: ci = cpu_lookup(i);
1.2 ad 741: if (ci == NULL)
742: continue;
743: if (ci->ci_data.cpu_idlelwp == NULL)
744: continue;
745: if ((ci->ci_flags & CPUF_PRESENT) == 0)
746: continue;
747: cpu_init_idle_lwp(ci);
748: }
749: }
750:
1.101 kiyohara 751: #ifdef MULTIPROCESSOR
1.2 ad 752: void
1.12 jmcneill 753: cpu_start_secondary(struct cpu_info *ci)
1.2 ad 754: {
1.38 ad 755: extern paddr_t mp_pdirpa;
756: u_long psl;
1.2 ad 757: int i;
758:
1.12 jmcneill 759: mp_pdirpa = pmap_init_tmp_pgtbl(mp_trampoline_paddr);
1.9 ad 760: atomic_or_32(&ci->ci_flags, CPUF_AP);
1.2 ad 761: ci->ci_curlwp = ci->ci_data.cpu_idlelwp;
1.45 ad 762: if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) {
1.25 ad 763: return;
1.45 ad 764: }
1.2 ad 765:
766: /*
1.50 ad 767: * Wait for it to become ready. Setting cpu_starting opens the
768: * initial gate and allows the AP to start soft initialization.
1.2 ad 769: */
1.50 ad 770: KASSERT(cpu_starting == NULL);
771: cpu_starting = ci;
1.26 cegger 772: for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) {
1.24 ad 773: #ifdef MPDEBUG
774: extern int cpu_trace[3];
775: static int otrace[3];
776: if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) {
1.26 cegger 777: aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n",
778: cpu_trace[0], cpu_trace[1], cpu_trace[2]);
1.24 ad 779: memcpy(otrace, cpu_trace, sizeof(otrace));
780: }
781: #endif
1.11 ad 782: i8254_delay(10);
1.2 ad 783: }
1.38 ad 784:
1.9 ad 785: if ((ci->ci_flags & CPUF_PRESENT) == 0) {
1.26 cegger 786: aprint_error_dev(ci->ci_dev, "failed to become ready\n");
1.2 ad 787: #if defined(MPDEBUG) && defined(DDB)
788: printf("dropping into debugger; continue from here to resume boot\n");
789: Debugger();
790: #endif
1.38 ad 791: } else {
792: /*
1.68 jym 793: * Synchronize time stamp counters. Invalidate cache and do
794: * twice to try and minimize possible cache effects. Disable
795: * interrupts to try and rule out any external interference.
1.38 ad 796: */
797: psl = x86_read_psl();
798: x86_disable_intr();
799: wbinvd();
800: tsc_sync_bp(ci);
801: x86_write_psl(psl);
1.2 ad 802: }
803:
804: CPU_START_CLEANUP(ci);
1.45 ad 805: cpu_starting = NULL;
1.2 ad 806: }
807:
808: void
1.12 jmcneill 809: cpu_boot_secondary(struct cpu_info *ci)
1.2 ad 810: {
1.38 ad 811: int64_t drift;
812: u_long psl;
1.2 ad 813: int i;
814:
1.9 ad 815: atomic_or_32(&ci->ci_flags, CPUF_GO);
1.26 cegger 816: for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) {
1.11 ad 817: i8254_delay(10);
1.2 ad 818: }
1.9 ad 819: if ((ci->ci_flags & CPUF_RUNNING) == 0) {
1.26 cegger 820: aprint_error_dev(ci->ci_dev, "failed to start\n");
1.2 ad 821: #if defined(MPDEBUG) && defined(DDB)
822: printf("dropping into debugger; continue from here to resume boot\n");
823: Debugger();
824: #endif
1.38 ad 825: } else {
826: /* Synchronize TSC again, check for drift. */
827: drift = ci->ci_data.cpu_cc_skew;
828: psl = x86_read_psl();
829: x86_disable_intr();
830: wbinvd();
831: tsc_sync_bp(ci);
832: x86_write_psl(psl);
833: drift -= ci->ci_data.cpu_cc_skew;
834: aprint_debug_dev(ci->ci_dev, "TSC skew=%lld drift=%lld\n",
835: (long long)ci->ci_data.cpu_cc_skew, (long long)drift);
836: tsc_sync_drift(drift);
1.2 ad 837: }
838: }
839:
840: /*
1.117 maxv 841: * The CPU ends up here when it's ready to run.
1.2 ad 842: * This is called from code in mptramp.s; at this point, we are running
843: * in the idle pcb/idle stack of the new CPU. When this function returns,
844: * this processor will enter the idle loop and start looking for work.
845: */
846: void
847: cpu_hatch(void *v)
848: {
849: struct cpu_info *ci = (struct cpu_info *)v;
1.65 rmind 850: struct pcb *pcb;
1.130 kre 851: int s, i;
1.2 ad 852:
1.12 jmcneill 853: cpu_init_msrs(ci, true);
1.40 ad 854: cpu_probe(ci);
1.46 ad 855:
856: ci->ci_data.cpu_cc_freq = cpu_info_primary.ci_data.cpu_cc_freq;
1.130.2.3 martin 857: /* cpu_get_tsc_freq(ci); */
1.38 ad 858:
1.8 ad 859: KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0);
1.38 ad 860:
861: /*
862: * Synchronize time stamp counters. Invalidate cache and do twice
863: * to try and minimize possible cache effects. Note that interrupts
864: * are off at this point.
865: */
866: wbinvd();
1.9 ad 867: atomic_or_32(&ci->ci_flags, CPUF_PRESENT);
1.38 ad 868: tsc_sync_ap(ci);
869:
870: /*
871: * Wait to be brought online. Use 'monitor/mwait' if available,
872: * in order to make the TSC drift as much as possible. so that
1.130.2.3 martin 873: * we can detect it later. If not available, try 'pause'.
1.38 ad 874: * We'd like to use 'hlt', but we have interrupts off.
875: */
1.6 ad 876: while ((ci->ci_flags & CPUF_GO) == 0) {
1.70 jym 877: if ((cpu_feature[1] & CPUID2_MONITOR) != 0) {
1.38 ad 878: x86_monitor(&ci->ci_flags, 0, 0);
879: if ((ci->ci_flags & CPUF_GO) != 0) {
880: continue;
881: }
882: x86_mwait(0, 0);
883: } else {
1.130.2.1 snj 884: /*
885: * XXX The loop repetition count could be a lot higher, but
886: * XXX currently qemu emulator takes a _very_long_time_ to
887: * XXX execute the pause instruction. So for now, use a low
888: * XXX value to allow the cpu to hatch before timing out.
889: */
890: for (i = 50; i != 0; i--) {
1.127 pgoyette 891: x86_pause();
892: }
1.38 ad 893: }
1.6 ad 894: }
1.5 ad 895:
1.26 cegger 896: /* Because the text may have been patched in x86_patch(). */
1.5 ad 897: wbinvd();
898: x86_flush();
1.88 rmind 899: tlbflushg();
1.5 ad 900:
1.8 ad 901: KASSERT((ci->ci_flags & CPUF_RUNNING) == 0);
1.2 ad 902:
1.73 jym 903: #ifdef PAE
904: pd_entry_t * l3_pd = ci->ci_pae_l3_pdir;
905: for (i = 0 ; i < PDP_SIZE; i++) {
906: l3_pd[i] = pmap_kernel()->pm_pdirpa[i] | PG_V;
907: }
908: lcr3(ci->ci_pae_l3_pdirpa);
909: #else
910: lcr3(pmap_pdirpa(pmap_kernel(), 0));
911: #endif
912:
1.65 rmind 913: pcb = lwp_getpcb(curlwp);
1.73 jym 914: pcb->pcb_cr3 = rcr3();
1.65 rmind 915: pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp);
916: lcr0(pcb->pcb_cr0);
917:
1.2 ad 918: cpu_init_idt();
1.8 ad 919: gdt_init_cpu(ci);
1.111 joerg 920: #if NLAPIC > 0
1.8 ad 921: lapic_enable();
1.2 ad 922: lapic_set_lvt();
1.8 ad 923: lapic_initclocks();
1.111 joerg 924: #endif
1.2 ad 925:
926: fpuinit(ci);
927: lldt(GSYSSEL(GLDT_SEL, SEL_KPL));
1.15 yamt 928: ltr(ci->ci_tss_sel);
1.2 ad 929:
930: cpu_init(ci);
1.7 ad 931: cpu_get_tsc_freq(ci);
1.2 ad 932:
933: s = splhigh();
1.124 nonaka 934: lapic_write_tpri(0);
1.3 ad 935: x86_enable_intr();
1.2 ad 936: splx(s);
1.6 ad 937: x86_errata();
1.2 ad 938:
1.42 ad 939: aprint_debug_dev(ci->ci_dev, "running\n");
1.98 rmind 940:
941: idle_loop(NULL);
942: KASSERT(false);
1.2 ad 943: }
1.101 kiyohara 944: #endif
1.2 ad 945:
946: #if defined(DDB)
947:
948: #include <ddb/db_output.h>
949: #include <machine/db_machdep.h>
950:
951: /*
952: * Dump CPU information from ddb.
953: */
954: void
955: cpu_debug_dump(void)
956: {
957: struct cpu_info *ci;
958: CPU_INFO_ITERATOR cii;
959:
1.107 christos 960: db_printf("addr dev id flags ipis curlwp fpcurlwp\n");
1.2 ad 961: for (CPU_INFO_FOREACH(cii, ci)) {
1.107 christos 962: db_printf("%p %s %ld %x %x %10p %10p\n",
1.2 ad 963: ci,
1.27 cegger 964: ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),
1.2 ad 965: (long)ci->ci_cpuid,
966: ci->ci_flags, ci->ci_ipis,
1.107 christos 967: ci->ci_curlwp,
968: ci->ci_fpcurlwp);
1.2 ad 969: }
970: }
971: #endif
972:
1.101 kiyohara 973: #if NLAPIC > 0
1.2 ad 974: static void
1.12 jmcneill 975: cpu_copy_trampoline(void)
1.2 ad 976: {
977: /*
978: * Copy boot code.
979: */
980: extern u_char cpu_spinup_trampoline[];
981: extern u_char cpu_spinup_trampoline_end[];
1.130.2.3 martin 982:
1.12 jmcneill 983: vaddr_t mp_trampoline_vaddr;
984:
985: mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
986: UVM_KMF_VAONLY);
987:
988: pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr,
1.64 cegger 989: VM_PROT_READ | VM_PROT_WRITE, 0);
1.2 ad 990: pmap_update(pmap_kernel());
1.12 jmcneill 991: memcpy((void *)mp_trampoline_vaddr,
1.2 ad 992: cpu_spinup_trampoline,
1.26 cegger 993: cpu_spinup_trampoline_end - cpu_spinup_trampoline);
1.12 jmcneill 994:
995: pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE);
996: pmap_update(pmap_kernel());
997: uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY);
1.2 ad 998: }
1.101 kiyohara 999: #endif
1.2 ad 1000:
1.101 kiyohara 1001: #ifdef MULTIPROCESSOR
1.2 ad 1002: int
1.14 joerg 1003: mp_cpu_start(struct cpu_info *ci, paddr_t target)
1.2 ad 1004: {
1.44 ad 1005: unsigned short dwordptr[2];
1.2 ad 1006: int error;
1.14 joerg 1007:
1008: /*
1009: * Bootstrap code must be addressable in real mode
1010: * and it must be page aligned.
1011: */
1012: KASSERT(target < 0x10000 && target % PAGE_SIZE == 0);
1.2 ad 1013:
1014: /*
1015: * "The BSP must initialize CMOS shutdown code to 0Ah ..."
1016: */
1017:
1018: outb(IO_RTC, NVRAM_RESET);
1019: outb(IO_RTC+1, NVRAM_RESET_JUMP);
1020:
1021: /*
1022: * "and the warm reset vector (DWORD based at 40:67) to point
1023: * to the AP startup code ..."
1024: */
1025:
1026: dwordptr[0] = 0;
1.14 joerg 1027: dwordptr[1] = target >> 4;
1.2 ad 1028:
1.111 joerg 1029: #if NLAPIC > 0
1.25 ad 1030: memcpy((uint8_t *)cmos_data_mapping + 0x467, dwordptr, 4);
1.111 joerg 1031: #endif
1.2 ad 1032:
1.70 jym 1033: if ((cpu_feature[0] & CPUID_APIC) == 0) {
1.25 ad 1034: aprint_error("mp_cpu_start: CPU does not have APIC\n");
1035: return ENODEV;
1036: }
1037:
1.2 ad 1038: /*
1.51 ad 1039: * ... prior to executing the following sequence:". We'll also add in
1040: * local cache flush, in case the BIOS has left the AP with its cache
1041: * disabled. It may not be able to cope with MP coherency.
1.2 ad 1042: */
1.51 ad 1043: wbinvd();
1.2 ad 1044:
1045: if (ci->ci_flags & CPUF_AP) {
1.42 ad 1046: error = x86_ipi_init(ci->ci_cpuid);
1.26 cegger 1047: if (error != 0) {
1048: aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n",
1.50 ad 1049: __func__);
1.2 ad 1050: return error;
1.25 ad 1051: }
1.11 ad 1052: i8254_delay(10000);
1.2 ad 1053:
1.50 ad 1054: error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE);
1.26 cegger 1055: if (error != 0) {
1056: aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n",
1.50 ad 1057: __func__);
1.25 ad 1058: return error;
1059: }
1060: i8254_delay(200);
1.2 ad 1061:
1.50 ad 1062: error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE);
1.26 cegger 1063: if (error != 0) {
1064: aprint_error_dev(ci->ci_dev, "%s: IPI not taken (3)\n",
1.50 ad 1065: __func__);
1.25 ad 1066: return error;
1.2 ad 1067: }
1.25 ad 1068: i8254_delay(200);
1.2 ad 1069: }
1.44 ad 1070:
1.2 ad 1071: return 0;
1072: }
1073:
1074: void
1075: mp_cpu_start_cleanup(struct cpu_info *ci)
1076: {
1077: /*
1078: * Ensure the NVRAM reset byte contains something vaguely sane.
1079: */
1080:
1081: outb(IO_RTC, NVRAM_RESET);
1082: outb(IO_RTC+1, NVRAM_RESET_RST);
1083: }
1.101 kiyohara 1084: #endif
1.2 ad 1085:
1086: #ifdef __x86_64__
1087: typedef void (vector)(void);
1.130.2.5 martin 1088: extern vector Xsyscall, Xsyscall32, Xsyscall_svs;
1.70 jym 1089: #endif
1.2 ad 1090:
1091: void
1.12 jmcneill 1092: cpu_init_msrs(struct cpu_info *ci, bool full)
1.2 ad 1093: {
1.70 jym 1094: #ifdef __x86_64__
1.2 ad 1095: wrmsr(MSR_STAR,
1096: ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
1097: ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48));
1098: wrmsr(MSR_LSTAR, (uint64_t)Xsyscall);
1099: wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32);
1.130.2.6 martin 1100: wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D|PSL_AC);
1.2 ad 1101:
1.130.2.5 martin 1102: #ifdef SVS
1103: if (svs_enabled)
1104: wrmsr(MSR_LSTAR, (uint64_t)Xsyscall_svs);
1105: #endif
1106:
1.12 jmcneill 1107: if (full) {
1108: wrmsr(MSR_FSBASE, 0);
1.27 cegger 1109: wrmsr(MSR_GSBASE, (uint64_t)ci);
1.12 jmcneill 1110: wrmsr(MSR_KERNELGSBASE, 0);
1111: }
1.70 jym 1112: #endif /* __x86_64__ */
1.2 ad 1113:
1.70 jym 1114: if (cpu_feature[2] & CPUID_NOX)
1.2 ad 1115: wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE);
1116: }
1.7 ad 1117:
1.107 christos 1118: void
1119: cpu_offline_md(void)
1120: {
1121: int s;
1122:
1123: s = splhigh();
1124: fpusave_cpu(true);
1125: splx(s);
1126: }
1127:
1.12 jmcneill 1128: /* XXX joerg restructure and restart CPUs individually */
1129: static bool
1.96 jruoho 1130: cpu_stop(device_t dv)
1.12 jmcneill 1131: {
1132: struct cpu_softc *sc = device_private(dv);
1133: struct cpu_info *ci = sc->sc_info;
1.18 joerg 1134: int err;
1.12 jmcneill 1135:
1.96 jruoho 1136: KASSERT((ci->ci_flags & CPUF_PRESENT) != 0);
1.93 jruoho 1137:
1138: if ((ci->ci_flags & CPUF_PRIMARY) != 0)
1139: return true;
1140:
1.12 jmcneill 1141: if (ci->ci_data.cpu_idlelwp == NULL)
1142: return true;
1143:
1.20 jmcneill 1144: sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE);
1.17 joerg 1145:
1.20 jmcneill 1146: if (sc->sc_wasonline) {
1147: mutex_enter(&cpu_lock);
1.58 rmind 1148: err = cpu_setstate(ci, false);
1.20 jmcneill 1149: mutex_exit(&cpu_lock);
1.79 jruoho 1150:
1.93 jruoho 1151: if (err != 0)
1.20 jmcneill 1152: return false;
1153: }
1.17 joerg 1154:
1155: return true;
1.12 jmcneill 1156: }
1157:
1158: static bool
1.96 jruoho 1159: cpu_suspend(device_t dv, const pmf_qual_t *qual)
1160: {
1161: struct cpu_softc *sc = device_private(dv);
1162: struct cpu_info *ci = sc->sc_info;
1163:
1164: if ((ci->ci_flags & CPUF_PRESENT) == 0)
1165: return true;
1166: else {
1167: cpufreq_suspend(ci);
1168: }
1169:
1170: return cpu_stop(dv);
1171: }
1172:
1173: static bool
1.69 dyoung 1174: cpu_resume(device_t dv, const pmf_qual_t *qual)
1.12 jmcneill 1175: {
1176: struct cpu_softc *sc = device_private(dv);
1177: struct cpu_info *ci = sc->sc_info;
1.20 jmcneill 1178: int err = 0;
1.12 jmcneill 1179:
1.93 jruoho 1180: if ((ci->ci_flags & CPUF_PRESENT) == 0)
1.12 jmcneill 1181: return true;
1.93 jruoho 1182:
1183: if ((ci->ci_flags & CPUF_PRIMARY) != 0)
1184: goto out;
1185:
1.12 jmcneill 1186: if (ci->ci_data.cpu_idlelwp == NULL)
1.93 jruoho 1187: goto out;
1.12 jmcneill 1188:
1.20 jmcneill 1189: if (sc->sc_wasonline) {
1190: mutex_enter(&cpu_lock);
1.58 rmind 1191: err = cpu_setstate(ci, true);
1.20 jmcneill 1192: mutex_exit(&cpu_lock);
1193: }
1.13 joerg 1194:
1.93 jruoho 1195: out:
1196: if (err != 0)
1197: return false;
1198:
1199: cpufreq_resume(ci);
1200:
1201: return true;
1.12 jmcneill 1202: }
1203:
1.79 jruoho 1204: static bool
1205: cpu_shutdown(device_t dv, int how)
1206: {
1.90 dyoung 1207: struct cpu_softc *sc = device_private(dv);
1208: struct cpu_info *ci = sc->sc_info;
1209:
1.96 jruoho 1210: if ((ci->ci_flags & CPUF_BSP) != 0)
1.90 dyoung 1211: return false;
1212:
1.96 jruoho 1213: if ((ci->ci_flags & CPUF_PRESENT) == 0)
1214: return true;
1215:
1216: return cpu_stop(dv);
1.79 jruoho 1217: }
1218:
1.7 ad 1219: void
1220: cpu_get_tsc_freq(struct cpu_info *ci)
1221: {
1222: uint64_t last_tsc;
1223:
1.70 jym 1224: if (cpu_hascounter()) {
1.80 bouyer 1225: last_tsc = cpu_counter_serializing();
1.7 ad 1226: i8254_delay(100000);
1.80 bouyer 1227: ci->ci_data.cpu_cc_freq =
1228: (cpu_counter_serializing() - last_tsc) * 10;
1.7 ad 1229: }
1230: }
1.37 joerg 1231:
1232: void
1233: x86_cpu_idle_mwait(void)
1234: {
1235: struct cpu_info *ci = curcpu();
1236:
1237: KASSERT(ci->ci_ilevel == IPL_NONE);
1238:
1239: x86_monitor(&ci->ci_want_resched, 0, 0);
1240: if (__predict_false(ci->ci_want_resched)) {
1241: return;
1242: }
1243: x86_mwait(0, 0);
1244: }
1245:
1246: void
1247: x86_cpu_idle_halt(void)
1248: {
1249: struct cpu_info *ci = curcpu();
1250:
1251: KASSERT(ci->ci_ilevel == IPL_NONE);
1252:
1253: x86_disable_intr();
1254: if (!__predict_false(ci->ci_want_resched)) {
1255: x86_stihlt();
1256: } else {
1257: x86_enable_intr();
1258: }
1259: }
1.73 jym 1260:
1261: /*
1262: * Loads pmap for the current CPU.
1263: */
1264: void
1.97 bouyer 1265: cpu_load_pmap(struct pmap *pmap, struct pmap *oldpmap)
1.73 jym 1266: {
1.130.2.5 martin 1267: #ifdef SVS
1.130.2.7! martin 1268: if (svs_enabled) {
! 1269: svs_pdir_switch(pmap);
! 1270: }
1.130.2.5 martin 1271: #endif
1272:
1.73 jym 1273: #ifdef PAE
1.99 yamt 1274: struct cpu_info *ci = curcpu();
1.116 nat 1275: bool interrupts_enabled;
1.99 yamt 1276: pd_entry_t *l3_pd = ci->ci_pae_l3_pdir;
1277: int i;
1.73 jym 1278:
1.99 yamt 1279: /*
1280: * disable interrupts to block TLB shootdowns, which can reload cr3.
1281: * while this doesn't block NMIs, it's probably ok as NMIs unlikely
1282: * reload cr3.
1283: */
1.116 nat 1284: interrupts_enabled = (x86_read_flags() & PSL_I) != 0;
1285: if (interrupts_enabled)
1286: x86_disable_intr();
1287:
1.73 jym 1288: for (i = 0 ; i < PDP_SIZE; i++) {
1289: l3_pd[i] = pmap->pm_pdirpa[i] | PG_V;
1290: }
1.130.2.3 martin 1291:
1.116 nat 1292: if (interrupts_enabled)
1293: x86_enable_intr();
1.73 jym 1294: tlbflush();
1295: #else /* PAE */
1296: lcr3(pmap_pdirpa(pmap, 0));
1297: #endif /* PAE */
1298: }
1.91 cherry 1299:
1300: /*
1301: * Notify all other cpus to halt.
1302: */
1303:
1304: void
1.92 cherry 1305: cpu_broadcast_halt(void)
1.91 cherry 1306: {
1307: x86_broadcast_ipi(X86_IPI_HALT);
1308: }
1309:
1310: /*
1311: * Send a dummy ipi to a cpu to force it to run splraise()/spllower()
1312: */
1313:
1314: void
1315: cpu_kick(struct cpu_info *ci)
1316: {
1317: x86_send_ipi(ci, 0);
1318: }
CVSweb <webmaster@jp.NetBSD.org>