Annotation of src/sys/arch/x86/x86/cpu.c, Revision 1.12.2.2
1.12.2.1 bouyer 1: /* $NetBSD$ */
1.2 ad 2:
3: /*-
1.7 ad 4: * Copyright (c) 2000, 2006, 2007 The NetBSD Foundation, Inc.
1.2 ad 5: * All rights reserved.
6: *
7: * This code is derived from software contributed to The NetBSD Foundation
1.11 ad 8: * by Bill Sommerfeld of RedBack Networks Inc, and by Andrew Doran.
1.2 ad 9: *
10: * Redistribution and use in source and binary forms, with or without
11: * modification, are permitted provided that the following conditions
12: * are met:
13: * 1. Redistributions of source code must retain the above copyright
14: * notice, this list of conditions and the following disclaimer.
15: * 2. Redistributions in binary form must reproduce the above copyright
16: * notice, this list of conditions and the following disclaimer in the
17: * documentation and/or other materials provided with the distribution.
18: * 3. All advertising materials mentioning features or use of this software
19: * must display the following acknowledgement:
20: * This product includes software developed by the NetBSD
21: * Foundation, Inc. and its contributors.
22: * 4. Neither the name of The NetBSD Foundation nor the names of its
23: * contributors may be used to endorse or promote products derived
24: * from this software without specific prior written permission.
25: *
26: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
27: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
28: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
30: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
31: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
32: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
33: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
34: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
35: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36: * POSSIBILITY OF SUCH DAMAGE.
37: */
38:
39: /*
40: * Copyright (c) 1999 Stefan Grefen
41: *
42: * Redistribution and use in source and binary forms, with or without
43: * modification, are permitted provided that the following conditions
44: * are met:
45: * 1. Redistributions of source code must retain the above copyright
46: * notice, this list of conditions and the following disclaimer.
47: * 2. Redistributions in binary form must reproduce the above copyright
48: * notice, this list of conditions and the following disclaimer in the
49: * documentation and/or other materials provided with the distribution.
50: * 3. All advertising materials mentioning features or use of this software
51: * must display the following acknowledgement:
52: * This product includes software developed by the NetBSD
53: * Foundation, Inc. and its contributors.
54: * 4. Neither the name of The NetBSD Foundation nor the names of its
55: * contributors may be used to endorse or promote products derived
56: * from this software without specific prior written permission.
57: *
58: * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY
59: * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
60: * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
61: * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE
62: * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
63: * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
64: * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
65: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
66: * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
67: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
68: * SUCH DAMAGE.
69: */
70:
71: #include <sys/cdefs.h>
1.12.2.1 bouyer 72: __KERNEL_RCSID(0, "$NetBSD$");
1.2 ad 73:
74: #include "opt_ddb.h"
75: #include "opt_multiprocessor.h"
76: #include "opt_mpbios.h" /* for MPDEBUG */
77: #include "opt_mtrr.h"
78:
79: #include "lapic.h"
80: #include "ioapic.h"
81:
82: #include <sys/param.h>
83: #include <sys/proc.h>
84: #include <sys/user.h>
85: #include <sys/systm.h>
86: #include <sys/device.h>
87: #include <sys/malloc.h>
1.9 ad 88: #include <sys/cpu.h>
89: #include <sys/atomic.h>
1.2 ad 90:
91: #include <uvm/uvm_extern.h>
92:
93: #include <machine/cpufunc.h>
94: #include <machine/cpuvar.h>
95: #include <machine/pmap.h>
96: #include <machine/vmparam.h>
97: #include <machine/mpbiosvar.h>
98: #include <machine/pcb.h>
99: #include <machine/specialreg.h>
100: #include <machine/segments.h>
101: #include <machine/gdt.h>
102: #include <machine/mtrr.h>
103: #include <machine/pio.h>
104:
105: #ifdef i386
106: #include <machine/tlog.h>
107: #endif
108:
109: #if NLAPIC > 0
110: #include <machine/apicvar.h>
111: #include <machine/i82489reg.h>
112: #include <machine/i82489var.h>
113: #endif
114:
115: #if NIOAPIC > 0
116: #include <machine/i82093var.h>
117: #endif
118:
119: #include <dev/ic/mc146818reg.h>
120: #include <i386/isa/nvram.h>
121: #include <dev/isa/isareg.h>
122:
123: int cpu_match(struct device *, struct cfdata *, void *);
124: void cpu_attach(struct device *, struct device *, void *);
125:
1.12 jmcneill 126: static bool cpu_suspend(device_t);
127: static bool cpu_resume(device_t);
128:
1.2 ad 129: struct cpu_softc {
130: struct device sc_dev; /* device tree glue */
131: struct cpu_info *sc_info; /* pointer to CPU info */
132: };
133:
1.12.2.1 bouyer 134: int mp_cpu_start(struct cpu_info *, paddr_t);
1.2 ad 135: void mp_cpu_start_cleanup(struct cpu_info *);
136: const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL,
137: mp_cpu_start_cleanup };
138:
139:
140: CFATTACH_DECL(cpu, sizeof(struct cpu_softc),
141: cpu_match, cpu_attach, NULL, NULL);
142:
143: /*
144: * Statically-allocated CPU info for the primary CPU (or the only
145: * CPU, on uniprocessors). The CPU info list is initialized to
146: * point at it.
147: */
148: #ifdef TRAPLOG
149: struct tlog tlog_primary;
150: #endif
151: struct cpu_info cpu_info_primary = {
152: .ci_dev = 0,
153: .ci_self = &cpu_info_primary,
154: .ci_idepth = -1,
155: .ci_curlwp = &lwp0,
156: #ifdef TRAPLOG
157: .ci_tlog_base = &tlog_primary,
158: #endif /* !TRAPLOG */
159: };
160:
161: struct cpu_info *cpu_info_list = &cpu_info_primary;
162:
1.12 jmcneill 163: static void cpu_set_tss_gates(struct cpu_info *);
1.2 ad 164:
165: #ifdef i386
1.12.2.2! bouyer 166: static void tss_init(struct i386tss *, void *, void *);
1.2 ad 167: #endif
168:
1.12 jmcneill 169: #ifdef MULTIPROCESSOR
170: static void cpu_init_idle_lwp(struct cpu_info *);
171: #endif
172:
1.2 ad 173: uint32_t cpus_attached = 0;
1.9 ad 174: uint32_t cpus_running = 0;
1.2 ad 175:
176: extern char x86_64_doubleflt_stack[];
177:
1.12 jmcneill 178: bool x86_mp_online;
179: paddr_t mp_trampoline_paddr = MP_TRAMPOLINE;
180:
1.12.2.1 bouyer 181: static vaddr_t cmos_data_mapping;
182:
1.2 ad 183: #ifdef MULTIPROCESSOR
184: /*
185: * Array of CPU info structures. Must be statically-allocated because
186: * curproc, etc. are used early.
187: */
188: struct cpu_info *cpu_info[X86_MAXPROCS] = { &cpu_info_primary, };
189:
190: void cpu_hatch(void *);
191: static void cpu_boot_secondary(struct cpu_info *ci);
192: static void cpu_start_secondary(struct cpu_info *ci);
193: static void cpu_copy_trampoline(void);
194:
195: /*
196: * Runs once per boot once multiprocessor goo has been detected and
197: * the local APIC on the boot processor has been mapped.
198: *
199: * Called from lapic_boot_init() (from mpbios_scan()).
200: */
201: void
1.9 ad 202: cpu_init_first(void)
1.2 ad 203: {
204: int cpunum = lapic_cpu_number();
205:
206: if (cpunum != 0) {
207: cpu_info[0] = NULL;
208: cpu_info[cpunum] = &cpu_info_primary;
209: }
210:
211: cpu_info_primary.ci_cpuid = cpunum;
212: cpu_copy_trampoline();
1.12.2.1 bouyer 213:
214: cmos_data_mapping = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, UVM_KMF_VAONLY);
215: if (cmos_data_mapping == 0)
216: panic("No KVA for page 0");
217: pmap_kenter_pa(cmos_data_mapping, 0, VM_PROT_READ|VM_PROT_WRITE);
218: pmap_update(pmap_kernel());
1.2 ad 219: }
220: #endif
221:
222: int
223: cpu_match(struct device *parent, struct cfdata *match,
224: void *aux)
225: {
226:
227: return 1;
228: }
229:
230: static void
231: cpu_vm_init(struct cpu_info *ci)
232: {
233: int ncolors = 2, i;
234:
235: for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) {
236: struct x86_cache_info *cai;
237: int tcolors;
238:
239: cai = &ci->ci_cinfo[i];
240:
241: tcolors = atop(cai->cai_totalsize);
242: switch(cai->cai_associativity) {
243: case 0xff:
244: tcolors = 1; /* fully associative */
245: break;
246: case 0:
247: case 1:
248: break;
249: default:
250: tcolors /= cai->cai_associativity;
251: }
252: ncolors = max(ncolors, tcolors);
253: }
254:
255: /*
256: * Knowing the size of the largest cache on this CPU, re-color
257: * our pages.
258: */
259: if (ncolors <= uvmexp.ncolors)
260: return;
261: aprint_verbose("%s: %d page colors\n", ci->ci_dev->dv_xname, ncolors);
262: uvm_page_recolor(ncolors);
263: }
264:
265:
266: void
267: cpu_attach(struct device *parent, struct device *self, void *aux)
268: {
269: struct cpu_softc *sc = (void *) self;
270: struct cpu_attach_args *caa = aux;
271: struct cpu_info *ci;
272: #if defined(MULTIPROCESSOR)
273: int cpunum = caa->cpu_number;
274: #endif
275:
276: /*
277: * If we're an Application Processor, allocate a cpu_info
278: * structure, otherwise use the primary's.
279: */
280: if (caa->cpu_role == CPU_ROLE_AP) {
281: aprint_naive(": Application Processor\n");
282: ci = malloc(sizeof(*ci), M_DEVBUF, M_WAITOK);
283: memset(ci, 0, sizeof(*ci));
284: #if defined(MULTIPROCESSOR)
285: if (cpu_info[cpunum] != NULL) {
286: printf("\n");
287: panic("cpu at apic id %d already attached?", cpunum);
288: }
289: cpu_info[cpunum] = ci;
290: #endif
291: #ifdef TRAPLOG
292: ci->ci_tlog_base = malloc(sizeof(struct tlog),
293: M_DEVBUF, M_WAITOK);
294: #endif
295: } else {
296: aprint_naive(": %s Processor\n",
297: caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot");
298: ci = &cpu_info_primary;
299: #if defined(MULTIPROCESSOR)
300: if (cpunum != lapic_cpu_number()) {
301: printf("\n");
302: panic("%s: running CPU is at apic %d"
303: " instead of at expected %d",
304: sc->sc_dev.dv_xname, lapic_cpu_number(), cpunum);
305: }
306: #endif
307: }
308:
309: ci->ci_self = ci;
310: sc->sc_info = ci;
311:
312: ci->ci_dev = self;
313: ci->ci_apicid = caa->cpu_number;
314: #ifdef MULTIPROCESSOR
315: ci->ci_cpuid = ci->ci_apicid;
316: #else
317: ci->ci_cpuid = 0; /* False for APs, but they're not used anyway */
318: #endif
319: ci->ci_cpumask = (1 << ci->ci_cpuid);
320: ci->ci_func = caa->cpu_func;
321:
322: if (caa->cpu_role == CPU_ROLE_AP) {
323: #ifdef MULTIPROCESSOR
324: int error;
325:
326: error = mi_cpu_attach(ci);
327: if (error != 0) {
328: aprint_normal("\n");
329: aprint_error("%s: mi_cpu_attach failed with %d\n",
330: sc->sc_dev.dv_xname, error);
331: return;
332: }
333: #endif
1.12.2.2! bouyer 334: cpu_init_tss(ci);
1.2 ad 335: } else {
336: KASSERT(ci->ci_data.cpu_idlelwp != NULL);
337: }
338:
339: pmap_reference(pmap_kernel());
340: ci->ci_pmap = pmap_kernel();
341: ci->ci_tlbstate = TLBSTATE_STALE;
342:
343: /* further PCB init done later. */
344:
345: switch (caa->cpu_role) {
346: case CPU_ROLE_SP:
347: aprint_normal(": (uniprocessor)\n");
1.9 ad 348: atomic_or_32(&ci->ci_flags,
349: CPUF_PRESENT | CPUF_SP | CPUF_PRIMARY);
1.2 ad 350: cpu_intr_init(ci);
351: identifycpu(ci);
352: cpu_init(ci);
353: cpu_set_tss_gates(ci);
354: pmap_cpu_init_late(ci);
1.6 ad 355: x86_errata();
1.2 ad 356: break;
357:
358: case CPU_ROLE_BP:
359: aprint_normal(": (boot processor)\n");
1.9 ad 360: atomic_or_32(&ci->ci_flags,
361: CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY);
1.2 ad 362: cpu_intr_init(ci);
363: identifycpu(ci);
364: cpu_init(ci);
365: cpu_set_tss_gates(ci);
366: pmap_cpu_init_late(ci);
367: #if NLAPIC > 0
368: /*
369: * Enable local apic
370: */
371: lapic_enable();
372: lapic_calibrate_timer(ci);
373: #endif
374: #if NIOAPIC > 0
375: ioapic_bsp_id = caa->cpu_number;
376: #endif
1.6 ad 377: x86_errata();
1.2 ad 378: break;
379:
380: case CPU_ROLE_AP:
381: /*
382: * report on an AP
383: */
384: aprint_normal(": (application processor)\n");
385:
386: #if defined(MULTIPROCESSOR)
387: cpu_intr_init(ci);
388: gdt_alloc_cpu(ci);
389: cpu_set_tss_gates(ci);
390: pmap_cpu_init_early(ci);
391: pmap_cpu_init_late(ci);
392: cpu_start_secondary(ci);
393: if (ci->ci_flags & CPUF_PRESENT) {
394: identifycpu(ci);
395: ci->ci_next = cpu_info_list->ci_next;
396: cpu_info_list->ci_next = ci;
397: }
398: #else
399: aprint_normal("%s: not started\n", sc->sc_dev.dv_xname);
400: #endif
401: break;
402:
403: default:
404: printf("\n");
405: panic("unknown processor type??\n");
406: }
407: cpu_vm_init(ci);
408:
409: cpus_attached |= ci->ci_cpumask;
410:
1.12 jmcneill 411: if (!pmf_device_register(self, cpu_suspend, cpu_resume))
412: aprint_error_dev(self, "couldn't establish power handler\n");
413:
1.2 ad 414: #if defined(MULTIPROCESSOR)
415: if (mp_verbose) {
416: struct lwp *l = ci->ci_data.cpu_idlelwp;
417:
418: aprint_verbose(
419: "%s: idle lwp at %p, idle sp at %p\n",
420: sc->sc_dev.dv_xname, l,
421: #ifdef i386
422: (void *)l->l_addr->u_pcb.pcb_esp
423: #else
424: (void *)l->l_addr->u_pcb.pcb_rsp
425: #endif
426: );
427: }
428: #endif
429: }
430:
431: /*
432: * Initialize the processor appropriately.
433: */
434:
435: void
1.9 ad 436: cpu_init(struct cpu_info *ci)
1.2 ad 437: {
438: /* configure the CPU if needed */
439: if (ci->cpu_setup != NULL)
440: (*ci->cpu_setup)(ci);
441:
442: #ifdef i386
443: /*
444: * On a 486 or above, enable ring 0 write protection.
445: */
446: if (ci->ci_cpu_class >= CPUCLASS_486)
447: lcr0(rcr0() | CR0_WP);
448: #else
449: lcr0(rcr0() | CR0_WP);
450: #endif
451:
452: /*
453: * On a P6 or above, enable global TLB caching if the
454: * hardware supports it.
455: */
456: if (cpu_feature & CPUID_PGE)
457: lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */
458:
459: /*
460: * If we have FXSAVE/FXRESTOR, use them.
461: */
462: if (cpu_feature & CPUID_FXSR) {
463: lcr4(rcr4() | CR4_OSFXSR);
464:
465: /*
466: * If we have SSE/SSE2, enable XMM exceptions.
467: */
468: if (cpu_feature & (CPUID_SSE|CPUID_SSE2))
469: lcr4(rcr4() | CR4_OSXMMEXCPT);
470: }
471:
472: #ifdef MTRR
473: /*
474: * On a P6 or above, initialize MTRR's if the hardware supports them.
475: */
476: if (cpu_feature & CPUID_MTRR) {
477: if ((ci->ci_flags & CPUF_AP) == 0)
478: i686_mtrr_init_first();
479: mtrr_init_cpu(ci);
480: }
481:
482: #ifdef i386
483: if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) {
484: /*
485: * Must be a K6-2 Step >= 7 or a K6-III.
486: */
487: if (CPUID2FAMILY(ci->ci_signature) == 5) {
488: if (CPUID2MODEL(ci->ci_signature) > 8 ||
489: (CPUID2MODEL(ci->ci_signature) == 8 &&
490: CPUID2STEPPING(ci->ci_signature) >= 7)) {
491: mtrr_funcs = &k6_mtrr_funcs;
492: k6_mtrr_init_first();
493: mtrr_init_cpu(ci);
494: }
495: }
496: }
497: #endif /* i386 */
498: #endif /* MTRR */
499:
1.9 ad 500: atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
501: atomic_or_32(&cpus_running, ci->ci_cpumask);
502:
503: #ifndef MULTIPROCESSOR
1.5 ad 504: /* XXX */
505: x86_patch();
1.2 ad 506: #endif
507: }
508:
509: #ifdef MULTIPROCESSOR
510: void
1.12 jmcneill 511: cpu_boot_secondary_processors(void)
1.2 ad 512: {
513: struct cpu_info *ci;
514: u_long i;
515:
1.5 ad 516: /* Now that we know the number of CPUs, patch the text segment. */
517: x86_patch();
518:
1.2 ad 519: for (i=0; i < X86_MAXPROCS; i++) {
520: ci = cpu_info[i];
521: if (ci == NULL)
522: continue;
523: if (ci->ci_data.cpu_idlelwp == NULL)
524: continue;
525: if ((ci->ci_flags & CPUF_PRESENT) == 0)
526: continue;
527: if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))
528: continue;
529: cpu_boot_secondary(ci);
530: }
531:
532: x86_mp_online = true;
533: }
534:
535: static void
536: cpu_init_idle_lwp(struct cpu_info *ci)
537: {
538: struct lwp *l = ci->ci_data.cpu_idlelwp;
539: struct pcb *pcb = &l->l_addr->u_pcb;
540:
541: pcb->pcb_cr0 = rcr0();
542: }
543:
544: void
1.12 jmcneill 545: cpu_init_idle_lwps(void)
1.2 ad 546: {
547: struct cpu_info *ci;
548: u_long i;
549:
550: for (i = 0; i < X86_MAXPROCS; i++) {
551: ci = cpu_info[i];
552: if (ci == NULL)
553: continue;
554: if (ci->ci_data.cpu_idlelwp == NULL)
555: continue;
556: if ((ci->ci_flags & CPUF_PRESENT) == 0)
557: continue;
558: cpu_init_idle_lwp(ci);
559: }
560: }
561:
562: void
1.12 jmcneill 563: cpu_start_secondary(struct cpu_info *ci)
1.2 ad 564: {
565: int i;
566: extern paddr_t mp_pdirpa;
567:
1.12 jmcneill 568: mp_pdirpa = pmap_init_tmp_pgtbl(mp_trampoline_paddr);
1.2 ad 569:
1.9 ad 570: atomic_or_32(&ci->ci_flags, CPUF_AP);
1.2 ad 571:
572: aprint_debug("%s: starting\n", ci->ci_dev->dv_xname);
573:
574: ci->ci_curlwp = ci->ci_data.cpu_idlelwp;
1.12.2.1 bouyer 575: CPU_STARTUP(ci, mp_trampoline_paddr);
1.2 ad 576:
577: /*
578: * wait for it to become ready
579: */
580: for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i>0;i--) {
1.11 ad 581: i8254_delay(10);
1.2 ad 582: }
1.9 ad 583: if ((ci->ci_flags & CPUF_PRESENT) == 0) {
1.2 ad 584: aprint_error("%s: failed to become ready\n",
585: ci->ci_dev->dv_xname);
586: #if defined(MPDEBUG) && defined(DDB)
587: printf("dropping into debugger; continue from here to resume boot\n");
588: Debugger();
589: #endif
590: }
591:
592: CPU_START_CLEANUP(ci);
593: }
594:
595: void
1.12 jmcneill 596: cpu_boot_secondary(struct cpu_info *ci)
1.2 ad 597: {
598: int i;
599:
1.9 ad 600: atomic_or_32(&ci->ci_flags, CPUF_GO);
1.2 ad 601: for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i>0;i--) {
1.11 ad 602: i8254_delay(10);
1.2 ad 603: }
1.9 ad 604: if ((ci->ci_flags & CPUF_RUNNING) == 0) {
1.2 ad 605: aprint_error("%s: failed to start\n", ci->ci_dev->dv_xname);
606: #if defined(MPDEBUG) && defined(DDB)
607: printf("dropping into debugger; continue from here to resume boot\n");
608: Debugger();
609: #endif
610: }
611: }
612:
613: /*
614: * The CPU ends up here when its ready to run
615: * This is called from code in mptramp.s; at this point, we are running
616: * in the idle pcb/idle stack of the new CPU. When this function returns,
617: * this processor will enter the idle loop and start looking for work.
618: */
619: void
620: cpu_hatch(void *v)
621: {
622: struct cpu_info *ci = (struct cpu_info *)v;
1.6 ad 623: int s, i;
1.2 ad 624:
625: #ifdef __x86_64__
1.12 jmcneill 626: cpu_init_msrs(ci, true);
1.2 ad 627: #endif
628: cpu_probe_features(ci);
629: cpu_feature &= ci->ci_feature_flags;
630: cpu_feature2 &= ci->ci_feature2_flags;
631:
1.8 ad 632: KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0);
1.9 ad 633: atomic_or_32(&ci->ci_flags, CPUF_PRESENT);
1.6 ad 634: while ((ci->ci_flags & CPUF_GO) == 0) {
635: /* Don't use delay, boot CPU may be patching the text. */
636: for (i = 10000; i != 0; i--)
637: x86_pause();
638: }
1.5 ad 639:
640: /* Beacuse the text may have been patched in x86_patch(). */
641: wbinvd();
642: x86_flush();
643:
1.8 ad 644: KASSERT((ci->ci_flags & CPUF_RUNNING) == 0);
1.2 ad 645:
1.12 jmcneill 646: lcr3(pmap_kernel()->pm_pdirpa);
647: curlwp->l_addr->u_pcb.pcb_cr3 = pmap_kernel()->pm_pdirpa;
1.2 ad 648: lcr0(ci->ci_data.cpu_idlelwp->l_addr->u_pcb.pcb_cr0);
649: cpu_init_idt();
1.8 ad 650: gdt_init_cpu(ci);
651: lapic_enable();
1.2 ad 652: lapic_set_lvt();
1.8 ad 653: lapic_initclocks();
1.2 ad 654:
655: #ifdef i386
656: npxinit(ci);
657: #else
658: fpuinit(ci);
1.4 yamt 659: #endif
1.2 ad 660: lldt(GSYSSEL(GLDT_SEL, SEL_KPL));
1.12.2.2! bouyer 661: ltr(ci->ci_tss_sel);
1.2 ad 662:
663: cpu_init(ci);
1.7 ad 664: cpu_get_tsc_freq(ci);
1.2 ad 665:
666: s = splhigh();
667: #ifdef i386
668: lapic_tpr = 0;
669: #else
670: lcr8(0);
671: #endif
1.3 ad 672: x86_enable_intr();
1.2 ad 673: splx(s);
1.6 ad 674: x86_errata();
1.2 ad 675:
676: aprint_debug("%s: CPU %ld running\n", ci->ci_dev->dv_xname,
677: (long)ci->ci_cpuid);
678: }
679:
680: #if defined(DDB)
681:
682: #include <ddb/db_output.h>
683: #include <machine/db_machdep.h>
684:
685: /*
686: * Dump CPU information from ddb.
687: */
688: void
689: cpu_debug_dump(void)
690: {
691: struct cpu_info *ci;
692: CPU_INFO_ITERATOR cii;
693:
694: db_printf("addr dev id flags ipis curproc fpcurproc\n");
695: for (CPU_INFO_FOREACH(cii, ci)) {
696: db_printf("%p %s %ld %x %x %10p %10p\n",
697: ci,
698: ci->ci_dev == NULL ? "BOOT" : ci->ci_dev->dv_xname,
699: (long)ci->ci_cpuid,
700: ci->ci_flags, ci->ci_ipis,
701: ci->ci_curlwp,
702: ci->ci_fpcurlwp);
703: }
704: }
705: #endif
706:
707: static void
1.12 jmcneill 708: cpu_copy_trampoline(void)
1.2 ad 709: {
710: /*
711: * Copy boot code.
712: */
713: extern u_char cpu_spinup_trampoline[];
714: extern u_char cpu_spinup_trampoline_end[];
1.12 jmcneill 715:
716: vaddr_t mp_trampoline_vaddr;
717:
718: mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0,
719: UVM_KMF_VAONLY);
720:
721: pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr,
722: VM_PROT_READ | VM_PROT_WRITE);
1.2 ad 723: pmap_update(pmap_kernel());
1.12 jmcneill 724: memcpy((void *)mp_trampoline_vaddr,
1.2 ad 725: cpu_spinup_trampoline,
726: cpu_spinup_trampoline_end-cpu_spinup_trampoline);
1.12 jmcneill 727:
728: pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE);
729: pmap_update(pmap_kernel());
730: uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY);
1.2 ad 731: }
732:
733: #endif
734:
735: #ifdef i386
736: static void
1.12.2.2! bouyer 737: tss_init(struct i386tss *tss, void *stack, void *func)
1.2 ad 738: {
739: memset(tss, 0, sizeof *tss);
740: tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16);
741: tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL);
742: tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL);
743: tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL);
744: tss->tss_gs = tss->__tss_es = tss->__tss_ds =
745: tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL);
746: tss->tss_cr3 = pmap_kernel()->pm_pdirpa;
747: tss->tss_esp = (int)((char *)stack + USPACE - 16);
748: tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL);
749: tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */
750: tss->__tss_eip = (int)func;
751: }
752:
753: /* XXX */
754: #define IDTVEC(name) __CONCAT(X, name)
755: typedef void (vector)(void);
756: extern vector IDTVEC(tss_trap08);
757: #ifdef DDB
758: extern vector Xintrddbipi;
759: extern int ddb_vec;
760: #endif
761:
762: static void
763: cpu_set_tss_gates(struct cpu_info *ci)
764: {
765: struct segment_descriptor sd;
766:
767: ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0,
768: UVM_KMF_WIRED);
1.12.2.2! bouyer 769: tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack,
1.2 ad 770: IDTVEC(tss_trap08));
771: setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1,
772: SDT_SYS386TSS, SEL_KPL, 0, 0);
773: ci->ci_gdt[GTRAPTSS_SEL].sd = sd;
774: setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
775: GSEL(GTRAPTSS_SEL, SEL_KPL));
776:
777: #if defined(DDB) && defined(MULTIPROCESSOR)
778: /*
779: * Set up separate handler for the DDB IPI, so that it doesn't
780: * stomp on a possibly corrupted stack.
781: *
782: * XXX overwriting the gate set in db_machine_init.
783: * Should rearrange the code so that it's set only once.
784: */
785: ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0,
786: UVM_KMF_WIRED);
1.12.2.2! bouyer 787: tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, Xintrddbipi);
1.2 ad 788:
789: setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1,
790: SDT_SYS386TSS, SEL_KPL, 0, 0);
791: ci->ci_gdt[GIPITSS_SEL].sd = sd;
792:
793: setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL,
794: GSEL(GIPITSS_SEL, SEL_KPL));
795: #endif
796: }
797: #else
798: static void
799: cpu_set_tss_gates(struct cpu_info *ci)
800: {
801:
802: }
803: #endif /* i386 */
804:
805: int
1.12.2.1 bouyer 806: mp_cpu_start(struct cpu_info *ci, paddr_t target)
1.2 ad 807: {
808: #if NLAPIC > 0
809: int error;
810: #endif
811: unsigned short dwordptr[2];
1.12.2.1 bouyer 812:
813: /*
814: * Bootstrap code must be addressable in real mode
815: * and it must be page aligned.
816: */
817: KASSERT(target < 0x10000 && target % PAGE_SIZE == 0);
1.2 ad 818:
819: /*
820: * "The BSP must initialize CMOS shutdown code to 0Ah ..."
821: */
822:
823: outb(IO_RTC, NVRAM_RESET);
824: outb(IO_RTC+1, NVRAM_RESET_JUMP);
825:
826: /*
827: * "and the warm reset vector (DWORD based at 40:67) to point
828: * to the AP startup code ..."
829: */
830:
831: dwordptr[0] = 0;
1.12.2.1 bouyer 832: dwordptr[1] = target >> 4;
1.2 ad 833:
1.12.2.1 bouyer 834: memcpy((uint8_t *)(cmos_data_mapping + 0x467), dwordptr, 4);
1.2 ad 835:
836: #if NLAPIC > 0
837: /*
838: * ... prior to executing the following sequence:"
839: */
840:
841: if (ci->ci_flags & CPUF_AP) {
842: if ((error = x86_ipi_init(ci->ci_apicid)) != 0)
843: return error;
844:
1.11 ad 845: i8254_delay(10000);
1.2 ad 846:
847: if (cpu_feature & CPUID_APIC) {
848:
1.12.2.1 bouyer 849: if ((error = x86_ipi(target / PAGE_SIZE,
1.2 ad 850: ci->ci_apicid,
851: LAPIC_DLMODE_STARTUP)) != 0)
852: return error;
1.11 ad 853: i8254_delay(200);
1.2 ad 854:
1.12.2.1 bouyer 855: if ((error = x86_ipi(target / PAGE_SIZE,
1.2 ad 856: ci->ci_apicid,
857: LAPIC_DLMODE_STARTUP)) != 0)
858: return error;
1.11 ad 859: i8254_delay(200);
1.2 ad 860: }
861: }
862: #endif
863: return 0;
864: }
865:
866: void
867: mp_cpu_start_cleanup(struct cpu_info *ci)
868: {
869: /*
870: * Ensure the NVRAM reset byte contains something vaguely sane.
871: */
872:
873: outb(IO_RTC, NVRAM_RESET);
874: outb(IO_RTC+1, NVRAM_RESET_RST);
875: }
876:
877: #ifdef __x86_64__
878: typedef void (vector)(void);
879: extern vector Xsyscall, Xsyscall32;
880:
881: void
1.12 jmcneill 882: cpu_init_msrs(struct cpu_info *ci, bool full)
1.2 ad 883: {
884: wrmsr(MSR_STAR,
885: ((uint64_t)GSEL(GCODE_SEL, SEL_KPL) << 32) |
886: ((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48));
887: wrmsr(MSR_LSTAR, (uint64_t)Xsyscall);
888: wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32);
889: wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C);
890:
1.12 jmcneill 891: if (full) {
892: wrmsr(MSR_FSBASE, 0);
893: wrmsr(MSR_GSBASE, (u_int64_t)ci);
894: wrmsr(MSR_KERNELGSBASE, 0);
895: }
1.2 ad 896:
897: if (cpu_feature & CPUID_NOX)
898: wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE);
899: }
900: #endif /* __x86_64__ */
1.7 ad 901:
1.12 jmcneill 902: /* XXX joerg restructure and restart CPUs individually */
903: static bool
904: cpu_suspend(device_t dv)
905: {
906: struct cpu_softc *sc = device_private(dv);
907: struct cpu_info *ci = sc->sc_info;
908: int err;
909:
1.12.2.1 bouyer 910: if (ci->ci_flags & CPUF_PRIMARY)
1.12 jmcneill 911: return true;
912: if (ci->ci_data.cpu_idlelwp == NULL)
913: return true;
914: if ((ci->ci_flags & CPUF_PRESENT) == 0)
915: return true;
916:
917: mutex_enter(&cpu_lock);
918: err = cpu_setonline(ci, false);
919: mutex_exit(&cpu_lock);
1.12.2.1 bouyer 920: return err == 0;
1.12 jmcneill 921: }
922:
923: static bool
924: cpu_resume(device_t dv)
925: {
926: struct cpu_softc *sc = device_private(dv);
927: struct cpu_info *ci = sc->sc_info;
928: int err;
929:
1.12.2.1 bouyer 930: if (ci->ci_flags & CPUF_PRIMARY)
1.12 jmcneill 931: return true;
932: if (ci->ci_data.cpu_idlelwp == NULL)
933: return true;
934: if ((ci->ci_flags & CPUF_PRESENT) == 0)
935: return true;
936:
937: mutex_enter(&cpu_lock);
938: err = cpu_setonline(ci, true);
939: mutex_exit(&cpu_lock);
1.12.2.1 bouyer 940:
941: return err == 0;
1.12 jmcneill 942: }
943:
1.7 ad 944: void
945: cpu_get_tsc_freq(struct cpu_info *ci)
946: {
947: uint64_t last_tsc;
948: u_int junk[4];
949:
950: if (ci->ci_feature_flags & CPUID_TSC) {
951: /* Serialize. */
952: x86_cpuid(0, junk);
953: last_tsc = rdtsc();
954: i8254_delay(100000);
955: ci->ci_tsc_freq = (rdtsc() - last_tsc) * 10;
956: }
957: }
CVSweb <webmaster@jp.NetBSD.org>