version 1.130, 2017/05/31 14:41:07 |
version 1.130.2.8, 2019/03/09 17:10:19 |
Line 68 __KERNEL_RCSID(0, "$NetBSD$"); |
|
Line 68 __KERNEL_RCSID(0, "$NetBSD$"); |
|
#include "opt_mpbios.h" /* for MPDEBUG */ |
#include "opt_mpbios.h" /* for MPDEBUG */ |
#include "opt_mtrr.h" |
#include "opt_mtrr.h" |
#include "opt_multiprocessor.h" |
#include "opt_multiprocessor.h" |
|
#include "opt_svs.h" |
|
|
#include "lapic.h" |
#include "lapic.h" |
#include "ioapic.h" |
#include "ioapic.h" |
Line 164 struct cpu_info cpu_info_primary __align |
|
Line 165 struct cpu_info cpu_info_primary __align |
|
.ci_curldt = -1, |
.ci_curldt = -1, |
#ifdef TRAPLOG |
#ifdef TRAPLOG |
.ci_tlog_base = &tlog_primary, |
.ci_tlog_base = &tlog_primary, |
#endif /* !TRAPLOG */ |
#endif |
}; |
}; |
|
|
struct cpu_info *cpu_info_list = &cpu_info_primary; |
struct cpu_info *cpu_info_list = &cpu_info_primary; |
|
|
static void cpu_set_tss_gates(struct cpu_info *); |
|
|
|
#ifdef i386 |
#ifdef i386 |
static void tss_init(struct i386tss *, void *, void *); |
void cpu_set_tss_gates(struct cpu_info *); |
#endif |
#endif |
|
|
static void cpu_init_idle_lwp(struct cpu_info *); |
static void cpu_init_idle_lwp(struct cpu_info *); |
Line 187 uint32_t cpu_feature[7] __read_mostly; / |
|
Line 186 uint32_t cpu_feature[7] __read_mostly; / |
|
* [6] structured extended features cpuid.7:%ecx |
* [6] structured extended features cpuid.7:%ecx |
*/ |
*/ |
|
|
extern char x86_64_doubleflt_stack[]; |
|
|
|
#ifdef MULTIPROCESSOR |
#ifdef MULTIPROCESSOR |
bool x86_mp_online; |
bool x86_mp_online; |
paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; |
paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; |
Line 236 cpu_match(device_t parent, cfdata_t matc |
|
Line 233 cpu_match(device_t parent, cfdata_t matc |
|
return 1; |
return 1; |
} |
} |
|
|
|
#ifdef __HAVE_PCPU_AREA |
|
void |
|
cpu_pcpuarea_init(struct cpu_info *ci) |
|
{ |
|
struct vm_page *pg; |
|
size_t i, npages; |
|
vaddr_t base, va; |
|
paddr_t pa; |
|
|
|
CTASSERT(sizeof(struct pcpu_entry) % PAGE_SIZE == 0); |
|
|
|
npages = sizeof(struct pcpu_entry) / PAGE_SIZE; |
|
base = (vaddr_t)&pcpuarea->ent[cpu_index(ci)]; |
|
|
|
for (i = 0; i < npages; i++) { |
|
pg = uvm_pagealloc(NULL, 0, NULL, UVM_PGA_ZERO); |
|
if (pg == NULL) { |
|
panic("failed to allocate pcpu PA"); |
|
} |
|
|
|
va = base + i * PAGE_SIZE; |
|
pa = VM_PAGE_TO_PHYS(pg); |
|
|
|
pmap_kenter_pa(va, pa, VM_PROT_READ|VM_PROT_WRITE, 0); |
|
} |
|
|
|
pmap_update(pmap_kernel()); |
|
} |
|
#endif |
|
|
static void |
static void |
cpu_vm_init(struct cpu_info *ci) |
cpu_vm_init(struct cpu_info *ci) |
{ |
{ |
Line 376 cpu_attach(device_t parent, device_t sel |
|
Line 403 cpu_attach(device_t parent, device_t sel |
|
"mi_cpu_attach failed with %d\n", error); |
"mi_cpu_attach failed with %d\n", error); |
return; |
return; |
} |
} |
|
#ifdef __HAVE_PCPU_AREA |
|
cpu_pcpuarea_init(ci); |
|
#endif |
cpu_init_tss(ci); |
cpu_init_tss(ci); |
} else { |
} else { |
KASSERT(ci->ci_data.cpu_idlelwp != NULL); |
KASSERT(ci->ci_data.cpu_idlelwp != NULL); |
} |
} |
|
|
|
#ifdef SVS |
|
cpu_svs_init(ci); |
|
#endif |
|
|
pmap_reference(pmap_kernel()); |
pmap_reference(pmap_kernel()); |
ci->ci_pmap = pmap_kernel(); |
ci->ci_pmap = pmap_kernel(); |
ci->ci_tlbstate = TLBSTATE_STALE; |
ci->ci_tlbstate = TLBSTATE_STALE; |
Line 395 cpu_attach(device_t parent, device_t sel |
|
Line 429 cpu_attach(device_t parent, device_t sel |
|
cpu_intr_init(ci); |
cpu_intr_init(ci); |
cpu_get_tsc_freq(ci); |
cpu_get_tsc_freq(ci); |
cpu_init(ci); |
cpu_init(ci); |
|
#ifdef i386 |
cpu_set_tss_gates(ci); |
cpu_set_tss_gates(ci); |
|
#endif |
pmap_cpu_init_late(ci); |
pmap_cpu_init_late(ci); |
#if NLAPIC > 0 |
#if NLAPIC > 0 |
if (caa->cpu_role != CPU_ROLE_SP) { |
if (caa->cpu_role != CPU_ROLE_SP) { |
Line 434 cpu_attach(device_t parent, device_t sel |
|
Line 470 cpu_attach(device_t parent, device_t sel |
|
*/ |
*/ |
cpu_intr_init(ci); |
cpu_intr_init(ci); |
gdt_alloc_cpu(ci); |
gdt_alloc_cpu(ci); |
|
#ifdef i386 |
cpu_set_tss_gates(ci); |
cpu_set_tss_gates(ci); |
|
#endif |
pmap_cpu_init_late(ci); |
pmap_cpu_init_late(ci); |
cpu_start_secondary(ci); |
cpu_start_secondary(ci); |
if (ci->ci_flags & CPUF_PRESENT) { |
if (ci->ci_flags & CPUF_PRESENT) { |
Line 501 cpu_rescan(device_t self, const char *if |
|
Line 539 cpu_rescan(device_t self, const char *if |
|
cfaa.ci = ci; |
cfaa.ci = ci; |
|
|
if (ifattr_match(ifattr, "cpufeaturebus")) { |
if (ifattr_match(ifattr, "cpufeaturebus")) { |
|
|
if (ci->ci_frequency == NULL) { |
if (ci->ci_frequency == NULL) { |
cfaa.name = "frequency"; |
cfaa.name = "frequency"; |
ci->ci_frequency = config_found_ia(self, |
ci->ci_frequency = config_found_ia(self, |
Line 565 cpu_init(struct cpu_info *ci) |
|
Line 602 cpu_init(struct cpu_info *ci) |
|
* hardware supports it. |
* hardware supports it. |
*/ |
*/ |
if (cpu_feature[0] & CPUID_PGE) |
if (cpu_feature[0] & CPUID_PGE) |
|
#ifdef SVS |
|
if (!svs_enabled) |
|
#endif |
cr4 |= CR4_PGE; /* enable global TLB caching */ |
cr4 |= CR4_PGE; /* enable global TLB caching */ |
|
|
/* |
/* |
Line 588 cpu_init(struct cpu_info *ci) |
|
Line 628 cpu_init(struct cpu_info *ci) |
|
if (cpu_feature[5] & CPUID_SEF_SMEP) |
if (cpu_feature[5] & CPUID_SEF_SMEP) |
cr4 |= CR4_SMEP; |
cr4 |= CR4_SMEP; |
|
|
|
#ifdef amd64 |
|
/* If SMAP is supported, enable it */ |
|
if (cpu_feature[5] & CPUID_SEF_SMAP) |
|
cr4 |= CR4_SMAP; |
|
#endif |
|
|
if (cr4) { |
if (cr4) { |
cr4 |= rcr4(); |
cr4 |= rcr4(); |
lcr4(cr4); |
lcr4(cr4); |
Line 733 cpu_start_secondary(struct cpu_info *ci) |
|
Line 779 cpu_start_secondary(struct cpu_info *ci) |
|
memcpy(otrace, cpu_trace, sizeof(otrace)); |
memcpy(otrace, cpu_trace, sizeof(otrace)); |
} |
} |
#endif |
#endif |
i8254_delay(10); |
x86_delay(10); |
} |
} |
|
|
if ((ci->ci_flags & CPUF_PRESENT) == 0) { |
if ((ci->ci_flags & CPUF_PRESENT) == 0) { |
Line 768 cpu_boot_secondary(struct cpu_info *ci) |
|
Line 814 cpu_boot_secondary(struct cpu_info *ci) |
|
|
|
atomic_or_32(&ci->ci_flags, CPUF_GO); |
atomic_or_32(&ci->ci_flags, CPUF_GO); |
for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { |
for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { |
i8254_delay(10); |
x86_delay(10); |
} |
} |
if ((ci->ci_flags & CPUF_RUNNING) == 0) { |
if ((ci->ci_flags & CPUF_RUNNING) == 0) { |
aprint_error_dev(ci->ci_dev, "failed to start\n"); |
aprint_error_dev(ci->ci_dev, "failed to start\n"); |
Line 808 cpu_hatch(void *v) |
|
Line 854 cpu_hatch(void *v) |
|
cpu_probe(ci); |
cpu_probe(ci); |
|
|
ci->ci_data.cpu_cc_freq = cpu_info_primary.ci_data.cpu_cc_freq; |
ci->ci_data.cpu_cc_freq = cpu_info_primary.ci_data.cpu_cc_freq; |
/* cpu_get_tsc_freq(ci); */ |
/* cpu_get_tsc_freq(ci); */ |
|
|
KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); |
KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); |
|
|
Line 824 cpu_hatch(void *v) |
|
Line 870 cpu_hatch(void *v) |
|
/* |
/* |
* Wait to be brought online. Use 'monitor/mwait' if available, |
* Wait to be brought online. Use 'monitor/mwait' if available, |
* in order to make the TSC drift as much as possible. so that |
* in order to make the TSC drift as much as possible. so that |
* we can detect it later. If not available, try 'pause'. |
* we can detect it later. If not available, try 'pause'. |
* We'd like to use 'hlt', but we have interrupts off. |
* We'd like to use 'hlt', but we have interrupts off. |
*/ |
*/ |
while ((ci->ci_flags & CPUF_GO) == 0) { |
while ((ci->ci_flags & CPUF_GO) == 0) { |
Line 835 cpu_hatch(void *v) |
|
Line 881 cpu_hatch(void *v) |
|
} |
} |
x86_mwait(0, 0); |
x86_mwait(0, 0); |
} else { |
} else { |
for (i = 100; i != 0; i--) { |
/* |
|
* XXX The loop repetition count could be a lot higher, but |
|
* XXX currently qemu emulator takes a _very_long_time_ to |
|
* XXX execute the pause instruction. So for now, use a low |
|
* XXX value to allow the cpu to hatch before timing out. |
|
*/ |
|
for (i = 50; i != 0; i--) { |
x86_pause(); |
x86_pause(); |
} |
} |
} |
} |
Line 927 cpu_copy_trampoline(void) |
|
Line 979 cpu_copy_trampoline(void) |
|
*/ |
*/ |
extern u_char cpu_spinup_trampoline[]; |
extern u_char cpu_spinup_trampoline[]; |
extern u_char cpu_spinup_trampoline_end[]; |
extern u_char cpu_spinup_trampoline_end[]; |
|
|
vaddr_t mp_trampoline_vaddr; |
vaddr_t mp_trampoline_vaddr; |
|
|
mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, |
mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, |
Line 946 cpu_copy_trampoline(void) |
|
Line 998 cpu_copy_trampoline(void) |
|
} |
} |
#endif |
#endif |
|
|
#ifdef i386 |
|
static void |
|
tss_init(struct i386tss *tss, void *stack, void *func) |
|
{ |
|
KASSERT(curcpu()->ci_pmap == pmap_kernel()); |
|
|
|
memset(tss, 0, sizeof *tss); |
|
tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16); |
|
tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); |
|
tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL); |
|
tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL); |
|
tss->tss_gs = tss->__tss_es = tss->__tss_ds = |
|
tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL); |
|
/* %cr3 contains the value associated to pmap_kernel */ |
|
tss->tss_cr3 = rcr3(); |
|
tss->tss_esp = (int)((char *)stack + USPACE - 16); |
|
tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL); |
|
tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */ |
|
tss->__tss_eip = (int)func; |
|
} |
|
|
|
/* XXX */ |
|
#define IDTVEC(name) __CONCAT(X, name) |
|
typedef void (vector)(void); |
|
extern vector IDTVEC(tss_trap08); |
|
#if defined(DDB) && defined(MULTIPROCESSOR) |
|
extern vector Xintrddbipi, Xx2apic_intrddbipi; |
|
extern int ddb_vec; |
|
#endif |
|
|
|
static void |
|
cpu_set_tss_gates(struct cpu_info *ci) |
|
{ |
|
struct segment_descriptor sd; |
|
|
|
ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, |
|
UVM_KMF_WIRED); |
|
tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack, |
|
IDTVEC(tss_trap08)); |
|
setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1, |
|
SDT_SYS386TSS, SEL_KPL, 0, 0); |
|
ci->ci_gdt[GTRAPTSS_SEL].sd = sd; |
|
setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL, |
|
GSEL(GTRAPTSS_SEL, SEL_KPL)); |
|
|
|
#if defined(DDB) && defined(MULTIPROCESSOR) |
|
/* |
|
* Set up separate handler for the DDB IPI, so that it doesn't |
|
* stomp on a possibly corrupted stack. |
|
* |
|
* XXX overwriting the gate set in db_machine_init. |
|
* Should rearrange the code so that it's set only once. |
|
*/ |
|
ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, |
|
UVM_KMF_WIRED); |
|
tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, |
|
x2apic_mode ? Xx2apic_intrddbipi : Xintrddbipi); |
|
|
|
setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1, |
|
SDT_SYS386TSS, SEL_KPL, 0, 0); |
|
ci->ci_gdt[GIPITSS_SEL].sd = sd; |
|
|
|
setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL, |
|
GSEL(GIPITSS_SEL, SEL_KPL)); |
|
#endif |
|
} |
|
#else |
|
static void |
|
cpu_set_tss_gates(struct cpu_info *ci) |
|
{ |
|
|
|
} |
|
#endif /* i386 */ |
|
|
|
#ifdef MULTIPROCESSOR |
#ifdef MULTIPROCESSOR |
int |
int |
mp_cpu_start(struct cpu_info *ci, paddr_t target) |
mp_cpu_start(struct cpu_info *ci, paddr_t target) |
Line 1071 mp_cpu_start(struct cpu_info *ci, paddr_ |
|
Line 1049 mp_cpu_start(struct cpu_info *ci, paddr_ |
|
__func__); |
__func__); |
return error; |
return error; |
} |
} |
i8254_delay(10000); |
x86_delay(10000); |
|
|
error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE); |
error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE); |
if (error != 0) { |
if (error != 0) { |
Line 1079 mp_cpu_start(struct cpu_info *ci, paddr_ |
|
Line 1057 mp_cpu_start(struct cpu_info *ci, paddr_ |
|
__func__); |
__func__); |
return error; |
return error; |
} |
} |
i8254_delay(200); |
x86_delay(200); |
|
|
error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE); |
error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE); |
if (error != 0) { |
if (error != 0) { |
Line 1087 mp_cpu_start(struct cpu_info *ci, paddr_ |
|
Line 1065 mp_cpu_start(struct cpu_info *ci, paddr_ |
|
__func__); |
__func__); |
return error; |
return error; |
} |
} |
i8254_delay(200); |
x86_delay(200); |
} |
} |
|
|
return 0; |
return 0; |
Line 1107 mp_cpu_start_cleanup(struct cpu_info *ci |
|
Line 1085 mp_cpu_start_cleanup(struct cpu_info *ci |
|
|
|
#ifdef __x86_64__ |
#ifdef __x86_64__ |
typedef void (vector)(void); |
typedef void (vector)(void); |
extern vector Xsyscall, Xsyscall32; |
extern vector Xsyscall, Xsyscall32, Xsyscall_svs; |
#endif |
#endif |
|
|
void |
void |
Line 1119 cpu_init_msrs(struct cpu_info *ci, bool |
|
Line 1097 cpu_init_msrs(struct cpu_info *ci, bool |
|
((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48)); |
((uint64_t)LSEL(LSYSRETBASE_SEL, SEL_UPL) << 48)); |
wrmsr(MSR_LSTAR, (uint64_t)Xsyscall); |
wrmsr(MSR_LSTAR, (uint64_t)Xsyscall); |
wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32); |
wrmsr(MSR_CSTAR, (uint64_t)Xsyscall32); |
wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C); |
wrmsr(MSR_SFMASK, PSL_NT|PSL_T|PSL_I|PSL_C|PSL_D|PSL_AC); |
|
|
|
#ifdef SVS |
|
if (svs_enabled) |
|
wrmsr(MSR_LSTAR, (uint64_t)Xsyscall_svs); |
|
#endif |
|
|
if (full) { |
if (full) { |
wrmsr(MSR_FSBASE, 0); |
wrmsr(MSR_FSBASE, 0); |
Line 1240 cpu_get_tsc_freq(struct cpu_info *ci) |
|
Line 1223 cpu_get_tsc_freq(struct cpu_info *ci) |
|
|
|
if (cpu_hascounter()) { |
if (cpu_hascounter()) { |
last_tsc = cpu_counter_serializing(); |
last_tsc = cpu_counter_serializing(); |
i8254_delay(100000); |
x86_delay(100000); |
ci->ci_data.cpu_cc_freq = |
ci->ci_data.cpu_cc_freq = |
(cpu_counter_serializing() - last_tsc) * 10; |
(cpu_counter_serializing() - last_tsc) * 10; |
} |
} |
Line 1281 x86_cpu_idle_halt(void) |
|
Line 1264 x86_cpu_idle_halt(void) |
|
void |
void |
cpu_load_pmap(struct pmap *pmap, struct pmap *oldpmap) |
cpu_load_pmap(struct pmap *pmap, struct pmap *oldpmap) |
{ |
{ |
|
#ifdef SVS |
|
if (svs_enabled) { |
|
svs_pdir_switch(pmap); |
|
} |
|
#endif |
|
|
#ifdef PAE |
#ifdef PAE |
struct cpu_info *ci = curcpu(); |
struct cpu_info *ci = curcpu(); |
bool interrupts_enabled; |
bool interrupts_enabled; |
Line 1299 cpu_load_pmap(struct pmap *pmap, struct |
|
Line 1288 cpu_load_pmap(struct pmap *pmap, struct |
|
for (i = 0 ; i < PDP_SIZE; i++) { |
for (i = 0 ; i < PDP_SIZE; i++) { |
l3_pd[i] = pmap->pm_pdirpa[i] | PG_V; |
l3_pd[i] = pmap->pm_pdirpa[i] | PG_V; |
} |
} |
|
|
if (interrupts_enabled) |
if (interrupts_enabled) |
x86_enable_intr(); |
x86_enable_intr(); |
tlbflush(); |
tlbflush(); |