Please note that diffs are not public domain; they are subject to the copyright notices on the relevant files. =================================================================== RCS file: /ftp/cvs/cvsroot/src/sys/arch/x86/x86/lapic.c,v rcsdiff: /ftp/cvs/cvsroot/src/sys/arch/x86/x86/lapic.c,v: warning: Unknown phrases like `commitid ...;' are present. retrieving revision 1.12 retrieving revision 1.12.2.7 diff -u -p -r1.12 -r1.12.2.7 --- src/sys/arch/x86/x86/lapic.c 2005/05/29 21:36:40 1.12 +++ src/sys/arch/x86/x86/lapic.c 2007/12/07 17:27:01 1.12.2.7 @@ -1,4 +1,4 @@ -/* $NetBSD: lapic.c,v 1.12 2005/05/29 21:36:40 christos Exp $ */ +/* $NetBSD: lapic.c,v 1.12.2.7 2007/12/07 17:27:01 yamt Exp $ */ /*- * Copyright (c) 2000 The NetBSD Foundation, Inc. @@ -39,11 +39,11 @@ */ #include -__KERNEL_RCSID(0, "$NetBSD: lapic.c,v 1.12 2005/05/29 21:36:40 christos Exp $"); +__KERNEL_RCSID(0, "$NetBSD: lapic.c,v 1.12.2.7 2007/12/07 17:27:01 yamt Exp $"); #include "opt_ddb.h" -#include "opt_multiprocessor.h" #include "opt_mpbios.h" /* for MPDEBUG */ +#include "opt_multiprocessor.h" #include "opt_ntp.h" #include @@ -51,12 +51,14 @@ __KERNEL_RCSID(0, "$NetBSD: lapic.c,v 1. #include #include #include +#include #include #include #include +#include #include #include #include @@ -65,16 +67,18 @@ __KERNEL_RCSID(0, "$NetBSD: lapic.c,v 1. #include #include #include +#include #include #include #include -void lapic_delay __P((int)); -void lapic_microtime __P((struct timeval *)); -static u_int32_t lapic_gettick __P((void)); -void lapic_clockintr __P((void *, struct intrframe)); -static void lapic_map __P((paddr_t)); +/* Referenced from vector.S */ +void lapic_clockintr(void *, struct intrframe *); + +static void lapic_delay(unsigned int); +static uint32_t lapic_gettick(void); +static void lapic_map(paddr_t); static void lapic_hwmask(struct pic *, int); static void lapic_hwunmask(struct pic *, int); @@ -95,14 +99,13 @@ struct pic local_pic = { }; static void -lapic_map(lapic_base) - paddr_t lapic_base; +lapic_map(paddr_t lapic_base) { int s; pt_entry_t *pte; vaddr_t va = (vaddr_t)&local_apic; - disable_intr(); + x86_disable_intr(); s = lapic_tpr; /* @@ -123,20 +126,20 @@ lapic_map(lapic_base) #endif lapic_tpr = s; - enable_intr(); + x86_enable_intr(); } /* * enable local apic */ void -lapic_enable() +lapic_enable(void) { i82489_writereg(LAPIC_SVR, LAPIC_SVR_ENABLE | LAPIC_SPURIOUS_VECTOR); } void -lapic_set_lvt() +lapic_set_lvt(void) { struct cpu_info *ci = curcpu(); int i; @@ -177,7 +180,7 @@ lapic_set_lvt() i82489_writereg(LAPIC_LVINT1, mpi->redir); } } - + #ifdef MULTIPROCESSOR if (mp_verbose) { apic_format_redir (ci->ci_dev->dv_xname, "timer", 0, 0, @@ -198,14 +201,17 @@ lapic_set_lvt() * Initialize fixed idt vectors for use by local apic. */ void -lapic_boot_init(lapic_base) - paddr_t lapic_base; +lapic_boot_init(paddr_t lapic_base) { lapic_map(lapic_base); #ifdef MULTIPROCESSOR idt_allocmap[LAPIC_IPI_VECTOR] = 1; idt_vec_set(LAPIC_IPI_VECTOR, Xintr_lapic_ipi); + idt_allocmap[LAPIC_TLB_MCAST_VECTOR] = 1; + idt_vec_set(LAPIC_TLB_MCAST_VECTOR, Xintr_lapic_tlb_mcast); + idt_allocmap[LAPIC_TLB_BCAST_VECTOR] = 1; + idt_vec_set(LAPIC_TLB_BCAST_VECTOR, Xintr_lapic_tlb_bcast); #endif idt_allocmap[LAPIC_SPURIOUS_VECTOR] = 1; idt_vec_set(LAPIC_SPURIOUS_VECTOR, Xintrspurious); @@ -214,7 +220,8 @@ lapic_boot_init(lapic_base) idt_vec_set(LAPIC_TIMER_VECTOR, Xintr_lapic_ltimer); } -static inline u_int32_t lapic_gettick() +static uint32_t +lapic_gettick(void) { return i82489_readreg(LAPIC_CCR_TIMER); } @@ -222,63 +229,158 @@ static inline u_int32_t lapic_gettick() #include /* for hz */ int lapic_timer = 0; -u_int32_t lapic_tval; +uint32_t lapic_tval; /* * this gets us up to a 4GHz busclock.... */ -u_int32_t lapic_per_second; -u_int32_t lapic_frac_usec_per_cycle; -u_int64_t lapic_frac_cycle_per_usec; -u_int32_t lapic_delaytab[26]; +uint32_t lapic_per_second; +uint32_t lapic_frac_usec_per_cycle; +uint64_t lapic_frac_cycle_per_usec; +uint32_t lapic_delaytab[26]; -void -lapic_clockintr(void *arg, struct intrframe frame) +static u_int +lapic_get_timecount(struct timecounter *tc) { -#if defined(I586_CPU) || defined(I686_CPU) || defined(__x86_64__) - static int microset_iter; /* call cc_microset once/sec */ - struct cpu_info *ci = curcpu(); + struct cpu_info *ci; + uint32_t cur_timer; + int s; - ci->ci_isources[LIR_TIMER]->is_evcnt.ev_count++; + s = splhigh(); + ci = curcpu(); /* - * If we have a cycle counter, do the microset thing. + * Check for a race against the clockinterrupt. + * The update of ci_lapic_counter is blocked by splhigh() and + * the check for a pending clockinterrupt compensates for that. + * + * If the current tick is almost the Initial Counter, explicitly + * check for the pending interrupt bit as the interrupt delivery + * could be asynchronious and compensate as well. + * + * This can't be done without splhigh() as the calling code might + * have masked the clockinterrupt already. + * + * This code assumes that clockinterrupts are not missed. */ - if (ci->ci_feature_flags & CPUID_TSC) { - if ( -#if defined(MULTIPROCESSOR) - CPU_IS_PRIMARY(ci) && -#endif - (microset_iter--) == 0) { - microset_iter = hz - 1; - cc_microset_time = time; -#if defined(MULTIPROCESSOR) - x86_broadcast_ipi(X86_IPI_MICROSET); + cur_timer = lapic_gettick(); + if (cur_timer >= lapic_tval - 1) { + uint16_t reg = LAPIC_IRR + LAPIC_TIMER_VECTOR / 32 * 16; + + if (i82489_readreg(reg) & (1 << (LAPIC_TIMER_VECTOR % 32))) { + cur_timer -= lapic_tval; + } + } else if (ci->ci_istate.ipending & (1 << LIR_TIMER)) + cur_timer = lapic_gettick() - lapic_tval; + cur_timer = ci->ci_lapic_counter - cur_timer; + splx(s); + + return cur_timer; +} + +static struct timecounter lapic_timecounter = { + lapic_get_timecount, + NULL, + ~0u, + 0, + "lapic", +#ifndef MULTIPROCESSOR + 2100, +#else + -100, /* per CPU state */ #endif - cc_microset(ci); + NULL, + NULL, +}; + +extern u_int i8254_get_timecount(struct timecounter *); + +void +lapic_clockintr(void *arg, struct intrframe *frame) +{ +#if defined(TIMECOUNTER_DEBUG) + static u_int last_count[X86_MAXPROCS], + last_delta[X86_MAXPROCS], + last_tsc[X86_MAXPROCS], + last_tscdelta[X86_MAXPROCS], + last_factor[X86_MAXPROCS]; +#endif /* TIMECOUNTER_DEBUG && __HAVE_TIMECOUNTER */ + struct cpu_info *ci = curcpu(); + + ci->ci_lapic_counter += lapic_tval; + ci->ci_isources[LIR_TIMER]->is_evcnt.ev_count++; + +#if defined(TIMECOUNTER_DEBUG) + { + int cid = ci->ci_cpuid; + extern u_int i8254_get_timecount(struct timecounter *); + u_int c_count = i8254_get_timecount(NULL); + u_int c_tsc = cpu_counter32(); + u_int delta, ddelta, tsc_delta, factor = 0; + int idelta; + + if (c_count > last_count[cid]) + delta = c_count - last_count[cid]; + else + delta = 0x100000000ULL - last_count[cid] + c_count; + + if (delta > last_delta[cid]) + ddelta = delta - last_delta[cid]; + else + ddelta = last_delta[cid] - delta; + + if (c_tsc > last_tsc[cid]) + tsc_delta = c_tsc - last_tsc[cid]; + else + tsc_delta = 0x100000000ULL - last_tsc[cid] + c_tsc; + + idelta = tsc_delta - last_tscdelta[cid]; + if (idelta < 0) + idelta = -idelta; + + if (delta) { + int fdelta = tsc_delta / delta - last_factor[cid]; + if (fdelta < 0) + fdelta = -fdelta; + + if (fdelta > last_factor[cid] / 10) { + printf("cpu%d: freq skew exceeds 10%%: delta %u, factor %u, last %u\n", cid, fdelta, tsc_delta / delta, last_factor[cid]); + } + factor = tsc_delta / delta; } + + if (ddelta > last_delta[cid] / 10) { + printf("cpu%d: tick delta exceeds 10%%: delta %u, last %u, tick %u, last %u, factor %u, last %u\n", + cid, ddelta, last_delta[cid], c_count, last_count[cid], factor, last_factor[cid]); + } + + if (last_count[cid] > c_count) { + printf("cpu%d: tick wrapped/lost: delta %u, tick %u, last %u\n", cid, last_count[cid] - c_count, c_count, last_count[cid]); + } + + if (idelta > last_tscdelta[cid] / 10) { + printf("cpu%d: TSC delta exceeds 10%%: delta %u, last %u, tsc %u, factor %u, last %u\n", cid, idelta, last_tscdelta[cid], last_tsc[cid], + factor, last_factor[cid]); + } + + last_factor[cid] = factor; + last_delta[cid] = delta; + last_count[cid] = c_count; + last_tsc[cid] = c_tsc; + last_tscdelta[cid] = tsc_delta; } -#endif +#endif /* TIMECOUNTER_DEBUG */ - hardclock((struct clockframe *)&frame); + hardclock((struct clockframe *)frame); } -#ifdef NTP +#if !defined(__HAVE_TIMECOUNTER) && defined(NTP) extern int fixtick; -#endif /* NTP */ +#endif /* !__HAVE_TIMECOUNTER && NTP */ void lapic_initclocks() { - -#ifdef NTP - /* - * we'll actually get (lapic_per_second/lapic_tval) interrupts/sec. - */ - fixtick = 1000000 - - ((int64_t)tick * lapic_per_second + lapic_tval / 2) / lapic_tval; -#endif /* NTP */ - /* * Start local apic countdown timer running, in repeated mode. * @@ -292,9 +394,9 @@ lapic_initclocks() i82489_writereg (LAPIC_LVTT, LAPIC_LVTT_TM|LAPIC_TIMER_VECTOR); } -extern int gettick __P((void)); /* XXX put in header file */ +extern unsigned int gettick(void); /* XXX put in header file */ extern int rtclock_tval; /* XXX put in header file */ -extern void (*initclock_func) __P((void)); /* XXX put in header file */ +extern void (*initclock_func)(void); /* XXX put in header file */ /* * Calibrate the local apic count-down timer (which is running at @@ -308,16 +410,15 @@ extern void (*initclock_func) __P((void) * We're actually using the IRQ0 timer. Hmm. */ void -lapic_calibrate_timer(ci) - struct cpu_info *ci; +lapic_calibrate_timer(struct cpu_info *ci) { unsigned int starttick, tick1, tick2, endtick; unsigned int startapic, apic1, apic2, endapic; - u_int64_t dtick, dapic, tmp; + uint64_t dtick, dapic, tmp; int i; char tbuf[9]; - printf("%s: calibrating local timer\n", ci->ci_dev->dv_xname); + aprint_verbose("%s: calibrating local timer\n", ci->ci_dev->dv_xname); /* * Configure timer to one-shot, interrupt masked, @@ -331,12 +432,12 @@ lapic_calibrate_timer(ci) startapic = lapic_gettick(); for (i=0; ici_dev->dv_xname, tbuf); + aprint_verbose("%s: apic clock running at %s\n", + ci->ci_dev->dv_xname, tbuf); if (lapic_per_second != 0) { /* @@ -377,7 +479,7 @@ lapic_calibrate_timer(ci) /* * Compute fixed-point ratios between cycles and * microseconds to avoid having to do any division - * in lapic_delay and lapic_microtime. + * in lapic_delay. */ tmp = (1000000 * (u_int64_t)1<<32) / lapic_per_second; @@ -400,6 +502,16 @@ lapic_calibrate_timer(ci) */ delay_func = lapic_delay; initclock_func = lapic_initclocks; + initrtclock(0); + + if (lapic_timecounter.tc_frequency == 0) { + /* + * Hook up time counter. + * This assume that all LAPICs have the same frequency. + */ + lapic_timecounter.tc_frequency = lapic_per_second; + tc_init(&lapic_timecounter); + } } } @@ -407,8 +519,8 @@ lapic_calibrate_timer(ci) * delay for N usec. */ -void lapic_delay(usec) - int usec; +static void +lapic_delay(unsigned int usec) { int32_t xtick, otick; int64_t deltat; /* XXX may want to be 64bit */ @@ -438,10 +550,8 @@ void lapic_delay(usec) * XXX the following belong mostly or partly elsewhere.. */ -static __inline void i82489_icr_wait(void); - -static __inline void -i82489_icr_wait() +static void +i82489_icr_wait(void) { #ifdef DIAGNOSTIC unsigned j = 100000; @@ -458,8 +568,7 @@ i82489_icr_wait() } int -x86_ipi_init(target) - int target; +x86_ipi_init(int target) { if ((target&LAPIC_DEST_MASK)==0) { @@ -467,14 +576,14 @@ x86_ipi_init(target) } i82489_writereg(LAPIC_ICRLO, (target & LAPIC_DEST_MASK) | - LAPIC_DLMODE_INIT | LAPIC_LVL_ASSERT ); + LAPIC_DLMODE_INIT | LAPIC_LEVEL_ASSERT ); i82489_icr_wait(); - delay(10000); + i8254_delay(10000); i82489_writereg(LAPIC_ICRLO, (target & LAPIC_DEST_MASK) | - LAPIC_DLMODE_INIT | LAPIC_LVL_TRIG | LAPIC_LVL_DEASSERT); + LAPIC_DLMODE_INIT | LAPIC_TRIGGER_LEVEL | LAPIC_LEVEL_DEASSERT); i82489_icr_wait(); @@ -482,8 +591,7 @@ x86_ipi_init(target) } int -x86_ipi(vec,target,dl) - int vec,target,dl; +x86_ipi(int vec, int target, int dl) { int result, s; @@ -495,12 +603,15 @@ x86_ipi(vec,target,dl) i82489_writereg(LAPIC_ICRHI, target << LAPIC_ID_SHIFT); i82489_writereg(LAPIC_ICRLO, - (target & LAPIC_DEST_MASK) | vec | dl | LAPIC_LVL_ASSERT); + (target & LAPIC_DEST_MASK) | vec | dl | LAPIC_LEVEL_ASSERT); +#ifdef DIAGNOSTIC i82489_icr_wait(); - result = (i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) ? EBUSY : 0; - +#else + /* Don't wait - if it doesn't go, we're in big trouble anyway. */ + result = 0; +#endif splx(s); return result; @@ -542,6 +653,7 @@ lapic_hwunmask(struct pic *pic, int pin) } static void -lapic_setup(struct pic *pic, struct cpu_info *ci, int pin, int idtvec, int type) +lapic_setup(struct pic *pic, struct cpu_info *ci, + int pin, int idtvec, int type) { }