Annotation of src/sys/arch/mips/mips/cpu_subr.c, Revision 1.27
1.27 ! matt 1: /* $NetBSD: cpu_subr.c,v 1.26 2015/06/11 15:50:17 matt Exp $ */
1.3 rmind 2:
1.2 matt 3: /*-
4: * Copyright (c) 2010 The NetBSD Foundation, Inc.
5: * All rights reserved.
6: *
7: * This code is derived from software contributed to The NetBSD Foundation
8: * by Matt Thomas of 3am Software Foundry.
9: *
10: * Redistribution and use in source and binary forms, with or without
11: * modification, are permitted provided that the following conditions
12: * are met:
13: * 1. Redistributions of source code must retain the above copyright
14: * notice, this list of conditions and the following disclaimer.
15: * 2. Redistributions in binary form must reproduce the above copyright
16: * notice, this list of conditions and the following disclaimer in the
17: * documentation and/or other materials provided with the distribution.
18: *
19: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29: * POSSIBILITY OF SUCH DAMAGE.
30: */
31:
32: #include <sys/cdefs.h>
1.27 ! matt 33: __KERNEL_RCSID(0, "$NetBSD: cpu_subr.c,v 1.26 2015/06/11 15:50:17 matt Exp $");
1.2 matt 34:
1.27 ! matt 35: #include "opt_ddb.h"
1.21 matt 36: #include "opt_cputype.h"
1.27 ! matt 37: #include "opt_modular.h"
1.2 matt 38: #include "opt_multiprocessor.h"
39:
40: #include <sys/param.h>
41: #include <sys/cpu.h>
42: #include <sys/intr.h>
43: #include <sys/atomic.h>
44: #include <sys/device.h>
45: #include <sys/lwp.h>
46: #include <sys/proc.h>
47: #include <sys/ras.h>
1.27 ! matt 48: #include <sys/module.h>
1.2 matt 49: #include <sys/bitops.h>
50: #include <sys/idle.h>
51: #include <sys/xcall.h>
1.24 matt 52: #include <sys/kernel.h>
1.17 rmind 53: #include <sys/ipi.h>
1.2 matt 54:
55: #include <uvm/uvm.h>
56:
57: #include <mips/locore.h>
58: #include <mips/regnum.h>
59: #include <mips/pcb.h>
60: #include <mips/cache.h>
61: #include <mips/frame.h>
62: #include <mips/userret.h>
63: #include <mips/pte.h>
64:
65: #if defined(DDB) || defined(KGDB)
66: #ifdef DDB
67: #include <mips/db_machdep.h>
68: #include <ddb/db_command.h>
69: #include <ddb/db_output.h>
70: #endif
71: #endif
72:
1.20 matt 73: #ifdef MIPS64_OCTEON
74: extern struct cpu_softc octeon_cpu0_softc;
75: #endif
76:
1.2 matt 77: struct cpu_info cpu_info_store
1.20 matt 78: #if defined(MULTIPROCESSOR) && !defined(MIPS64_OCTEON)
1.2 matt 79: __section(".data1")
80: __aligned(1LU << ilog2((2*sizeof(struct cpu_info)-1)))
81: #endif
82: = {
83: .ci_curlwp = &lwp0,
84: .ci_tlb_info = &pmap_tlb0_info,
1.27 ! matt 85: .ci_pmap_kern_segtab = &pmap_kern_segtab,
! 86: .ci_pmap_user_segtab = NULL,
1.2 matt 87: #ifdef _LP64
1.27 ! matt 88: .ci_pmap_user_seg0tab = NULL,
1.2 matt 89: #endif
90: .ci_cpl = IPL_HIGH,
91: .ci_tlb_slot = -1,
92: #ifdef MULTIPROCESSOR
93: .ci_flags = CPUF_PRIMARY|CPUF_PRESENT|CPUF_RUNNING,
94: #endif
1.20 matt 95: #ifdef MIPS64_OCTEON
96: .ci_softc = &octeon_cpu0_softc,
97: #endif
1.2 matt 98: };
99:
1.14 matt 100: const pcu_ops_t * const pcu_ops_md_defs[PCU_UNIT_COUNT] = {
101: [PCU_FPU] = &mips_fpu_ops,
102: #if (MIPS32R2 + MIPS64R2) > 0
103: [PCU_DSP] = &mips_dsp_ops,
104: #endif
105: };
106:
1.2 matt 107: #ifdef MULTIPROCESSOR
1.20 matt 108: struct cpu_info * cpuid_infos[MAXCPUS] = {
109: [0] = &cpu_info_store,
110: };
1.2 matt 111:
1.24 matt 112: kcpuset_t *cpus_halted;
113: kcpuset_t *cpus_hatched;
114: kcpuset_t *cpus_paused;
115: kcpuset_t *cpus_resumed;
116: kcpuset_t *cpus_running;
1.2 matt 117:
1.24 matt 118: static void cpu_ipi_wait(const char *, const kcpuset_t *, const kcpuset_t *);
1.2 matt 119:
120: struct cpu_info *
1.6 matt 121: cpu_info_alloc(struct pmap_tlb_info *ti, cpuid_t cpu_id, cpuid_t cpu_package_id,
1.2 matt 122: cpuid_t cpu_core_id, cpuid_t cpu_smt_id)
123: {
1.20 matt 124: KASSERT(cpu_id < MAXCPUS);
125:
1.21 matt 126: #ifdef MIPS64_OCTEON
1.20 matt 127: vaddr_t exc_page = MIPS_UTLB_MISS_EXC_VEC + 0x1000*cpu_id;
1.23 matt 128: __CTASSERT(sizeof(struct cpu_info) + sizeof(struct pmap_tlb_info) <= 0x1000 - 0x280);
1.20 matt 129:
1.23 matt 130: struct cpu_info * const ci = ((struct cpu_info *)(exc_page + 0x1000)) - 1;
1.21 matt 131: memset((void *)exc_page, 0, PAGE_SIZE);
1.20 matt 132:
133: if (ti == NULL) {
134: ti = ((struct pmap_tlb_info *)ci) - 1;
135: pmap_tlb_info_init(ti);
136: }
137: #else
138: const vaddr_t cpu_info_offset = (vaddr_t)&cpu_info_store & PAGE_MASK;
1.21 matt 139: struct pglist pglist;
140: int error;
1.20 matt 141:
1.2 matt 142: /*
143: * Grab a page from the first 512MB (mappable by KSEG0) to use to store
144: * exception vectors and cpu_info for this cpu.
145: */
146: error = uvm_pglistalloc(PAGE_SIZE,
147: 0, MIPS_KSEG1_START - MIPS_KSEG0_START,
148: PAGE_SIZE, PAGE_SIZE, &pglist, 1, false);
149: if (error)
150: return NULL;
151:
152: const paddr_t pa = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist));
153: const vaddr_t va = MIPS_PHYS_TO_KSEG0(pa);
154: struct cpu_info * const ci = (void *) (va + cpu_info_offset);
155: memset((void *)va, 0, PAGE_SIZE);
156:
157: /*
158: * If we weren't passed a pmap_tlb_info to use, the caller wants us
159: * to take care of that for him. Since we have room left over in the
160: * page we just allocated, just use a piece of that for it.
161: */
162: if (ti == NULL) {
163: if (cpu_info_offset >= sizeof(*ti)) {
164: ti = (void *) va;
165: } else {
166: KASSERT(PAGE_SIZE - cpu_info_offset + sizeof(*ci) >= sizeof(*ti));
167: ti = (struct pmap_tlb_info *)(va + PAGE_SIZE) - 1;
168: }
169: pmap_tlb_info_init(ti);
170: }
171:
1.20 matt 172: /*
173: * Attach its TLB info (which must be direct-mapped)
174: */
175: #ifdef _LP64
176: KASSERT(MIPS_KSEG0_P(ti) || MIPS_XKPHYS_P(ti));
177: #else
178: KASSERT(MIPS_KSEG0_P(ti));
179: #endif
180: #endif /* MIPS64_OCTEON */
181:
1.21 matt 182: KASSERT(cpu_id != 0);
1.2 matt 183: ci->ci_cpuid = cpu_id;
1.6 matt 184: ci->ci_data.cpu_package_id = cpu_package_id;
1.2 matt 185: ci->ci_data.cpu_core_id = cpu_core_id;
186: ci->ci_data.cpu_smt_id = cpu_smt_id;
187: ci->ci_cpu_freq = cpu_info_store.ci_cpu_freq;
188: ci->ci_cctr_freq = cpu_info_store.ci_cctr_freq;
189: ci->ci_cycles_per_hz = cpu_info_store.ci_cycles_per_hz;
190: ci->ci_divisor_delay = cpu_info_store.ci_divisor_delay;
191: ci->ci_divisor_recip = cpu_info_store.ci_divisor_recip;
1.11 matt 192: ci->ci_cpuwatch_count = cpu_info_store.ci_cpuwatch_count;
1.2 matt 193:
1.27 ! matt 194: pmap_md_alloc_ephemeral_address_space(ci);
1.2 matt 195:
196: mi_cpu_attach(ci);
197:
198: pmap_tlb_info_attach(ti, ci);
199:
200: return ci;
201: }
202: #endif /* MULTIPROCESSOR */
203:
1.5 matt 204: static void
205: cpu_hwrena_setup(void)
206: {
207: #if (MIPS32R2 + MIPS64R2) > 0
208: const int cp0flags = mips_options.mips_cpu->cpu_cp0flags;
209: if ((cp0flags & MIPS_CP0FL_USE) == 0)
210: return;
211:
212: if (cp0flags & MIPS_CP0FL_HWRENA) {
213: mipsNN_cp0_hwrena_write(
214: MIPS_HWRENA_UL
215: |MIPS_HWRENA_CCRES
216: |MIPS_HWRENA_CC
217: |MIPS_HWRENA_SYNCI_STEP
218: |MIPS_HWRENA_CPUNUM);
219: if (cp0flags & MIPS_CP0FL_USERLOCAL) {
220: mipsNN_cp0_userlocal_write(curlwp->l_private);
221: }
222: }
223: #endif
224: }
225:
1.2 matt 226: void
227: cpu_attach_common(device_t self, struct cpu_info *ci)
228: {
1.6 matt 229: const char * const xname = device_xname(self);
230:
1.2 matt 231: /*
232: * Cross link cpu_info and its device together
233: */
234: ci->ci_dev = self;
235: self->dv_private = ci;
236: KASSERT(ci->ci_idepth == 0);
237:
238: evcnt_attach_dynamic(&ci->ci_ev_count_compare,
1.6 matt 239: EVCNT_TYPE_INTR, NULL, xname,
1.2 matt 240: "int 5 (clock)");
241: evcnt_attach_dynamic(&ci->ci_ev_count_compare_missed,
1.6 matt 242: EVCNT_TYPE_INTR, NULL, xname,
1.2 matt 243: "int 5 (clock) missed");
244: evcnt_attach_dynamic(&ci->ci_ev_fpu_loads,
1.6 matt 245: EVCNT_TYPE_MISC, NULL, xname,
1.2 matt 246: "fpu loads");
247: evcnt_attach_dynamic(&ci->ci_ev_fpu_saves,
1.6 matt 248: EVCNT_TYPE_MISC, NULL, xname,
1.2 matt 249: "fpu saves");
1.14 matt 250: evcnt_attach_dynamic(&ci->ci_ev_dsp_loads,
251: EVCNT_TYPE_MISC, NULL, xname,
252: "dsp loads");
253: evcnt_attach_dynamic(&ci->ci_ev_dsp_saves,
254: EVCNT_TYPE_MISC, NULL, xname,
255: "dsp saves");
1.2 matt 256: evcnt_attach_dynamic(&ci->ci_ev_tlbmisses,
1.6 matt 257: EVCNT_TYPE_TRAP, NULL, xname,
1.2 matt 258: "tlb misses");
259:
260: #ifdef MULTIPROCESSOR
261: if (ci != &cpu_info_store) {
262: /*
263: * Tail insert this onto the list of cpu_info's.
264: */
1.20 matt 265: KASSERT(cpuid_infos[ci->ci_cpuid] == NULL);
266: cpuid_infos[ci->ci_cpuid] = ci;
267: membar_producer();
1.2 matt 268: }
1.21 matt 269: KASSERT(cpuid_infos[ci->ci_cpuid] != NULL);
1.2 matt 270: evcnt_attach_dynamic(&ci->ci_evcnt_synci_activate_rqst,
1.6 matt 271: EVCNT_TYPE_MISC, NULL, xname,
1.2 matt 272: "syncicache activate request");
273: evcnt_attach_dynamic(&ci->ci_evcnt_synci_deferred_rqst,
1.6 matt 274: EVCNT_TYPE_MISC, NULL, xname,
1.2 matt 275: "syncicache deferred request");
276: evcnt_attach_dynamic(&ci->ci_evcnt_synci_ipi_rqst,
1.6 matt 277: EVCNT_TYPE_MISC, NULL, xname,
1.2 matt 278: "syncicache ipi request");
279: evcnt_attach_dynamic(&ci->ci_evcnt_synci_onproc_rqst,
1.6 matt 280: EVCNT_TYPE_MISC, NULL, xname,
1.2 matt 281: "syncicache onproc request");
282:
283: /*
284: * Initialize IPI framework for this cpu instance
285: */
286: ipi_init(ci);
287: #endif
288: }
289:
290: void
291: cpu_startup_common(void)
292: {
293: vaddr_t minaddr, maxaddr;
294: char pbuf[9]; /* "99999 MB" */
295:
296: pmap_tlb_info_evcnt_attach(&pmap_tlb0_info);
297:
1.24 matt 298: #ifdef MULTIPROCESSOR
299: kcpuset_create(&cpus_halted, true);
300: KASSERT(cpus_halted != NULL);
301: kcpuset_create(&cpus_hatched, true);
302: KASSERT(cpus_hatched != NULL);
303: kcpuset_create(&cpus_paused, true);
304: KASSERT(cpus_paused != NULL);
305: kcpuset_create(&cpus_resumed, true);
306: KASSERT(cpus_resumed != NULL);
307: kcpuset_create(&cpus_running, true);
308: KASSERT(cpus_running != NULL);
309: kcpuset_set(cpus_hatched, cpu_number());
310: kcpuset_set(cpus_running, cpu_number());
311: #endif
312:
1.5 matt 313: cpu_hwrena_setup();
314:
1.2 matt 315: /*
316: * Good {morning,afternoon,evening,night}.
317: */
318: printf("%s%s", copyright, version);
1.18 matt 319: printf("%s\n", cpu_getmodel());
1.2 matt 320: format_bytes(pbuf, sizeof(pbuf), ctob(physmem));
321: printf("total memory = %s\n", pbuf);
322:
323: minaddr = 0;
324: /*
325: * Allocate a submap for physio.
326: */
327: phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
328: VM_PHYS_SIZE, 0, FALSE, NULL);
329:
330: /*
331: * (No need to allocate an mbuf cluster submap. Mbuf clusters
332: * are allocated via the pool allocator, and we use KSEG/XKPHYS to
333: * map those pages.)
334: */
335:
336: format_bytes(pbuf, sizeof(pbuf), ptoa(uvmexp.free));
337: printf("avail memory = %s\n", pbuf);
1.27 ! matt 338:
! 339: #if defined(__mips_n32)
! 340: module_machine = "mips-n32";
! 341: #endif
1.2 matt 342: }
343:
344: void
345: cpu_getmcontext(struct lwp *l, mcontext_t *mcp, unsigned int *flags)
346: {
347: const struct trapframe *tf = l->l_md.md_utf;
348: __greg_t *gr = mcp->__gregs;
349: __greg_t ras_pc;
350:
351: /* Save register context. Dont copy R0 - it is always 0 */
352: memcpy(&gr[_REG_AT], &tf->tf_regs[_R_AST], sizeof(mips_reg_t) * 31);
353:
354: gr[_REG_MDLO] = tf->tf_regs[_R_MULLO];
355: gr[_REG_MDHI] = tf->tf_regs[_R_MULHI];
356: gr[_REG_CAUSE] = tf->tf_regs[_R_CAUSE];
357: gr[_REG_EPC] = tf->tf_regs[_R_PC];
358: gr[_REG_SR] = tf->tf_regs[_R_SR];
1.6 matt 359: mcp->_mc_tlsbase = (intptr_t)l->l_private;
1.2 matt 360:
361: if ((ras_pc = (intptr_t)ras_lookup(l->l_proc,
362: (void *) (intptr_t)gr[_REG_EPC])) != -1)
363: gr[_REG_EPC] = ras_pc;
364:
1.6 matt 365: *flags |= _UC_CPU | _UC_TLSBASE;
1.4 joerg 366:
1.2 matt 367: /* Save floating point register context, if any. */
1.13 rmind 368: KASSERT(l == curlwp);
369: if (fpu_used_p()) {
1.2 matt 370: size_t fplen;
371: /*
372: * If this process is the current FP owner, dump its
373: * context to the PCB first.
374: */
375: fpu_save();
376:
377: /*
378: * The PCB FP regs struct includes the FP CSR, so use the
379: * size of __fpregs.__fp_r when copying.
380: */
381: #if !defined(__mips_o32)
382: if (_MIPS_SIM_NEWABI_P(l->l_proc->p_md.md_abi)) {
383: #endif
384: fplen = sizeof(struct fpreg);
385: #if !defined(__mips_o32)
386: } else {
387: fplen = sizeof(struct fpreg_oabi);
388: }
389: #endif
390: struct pcb * const pcb = lwp_getpcb(l);
391: memcpy(&mcp->__fpregs, &pcb->pcb_fpregs, fplen);
392: *flags |= _UC_FPU;
393: }
394: }
395:
396: int
1.16 martin 397: cpu_mcontext_validate(struct lwp *l, const mcontext_t *mcp)
398: {
399: /* XXX: Do we validate the addresses?? */
400: return 0;
401: }
402:
403: int
1.2 matt 404: cpu_setmcontext(struct lwp *l, const mcontext_t *mcp, unsigned int flags)
405: {
406: struct trapframe *tf = l->l_md.md_utf;
407: struct proc *p = l->l_proc;
408: const __greg_t *gr = mcp->__gregs;
1.16 martin 409: int error;
1.2 matt 410:
411: /* Restore register context, if any. */
412: if (flags & _UC_CPU) {
1.16 martin 413: error = cpu_mcontext_validate(l, mcp);
414: if (error)
415: return error;
416:
1.2 matt 417: /* Save register context. */
1.16 martin 418:
1.2 matt 419: #ifdef __mips_n32
420: CTASSERT(_R_AST == _REG_AT);
421: if (__predict_false(p->p_md.md_abi == _MIPS_BSD_API_O32)) {
422: const mcontext_o32_t *mcp32 = (const mcontext_o32_t *)mcp;
423: const __greg32_t *gr32 = mcp32->__gregs;
424: for (size_t i = _R_AST; i < 32; i++) {
425: tf->tf_regs[i] = gr32[i];
426: }
427: } else
428: #endif
429: memcpy(&tf->tf_regs[_R_AST], &gr[_REG_AT],
430: sizeof(mips_reg_t) * 31);
431:
432: tf->tf_regs[_R_MULLO] = gr[_REG_MDLO];
433: tf->tf_regs[_R_MULHI] = gr[_REG_MDHI];
434: tf->tf_regs[_R_CAUSE] = gr[_REG_CAUSE];
435: tf->tf_regs[_R_PC] = gr[_REG_EPC];
436: /* Do not restore SR. */
437: }
438:
1.6 matt 439: /* Restore the private thread context */
440: if (flags & _UC_TLSBASE) {
441: lwp_setprivate(l, (void *)(intptr_t)mcp->_mc_tlsbase);
442: }
443:
1.2 matt 444: /* Restore floating point register context, if any. */
445: if (flags & _UC_FPU) {
446: size_t fplen;
447:
448: /* Disable the FPU contents. */
449: fpu_discard();
450:
451: #if !defined(__mips_o32)
452: if (_MIPS_SIM_NEWABI_P(l->l_proc->p_md.md_abi)) {
453: #endif
454: fplen = sizeof(struct fpreg);
455: #if !defined(__mips_o32)
456: } else {
457: fplen = sizeof(struct fpreg_oabi);
458: }
459: #endif
460: /*
461: * The PCB FP regs struct includes the FP CSR, so use the
462: * proper size of fpreg when copying.
463: */
464: struct pcb * const pcb = lwp_getpcb(l);
465: memcpy(&pcb->pcb_fpregs, &mcp->__fpregs, fplen);
466: }
467:
468: mutex_enter(p->p_lock);
469: if (flags & _UC_SETSTACK)
470: l->l_sigstk.ss_flags |= SS_ONSTACK;
471: if (flags & _UC_CLRSTACK)
472: l->l_sigstk.ss_flags &= ~SS_ONSTACK;
473: mutex_exit(p->p_lock);
474:
475: return (0);
476: }
477:
478: void
479: cpu_need_resched(struct cpu_info *ci, int flags)
480: {
481: struct lwp * const l = ci->ci_data.cpu_onproc;
482: #ifdef MULTIPROCESSOR
483: struct cpu_info * const cur_ci = curcpu();
484: #endif
485:
486: KASSERT(kpreempt_disabled());
487:
488: ci->ci_want_resched |= flags;
489:
490: if (__predict_false((l->l_pflag & LP_INTR) != 0)) {
491: /*
492: * No point doing anything, it will switch soon.
493: * Also here to prevent an assertion failure in
494: * kpreempt() due to preemption being set on a
495: * soft interrupt LWP.
496: */
497: return;
498: }
499:
500: if (__predict_false(l == ci->ci_data.cpu_idlelwp)) {
501: #ifdef MULTIPROCESSOR
502: /*
503: * If the other CPU is idling, it must be waiting for an
504: * interrupt. So give it one.
505: */
506: if (__predict_false(ci != cur_ci))
507: cpu_send_ipi(ci, IPI_NOP);
508: #endif
509: return;
510: }
511:
512: #ifdef MULTIPROCESSOR
513: atomic_or_uint(&ci->ci_want_resched, flags);
514: #else
515: ci->ci_want_resched |= flags;
516: #endif
517:
518: if (flags & RESCHED_KPREEMPT) {
519: #ifdef __HAVE_PREEMPTION
520: atomic_or_uint(&l->l_dopreempt, DOPREEMPT_ACTIVE);
521: if (ci == cur_ci) {
522: softint_trigger(SOFTINT_KPREEMPT);
523: } else {
524: cpu_send_ipi(ci, IPI_KPREEMPT);
525: }
526: #endif
527: return;
528: }
529: l->l_md.md_astpending = 1; /* force call to ast() */
530: #ifdef MULTIPROCESSOR
531: if (ci != cur_ci && (flags & RESCHED_IMMED)) {
532: cpu_send_ipi(ci, IPI_AST);
533: }
534: #endif
535: }
536:
1.27 ! matt 537: uint32_t
! 538: cpu_clkf_usermode_mask(void)
! 539: {
! 540: return CPUISMIPS3 ? MIPS_SR_KSU_USER : MIPS_SR_KU_PREV;
! 541: }
! 542:
1.2 matt 543: void
544: cpu_signotify(struct lwp *l)
545: {
546: KASSERT(kpreempt_disabled());
547: #ifdef __HAVE_FAST_SOFTINTS
548: KASSERT(lwp_locked(l, NULL));
549: #endif
550: KASSERT(l->l_stat == LSONPROC || l->l_stat == LSRUN || l->l_stat == LSSTOP);
551:
552: l->l_md.md_astpending = 1; /* force call to ast() */
553: }
554:
555: void
556: cpu_need_proftick(struct lwp *l)
557: {
558: KASSERT(kpreempt_disabled());
559: KASSERT(l->l_cpu == curcpu());
560:
561: l->l_pflag |= LP_OWEUPC;
562: l->l_md.md_astpending = 1; /* force call to ast() */
563: }
564:
565: void
566: cpu_set_curpri(int pri)
567: {
568: kpreempt_disable();
569: curcpu()->ci_schedstate.spc_curpriority = pri;
570: kpreempt_enable();
571: }
572:
573:
574: #ifdef __HAVE_PREEMPTION
575: bool
576: cpu_kpreempt_enter(uintptr_t where, int s)
577: {
578: KASSERT(kpreempt_disabled());
579:
580: #if 0
581: if (where == (intptr_t)-2) {
582: KASSERT(curcpu()->ci_mtx_count == 0);
583: /*
584: * We must be called via kern_intr (which already checks for
585: * IPL_NONE so of course we call be preempted).
586: */
587: return true;
588: }
589: /*
590: * We are called from KPREEMPT_ENABLE(). If we are at IPL_NONE,
591: * of course we can be preempted. If we aren't, ask for a
592: * softint so that kern_intr can call kpreempt.
593: */
594: if (s == IPL_NONE) {
595: KASSERT(curcpu()->ci_mtx_count == 0);
596: return true;
597: }
598: softint_trigger(SOFTINT_KPREEMPT);
599: #endif
600: return false;
601: }
602:
603: void
604: cpu_kpreempt_exit(uintptr_t where)
605: {
606:
607: /* do nothing */
608: }
609:
610: /*
611: * Return true if preemption is disabled for MD reasons. Must be called
612: * with preemption disabled, and thus is only for diagnostic checks.
613: */
614: bool
615: cpu_kpreempt_disabled(void)
616: {
617: /*
618: * Any elevated IPL disables preemption.
619: */
620: return curcpu()->ci_cpl > IPL_NONE;
621: }
622: #endif /* __HAVE_PREEMPTION */
623:
624: void
625: cpu_idle(void)
626: {
627: void (*const mach_idle)(void) = mips_locoresw.lsw_cpu_idle;
628: struct cpu_info * const ci = curcpu();
629:
630: while (!ci->ci_want_resched) {
631: #ifdef __HAVE_FAST_SOFTINTS
632: KASSERT(ci->ci_data.cpu_softints == 0);
633: #endif
634: (*mach_idle)();
635: }
636: }
637:
638: bool
639: cpu_intr_p(void)
640: {
641: bool rv;
642: kpreempt_disable();
643: rv = (curcpu()->ci_idepth != 0);
644: kpreempt_enable();
645: return rv;
646: }
647:
648: #ifdef MULTIPROCESSOR
649:
650: void
651: cpu_broadcast_ipi(int tag)
652: {
1.24 matt 653: // No reason to remove ourselves since multicast_ipi will do that for us
654: cpu_multicast_ipi(cpus_running, tag);
1.2 matt 655: }
656:
657: void
1.24 matt 658: cpu_multicast_ipi(const kcpuset_t *kcp, int tag)
1.2 matt 659: {
1.24 matt 660: struct cpu_info * const ci = curcpu();
661: kcpuset_t *kcp2;
1.2 matt 662:
1.24 matt 663: if (kcpuset_match(cpus_running, ci->ci_data.cpu_kcpuset))
1.2 matt 664: return;
665:
1.24 matt 666: kcpuset_clone(&kcp2, kcp);
667: kcpuset_remove(kcp2, ci->ci_data.cpu_kcpuset);
668: for (cpuid_t cii; (cii = kcpuset_ffs(kcp2)) != 0; ) {
669: kcpuset_clear(kcp2, --cii);
670: (void)cpu_send_ipi(cpu_lookup(cii), tag);
1.2 matt 671: }
1.24 matt 672: kcpuset_destroy(kcp2);
1.2 matt 673: }
674:
675: int
676: cpu_send_ipi(struct cpu_info *ci, int tag)
677: {
678:
679: return (*mips_locoresw.lsw_send_ipi)(ci, tag);
680: }
681:
682: static void
1.24 matt 683: cpu_ipi_wait(const char *s, const kcpuset_t *watchset, const kcpuset_t *wanted)
1.2 matt 684: {
1.24 matt 685: bool done = false;
686: kcpuset_t *kcp;
687: kcpuset_create(&kcp, false);
688:
689: /* some finite amount of time */
690:
691: for (u_long limit = curcpu()->ci_cpu_freq/10; !done && limit--; ) {
692: kcpuset_copy(kcp, watchset);
693: kcpuset_intersect(kcp, wanted);
694: done = kcpuset_match(kcp, wanted);
695: }
696:
697: if (!done) {
698: cpuid_t cii;
699: kcpuset_copy(kcp, wanted);
700: kcpuset_remove(kcp, watchset);
701: if ((cii = kcpuset_ffs(kcp)) != 0) {
702: printf("Failed to %s:", s);
703: do {
704: kcpuset_clear(kcp, --cii);
705: printf(" cpu%lu", cii);
706: } while ((cii = kcpuset_ffs(kcp)) != 0);
707: printf("\n");
708: }
1.2 matt 709: }
710:
1.24 matt 711: kcpuset_destroy(kcp);
1.2 matt 712: }
713:
714: /*
715: * Halt this cpu
716: */
717: void
718: cpu_halt(void)
719: {
1.24 matt 720: cpuid_t cii = cpu_index(curcpu());
1.2 matt 721:
1.24 matt 722: printf("cpu%lu: shutting down\n", cii);
723: kcpuset_atomic_set(cpus_halted, cii);
1.2 matt 724: spl0(); /* allow interrupts e.g. further ipi ? */
725: for (;;) ; /* spin */
726:
727: /* NOTREACHED */
728: }
729:
730: /*
731: * Halt all running cpus, excluding current cpu.
732: */
733: void
734: cpu_halt_others(void)
735: {
1.24 matt 736: kcpuset_t *kcp;
1.2 matt 737:
1.24 matt 738: // If we are the only CPU running, there's nothing to do.
739: if (kcpuset_match(cpus_running, curcpu()->ci_data.cpu_kcpuset))
1.2 matt 740: return;
741:
1.24 matt 742: // Get all running CPUs
743: kcpuset_clone(&kcp, cpus_running);
744: // Remove ourself
745: kcpuset_remove(kcp, curcpu()->ci_data.cpu_kcpuset);
746: // Remove any halted CPUs
747: kcpuset_remove(kcp, cpus_halted);
748: // If there are CPUs left, send the IPIs
749: if (!kcpuset_iszero(kcp)) {
750: cpu_multicast_ipi(kcp, IPI_HALT);
751: cpu_ipi_wait("halt", cpus_halted, kcp);
752: }
753: kcpuset_destroy(kcp);
1.2 matt 754:
755: /*
756: * TBD
757: * Depending on available firmware methods, other cpus will
1.24 matt 758: * either shut down themselves, or spin and wait for us to
1.2 matt 759: * stop them.
760: */
761: }
762:
763: /*
764: * Pause this cpu
765: */
766: void
767: cpu_pause(struct reg *regsp)
768: {
769: int s = splhigh();
1.24 matt 770: cpuid_t cii = cpu_index(curcpu());
1.2 matt 771:
1.24 matt 772: if (__predict_false(cold))
773: return;
774:
775: do {
776: kcpuset_atomic_set(cpus_paused, cii);
1.2 matt 777: do {
778: ;
1.24 matt 779: } while (kcpuset_isset(cpus_paused, cii));
780: kcpuset_atomic_set(cpus_resumed, cii);
1.2 matt 781: #if defined(DDB)
1.6 matt 782: if (ddb_running_on_this_cpu_p())
1.2 matt 783: cpu_Debugger();
1.6 matt 784: if (ddb_running_on_any_cpu_p())
1.2 matt 785: continue;
786: #endif
1.24 matt 787: } while (false);
1.2 matt 788:
789: splx(s);
790: }
791:
792: /*
793: * Pause all running cpus, excluding current cpu.
794: */
795: void
796: cpu_pause_others(void)
797: {
1.24 matt 798: struct cpu_info * const ci = curcpu();
799: kcpuset_t *kcp;
1.2 matt 800:
1.24 matt 801: if (cold || kcpuset_match(cpus_running, ci->ci_data.cpu_kcpuset))
802: return;
803:
804: kcpuset_clone(&kcp, cpus_running);
805: kcpuset_remove(kcp, ci->ci_data.cpu_kcpuset);
806: kcpuset_remove(kcp, cpus_paused);
1.2 matt 807:
1.24 matt 808: cpu_broadcast_ipi(IPI_SUSPEND);
809: cpu_ipi_wait("pause", cpus_paused, kcp);
1.2 matt 810:
1.24 matt 811: kcpuset_destroy(kcp);
1.2 matt 812: }
813:
814: /*
815: * Resume a single cpu
816: */
817: void
1.24 matt 818: cpu_resume(cpuid_t cii)
1.2 matt 819: {
1.24 matt 820: kcpuset_t *kcp;
821:
822: if (__predict_false(cold))
823: return;
824:
825: kcpuset_create(&kcp, true);
826: kcpuset_set(kcp, cii);
827: kcpuset_atomicly_remove(cpus_resumed, cpus_resumed);
828: kcpuset_atomic_clear(cpus_paused, cii);
829:
830: cpu_ipi_wait("resume", cpus_resumed, kcp);
1.2 matt 831:
1.24 matt 832: kcpuset_destroy(kcp);
1.2 matt 833: }
834:
835: /*
836: * Resume all paused cpus.
837: */
838: void
839: cpu_resume_others(void)
840: {
1.24 matt 841: kcpuset_t *kcp;
842:
843: if (__predict_false(cold))
844: return;
1.2 matt 845:
1.24 matt 846: kcpuset_atomicly_remove(cpus_resumed, cpus_resumed);
847: kcpuset_clone(&kcp, cpus_paused);
848: kcpuset_atomicly_remove(cpus_paused, cpus_paused);
1.2 matt 849:
850: /* CPUs awake on cpus_paused clear */
1.24 matt 851: cpu_ipi_wait("resume", cpus_resumed, kcp);
852:
853: kcpuset_destroy(kcp);
1.2 matt 854: }
855:
1.24 matt 856: bool
857: cpu_is_paused(cpuid_t cii)
1.2 matt 858: {
859:
1.24 matt 860: return !cold && kcpuset_isset(cpus_paused, cii);
1.2 matt 861: }
862:
1.7 cliff 863: #ifdef DDB
1.2 matt 864: void
865: cpu_debug_dump(void)
866: {
867: CPU_INFO_ITERATOR cii;
868: struct cpu_info *ci;
869: char running, hatched, paused, resumed, halted;
870:
871: db_printf("CPU CPUID STATE CPUINFO CPL INT MTX IPIS\n");
872: for (CPU_INFO_FOREACH(cii, ci)) {
1.24 matt 873: hatched = (kcpuset_isset(cpus_hatched, cpu_index(ci)) ? 'H' : '-');
874: running = (kcpuset_isset(cpus_running, cpu_index(ci)) ? 'R' : '-');
875: paused = (kcpuset_isset(cpus_paused, cpu_index(ci)) ? 'P' : '-');
876: resumed = (kcpuset_isset(cpus_resumed, cpu_index(ci)) ? 'r' : '-');
877: halted = (kcpuset_isset(cpus_halted, cpu_index(ci)) ? 'h' : '-');
1.2 matt 878: db_printf("%3d 0x%03lx %c%c%c%c%c %p "
879: "%3d %3d %3d "
880: "0x%02" PRIx64 "/0x%02" PRIx64 "\n",
881: cpu_index(ci), ci->ci_cpuid,
882: running, hatched, paused, resumed, halted,
883: ci, ci->ci_cpl, ci->ci_idepth, ci->ci_mtx_count,
884: ci->ci_active_ipis, ci->ci_request_ipis);
885: }
886: }
1.7 cliff 887: #endif
1.2 matt 888:
889: void
890: cpu_hatch(struct cpu_info *ci)
891: {
892: struct pmap_tlb_info * const ti = ci->ci_tlb_info;
893:
894: /*
895: * Invalidate all the TLB enties (even wired ones) and then reserve
896: * space for the wired TLB entries.
897: */
898: mips3_cp0_wired_write(0);
899: tlb_invalidate_all();
900: mips3_cp0_wired_write(ti->ti_wired);
901:
902: /*
1.5 matt 903: * Setup HWRENA and USERLOCAL COP0 registers (MIPSxxR2).
904: */
905: cpu_hwrena_setup();
906:
907: /*
1.2 matt 908: * If we are using register zero relative addressing to access cpu_info
909: * in the exception vectors, enter that mapping into TLB now.
910: */
911: if (ci->ci_tlb_slot >= 0) {
912: const uint32_t tlb_lo = MIPS3_PG_G|MIPS3_PG_V
913: | mips3_paddr_to_tlbpfn((vaddr_t)ci);
1.27 ! matt 914: const struct tlbmask tlbmask = {
! 915: .tlb_hi = -PAGE_SIZE | KERNEL_PID,
! 916: #if (PGSHIFT & 1)
! 917: .tlb_lo0 = tlb_lo,
! 918: .tlb_lo1 = tlb_lo + MIPS3_PG_NEXT,
! 919: #else
! 920: .tlb_lo0 = 0,
! 921: .tlb_lo1 = tlb_lo,
! 922: #endif
! 923: .tlb_mask = -1,
! 924: };
1.2 matt 925:
1.27 ! matt 926: tlb_invalidate_addr(tlbmask.tlb_hi, KERNEL_PID);
! 927: tlb_write_entry(ci->ci_tlb_slot, &tlbmask);
1.2 matt 928: }
929:
930: /*
931: * Flush the icache just be sure.
932: */
933: mips_icache_sync_all();
934:
935: /*
936: * Let this CPU do its own initialization (for things that have to be
937: * done on the local CPU).
938: */
939: (*mips_locoresw.lsw_cpu_init)(ci);
940:
1.21 matt 941: // Show this CPU as present.
942: atomic_or_ulong(&ci->ci_flags, CPUF_PRESENT);
943:
1.2 matt 944: /*
945: * Announce we are hatched
946: */
1.24 matt 947: kcpuset_atomic_set(cpus_hatched, cpu_index(ci));
1.2 matt 948:
949: /*
950: * Now wait to be set free!
951: */
1.24 matt 952: while (! kcpuset_isset(cpus_running, cpu_index(ci))) {
1.2 matt 953: /* spin, spin, spin */
954: }
955:
956: /*
957: * initialize the MIPS count/compare clock
958: */
959: mips3_cp0_count_write(ci->ci_data.cpu_cc_skew);
960: KASSERT(ci->ci_cycles_per_hz != 0);
961: ci->ci_next_cp0_clk_intr = ci->ci_data.cpu_cc_skew + ci->ci_cycles_per_hz;
962: mips3_cp0_compare_write(ci->ci_next_cp0_clk_intr);
963: ci->ci_data.cpu_cc_skew = 0;
964:
965: /*
1.7 cliff 966: * Let this CPU do its own post-running initialization
967: * (for things that have to be done on the local CPU).
968: */
1.8 cliff 969: (*mips_locoresw.lsw_cpu_run)(ci);
1.7 cliff 970:
971: /*
1.23 matt 972: * Now turn on interrupts (and verify they are on).
1.2 matt 973: */
974: spl0();
1.23 matt 975: KASSERTMSG(ci->ci_cpl == IPL_NONE, "cpl %d", ci->ci_cpl);
976: KASSERT(mips_cp0_status_read() & MIPS_SR_INT_IE);
1.2 matt 977:
1.24 matt 978: kcpuset_atomic_set(pmap_kernel()->pm_onproc, cpu_index(ci));
979: kcpuset_atomic_set(pmap_kernel()->pm_active, cpu_index(ci));
980:
1.2 matt 981: /*
982: * And do a tail call to idle_loop
983: */
984: idle_loop(NULL);
985: }
986:
987: void
988: cpu_boot_secondary_processors(void)
989: {
1.20 matt 990: CPU_INFO_ITERATOR cii;
991: struct cpu_info *ci;
992: for (CPU_INFO_FOREACH(cii, ci)) {
993: if (CPU_IS_PRIMARY(ci))
994: continue;
1.2 matt 995: KASSERT(ci->ci_data.cpu_idlelwp);
996:
997: /*
998: * Skip this CPU if it didn't sucessfully hatch.
999: */
1.24 matt 1000: if (!kcpuset_isset(cpus_hatched, cpu_index(ci)))
1.2 matt 1001: continue;
1002:
1003: ci->ci_data.cpu_cc_skew = mips3_cp0_count_read();
1004: atomic_or_ulong(&ci->ci_flags, CPUF_RUNNING);
1.24 matt 1005: kcpuset_set(cpus_running, cpu_index(ci));
1.23 matt 1006: // Spin until the cpu calls idle_loop
1007: for (u_int i = 0; i < 100; i++) {
1.24 matt 1008: if (kcpuset_isset(cpus_running, cpu_index(ci)))
1.23 matt 1009: break;
1010: delay(1000);
1011: }
1.2 matt 1012: }
1013: }
1014:
1015: void
1016: xc_send_ipi(struct cpu_info *ci)
1017: {
1018:
1.12 matt 1019: (*mips_locoresw.lsw_send_ipi)(ci, IPI_XCALL);
1.2 matt 1020: }
1.17 rmind 1021:
1022: void
1023: cpu_ipi(struct cpu_info *ci)
1024: {
1025: (*mips_locoresw.lsw_send_ipi)(ci, IPI_GENERIC);
1026: }
1027:
1.2 matt 1028: #endif /* MULTIPROCESSOR */
1029:
1030: void
1031: cpu_offline_md(void)
1032: {
1033:
1034: (*mips_locoresw.lsw_cpu_offline_md)();
1035: }
1036:
1037: #ifdef _LP64
1038: void
1039: cpu_vmspace_exec(lwp_t *l, vaddr_t start, vaddr_t end)
1040: {
1041: /*
1042: * We need to turn on/off UX so that copyout/copyin will work
1043: * well before setreg gets called.
1044: */
1045: uint32_t sr = mips_cp0_status_read();
1046: if (end != (uint32_t) end) {
1047: mips_cp0_status_write(sr | MIPS3_SR_UX);
1048: } else {
1049: mips_cp0_status_write(sr & ~MIPS3_SR_UX);
1050: }
1051: }
1052: #endif
1.5 matt 1053:
1054: int
1055: cpu_lwp_setprivate(lwp_t *l, void *v)
1056: {
1057: #if (MIPS32R2 + MIPS64R2) > 0
1.10 matt 1058: if (l == curlwp
1059: && (mips_options.mips_cpu->cpu_cp0flags & MIPS_CP0FL_USERLOCAL)) {
1.5 matt 1060: mipsNN_cp0_userlocal_write(v);
1061: }
1062: #endif
1063: return 0;
1064: }
1.7 cliff 1065:
1066:
1.9 cliff 1067: #if (MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0
1068:
1.7 cliff 1069: #if (CPUWATCH_MAX != 8)
1070: # error CPUWATCH_MAX
1071: #endif
1072:
1073: /*
1074: * cpuwatch_discover - determine how many COP0 watchpoints this CPU supports
1075: */
1076: u_int
1077: cpuwatch_discover(void)
1078: {
1079: int i;
1080:
1081: for (i=0; i < CPUWATCH_MAX; i++) {
1082: uint32_t watchhi = mipsNN_cp0_watchhi_read(i);
1083: if ((watchhi & __BIT(31)) == 0) /* test 'M' bit */
1084: break;
1085: }
1086: return i + 1;
1087: }
1088:
1089: void
1090: cpuwatch_free(cpu_watchpoint_t *cwp)
1091: {
1092: #ifdef DIAGNOSTIC
1093: struct cpu_info * const ci = curcpu();
1094: KASSERT(cwp >= &ci->ci_cpuwatch_tab[0] &&
1095: cwp <= &ci->ci_cpuwatch_tab[ci->ci_cpuwatch_count-1]);
1096: #endif
1097: cwp->cw_mode = 0;
1098: cwp->cw_asid = 0;
1099: cwp->cw_addr = 0;
1100: cpuwatch_clr(cwp);
1101: }
1102:
1103: /*
1104: * cpuwatch_alloc
1105: * find an empty slot
1106: * no locking for the table since it is CPU private
1107: */
1108: cpu_watchpoint_t *
1109: cpuwatch_alloc(void)
1110: {
1111: struct cpu_info * const ci = curcpu();
1112: cpu_watchpoint_t *cwp;
1113:
1114: for (int i=0; i < ci->ci_cpuwatch_count; i++) {
1115: cwp = &ci->ci_cpuwatch_tab[i];
1116: if ((cwp->cw_mode & CPUWATCH_RWX) == 0)
1117: return cwp;
1118: }
1119: return NULL;
1120: }
1121:
1122:
1123: void
1124: cpuwatch_set_all(void)
1125: {
1126: struct cpu_info * const ci = curcpu();
1127: cpu_watchpoint_t *cwp;
1128: int i;
1129:
1130: for (i=0; i < ci->ci_cpuwatch_count; i++) {
1131: cwp = &ci->ci_cpuwatch_tab[i];
1132: if ((cwp->cw_mode & CPUWATCH_RWX) != 0)
1133: cpuwatch_set(cwp);
1134: }
1135: }
1136:
1137: void
1138: cpuwatch_clr_all(void)
1139: {
1140: struct cpu_info * const ci = curcpu();
1141: cpu_watchpoint_t *cwp;
1142: int i;
1143:
1144: for (i=0; i < ci->ci_cpuwatch_count; i++) {
1145: cwp = &ci->ci_cpuwatch_tab[i];
1146: if ((cwp->cw_mode & CPUWATCH_RWX) != 0)
1147: cpuwatch_clr(cwp);
1148: }
1149: }
1150:
1151: /*
1152: * cpuwatch_set - establish a MIPS COP0 watchpoint
1153: */
1154: void
1155: cpuwatch_set(cpu_watchpoint_t *cwp)
1156: {
1157: struct cpu_info * const ci = curcpu();
1158: uint32_t watchhi;
1159: register_t watchlo;
1160: int cwnum = cwp - &ci->ci_cpuwatch_tab[0];
1161:
1162: KASSERT(cwp >= &ci->ci_cpuwatch_tab[0] &&
1163: cwp <= &ci->ci_cpuwatch_tab[ci->ci_cpuwatch_count-1]);
1164:
1165: watchlo = cwp->cw_addr;
1166: if (cwp->cw_mode & CPUWATCH_WRITE)
1167: watchlo |= __BIT(0);
1168: if (cwp->cw_mode & CPUWATCH_READ)
1169: watchlo |= __BIT(1);
1170: if (cwp->cw_mode & CPUWATCH_EXEC)
1171: watchlo |= __BIT(2);
1172:
1173: if (cwp->cw_mode & CPUWATCH_ASID)
1174: watchhi = cwp->cw_asid << 16; /* addr qualified by asid */
1175: else
1176: watchhi = __BIT(30); /* addr not qual. by asid (Global) */
1177: if (cwp->cw_mode & CPUWATCH_MASK)
1178: watchhi |= cwp->cw_mask; /* set "dont care" addr match bits */
1179:
1180: mipsNN_cp0_watchhi_write(cwnum, watchhi);
1181: mipsNN_cp0_watchlo_write(cwnum, watchlo);
1182: }
1183:
1184: /*
1185: * cpuwatch_clr - disestablish a MIPS COP0 watchpoint
1186: */
1187: void
1188: cpuwatch_clr(cpu_watchpoint_t *cwp)
1189: {
1190: struct cpu_info * const ci = curcpu();
1191: int cwnum = cwp - &ci->ci_cpuwatch_tab[0];
1192:
1193: KASSERT(cwp >= &ci->ci_cpuwatch_tab[0] &&
1194: cwp <= &ci->ci_cpuwatch_tab[ci->ci_cpuwatch_count-1]);
1195:
1196: mipsNN_cp0_watchhi_write(cwnum, 0);
1197: mipsNN_cp0_watchlo_write(cwnum, 0);
1198: }
1199:
1.9 cliff 1200: #endif /* (MIPS32 + MIPS32R2 + MIPS64 + MIPS64R2) > 0 */
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to cpu_subr.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / arch / mips / mips