Annotation of src/sys/kern/subr_ipi.c, Revision 1.3
1.3 ! rmind 1: /* $NetBSD: subr_ipi.c,v 1.2 2014/05/25 15:34:19 rmind Exp $ */
1.1 rmind 2:
3: /*-
4: * Copyright (c) 2014 The NetBSD Foundation, Inc.
5: * All rights reserved.
6: *
7: * This code is derived from software contributed to The NetBSD Foundation
8: * by Mindaugas Rasiukevicius.
9: *
10: * Redistribution and use in source and binary forms, with or without
11: * modification, are permitted provided that the following conditions
12: * are met:
13: * 1. Redistributions of source code must retain the above copyright
14: * notice, this list of conditions and the following disclaimer.
15: * 2. Redistributions in binary form must reproduce the above copyright
16: * notice, this list of conditions and the following disclaimer in the
17: * documentation and/or other materials provided with the distribution.
18: *
19: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29: * POSSIBILITY OF SUCH DAMAGE.
30: */
31:
32: /*
1.2 rmind 33: * Inter-processor interrupt (IPI) interface: asynchronous IPIs to
34: * invoke functions with a constant argument and synchronous IPIs
35: * with the cross-call support.
1.1 rmind 36: */
37:
38: #include <sys/cdefs.h>
1.3 ! rmind 39: __KERNEL_RCSID(0, "$NetBSD: subr_ipi.c,v 1.2 2014/05/25 15:34:19 rmind Exp $");
1.1 rmind 40:
41: #include <sys/param.h>
42: #include <sys/types.h>
43:
44: #include <sys/atomic.h>
45: #include <sys/evcnt.h>
46: #include <sys/cpu.h>
47: #include <sys/ipi.h>
1.3 ! rmind 48: #include <sys/intr.h>
1.1 rmind 49: #include <sys/kcpuset.h>
50: #include <sys/kmem.h>
51: #include <sys/lock.h>
1.2 rmind 52: #include <sys/mutex.h>
53:
54: /*
55: * An array of the IPI handlers used for asynchronous invocation.
56: * The lock protects the slot allocation.
57: */
58:
59: typedef struct {
60: ipi_func_t func;
61: void * arg;
62: } ipi_intr_t;
63:
64: static kmutex_t ipi_mngmt_lock;
65: static ipi_intr_t ipi_intrs[IPI_MAXREG] __cacheline_aligned;
1.1 rmind 66:
67: /*
68: * Per-CPU mailbox for IPI messages: it is a single cache line storing
1.2 rmind 69: * up to IPI_MSG_MAX messages. This interface is built on top of the
70: * synchronous IPIs.
1.1 rmind 71: */
72:
73: #define IPI_MSG_SLOTS (CACHE_LINE_SIZE / sizeof(ipi_msg_t *))
74: #define IPI_MSG_MAX IPI_MSG_SLOTS
75:
76: typedef struct {
77: ipi_msg_t * msg[IPI_MSG_SLOTS];
78: } ipi_mbox_t;
79:
1.2 rmind 80:
81: /* Mailboxes for the synchronous IPIs. */
1.1 rmind 82: static ipi_mbox_t * ipi_mboxes __read_mostly;
83: static struct evcnt ipi_mboxfull_ev __cacheline_aligned;
1.2 rmind 84: static void ipi_msg_cpu_handler(void *);
85:
86: /* Handler for the synchronous IPIs - it must be zero. */
87: #define IPI_SYNCH_ID 0
1.1 rmind 88:
89: #ifndef MULTIPROCESSOR
90: #define cpu_ipi(ci) KASSERT(ci == NULL)
91: #endif
92:
93: void
94: ipi_sysinit(void)
95: {
96: const size_t len = ncpu * sizeof(ipi_mbox_t);
97:
1.2 rmind 98: /* Initialise the per-CPU bit fields. */
99: for (u_int i = 0; i < ncpu; i++) {
100: struct cpu_info *ci = cpu_lookup(i);
101: memset(&ci->ci_ipipend, 0, sizeof(ci->ci_ipipend));
102: }
103: mutex_init(&ipi_mngmt_lock, MUTEX_DEFAULT, IPL_NONE);
104: memset(ipi_intrs, 0, sizeof(ipi_intrs));
105:
1.1 rmind 106: /* Allocate per-CPU IPI mailboxes. */
107: ipi_mboxes = kmem_zalloc(len, KM_SLEEP);
108: KASSERT(ipi_mboxes != NULL);
109:
1.2 rmind 110: /*
111: * Register the handler for synchronous IPIs. This mechanism
112: * is built on top of the asynchronous interface. Slot zero is
113: * reserved permanently; it is also handy to use zero as a failure
114: * for other registers (as it is potentially less error-prone).
115: */
116: ipi_intrs[IPI_SYNCH_ID].func = ipi_msg_cpu_handler;
117:
1.1 rmind 118: evcnt_attach_dynamic(&ipi_mboxfull_ev, EVCNT_TYPE_MISC, NULL,
119: "ipi", "full");
120: }
121:
122: /*
1.2 rmind 123: * ipi_register: register an asynchronous IPI handler.
124: *
125: * => Returns IPI ID which is greater than zero; on failure - zero.
126: */
127: u_int
128: ipi_register(ipi_func_t func, void *arg)
129: {
130: mutex_enter(&ipi_mngmt_lock);
131: for (u_int i = 0; i < IPI_MAXREG; i++) {
132: if (ipi_intrs[i].func == NULL) {
133: /* Register the function. */
134: ipi_intrs[i].func = func;
135: ipi_intrs[i].arg = arg;
136: mutex_exit(&ipi_mngmt_lock);
137:
138: KASSERT(i != IPI_SYNCH_ID);
139: return i;
140: }
141: }
142: mutex_exit(&ipi_mngmt_lock);
143: printf("WARNING: ipi_register: table full, increase IPI_MAXREG\n");
144: return 0;
145: }
146:
147: /*
148: * ipi_unregister: release the IPI handler given the ID.
149: */
150: void
151: ipi_unregister(u_int ipi_id)
152: {
153: ipi_msg_t ipimsg = { .func = (ipi_func_t)nullop };
154:
155: KASSERT(ipi_id != IPI_SYNCH_ID);
156: KASSERT(ipi_id < IPI_MAXREG);
157:
158: /* Release the slot. */
159: mutex_enter(&ipi_mngmt_lock);
160: KASSERT(ipi_intrs[ipi_id].func != NULL);
161: ipi_intrs[ipi_id].func = NULL;
162:
163: /* Ensure that there are no IPIs in flight. */
164: kpreempt_disable();
165: ipi_broadcast(&ipimsg);
166: ipi_wait(&ipimsg);
167: kpreempt_enable();
168: mutex_exit(&ipi_mngmt_lock);
169: }
170:
171: /*
172: * ipi_trigger: asynchronously send an IPI to the specified CPU.
173: */
174: void
175: ipi_trigger(u_int ipi_id, struct cpu_info *ci)
176: {
177: const u_int i = ipi_id >> IPI_BITW_SHIFT;
178: const uint32_t bitm = 1U << (ipi_id & IPI_BITW_MASK);
179:
180: KASSERT(ipi_id < IPI_MAXREG);
181: KASSERT(kpreempt_disabled());
182: KASSERT(curcpu() != ci);
183:
184: /* Mark as pending and send an IPI. */
185: if (membar_consumer(), (ci->ci_ipipend[i] & bitm) == 0) {
186: atomic_or_32(&ci->ci_ipipend[i], bitm);
187: cpu_ipi(ci);
188: }
189: }
190:
191: /*
1.3 ! rmind 192: * ipi_trigger_multi: same as ipi_trigger() but sends to the multiple
! 193: * CPUs given the target CPU set.
! 194: */
! 195: void
! 196: ipi_trigger_multi(u_int ipi_id, const kcpuset_t *target)
! 197: {
! 198: const cpuid_t selfid = cpu_index(curcpu());
! 199: CPU_INFO_ITERATOR cii;
! 200: struct cpu_info *ci;
! 201:
! 202: KASSERT(kpreempt_disabled());
! 203: KASSERT(target != NULL);
! 204:
! 205: for (CPU_INFO_FOREACH(cii, ci)) {
! 206: const cpuid_t cpuid = cpu_index(ci);
! 207:
! 208: if (!kcpuset_isset(target, cpuid) || cpuid == selfid) {
! 209: continue;
! 210: }
! 211: ipi_trigger(ipi_id, ci);
! 212: }
! 213: if (kcpuset_isset(target, selfid)) {
! 214: int s = splhigh();
! 215: ipi_cpu_handler();
! 216: splx(s);
! 217: }
! 218: }
! 219:
! 220: /*
1.1 rmind 221: * put_msg: insert message into the mailbox.
222: */
223: static inline void
224: put_msg(ipi_mbox_t *mbox, ipi_msg_t *msg)
225: {
226: int count = SPINLOCK_BACKOFF_MIN;
227: again:
228: for (u_int i = 0; i < IPI_MSG_MAX; i++) {
229: if (__predict_true(mbox->msg[i] == NULL) &&
230: atomic_cas_ptr(&mbox->msg[i], NULL, msg) == NULL) {
231: return;
232: }
233: }
234:
235: /* All slots are full: we have to spin-wait. */
236: ipi_mboxfull_ev.ev_count++;
237: SPINLOCK_BACKOFF(count);
238: goto again;
239: }
240:
241: /*
242: * ipi_cpu_handler: the IPI handler.
243: */
244: void
245: ipi_cpu_handler(void)
246: {
1.2 rmind 247: struct cpu_info * const ci = curcpu();
248:
249: /*
250: * Handle asynchronous IPIs: inspect per-CPU bit field, extract
251: * IPI ID numbers and execute functions in those slots.
252: */
253: for (u_int i = 0; i < IPI_BITWORDS; i++) {
254: uint32_t pending, bit;
255:
256: if (ci->ci_ipipend[i] == 0) {
257: continue;
258: }
259: pending = atomic_swap_32(&ci->ci_ipipend[i], 0);
260: #ifndef __HAVE_ATOMIC_AS_MEMBAR
261: membar_producer();
262: #endif
263: while ((bit = ffs(pending)) != 0) {
264: const u_int ipi_id = (i << IPI_BITW_SHIFT) | --bit;
265: ipi_intr_t *ipi_hdl = &ipi_intrs[ipi_id];
266:
267: pending &= ~(1U << bit);
268: KASSERT(ipi_hdl->func != NULL);
269: ipi_hdl->func(ipi_hdl->arg);
270: }
271: }
272: }
273:
274: /*
275: * ipi_msg_cpu_handler: handle synchronous IPIs - iterate mailbox,
276: * execute the passed functions and acknowledge the messages.
277: */
278: static void
279: ipi_msg_cpu_handler(void *arg __unused)
280: {
1.1 rmind 281: const struct cpu_info * const ci = curcpu();
282: ipi_mbox_t *mbox = &ipi_mboxes[cpu_index(ci)];
283:
284: for (u_int i = 0; i < IPI_MSG_MAX; i++) {
285: ipi_msg_t *msg;
286:
287: /* Get the message. */
288: if ((msg = mbox->msg[i]) == NULL) {
289: continue;
290: }
291: mbox->msg[i] = NULL;
292:
293: /* Execute the handler. */
294: KASSERT(msg->func);
295: msg->func(msg->arg);
296:
297: /* Ack the request. */
298: atomic_dec_uint(&msg->_pending);
299: }
300: }
301:
302: /*
303: * ipi_unicast: send an IPI to a single CPU.
304: *
305: * => The CPU must be remote; must not be local.
306: * => The caller must ipi_wait() on the message for completion.
307: */
308: void
309: ipi_unicast(ipi_msg_t *msg, struct cpu_info *ci)
310: {
311: const cpuid_t id = cpu_index(ci);
312:
313: KASSERT(msg->func != NULL);
314: KASSERT(kpreempt_disabled());
315: KASSERT(curcpu() != ci);
316:
317: msg->_pending = 1;
318: membar_producer();
319:
320: put_msg(&ipi_mboxes[id], msg);
1.2 rmind 321: ipi_trigger(IPI_SYNCH_ID, ci);
1.1 rmind 322: }
323:
324: /*
325: * ipi_multicast: send an IPI to each CPU in the specified set.
326: *
327: * => The caller must ipi_wait() on the message for completion.
328: */
329: void
330: ipi_multicast(ipi_msg_t *msg, const kcpuset_t *target)
331: {
332: const struct cpu_info * const self = curcpu();
333: CPU_INFO_ITERATOR cii;
334: struct cpu_info *ci;
335: u_int local;
336:
337: KASSERT(msg->func != NULL);
338: KASSERT(kpreempt_disabled());
339:
340: local = !!kcpuset_isset(target, cpu_index(self));
341: msg->_pending = kcpuset_countset(target) - local;
342: membar_producer();
343:
344: for (CPU_INFO_FOREACH(cii, ci)) {
345: cpuid_t id;
346:
347: if (__predict_false(ci == self)) {
348: continue;
349: }
350: id = cpu_index(ci);
351: if (!kcpuset_isset(target, id)) {
352: continue;
353: }
354: put_msg(&ipi_mboxes[id], msg);
1.2 rmind 355: ipi_trigger(IPI_SYNCH_ID, ci);
1.1 rmind 356: }
357: if (local) {
358: msg->func(msg->arg);
359: }
360: }
361:
362: /*
363: * ipi_broadcast: send an IPI to all CPUs.
364: *
365: * => The caller must ipi_wait() on the message for completion.
366: */
367: void
368: ipi_broadcast(ipi_msg_t *msg)
369: {
370: const struct cpu_info * const self = curcpu();
371: CPU_INFO_ITERATOR cii;
372: struct cpu_info *ci;
373:
374: KASSERT(msg->func != NULL);
375: KASSERT(kpreempt_disabled());
376:
377: msg->_pending = ncpu - 1;
378: membar_producer();
379:
380: /* Broadcast IPIs for remote CPUs. */
381: for (CPU_INFO_FOREACH(cii, ci)) {
382: cpuid_t id;
383:
384: if (__predict_false(ci == self)) {
385: continue;
386: }
387: id = cpu_index(ci);
388: put_msg(&ipi_mboxes[id], msg);
1.2 rmind 389: ipi_trigger(IPI_SYNCH_ID, ci);
1.1 rmind 390: }
391:
392: /* Finally, execute locally. */
393: msg->func(msg->arg);
394: }
395:
396: /*
397: * ipi_wait: spin-wait until the message is processed.
398: */
399: void
400: ipi_wait(ipi_msg_t *msg)
401: {
402: int count = SPINLOCK_BACKOFF_MIN;
403:
404: while (msg->_pending) {
405: KASSERT(msg->_pending < ncpu);
406: SPINLOCK_BACKOFF(count);
407: }
408: }
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to subr_ipi.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / kern