Annotation of src/sys/arch/i386/i386/lock_stubs.S, Revision 1.23.2.1
1.23.2.1! uebayasi 1: /* $NetBSD$ */
1.2 ad 2:
3: /*-
1.23 ad 4: * Copyright (c) 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc.
1.2 ad 5: * All rights reserved.
6: *
7: * This code is derived from software contributed to The NetBSD Foundation
8: * by Andrew Doran.
9: *
10: * Redistribution and use in source and binary forms, with or without
11: * modification, are permitted provided that the following conditions
12: * are met:
13: * 1. Redistributions of source code must retain the above copyright
14: * notice, this list of conditions and the following disclaimer.
15: * 2. Redistributions in binary form must reproduce the above copyright
16: * notice, this list of conditions and the following disclaimer in the
17: * documentation and/or other materials provided with the distribution.
18: *
19: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29: * POSSIBILITY OF SUCH DAMAGE.
30: */
31:
32: /*
1.23.2.1! uebayasi 33: * Where possible we make each routine fit into an assumed 64-byte cache
! 34: * line. Please check alignment with 'objdump -d' after making changes.
1.2 ad 35: */
36:
1.11 lukem 37: #include <machine/asm.h>
1.23.2.1! uebayasi 38: __KERNEL_RCSID(0, "$NetBSD$");
1.11 lukem 39:
1.2 ad 40: #include "opt_lockdebug.h"
41:
42: #include <machine/cputypes.h>
43:
44: #include "assym.h"
45:
46: #define ALIGN64 .align 64
47: #define ALIGN32 .align 32
1.12 ad 48: #define LOCK(num) .Lpatch/**/num: lock
1.23 ad 49: #define RET(num) .Lret/**/num: ret; nop; nop; ret
1.16 yamt 50: #define ENDLABEL(name,a) .align a; LABEL(name)
1.2 ad 51:
52: #if !defined(LOCKDEBUG)
53:
54: /*
55: * void mutex_enter(kmutex_t *mtx);
56: *
57: * Acquire a mutex and post a load fence.
58: */
59: ALIGN64
60:
1.7 ad 61: ENTRY(mutex_enter)
62: movl 4(%esp), %edx
1.2 ad 63: xorl %eax, %eax
1.23 ad 64: movl %fs:CPU_INFO_CURLWP(%eax), %ecx
1.12 ad 65: LOCK(1)
1.22 ad 66: cmpxchgl %ecx, (%edx)
1.14 ad 67: jnz 1f
1.23 ad 68: RET(1)
1.14 ad 69: 1:
70: jmp _C_LABEL(mutex_vector_enter)
1.17 yamt 71: END(mutex_enter)
1.2 ad 72:
73: /*
74: * void mutex_exit(kmutex_t *mtx);
75: *
76: * Release a mutex and post a load fence.
77: *
78: * See comments in mutex_vector_enter() about doing this operation unlocked
79: * on multiprocessor systems, and comments in arch/x86/include/lock.h about
80: * memory ordering on Intel x86 systems.
81: */
1.7 ad 82: ENTRY(mutex_exit)
83: movl 4(%esp), %edx
1.2 ad 84: xorl %ecx, %ecx
1.23 ad 85: movl %fs:CPU_INFO_CURLWP(%ecx), %eax
1.22 ad 86: cmpxchgl %ecx, (%edx)
1.14 ad 87: jnz 1f
1.2 ad 88: ret
1.14 ad 89: 1:
90: jmp _C_LABEL(mutex_vector_exit)
1.17 yamt 91: END(mutex_exit)
1.2 ad 92:
93: /*
94: * void rw_enter(krwlock_t *rwl, krw_t op);
95: *
96: * Acquire one hold on a RW lock.
97: */
1.7 ad 98: ENTRY(rw_enter)
99: movl 4(%esp), %edx
1.2 ad 100: cmpl $RW_READER, 8(%esp)
101: jne 2f
102:
103: /*
104: * Reader
105: */
1.22 ad 106: movl (%edx), %eax
107: 0:
1.2 ad 108: testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
1.22 ad 109: jnz 3f
1.2 ad 110: leal RW_READ_INCR(%eax), %ecx
1.12 ad 111: LOCK(2)
1.22 ad 112: cmpxchgl %ecx, (%edx)
113: jnz 1f
1.23 ad 114: RET(2)
1.22 ad 115: 1:
116: jmp 0b
1.2 ad 117:
118: /*
119: * Writer
120: */
1.23 ad 121: 2: xorl %eax, %eax
122: movl %fs:CPU_INFO_CURLWP(%eax), %ecx
1.2 ad 123: orl $RW_WRITE_LOCKED, %ecx
1.12 ad 124: LOCK(3)
1.22 ad 125: cmpxchgl %ecx, (%edx)
1.14 ad 126: jnz 3f
1.23 ad 127: RET(3)
1.14 ad 128: 3:
129: jmp _C_LABEL(rw_vector_enter)
1.17 yamt 130: END(rw_enter)
1.2 ad 131:
132: /*
133: * void rw_exit(krwlock_t *rwl);
134: *
135: * Release one hold on a RW lock.
136: */
1.7 ad 137: ENTRY(rw_exit)
138: movl 4(%esp), %edx
1.22 ad 139: movl (%edx), %eax
1.2 ad 140: testb $RW_WRITE_LOCKED, %al
141: jnz 2f
142:
143: /*
144: * Reader
145: */
1.22 ad 146: 0: testb $RW_HAS_WAITERS, %al
1.14 ad 147: jnz 3f
1.2 ad 148: cmpl $RW_READ_INCR, %eax
1.22 ad 149: jb 3f
1.2 ad 150: leal -RW_READ_INCR(%eax), %ecx
1.12 ad 151: LOCK(4)
1.22 ad 152: cmpxchgl %ecx, (%edx)
153: jnz 1f
1.2 ad 154: ret
1.22 ad 155: 1:
156: jmp 0b
1.2 ad 157:
158: /*
159: * Writer
160: */
161: 2: leal -RW_WRITE_LOCKED(%eax), %ecx
162: subl CPUVAR(CURLWP), %ecx
1.14 ad 163: jnz 3f
1.12 ad 164: LOCK(5)
1.22 ad 165: cmpxchgl %ecx, (%edx)
1.14 ad 166: jnz 3f
1.2 ad 167: ret
168:
169: /*
170: * Slow path.
171: */
172: 3: jmp _C_LABEL(rw_vector_exit)
1.17 yamt 173: END(rw_exit)
1.2 ad 174:
1.13 ad 175: /*
176: * int rw_tryenter(krwlock_t *rwl, krw_t op);
177: *
178: * Try to acquire one hold on a RW lock.
179: */
180: ENTRY(rw_tryenter)
181: movl 4(%esp), %edx
182: cmpl $RW_READER, 8(%esp)
183: jne 2f
184:
185: /*
186: * Reader
187: */
1.22 ad 188: movl (%edx), %eax
189: 0:
1.13 ad 190: testb $(RW_WRITE_LOCKED|RW_WRITE_WANTED), %al
1.23 ad 191: jnz 4f
1.22 ad 192: leal RW_READ_INCR(%eax), %ecx
1.13 ad 193: LOCK(12)
1.22 ad 194: cmpxchgl %ecx, (%edx)
195: jnz 1f
1.18 ad 196: movl %edx, %eax /* nonzero */
1.23 ad 197: RET(4)
1.22 ad 198: 1:
199: jmp 0b
1.13 ad 200:
201: /*
202: * Writer
203: */
1.18 ad 204: 2:
1.23 ad 205: xorl %eax, %eax
206: movl %fs:CPU_INFO_CURLWP(%eax), %ecx
1.18 ad 207: orl $RW_WRITE_LOCKED, %ecx
1.13 ad 208: LOCK(13)
1.22 ad 209: cmpxchgl %ecx, (%edx)
1.19 ad 210: movl $0, %eax
1.13 ad 211: setz %al
1.18 ad 212: 3:
1.23 ad 213: RET(5)
214: 4:
1.18 ad 215: xorl %eax, %eax
1.23 ad 216: jmp 3b
1.17 yamt 217: END(rw_tryenter)
1.13 ad 218:
1.2 ad 219: #ifndef __XEN__
220:
221: /*
222: * void mutex_spin_enter(kmutex_t *mtx);
223: *
224: * Acquire a spin mutex and post a load fence.
225: */
1.7 ad 226: ENTRY(mutex_spin_enter)
227: movl 4(%esp), %edx
1.14 ad 228: movb CPUVAR(ILEVEL), %cl
229: movb MTX_IPL(%edx), %ch
1.21 ad 230: movl $0x01, %eax
1.2 ad 231: cmpb %ch, %cl
1.14 ad 232: jg 1f
1.7 ad 233: movb %ch, CPUVAR(ILEVEL) /* splraiseipl() */
1.14 ad 234: 1:
1.21 ad 235: subl %eax, CPUVAR(MTX_COUNT) /* decl does not set CF */
1.14 ad 236: jnc 2f
237: movb %cl, CPUVAR(MTX_OLDSPL)
1.2 ad 238: 2:
1.21 ad 239: xchgb %al, MTX_LOCK(%edx) /* lock it */
240: testb %al, %al
1.14 ad 241: jnz 3f
1.23 ad 242: RET(6)
1.14 ad 243: 3:
244: jmp _C_LABEL(mutex_spin_retry)
1.2 ad 245:
246: ALIGN64
247: LABEL(mutex_spin_enter_end)
1.17 yamt 248: END(mutex_spin_enter)
1.2 ad 249:
250: /*
251: * Release a spin mutex and post a store fence.
252: */
1.7 ad 253: ENTRY(mutex_spin_exit)
254: movl 4(%esp), %edx
255: movl CPUVAR(MTX_OLDSPL), %ecx
256: incl CPUVAR(MTX_COUNT)
257: movb $0, MTX_LOCK(%edx) /* zero */
1.2 ad 258: jnz 1f
1.7 ad 259: movl %fs:CPU_INFO_IUNMASK(,%ecx,4), %edx
260: cli
261: testl CPUVAR(IPENDING), %edx
1.2 ad 262: movl %ecx, 4(%esp)
263: jnz _C_LABEL(Xspllower) /* does sti */
1.7 ad 264: movl %ecx, CPUVAR(ILEVEL)
1.2 ad 265: sti
266: 1: ret
1.4 ad 267: nop /* XXX round up */
268: .align 32
1.2 ad 269: LABEL(mutex_spin_exit_end)
1.17 yamt 270: END(mutex_spin_exit)
1.2 ad 271:
272: /*
1.20 chs 273: * Patch for i686 CPUs where cli/sti is prohibitively expensive.
1.2 ad 274: * Must be the same size as mutex_spin_exit().
275: */
1.4 ad 276: ENTRY(i686_mutex_spin_exit)
1.2 ad 277: mov 4(%esp),%edx
1.7 ad 278: movl CPUVAR(MTX_OLDSPL), %ecx
279: incl CPUVAR(MTX_COUNT)
280: movb %ch, MTX_LOCK(%edx) /* zero */
281: jnz 1f
1.2 ad 282: pushl %ebx
1.3 yamt 283: 0:
1.7 ad 284: movl CPUVAR(IPENDING), %eax
285: testl %eax, %fs:CPU_INFO_IUNMASK(,%ecx,4)
1.22 ad 286: jnz 2f
1.2 ad 287: movl %eax, %ebx
1.7 ad 288: cmpxchg8b CPUVAR(ISTATE) /* swap in new ilevel */
289: jnz 0b
290: popl %ebx
1.2 ad 291: 1:
292: ret
293: 2:
294: popl %ebx
295: movl %ecx,4(%esp)
296: LABEL(i686_mutex_spin_exit_patch)
297: jmp _C_LABEL(Xspllower)
1.4 ad 298: .align 32
1.2 ad 299: LABEL(i686_mutex_spin_exit_end)
1.17 yamt 300: END(i686_mutex_spin_exit)
1.2 ad 301:
302: #else /* !__XEN__ */
303:
304: /* For now; strong alias not working for some reason. */
1.5 ad 305: ENTRY(mutex_spin_enter)
1.2 ad 306: jmp _C_LABEL(mutex_vector_enter)
307:
1.5 ad 308: ENTRY(mutex_spin_exit)
1.2 ad 309: jmp _C_LABEL(mutex_vector_exit)
310:
311: #endif /* !__XEN__ */
312:
313: #endif /* !LOCKDEBUG */
314:
315: /*
1.12 ad 316: * Spinlocks.
1.2 ad 317: */
1.20 chs 318: ENTRY(__cpu_simple_lock_init)
1.12 ad 319: movl 4(%esp), %edx
320: movb $0, (%edx)
1.2 ad 321: ret
1.17 yamt 322: END(__cpu_simple_lock_init)
1.14 ad 323:
1.12 ad 324: NENTRY(__cpu_simple_lock)
325: movl 4(%esp), %edx
326: movl $0x0100, %eax
327: 1:
328: LOCK(6)
329: cmpxchgb %ah, (%edx)
330: jnz 2f
1.23 ad 331: RET(7)
1.12 ad 332: 2:
333: movl $0x0100, %eax
334: pause
335: nop
1.7 ad 336: nop
1.12 ad 337: cmpb $0, (%edx)
338: je 1b
339: jmp 2b
1.17 yamt 340: END(__cpu_simple_lock)
1.2 ad 341:
1.20 chs 342: ENTRY(__cpu_simple_unlock)
1.12 ad 343: movl 4(%esp), %edx
344: movb $0, (%edx)
1.2 ad 345: ret
1.17 yamt 346: END(__cpu_simple_unlock)
1.2 ad 347:
1.20 chs 348: ENTRY(__cpu_simple_lock_try)
1.12 ad 349: movl 4(%esp), %edx
350: movl $0x0100, %eax
351: LOCK(7)
352: cmpxchgb %ah, (%edx)
353: movl $0, %eax
354: setz %al
1.23 ad 355: RET(8)
1.17 yamt 356: END(__cpu_simple_lock_try)
1.2 ad 357:
358: /*
1.7 ad 359: * Patchpoints to replace with NOP when ncpu == 1.
360: */
1.12 ad 361: .data
362:
1.7 ad 363: #ifndef LOCKDEBUG
364: LABEL(x86_lockpatch)
1.12 ad 365: .long .Lpatch1, .Lpatch2, .Lpatch3, .Lpatch4
1.13 ad 366: .long .Lpatch5, .Lpatch6, .Lpatch7, .Lpatch12
367: .long .Lpatch13
1.7 ad 368: .long 0
369: #endif
1.23 ad 370:
371: LABEL(x86_retpatch)
372: #ifndef LOCKDEBUG
373: .long .Lret1, .Lret2, .Lret3, .Lret4, .Lret5
374: #ifndef __XEN__
375: .long .Lret6
376: #endif
377: #endif
378: .long .Lret7, .Lret8
379: .long 0
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to lock_stubs.S CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / arch / i386 / i386