Annotation of src/sys/arch/x86/x86/patch.c, Revision 1.9.2.1
1.9.2.1 ! ad 1: /* $NetBSD: patch.c,v 1.11 2007/12/20 23:46:11 ad Exp $ */
1.2 ad 2:
3: /*-
4: * Copyright (c) 2007 The NetBSD Foundation, Inc.
5: * All rights reserved.
6: *
7: * This code is derived from software contributed to The NetBSD Foundation
8: * by Andrew Doran.
9: *
10: * Redistribution and use in source and binary forms, with or without
11: * modification, are permitted provided that the following conditions
12: * are met:
13: * 1. Redistributions of source code must retain the above copyright
14: * notice, this list of conditions and the following disclaimer.
15: * 2. Redistributions in binary form must reproduce the above copyright
16: * notice, this list of conditions and the following disclaimer in the
17: * documentation and/or other materials provided with the distribution.
18: * 3. All advertising materials mentioning features or use of this software
19: * must display the following acknowledgement:
20: * This product includes software developed by the NetBSD
21: * Foundation, Inc. and its contributors.
22: * 4. Neither the name of The NetBSD Foundation nor the names of its
23: * contributors may be used to endorse or promote products derived
24: * from this software without specific prior written permission.
25: *
26: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
27: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
28: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
30: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
31: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
32: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
33: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
34: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
35: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36: * POSSIBILITY OF SUCH DAMAGE.
37: */
38:
39: /*
40: * Patch kernel code at boot time, depending on available CPU features.
41: */
42:
43: #include <sys/cdefs.h>
1.9.2.1 ! ad 44: __KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.11 2007/12/20 23:46:11 ad Exp $");
1.2 ad 45:
46: #include "opt_lockdebug.h"
47:
48: #include <sys/types.h>
49: #include <sys/systm.h>
50:
51: #include <machine/cpu.h>
52: #include <machine/cpufunc.h>
53: #include <machine/specialreg.h>
54:
55: #include <x86/cpuvar.h>
56: #include <x86/cputypes.h>
57:
58: void spllower(int);
59: void spllower_end(void);
1.6 ad 60: void cx8_spllower(int);
61: void cx8_spllower_end(void);
62: void cx8_spllower_patch(void);
1.2 ad 63:
64: void mutex_spin_exit_end(void);
65: void i686_mutex_spin_exit(int);
66: void i686_mutex_spin_exit_end(void);
67: void i686_mutex_spin_exit_patch(void);
68:
1.8 ad 69: void membar_consumer(void);
70: void membar_consumer_end(void);
71: void membar_sync(void);
72: void membar_sync_end(void);
73: void sse2_lfence(void);
74: void sse2_lfence_end(void);
75: void sse2_mfence(void);
76: void sse2_mfence_end(void);
77:
1.9.2.1 ! ad 78: void _atomic_cas_64(void);
! 79: void _atomic_cas_64_end(void);
! 80: void _atomic_cas_cx8(void);
! 81: void _atomic_cas_cx8_end(void);
1.2 ad 82:
1.6 ad 83: extern void *x86_lockpatch[];
1.8 ad 84: extern void *atomic_lockpatch[];
1.6 ad 85:
1.2 ad 86: #define X86_NOP 0x90
87: #define X86_REP 0xf3
88: #define X86_RET 0xc3
89: #define X86_CS 0x2e
90: #define X86_DS 0x3e
91: #define X86_GROUP_0F 0x0f
92:
93: static void __attribute__ ((__unused__))
94: patchfunc(void *from_s, void *from_e, void *to_s, void *to_e,
95: void *pcrel)
96: {
97: uint8_t *ptr;
98:
99: if ((uintptr_t)from_e - (uintptr_t)from_s !=
100: (uintptr_t)to_e - (uintptr_t)to_s)
101: panic("patchfunc: sizes do not match (from=%p)", from_s);
102:
103: memcpy(to_s, from_s, (uintptr_t)to_e - (uintptr_t)to_s);
104: if (pcrel != NULL) {
105: ptr = pcrel;
106: /* Branch hints */
107: if (ptr[0] == X86_CS || ptr[0] == X86_DS)
108: ptr++;
109: /* Conditional jumps */
110: if (ptr[0] == X86_GROUP_0F)
111: ptr++;
112: /* 4-byte relative jump or call */
113: *(uint32_t *)(ptr + 1 - (uintptr_t)from_s + (uintptr_t)to_s) +=
114: ((uint32_t)(uintptr_t)from_s - (uint32_t)(uintptr_t)to_s);
115: }
116: }
117:
118: static inline void __attribute__ ((__unused__))
119: patchbytes(void *addr, const int byte1, const int byte2)
120: {
1.6 ad 121:
1.2 ad 122: ((uint8_t *)addr)[0] = (uint8_t)byte1;
123: if (byte2 != -1)
124: ((uint8_t *)addr)[1] = (uint8_t)byte2;
125: }
126:
127: void
128: x86_patch(void)
129: {
1.9.2.1 ! ad 130: #if !defined(GPROF)
1.2 ad 131: static int again;
1.6 ad 132: u_long psl;
1.2 ad 133: u_long cr0;
134:
135: if (again)
136: return;
137: again = 1;
138:
1.6 ad 139: /* Disable interrupts. */
140: psl = x86_read_psl();
141: x86_disable_intr();
142:
1.2 ad 143: /* Disable write protection in supervisor mode. */
144: cr0 = rcr0();
145: lcr0(cr0 & ~CR0_WP);
146:
1.6 ad 147: if (ncpu == 1) {
1.9.2.1 ! ad 148: #ifndef LOCKDEBUG
1.6 ad 149: int i;
150:
151: /* Uniprocessor: kill LOCK prefixes. */
152: for (i = 0; x86_lockpatch[i] != 0; i++)
153: patchbytes(x86_lockpatch[i], X86_NOP, -1);
1.8 ad 154: for (i = 0; atomic_lockpatch[i] != 0; i++)
155: patchbytes(atomic_lockpatch[i], X86_NOP, -1);
1.7 ad 156: /*
157: * Uniprocessor: kill kernel_lock. Fill another
158: * 14 bytes of NOPs so not to confuse the decoder.
159: */
160: patchbytes(_kernel_lock, X86_NOP, X86_RET);
161: patchbytes(_kernel_unlock, X86_NOP, X86_RET);
162: for (i = 2; i < 16; i++) {
163: patchbytes((char *)_kernel_lock + i, X86_NOP, -1);
164: patchbytes((char *)_kernel_unlock + i, X86_NOP, -1);
165: }
1.9.2.1 ! ad 166: #endif
1.6 ad 167: } else if ((cpu_feature & CPUID_SSE2) != 0) {
168: /* Faster memory barriers. */
1.2 ad 169: patchfunc(
1.8 ad 170: sse2_lfence, sse2_lfence_end,
171: membar_consumer, membar_consumer_end,
172: NULL
173: );
174: patchfunc(
175: sse2_mfence, sse2_mfence_end,
176: membar_sync, membar_sync_end,
177: NULL
178: );
1.2 ad 179: }
180:
1.6 ad 181: if ((cpu_feature & CPUID_CX8) != 0) {
182: /* Faster splx(), mutex_spin_exit(). */
183: patchfunc(
184: cx8_spllower, cx8_spllower_end,
185: spllower, spllower_end,
186: cx8_spllower_patch
187: );
1.9 ad 188: #if defined(i386)
1.9.2.1 ! ad 189: #ifndef LOCKDEBUG
1.6 ad 190: patchfunc(
191: i686_mutex_spin_exit, i686_mutex_spin_exit_end,
192: mutex_spin_exit, mutex_spin_exit_end,
193: i686_mutex_spin_exit_patch
194: );
195: #endif
1.9.2.1 ! ad 196: patchfunc(
! 197: _atomic_cas_cx8, _atomic_cas_cx8_end,
! 198: _atomic_cas_64, _atomic_cas_64_end,
! 199: NULL
! 200: );
! 201: #endif
1.6 ad 202: }
203:
204: /* Write back and invalidate cache, flush pipelines. */
205: wbinvd();
206: x86_flush();
207: x86_write_psl(psl);
1.2 ad 208:
209: /* Re-enable write protection. */
210: lcr0(cr0);
211: #endif /* GPROF */
212: }
CVSweb <webmaster@jp.NetBSD.org>