Annotation of src/sys/arch/x86/x86/patch.c, Revision 1.2.4.6
1.2.4.6 ! yamt 1: /* $NetBSD: patch.c,v 1.2.4.5 2007/12/07 17:27:02 yamt Exp $ */
1.2.4.2 yamt 2:
3: /*-
4: * Copyright (c) 2007 The NetBSD Foundation, Inc.
5: * All rights reserved.
6: *
7: * This code is derived from software contributed to The NetBSD Foundation
8: * by Andrew Doran.
9: *
10: * Redistribution and use in source and binary forms, with or without
11: * modification, are permitted provided that the following conditions
12: * are met:
13: * 1. Redistributions of source code must retain the above copyright
14: * notice, this list of conditions and the following disclaimer.
15: * 2. Redistributions in binary form must reproduce the above copyright
16: * notice, this list of conditions and the following disclaimer in the
17: * documentation and/or other materials provided with the distribution.
18: * 3. All advertising materials mentioning features or use of this software
19: * must display the following acknowledgement:
20: * This product includes software developed by the NetBSD
21: * Foundation, Inc. and its contributors.
22: * 4. Neither the name of The NetBSD Foundation nor the names of its
23: * contributors may be used to endorse or promote products derived
24: * from this software without specific prior written permission.
25: *
26: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
27: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
28: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
30: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
31: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
32: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
33: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
34: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
35: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36: * POSSIBILITY OF SUCH DAMAGE.
37: */
38:
39: /*
40: * Patch kernel code at boot time, depending on available CPU features.
41: */
42:
43: #include <sys/cdefs.h>
1.2.4.6 ! yamt 44: __KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.2.4.5 2007/12/07 17:27:02 yamt Exp $");
1.2.4.2 yamt 45:
46: #include "opt_lockdebug.h"
47:
48: #include <sys/types.h>
49: #include <sys/systm.h>
50:
51: #include <machine/cpu.h>
52: #include <machine/cpufunc.h>
53: #include <machine/specialreg.h>
54:
55: #include <x86/cpuvar.h>
56: #include <x86/cputypes.h>
57:
58: void spllower(int);
59: void spllower_end(void);
1.2.4.5 yamt 60: void cx8_spllower(int);
61: void cx8_spllower_end(void);
62: void cx8_spllower_patch(void);
1.2.4.2 yamt 63:
64: void mutex_spin_exit_end(void);
65: void i686_mutex_spin_exit(int);
66: void i686_mutex_spin_exit_end(void);
67: void i686_mutex_spin_exit_patch(void);
68:
1.2.4.6 ! yamt 69: void membar_consumer(void);
! 70: void membar_consumer_end(void);
! 71: void membar_sync(void);
! 72: void membar_sync_end(void);
! 73: void sse2_lfence(void);
! 74: void sse2_lfence_end(void);
! 75: void sse2_mfence(void);
! 76: void sse2_mfence_end(void);
! 77:
1.2.4.2 yamt 78: void mb_read(void);
79: void mb_read_end(void);
80: void mb_memory(void);
81: void mb_memory_end(void);
82: void sse2_mb_read(void);
83: void sse2_mb_read_end(void);
84: void sse2_mb_memory(void);
85: void sse2_mb_memory_end(void);
86:
1.2.4.5 yamt 87: extern void *x86_lockpatch[];
1.2.4.6 ! yamt 88: extern void *atomic_lockpatch[];
1.2.4.5 yamt 89:
1.2.4.2 yamt 90: #define X86_NOP 0x90
91: #define X86_REP 0xf3
92: #define X86_RET 0xc3
93: #define X86_CS 0x2e
94: #define X86_DS 0x3e
95: #define X86_GROUP_0F 0x0f
96:
97: static void __attribute__ ((__unused__))
98: patchfunc(void *from_s, void *from_e, void *to_s, void *to_e,
99: void *pcrel)
100: {
101: uint8_t *ptr;
102:
103: if ((uintptr_t)from_e - (uintptr_t)from_s !=
104: (uintptr_t)to_e - (uintptr_t)to_s)
105: panic("patchfunc: sizes do not match (from=%p)", from_s);
106:
107: memcpy(to_s, from_s, (uintptr_t)to_e - (uintptr_t)to_s);
108: if (pcrel != NULL) {
109: ptr = pcrel;
110: /* Branch hints */
111: if (ptr[0] == X86_CS || ptr[0] == X86_DS)
112: ptr++;
113: /* Conditional jumps */
114: if (ptr[0] == X86_GROUP_0F)
115: ptr++;
116: /* 4-byte relative jump or call */
117: *(uint32_t *)(ptr + 1 - (uintptr_t)from_s + (uintptr_t)to_s) +=
118: ((uint32_t)(uintptr_t)from_s - (uint32_t)(uintptr_t)to_s);
119: }
120: }
121:
122: static inline void __attribute__ ((__unused__))
123: patchbytes(void *addr, const int byte1, const int byte2)
124: {
1.2.4.5 yamt 125:
1.2.4.2 yamt 126: ((uint8_t *)addr)[0] = (uint8_t)byte1;
127: if (byte2 != -1)
128: ((uint8_t *)addr)[1] = (uint8_t)byte2;
129: }
130:
131: void
132: x86_patch(void)
133: {
1.2.4.6 ! yamt 134: #if !defined(GPROF) && !defined(LOCKDEBUG)
1.2.4.2 yamt 135: static int again;
1.2.4.5 yamt 136: u_long psl;
1.2.4.2 yamt 137: u_long cr0;
138:
139: if (again)
140: return;
141: again = 1;
142:
1.2.4.5 yamt 143: /* Disable interrupts. */
144: psl = x86_read_psl();
145: x86_disable_intr();
146:
1.2.4.2 yamt 147: /* Disable write protection in supervisor mode. */
148: cr0 = rcr0();
149: lcr0(cr0 & ~CR0_WP);
150:
1.2.4.5 yamt 151: if (ncpu == 1) {
152: int i;
153:
154: /* Uniprocessor: kill LOCK prefixes. */
155: for (i = 0; x86_lockpatch[i] != 0; i++)
156: patchbytes(x86_lockpatch[i], X86_NOP, -1);
1.2.4.6 ! yamt 157: for (i = 0; atomic_lockpatch[i] != 0; i++)
! 158: patchbytes(atomic_lockpatch[i], X86_NOP, -1);
1.2.4.5 yamt 159: /*
160: * Uniprocessor: kill kernel_lock. Fill another
161: * 14 bytes of NOPs so not to confuse the decoder.
162: */
163: patchbytes(_kernel_lock, X86_NOP, X86_RET);
164: patchbytes(_kernel_unlock, X86_NOP, X86_RET);
165: for (i = 2; i < 16; i++) {
166: patchbytes((char *)_kernel_lock + i, X86_NOP, -1);
167: patchbytes((char *)_kernel_unlock + i, X86_NOP, -1);
168: }
169: } else if ((cpu_feature & CPUID_SSE2) != 0) {
170: /* Faster memory barriers. */
1.2.4.2 yamt 171: patchfunc(
172: sse2_mb_read, sse2_mb_read_end,
173: mb_read, mb_read_end,
174: NULL
175: );
176: patchfunc(
177: sse2_mb_memory, sse2_mb_memory_end,
178: mb_memory, mb_memory_end,
179: NULL
180: );
1.2.4.6 ! yamt 181: patchfunc(
! 182: sse2_lfence, sse2_lfence_end,
! 183: membar_consumer, membar_consumer_end,
! 184: NULL
! 185: );
! 186: patchfunc(
! 187: sse2_mfence, sse2_mfence_end,
! 188: membar_sync, membar_sync_end,
! 189: NULL
! 190: );
1.2.4.2 yamt 191: }
192:
1.2.4.5 yamt 193: if ((cpu_feature & CPUID_CX8) != 0) {
194: /* Faster splx(), mutex_spin_exit(). */
195: patchfunc(
196: cx8_spllower, cx8_spllower_end,
197: spllower, spllower_end,
198: cx8_spllower_patch
199: );
1.2.4.6 ! yamt 200: #if defined(i386)
1.2.4.5 yamt 201: patchfunc(
202: i686_mutex_spin_exit, i686_mutex_spin_exit_end,
203: mutex_spin_exit, mutex_spin_exit_end,
204: i686_mutex_spin_exit_patch
205: );
206: #endif
207: }
208:
209: /* Write back and invalidate cache, flush pipelines. */
210: wbinvd();
211: x86_flush();
212: x86_write_psl(psl);
1.2.4.2 yamt 213:
214: /* Re-enable write protection. */
215: lcr0(cr0);
216: #endif /* GPROF */
217: }
CVSweb <webmaster@jp.NetBSD.org>