Annotation of src/sys/arch/x86/x86/patch.c, Revision 1.32
1.32 ! maxv 1: /* $NetBSD: patch.c,v 1.31 2018/01/27 09:33:25 maxv Exp $ */
1.2 ad 2:
3: /*-
1.16 ad 4: * Copyright (c) 2007, 2008, 2009 The NetBSD Foundation, Inc.
1.2 ad 5: * All rights reserved.
6: *
7: * This code is derived from software contributed to The NetBSD Foundation
8: * by Andrew Doran.
9: *
10: * Redistribution and use in source and binary forms, with or without
11: * modification, are permitted provided that the following conditions
12: * are met:
13: * 1. Redistributions of source code must retain the above copyright
14: * notice, this list of conditions and the following disclaimer.
15: * 2. Redistributions in binary form must reproduce the above copyright
16: * notice, this list of conditions and the following disclaimer in the
17: * documentation and/or other materials provided with the distribution.
18: *
19: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29: * POSSIBILITY OF SUCH DAMAGE.
30: */
31:
32: /*
33: * Patch kernel code at boot time, depending on available CPU features.
34: */
35:
36: #include <sys/cdefs.h>
1.32 ! maxv 37: __KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.31 2018/01/27 09:33:25 maxv Exp $");
1.2 ad 38:
39: #include "opt_lockdebug.h"
1.20 dyoung 40: #ifdef i386
1.19 dyoung 41: #include "opt_spldebug.h"
1.20 dyoung 42: #endif
1.2 ad 43:
44: #include <sys/types.h>
45: #include <sys/systm.h>
46:
47: #include <machine/cpu.h>
48: #include <machine/cpufunc.h>
49: #include <machine/specialreg.h>
1.26 maxv 50: #include <machine/frameasm.h>
1.2 ad 51:
52: #include <x86/cpuvar.h>
53: #include <x86/cputypes.h>
54:
1.26 maxv 55: struct hotpatch {
56: uint8_t name;
57: uint8_t size;
58: void *addr;
59: } __packed;
60:
1.2 ad 61: void spllower(int);
62: void spllower_end(void);
1.6 ad 63: void cx8_spllower(int);
64: void cx8_spllower_end(void);
65: void cx8_spllower_patch(void);
1.2 ad 66:
67: void mutex_spin_exit_end(void);
68: void i686_mutex_spin_exit(int);
69: void i686_mutex_spin_exit_end(void);
70: void i686_mutex_spin_exit_patch(void);
71:
1.8 ad 72: void membar_consumer(void);
73: void membar_consumer_end(void);
74: void membar_sync(void);
75: void membar_sync_end(void);
76: void sse2_lfence(void);
77: void sse2_lfence_end(void);
78: void sse2_mfence(void);
79: void sse2_mfence_end(void);
80:
1.10 ad 81: void _atomic_cas_64(void);
82: void _atomic_cas_64_end(void);
83: void _atomic_cas_cx8(void);
84: void _atomic_cas_cx8_end(void);
85:
1.8 ad 86: extern void *atomic_lockpatch[];
1.6 ad 87:
1.2 ad 88: #define X86_NOP 0x90
89: #define X86_REP 0xf3
90: #define X86_RET 0xc3
91: #define X86_CS 0x2e
92: #define X86_DS 0x3e
93: #define X86_GROUP_0F 0x0f
94:
1.17 enami 95: static void
96: adjust_jumpoff(uint8_t *ptr, void *from_s, void *to_s)
97: {
98:
99: /* Branch hints */
100: if (ptr[0] == X86_CS || ptr[0] == X86_DS)
101: ptr++;
102: /* Conditional jumps */
103: if (ptr[0] == X86_GROUP_0F)
104: ptr++;
105: /* 4-byte relative jump or call */
106: *(uint32_t *)(ptr + 1 - (uintptr_t)from_s + (uintptr_t)to_s) +=
107: ((uint32_t)(uintptr_t)from_s - (uint32_t)(uintptr_t)to_s);
108: }
109:
1.14 gmcgarry 110: static void __unused
1.2 ad 111: patchfunc(void *from_s, void *from_e, void *to_s, void *to_e,
112: void *pcrel)
113: {
114:
115: if ((uintptr_t)from_e - (uintptr_t)from_s !=
116: (uintptr_t)to_e - (uintptr_t)to_s)
117: panic("patchfunc: sizes do not match (from=%p)", from_s);
118:
119: memcpy(to_s, from_s, (uintptr_t)to_e - (uintptr_t)to_s);
1.17 enami 120: if (pcrel != NULL)
121: adjust_jumpoff(pcrel, from_s, to_s);
122:
123: #ifdef GPROF
124: #ifdef i386
125: #define MCOUNT_CALL_OFFSET 3
126: #endif
127: #ifdef __x86_64__
128: #define MCOUNT_CALL_OFFSET 5
129: #endif
130: /* Patch mcount call offset */
131: adjust_jumpoff((uint8_t *)from_s + MCOUNT_CALL_OFFSET, from_s, to_s);
132: #endif
1.2 ad 133: }
134:
1.14 gmcgarry 135: static inline void __unused
1.25 maxv 136: patchbytes(void *addr, const uint8_t *bytes, size_t size)
1.2 ad 137: {
1.25 maxv 138: uint8_t *ptr = (uint8_t *)addr;
139: size_t i;
1.6 ad 140:
1.25 maxv 141: for (i = 0; i < size; i++) {
142: ptr[i] = bytes[i];
143: }
1.2 ad 144: }
145:
1.26 maxv 146: static void
147: x86_hotpatch(uint32_t name, const uint8_t *bytes, size_t size)
148: {
149: extern char __rodata_hotpatch_start;
150: extern char __rodata_hotpatch_end;
151: struct hotpatch *hps, *hpe, *hp;
152:
153: hps = (struct hotpatch *)&__rodata_hotpatch_start;
154: hpe = (struct hotpatch *)&__rodata_hotpatch_end;
155:
156: for (hp = hps; hp < hpe; hp++) {
157: if (hp->name != name) {
158: continue;
159: }
160: if (hp->size != size) {
161: panic("x86_hotpatch: incorrect size");
162: }
163: patchbytes(hp->addr, bytes, size);
164: }
165: }
166:
1.2 ad 167: void
1.15 ad 168: x86_patch(bool early)
1.2 ad 169: {
1.16 ad 170: static bool first, second;
1.6 ad 171: u_long psl;
1.2 ad 172: u_long cr0;
173:
1.16 ad 174: if (early) {
175: if (first)
176: return;
177: first = true;
178: } else {
179: if (second)
180: return;
181: second = true;
182: }
1.2 ad 183:
1.6 ad 184: /* Disable interrupts. */
185: psl = x86_read_psl();
186: x86_disable_intr();
187:
1.2 ad 188: /* Disable write protection in supervisor mode. */
189: cr0 = rcr0();
190: lcr0(cr0 & ~CR0_WP);
191:
1.15 ad 192: #if !defined(GPROF)
193: if (!early && ncpu == 1) {
1.10 ad 194: #ifndef LOCKDEBUG
1.27 maxv 195: /*
196: * Uniprocessor: kill LOCK prefixes.
197: */
1.25 maxv 198: const uint8_t bytes[] = {
199: X86_NOP
200: };
201:
1.27 maxv 202: /* lock -> nop */
203: x86_hotpatch(HP_NAME_NOLOCK, bytes, sizeof(bytes));
1.30 christos 204: for (int i = 0; atomic_lockpatch[i] != 0; i++)
1.28 maxv 205: patchbytes(atomic_lockpatch[i], bytes, sizeof(bytes));
1.27 maxv 206: #endif
1.15 ad 207: }
1.27 maxv 208:
1.21 jym 209: if (!early && (cpu_feature[0] & CPUID_SSE2) != 0) {
1.24 riastrad 210: /*
211: * Faster memory barriers. We do not need to patch
212: * membar_producer to use SFENCE because on x86
213: * ordinary non-temporal stores are always issued in
214: * program order to main memory and to other CPUs.
215: */
1.2 ad 216: patchfunc(
1.8 ad 217: sse2_lfence, sse2_lfence_end,
218: membar_consumer, membar_consumer_end,
219: NULL
220: );
221: patchfunc(
222: sse2_mfence, sse2_mfence_end,
223: membar_sync, membar_sync_end,
224: NULL
225: );
1.2 ad 226: }
1.15 ad 227: #endif /* GPROF */
1.2 ad 228:
1.15 ad 229: #ifdef i386
230: /*
231: * Patch early and late. Second time around the 'lock' prefix
232: * may be gone.
233: */
1.21 jym 234: if ((cpu_feature[0] & CPUID_CX8) != 0) {
1.15 ad 235: patchfunc(
236: _atomic_cas_cx8, _atomic_cas_cx8_end,
237: _atomic_cas_64, _atomic_cas_64_end,
238: NULL
239: );
240: }
241: #endif /* i386 */
242:
1.19 dyoung 243: #if !defined(SPLDEBUG)
1.21 jym 244: if (!early && (cpu_feature[0] & CPUID_CX8) != 0) {
1.6 ad 245: /* Faster splx(), mutex_spin_exit(). */
246: patchfunc(
247: cx8_spllower, cx8_spllower_end,
248: spllower, spllower_end,
249: cx8_spllower_patch
250: );
1.15 ad 251: #if defined(i386) && !defined(LOCKDEBUG)
1.6 ad 252: patchfunc(
253: i686_mutex_spin_exit, i686_mutex_spin_exit_end,
254: mutex_spin_exit, mutex_spin_exit_end,
255: i686_mutex_spin_exit_patch
256: );
1.19 dyoung 257: #endif /* i386 && !LOCKDEBUG */
1.6 ad 258: }
1.19 dyoung 259: #endif /* !SPLDEBUG */
1.6 ad 260:
1.18 ad 261: /*
262: * On some Opteron revisions, locked operations erroneously
263: * allow memory references to be `bled' outside of critical
264: * sections. Apply workaround.
265: */
266: if (cpu_vendor == CPUVENDOR_AMD &&
1.22 msaitoh 267: (CPUID_TO_FAMILY(cpu_info_primary.ci_signature) == 0xe ||
268: (CPUID_TO_FAMILY(cpu_info_primary.ci_signature) == 0xf &&
269: CPUID_TO_EXTMODEL(cpu_info_primary.ci_signature) < 0x4))) {
1.25 maxv 270: const uint8_t bytes[] = {
271: 0x0F, 0xAE, 0xE8 /* lfence */
272: };
273:
1.29 maxv 274: /* ret,nop,nop -> lfence */
275: x86_hotpatch(HP_NAME_RETFENCE, bytes, sizeof(bytes));
1.18 ad 276: }
277:
1.23 maxv 278: /*
279: * If SMAP is present then patch the prepared holes with clac/stac
280: * instructions.
281: *
282: * clac = 0x0f, 0x01, 0xca
283: * stac = 0x0f, 0x01, 0xcb
284: */
285: if (!early && cpu_feature[5] & CPUID_SEF_SMAP) {
286: KASSERT(rcr4() & CR4_SMAP);
1.25 maxv 287: const uint8_t clac_bytes[] = {
288: 0x0F, 0x01, 0xCA /* clac */
289: };
290: const uint8_t stac_bytes[] = {
291: 0x0F, 0x01, 0xCB /* stac */
292: };
293:
1.26 maxv 294: /* nop,nop,nop -> clac */
295: x86_hotpatch(HP_NAME_CLAC, clac_bytes, sizeof(clac_bytes));
296:
297: /* nop,nop,nop -> stac */
298: x86_hotpatch(HP_NAME_STAC, stac_bytes, sizeof(stac_bytes));
1.23 maxv 299: }
300:
1.32 ! maxv 301: #ifdef SVS
! 302: if (early && cpu_vendor == CPUVENDOR_INTEL) {
! 303: extern uint8_t svs_enter, svs_enter_end;
! 304: extern uint8_t svs_enter_altstack, svs_enter_altstack_end;
! 305: extern uint8_t svs_leave, svs_leave_end;
! 306: extern uint8_t svs_leave_altstack, svs_leave_altstack_end;
! 307: extern bool svs_enabled;
! 308: uint8_t *bytes;
! 309: size_t size;
! 310:
! 311: svs_enabled = true;
! 312:
! 313: bytes = &svs_enter;
! 314: size = (size_t)&svs_enter_end - (size_t)&svs_enter;
! 315: x86_hotpatch(HP_NAME_SVS_ENTER, bytes, size);
! 316:
! 317: bytes = &svs_enter_altstack;
! 318: size = (size_t)&svs_enter_altstack_end -
! 319: (size_t)&svs_enter_altstack;
! 320: x86_hotpatch(HP_NAME_SVS_ENTER_ALT, bytes, size);
! 321:
! 322: bytes = &svs_leave;
! 323: size = (size_t)&svs_leave_end - (size_t)&svs_leave;
! 324: x86_hotpatch(HP_NAME_SVS_LEAVE, bytes, size);
! 325:
! 326: bytes = &svs_leave_altstack;
! 327: size = (size_t)&svs_leave_altstack_end -
! 328: (size_t)&svs_leave_altstack;
! 329: x86_hotpatch(HP_NAME_SVS_LEAVE_ALT, bytes, size);
! 330: }
! 331: #endif
! 332:
1.6 ad 333: /* Write back and invalidate cache, flush pipelines. */
334: wbinvd();
335: x86_flush();
336: x86_write_psl(psl);
1.2 ad 337:
338: /* Re-enable write protection. */
339: lcr0(cr0);
340: }
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to patch.c CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / arch / x86 / x86