[BACK]Return to patch.c CVS log [TXT][DIR] Up to [cvs.NetBSD.org] / src / sys / arch / x86 / x86

Annotation of src/sys/arch/x86/x86/patch.c, Revision 1.2.4.6

1.2.4.6 ! yamt        1: /*     $NetBSD: patch.c,v 1.2.4.5 2007/12/07 17:27:02 yamt Exp $       */
1.2.4.2   yamt        2:
                      3: /*-
                      4:  * Copyright (c) 2007 The NetBSD Foundation, Inc.
                      5:  * All rights reserved.
                      6:  *
                      7:  * This code is derived from software contributed to The NetBSD Foundation
                      8:  * by Andrew Doran.
                      9:  *
                     10:  * Redistribution and use in source and binary forms, with or without
                     11:  * modification, are permitted provided that the following conditions
                     12:  * are met:
                     13:  * 1. Redistributions of source code must retain the above copyright
                     14:  *    notice, this list of conditions and the following disclaimer.
                     15:  * 2. Redistributions in binary form must reproduce the above copyright
                     16:  *    notice, this list of conditions and the following disclaimer in the
                     17:  *    documentation and/or other materials provided with the distribution.
                     18:  * 3. All advertising materials mentioning features or use of this software
                     19:  *    must display the following acknowledgement:
                     20:  *     This product includes software developed by the NetBSD
                     21:  *     Foundation, Inc. and its contributors.
                     22:  * 4. Neither the name of The NetBSD Foundation nor the names of its
                     23:  *    contributors may be used to endorse or promote products derived
                     24:  *    from this software without specific prior written permission.
                     25:  *
                     26:  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
                     27:  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
                     28:  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
                     29:  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
                     30:  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
                     31:  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
                     32:  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
                     33:  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
                     34:  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
                     35:  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
                     36:  * POSSIBILITY OF SUCH DAMAGE.
                     37:  */
                     38:
                     39: /*
                     40:  * Patch kernel code at boot time, depending on available CPU features.
                     41:  */
                     42:
                     43: #include <sys/cdefs.h>
1.2.4.6 ! yamt       44: __KERNEL_RCSID(0, "$NetBSD: patch.c,v 1.2.4.5 2007/12/07 17:27:02 yamt Exp $");
1.2.4.2   yamt       45:
                     46: #include "opt_lockdebug.h"
                     47:
                     48: #include <sys/types.h>
                     49: #include <sys/systm.h>
                     50:
                     51: #include <machine/cpu.h>
                     52: #include <machine/cpufunc.h>
                     53: #include <machine/specialreg.h>
                     54:
                     55: #include <x86/cpuvar.h>
                     56: #include <x86/cputypes.h>
                     57:
                     58: void   spllower(int);
                     59: void   spllower_end(void);
1.2.4.5   yamt       60: void   cx8_spllower(int);
                     61: void   cx8_spllower_end(void);
                     62: void   cx8_spllower_patch(void);
1.2.4.2   yamt       63:
                     64: void   mutex_spin_exit_end(void);
                     65: void   i686_mutex_spin_exit(int);
                     66: void   i686_mutex_spin_exit_end(void);
                     67: void   i686_mutex_spin_exit_patch(void);
                     68:
1.2.4.6 ! yamt       69: void   membar_consumer(void);
        !            70: void   membar_consumer_end(void);
        !            71: void   membar_sync(void);
        !            72: void   membar_sync_end(void);
        !            73: void   sse2_lfence(void);
        !            74: void   sse2_lfence_end(void);
        !            75: void   sse2_mfence(void);
        !            76: void   sse2_mfence_end(void);
        !            77:
1.2.4.2   yamt       78: void   mb_read(void);
                     79: void   mb_read_end(void);
                     80: void   mb_memory(void);
                     81: void   mb_memory_end(void);
                     82: void   sse2_mb_read(void);
                     83: void   sse2_mb_read_end(void);
                     84: void   sse2_mb_memory(void);
                     85: void   sse2_mb_memory_end(void);
                     86:
1.2.4.5   yamt       87: extern void    *x86_lockpatch[];
1.2.4.6 ! yamt       88: extern void    *atomic_lockpatch[];
1.2.4.5   yamt       89:
1.2.4.2   yamt       90: #define        X86_NOP         0x90
                     91: #define        X86_REP         0xf3
                     92: #define        X86_RET         0xc3
                     93: #define        X86_CS          0x2e
                     94: #define        X86_DS          0x3e
                     95: #define        X86_GROUP_0F    0x0f
                     96:
                     97: static void __attribute__ ((__unused__))
                     98: patchfunc(void *from_s, void *from_e, void *to_s, void *to_e,
                     99:          void *pcrel)
                    100: {
                    101:        uint8_t *ptr;
                    102:
                    103:        if ((uintptr_t)from_e - (uintptr_t)from_s !=
                    104:            (uintptr_t)to_e - (uintptr_t)to_s)
                    105:                panic("patchfunc: sizes do not match (from=%p)", from_s);
                    106:
                    107:        memcpy(to_s, from_s, (uintptr_t)to_e - (uintptr_t)to_s);
                    108:        if (pcrel != NULL) {
                    109:                ptr = pcrel;
                    110:                /* Branch hints */
                    111:                if (ptr[0] == X86_CS || ptr[0] == X86_DS)
                    112:                        ptr++;
                    113:                /* Conditional jumps */
                    114:                if (ptr[0] == X86_GROUP_0F)
                    115:                        ptr++;
                    116:                /* 4-byte relative jump or call */
                    117:                *(uint32_t *)(ptr + 1 - (uintptr_t)from_s + (uintptr_t)to_s) +=
                    118:                    ((uint32_t)(uintptr_t)from_s - (uint32_t)(uintptr_t)to_s);
                    119:        }
                    120: }
                    121:
                    122: static inline void  __attribute__ ((__unused__))
                    123: patchbytes(void *addr, const int byte1, const int byte2)
                    124: {
1.2.4.5   yamt      125:
1.2.4.2   yamt      126:        ((uint8_t *)addr)[0] = (uint8_t)byte1;
                    127:        if (byte2 != -1)
                    128:                ((uint8_t *)addr)[1] = (uint8_t)byte2;
                    129: }
                    130:
                    131: void
                    132: x86_patch(void)
                    133: {
1.2.4.6 ! yamt      134: #if !defined(GPROF) && !defined(LOCKDEBUG)
1.2.4.2   yamt      135:        static int again;
1.2.4.5   yamt      136:        u_long psl;
1.2.4.2   yamt      137:        u_long cr0;
                    138:
                    139:        if (again)
                    140:                return;
                    141:        again = 1;
                    142:
1.2.4.5   yamt      143:        /* Disable interrupts. */
                    144:        psl = x86_read_psl();
                    145:        x86_disable_intr();
                    146:
1.2.4.2   yamt      147:        /* Disable write protection in supervisor mode. */
                    148:        cr0 = rcr0();
                    149:        lcr0(cr0 & ~CR0_WP);
                    150:
1.2.4.5   yamt      151:        if (ncpu == 1) {
                    152:                int i;
                    153:
                    154:                /* Uniprocessor: kill LOCK prefixes. */
                    155:                for (i = 0; x86_lockpatch[i] != 0; i++)
                    156:                        patchbytes(x86_lockpatch[i], X86_NOP, -1);
1.2.4.6 ! yamt      157:                for (i = 0; atomic_lockpatch[i] != 0; i++)
        !           158:                        patchbytes(atomic_lockpatch[i], X86_NOP, -1);
1.2.4.5   yamt      159:                /*
                    160:                 * Uniprocessor: kill kernel_lock.  Fill another
                    161:                 * 14 bytes of NOPs so not to confuse the decoder.
                    162:                 */
                    163:                patchbytes(_kernel_lock, X86_NOP, X86_RET);
                    164:                patchbytes(_kernel_unlock, X86_NOP, X86_RET);
                    165:                for (i = 2; i < 16; i++) {
                    166:                        patchbytes((char *)_kernel_lock + i, X86_NOP, -1);
                    167:                        patchbytes((char *)_kernel_unlock + i, X86_NOP, -1);
                    168:                }
                    169:        } else if ((cpu_feature & CPUID_SSE2) != 0) {
                    170:                /* Faster memory barriers. */
1.2.4.2   yamt      171:                patchfunc(
                    172:                    sse2_mb_read, sse2_mb_read_end,
                    173:                    mb_read, mb_read_end,
                    174:                    NULL
                    175:                );
                    176:                patchfunc(
                    177:                    sse2_mb_memory, sse2_mb_memory_end,
                    178:                    mb_memory, mb_memory_end,
                    179:                    NULL
                    180:                );
1.2.4.6 ! yamt      181:                patchfunc(
        !           182:                    sse2_lfence, sse2_lfence_end,
        !           183:                    membar_consumer, membar_consumer_end,
        !           184:                    NULL
        !           185:                );
        !           186:                patchfunc(
        !           187:                    sse2_mfence, sse2_mfence_end,
        !           188:                    membar_sync, membar_sync_end,
        !           189:                    NULL
        !           190:                );
1.2.4.2   yamt      191:        }
                    192:
1.2.4.5   yamt      193:        if ((cpu_feature & CPUID_CX8) != 0) {
                    194:                /* Faster splx(), mutex_spin_exit(). */
                    195:                patchfunc(
                    196:                    cx8_spllower, cx8_spllower_end,
                    197:                    spllower, spllower_end,
                    198:                    cx8_spllower_patch
                    199:                );
1.2.4.6 ! yamt      200: #if defined(i386)
1.2.4.5   yamt      201:                patchfunc(
                    202:                    i686_mutex_spin_exit, i686_mutex_spin_exit_end,
                    203:                    mutex_spin_exit, mutex_spin_exit_end,
                    204:                    i686_mutex_spin_exit_patch
                    205:                );
                    206: #endif
                    207:        }
                    208:
                    209:        /* Write back and invalidate cache, flush pipelines. */
                    210:        wbinvd();
                    211:        x86_flush();
                    212:        x86_write_psl(psl);
1.2.4.2   yamt      213:
                    214:        /* Re-enable write protection. */
                    215:        lcr0(cr0);
                    216: #endif /* GPROF */
                    217: }

CVSweb <webmaster@jp.NetBSD.org>