Annotation of src/sys/arch/amd64/amd64/cpufunc.S, Revision 1.57
1.57 ! ad 1: /* $NetBSD: cpufunc.S,v 1.56 2020/05/20 18:52:48 maxv Exp $ */
1.1 ad 2:
1.30 maxv 3: /*
1.53 ad 4: * Copyright (c) 1998, 2007, 2008, 2020 The NetBSD Foundation, Inc.
1.1 ad 5: * All rights reserved.
6: *
7: * This code is derived from software contributed to The NetBSD Foundation
8: * by Charles M. Hannum, and by Andrew Doran.
9: *
10: * Redistribution and use in source and binary forms, with or without
11: * modification, are permitted provided that the following conditions
12: * are met:
13: * 1. Redistributions of source code must retain the above copyright
14: * notice, this list of conditions and the following disclaimer.
15: * 2. Redistributions in binary form must reproduce the above copyright
16: * notice, this list of conditions and the following disclaimer in the
17: * documentation and/or other materials provided with the distribution.
18: *
19: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29: * POSSIBILITY OF SUCH DAMAGE.
30: */
31:
1.20 jym 32: #include <sys/errno.h>
33:
1.1 ad 34: #include <machine/asm.h>
1.17 chs 35: #include <machine/frameasm.h>
1.1 ad 36: #include <machine/specialreg.h>
37: #include <machine/segments.h>
38:
1.3 bouyer 39: #include "opt_xen.h"
1.41 maxv 40: #include "opt_svs.h"
1.3 bouyer 41:
1.1 ad 42: #include "assym.h"
43:
44: /* Small and slow, so align less. */
45: #undef _ALIGN_TEXT
46: #define _ALIGN_TEXT .align 8
47:
1.14 chs 48: ENTRY(x86_lfence)
1.1 ad 49: lfence
50: ret
1.30 maxv 51: END(x86_lfence)
1.1 ad 52:
1.14 chs 53: ENTRY(x86_sfence)
1.1 ad 54: sfence
55: ret
1.30 maxv 56: END(x86_sfence)
1.1 ad 57:
1.14 chs 58: ENTRY(x86_mfence)
1.1 ad 59: mfence
60: ret
1.30 maxv 61: END(x86_mfence)
1.1 ad 62:
1.36 cherry 63: #ifndef XENPV
1.35 cherry 64: ENTRY(invlpg)
1.41 maxv 65: #ifdef SVS
66: movb _C_LABEL(svs_pcid),%al
67: testb %al,%al
68: jz 1f
69: pushq %rdi
70: pushq $PMAP_PCID_USER
71: movq $INVPCID_ADDRESS,%rax
72: invpcid (%rsp),%rax
73: addq $16,%rsp
74: 1: /* FALLTHROUGH */
75: #endif
1.1 ad 76: invlpg (%rdi)
77: ret
1.35 cherry 78: END(invlpg)
1.1 ad 79:
1.45 maxv 80: ENTRY(lgdt)
81: /* Reload the descriptor table. */
82: movq %rdi,%rax
83: lgdt (%rax)
84: /* Flush the prefetch queue. */
85: jmp 1f
86: nop
87: 1: jmp _C_LABEL(lgdt_finish)
88: END(lgdt)
89:
1.35 cherry 90: ENTRY(lidt)
1.1 ad 91: lidt (%rdi)
92: ret
1.35 cherry 93: END(lidt)
1.1 ad 94:
1.35 cherry 95: ENTRY(lldt)
1.13 ad 96: cmpl %edi, CPUVAR(CURLDT)
97: jne 1f
98: ret
99: 1:
100: movl %edi, CPUVAR(CURLDT)
1.1 ad 101: lldt %di
102: ret
1.35 cherry 103: END(lldt)
1.1 ad 104:
1.35 cherry 105: ENTRY(ltr)
1.1 ad 106: ltr %di
107: ret
1.35 cherry 108: END(ltr)
1.1 ad 109:
1.35 cherry 110: ENTRY(tlbflushg)
1.1 ad 111: movq %cr4, %rax
1.19 rmind 112: testq $CR4_PGE, %rax
1.41 maxv 113: jz tlbflush
1.1 ad 114: movq %rax, %rdx
115: andq $~CR4_PGE, %rdx
116: movq %rdx, %cr4
117: movq %rax, %cr4
118: ret
1.35 cherry 119: END(tlbflushg)
1.1 ad 120:
1.35 cherry 121: ENTRY(tlbflush)
1.41 maxv 122: #ifdef SVS
123: movb _C_LABEL(svs_pcid),%al
124: testb %al,%al
125: jz 1f
126: xorq %rax,%rax
127: pushq %rax
128: pushq %rax
129: movq $INVPCID_ALL_NONGLOBAL,%rax
130: invpcid (%rsp),%rax
131: addq $16,%rsp
132: ret
133: #endif
134: 1: movq %cr3, %rax
1.1 ad 135: movq %rax, %cr3
136: ret
1.35 cherry 137: END(tlbflush)
1.1 ad 138:
1.45 maxv 139: ENTRY(wbinvd)
140: wbinvd
141: ret
142: END(wbinvd)
143:
144: ENTRY(setusergs)
145: CLI(ax)
146: swapgs
147: movw %di, %gs
148: swapgs
149: STI(ax)
150: ret
151: END(setusergs)
152:
1.14 chs 153: ENTRY(x86_read_flags)
1.1 ad 154: pushfq
155: popq %rax
1.47 maxv 156: KMSAN_INIT_RET(8)
1.1 ad 157: ret
1.30 maxv 158: END(x86_read_flags)
1.1 ad 159:
160: STRONG_ALIAS(x86_read_psl,x86_read_flags)
161:
1.14 chs 162: ENTRY(x86_write_flags)
1.1 ad 163: pushq %rdi
164: popfq
165: ret
1.30 maxv 166: END(x86_write_flags)
1.1 ad 167:
168: STRONG_ALIAS(x86_write_psl,x86_write_flags)
169:
1.51 bouyer 170: /*
171: * %rdi = name
172: * %rsi = sel
173: */
174: ENTRY(x86_hotpatch)
175: /* save RFLAGS, and disable intrs */
176: pushfq
177: cli
178:
179: /* save CR0, and disable WP */
180: movq %cr0,%rcx
181: pushq %rcx
182: andq $~CR0_WP,%rcx
183: movq %rcx,%cr0
184:
185: callq _C_LABEL(x86_hotpatch_apply)
186:
187: /* write back and invalidate cache */
188: wbinvd
189:
190: /* restore CR0 */
191: popq %rcx
192: movq %rcx,%cr0
193:
194: /* flush instruction pipeline */
195: pushq %rax
196: callq x86_flush
197: popq %rax
198:
199: /* clean up */
200: movq %rax,%rdi
201: callq _C_LABEL(x86_hotpatch_cleanup)
202:
203: /* restore RFLAGS */
204: popfq
205: ret
206: END(x86_hotpatch)
1.45 maxv 207: #endif /* !XENPV */
208:
1.56 maxv 209: /* Could be exact same as cpu_counter, but KMSAN needs to have the correct
210: * size of the return value. */
1.55 ad 211: ENTRY(cpu_counter32)
212: movq CPUVAR(CURLWP), %rcx
213: 1:
214: movq L_NCSW(%rcx), %rdi
215: rdtsc
216: addl CPUVAR(CC_SKEW), %eax
217: cmpq %rdi, L_NCSW(%rcx)
218: jne 2f
219: KMSAN_INIT_RET(4)
220: ret
221: 2:
222: jmp 1b
223: END(cpu_counter32)
224: STRONG_ALIAS(tsc_get_timecount, cpu_counter32)
225:
226: ENTRY(cpu_counter)
1.54 ad 227: movq CPUVAR(CURLWP), %rcx
228: 1:
229: movq L_NCSW(%rcx), %rdi
230: rdtsc
231: shlq $32, %rdx
232: orq %rdx, %rax
233: addq CPUVAR(CC_SKEW), %rax
234: cmpq %rdi, L_NCSW(%rcx)
235: jne 2f
1.55 ad 236: KMSAN_INIT_RET(8)
1.54 ad 237: ret
238: 2:
239: jmp 1b
1.55 ad 240: END(cpu_counter)
1.54 ad 241:
1.20 jym 242: ENTRY(rdmsr_safe)
243: movq CPUVAR(CURLWP), %r8
244: movq L_PCB(%r8), %r8
245: movq $_C_LABEL(msr_onfault), PCB_ONFAULT(%r8)
246:
1.45 maxv 247: movl %edi, %ecx
248: rdmsr
249: salq $32, %rdx
1.20 jym 250: movl %eax, %eax /* zero-extend %eax -> %rax */
251: orq %rdx, %rax
1.45 maxv 252: movq %rax, (%rsi)
1.20 jym 253:
1.45 maxv 254: xorq %rax, %rax
1.20 jym 255: movq %rax, PCB_ONFAULT(%r8)
1.47 maxv 256: #ifdef KMSAN
257: movq %rsi,%rdi
258: movq $8,%rsi
259: xorq %rdx,%rdx
260: callq _C_LABEL(kmsan_mark)
261: #endif
262: KMSAN_INIT_RET(4)
1.20 jym 263: ret
1.30 maxv 264: END(rdmsr_safe)
1.20 jym 265:
1.32 maxv 266: ENTRY(msr_onfault)
1.20 jym 267: movq CPUVAR(CURLWP), %r8
268: movq L_PCB(%r8), %r8
269: movq $0, PCB_ONFAULT(%r8)
270: movl $EFAULT, %eax
1.21 jym 271: ret
1.30 maxv 272: END(msr_onfault)
1.20 jym 273:
1.14 chs 274: ENTRY(breakpoint)
1.7 ad 275: pushq %rbp
276: movq %rsp, %rbp
1.1 ad 277: int $0x03 /* paranoid, not 'int3' */
1.7 ad 278: leave
1.1 ad 279: ret
1.30 maxv 280: END(breakpoint)
1.1 ad 281:
1.14 chs 282: ENTRY(x86_curcpu)
1.1 ad 283: movq %gs:(CPU_INFO_SELF), %rax
1.47 maxv 284: KMSAN_INIT_RET(8)
1.1 ad 285: ret
1.30 maxv 286: END(x86_curcpu)
1.1 ad 287:
1.14 chs 288: ENTRY(x86_curlwp)
1.1 ad 289: movq %gs:(CPU_INFO_CURLWP), %rax
1.47 maxv 290: KMSAN_INIT_RET(8)
1.1 ad 291: ret
1.30 maxv 292: END(x86_curlwp)
1.1 ad 293:
1.14 chs 294: ENTRY(__byte_swap_u32_variable)
1.1 ad 295: movl %edi, %eax
296: bswapl %eax
1.47 maxv 297: KMSAN_INIT_RET(4)
1.1 ad 298: ret
1.30 maxv 299: END(__byte_swap_u32_variable)
1.1 ad 300:
1.14 chs 301: ENTRY(__byte_swap_u16_variable)
1.1 ad 302: movl %edi, %eax
303: xchgb %al, %ah
1.47 maxv 304: KMSAN_INIT_RET(2)
1.1 ad 305: ret
1.30 maxv 306: END(__byte_swap_u16_variable)
1.1 ad 307:
308: /*
1.45 maxv 309: * Reload segments after a GDT change.
1.3 bouyer 310: */
1.14 chs 311: ENTRY(lgdt_finish)
1.1 ad 312: movl $GSEL(GDATA_SEL, SEL_KPL),%eax
313: movl %eax,%ds
314: movl %eax,%es
315: movl %eax,%ss
1.31 maxv 316: jmp _C_LABEL(x86_flush)
317: END(lgdt_finish)
1.1 ad 318:
319: /*
320: * Flush instruction pipelines by doing an intersegment (far) return.
321: */
1.14 chs 322: ENTRY(x86_flush)
1.1 ad 323: popq %rax
324: pushq $GSEL(GCODE_SEL, SEL_KPL)
325: pushq %rax
326: lretq
1.30 maxv 327: END(x86_flush)
1.1 ad 328:
329: /* Waits - set up stack frame. */
1.14 chs 330: ENTRY(x86_hlt)
1.1 ad 331: pushq %rbp
332: movq %rsp, %rbp
333: hlt
334: leave
335: ret
1.30 maxv 336: END(x86_hlt)
1.1 ad 337:
338: /* Waits - set up stack frame. */
1.14 chs 339: ENTRY(x86_stihlt)
1.1 ad 340: pushq %rbp
341: movq %rsp, %rbp
342: sti
343: hlt
344: leave
345: ret
1.30 maxv 346: END(x86_stihlt)
1.1 ad 347:
1.14 chs 348: ENTRY(x86_monitor)
1.1 ad 349: movq %rdi, %rax
350: movq %rsi, %rcx
1.16 skrll 351: monitor %rax, %rcx, %rdx
1.1 ad 352: ret
1.30 maxv 353: END(x86_monitor)
1.1 ad 354:
355: /* Waits - set up stack frame. */
1.30 maxv 356: ENTRY(x86_mwait)
1.1 ad 357: pushq %rbp
358: movq %rsp, %rbp
359: movq %rdi, %rax
360: movq %rsi, %rcx
1.16 skrll 361: mwait %rax, %rcx
1.1 ad 362: leave
363: ret
1.30 maxv 364: END(x86_mwait)
1.1 ad 365:
366: ENTRY(stts)
367: movq %cr0, %rax
368: orq $CR0_TS, %rax
369: movq %rax, %cr0
370: ret
1.30 maxv 371: END(stts)
1.1 ad 372:
1.14 chs 373: ENTRY(fldummy)
1.1 ad 374: ffree %st(7)
1.25 dsl 375: fldz
1.1 ad 376: ret
1.30 maxv 377: END(fldummy)
1.1 ad 378:
1.14 chs 379: ENTRY(inb)
1.1 ad 380: movq %rdi, %rdx
381: xorq %rax, %rax
382: inb %dx, %al
1.47 maxv 383: KMSAN_INIT_RET(1)
1.1 ad 384: ret
1.30 maxv 385: END(inb)
1.1 ad 386:
1.14 chs 387: ENTRY(inw)
1.1 ad 388: movq %rdi, %rdx
389: xorq %rax, %rax
390: inw %dx, %ax
1.47 maxv 391: KMSAN_INIT_RET(2)
1.1 ad 392: ret
1.30 maxv 393: END(inw)
1.1 ad 394:
1.14 chs 395: ENTRY(inl)
1.1 ad 396: movq %rdi, %rdx
397: xorq %rax, %rax
398: inl %dx, %eax
1.47 maxv 399: KMSAN_INIT_RET(4)
1.1 ad 400: ret
1.30 maxv 401: END(inl)
1.1 ad 402:
1.14 chs 403: ENTRY(outb)
1.1 ad 404: movq %rdi, %rdx
405: movq %rsi, %rax
406: outb %al, %dx
407: ret
1.30 maxv 408: END(outb)
1.1 ad 409:
1.14 chs 410: ENTRY(outw)
1.1 ad 411: movq %rdi, %rdx
412: movq %rsi, %rax
413: outw %ax, %dx
414: ret
1.30 maxv 415: END(outw)
1.1 ad 416:
1.14 chs 417: ENTRY(outl)
1.1 ad 418: movq %rdi, %rdx
419: movq %rsi, %rax
420: outl %eax, %dx
421: ret
1.30 maxv 422: END(outl)
1.57 ! ad 423:
! 424: ENTRY(x86_stos)
! 425: pushq %rbp
! 426: movq %rsp,%rbp
! 427: movq %rsi,%rax
! 428: movq %rdx,%rcx
! 429: rep
! 430: stosq
! 431: leave
! 432: ret
! 433: END(x86_stos)
! 434:
! 435: ENTRY(x86_movs)
! 436: pushq %rbp
! 437: movq %rsp,%rbp
! 438: movq %rdx,%rcx
! 439: rep
! 440: movsq
! 441: leave
! 442: ret
! 443: END(x86_stos)
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to cpufunc.S CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / arch / amd64 / amd64