Annotation of src/sys/arch/amd64/amd64/locore.S, Revision 1.68
1.68 ! jym 1: /* $NetBSD: locore.S,v 1.67 2012/02/24 08:06:07 cherry Exp $ */
1.1 fvdl 2:
3: /*
4: * Copyright-o-rama!
5: */
6:
7: /*
1.32 bouyer 8: * Copyright (c) 2007 Manuel Bouyer.
9: *
10: * Redistribution and use in source and binary forms, with or without
11: * modification, are permitted provided that the following conditions
12: * are met:
13: * 1. Redistributions of source code must retain the above copyright
14: * notice, this list of conditions and the following disclaimer.
15: * 2. Redistributions in binary form must reproduce the above copyright
16: * notice, this list of conditions and the following disclaimer in the
17: * documentation and/or other materials provided with the distribution.
18: *
19: * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20: * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21: * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22: * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23: * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24: * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25: * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26: * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27: * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28: * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29: *
30: */
31:
32: /*
33: * Copyright (c) 2006 Mathieu Ropert <mro@adviseo.fr>
34: *
35: * Permission to use, copy, modify, and distribute this software for any
36: * purpose with or without fee is hereby granted, provided that the above
37: * copyright notice and this permission notice appear in all copies.
38: *
39: * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
40: * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
41: * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
42: * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
43: * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
44: * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
45: * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
46: */
47:
48: /*
1.1 fvdl 49: * Copyright (c) 2001 Wasabi Systems, Inc.
50: * All rights reserved.
51: *
52: * Written by Frank van der Linden for Wasabi Systems, Inc.
53: *
54: * Redistribution and use in source and binary forms, with or without
55: * modification, are permitted provided that the following conditions
56: * are met:
57: * 1. Redistributions of source code must retain the above copyright
58: * notice, this list of conditions and the following disclaimer.
59: * 2. Redistributions in binary form must reproduce the above copyright
60: * notice, this list of conditions and the following disclaimer in the
61: * documentation and/or other materials provided with the distribution.
62: * 3. All advertising materials mentioning features or use of this software
63: * must display the following acknowledgement:
64: * This product includes software developed for the NetBSD Project by
65: * Wasabi Systems, Inc.
66: * 4. The name of Wasabi Systems, Inc. may not be used to endorse
67: * or promote products derived from this software without specific prior
68: * written permission.
69: *
70: * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
71: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
72: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
73: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
74: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
75: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
76: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
77: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
78: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
79: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
80: * POSSIBILITY OF SUCH DAMAGE.
81: */
82:
83:
84: /*-
1.41 ad 85: * Copyright (c) 1998, 2000, 2007, 2008 The NetBSD Foundation, Inc.
1.1 fvdl 86: * All rights reserved.
87: *
88: * This code is derived from software contributed to The NetBSD Foundation
89: * by Charles M. Hannum.
90: *
91: * Redistribution and use in source and binary forms, with or without
92: * modification, are permitted provided that the following conditions
93: * are met:
94: * 1. Redistributions of source code must retain the above copyright
95: * notice, this list of conditions and the following disclaimer.
96: * 2. Redistributions in binary form must reproduce the above copyright
97: * notice, this list of conditions and the following disclaimer in the
98: * documentation and/or other materials provided with the distribution.
99: *
100: * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
101: * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
102: * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
103: * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
104: * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
105: * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
106: * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
107: * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
108: * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
109: * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
110: * POSSIBILITY OF SUCH DAMAGE.
111: */
112:
113: /*-
114: * Copyright (c) 1990 The Regents of the University of California.
115: * All rights reserved.
116: *
117: * This code is derived from software contributed to Berkeley by
118: * William Jolitz.
119: *
120: * Redistribution and use in source and binary forms, with or without
121: * modification, are permitted provided that the following conditions
122: * are met:
123: * 1. Redistributions of source code must retain the above copyright
124: * notice, this list of conditions and the following disclaimer.
125: * 2. Redistributions in binary form must reproduce the above copyright
126: * notice, this list of conditions and the following disclaimer in the
127: * documentation and/or other materials provided with the distribution.
1.5 agc 128: * 3. Neither the name of the University nor the names of its contributors
1.1 fvdl 129: * may be used to endorse or promote products derived from this software
130: * without specific prior written permission.
131: *
132: * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
133: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
134: * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
135: * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
136: * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
137: * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
138: * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
139: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
140: * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
141: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
142: * SUCH DAMAGE.
143: *
144: * @(#)locore.s 7.3 (Berkeley) 5/13/91
145: */
146:
1.34 dsl 147: /*
148: * override user-land alignment before including asm.h
149: */
150: #define ALIGN_DATA .align 8
151: #define ALIGN_TEXT .align 16,0x90
152: #define _ALIGN_TEXT ALIGN_TEXT
153:
154: #include <machine/asm.h>
155:
1.1 fvdl 156: #include "opt_ddb.h"
157: #include "opt_ddbparam.h"
1.51 apb 158: #include "opt_modular.h"
1.1 fvdl 159: #include "opt_realmem.h"
160:
1.12 drochner 161: #include "opt_compat_netbsd.h"
162: #include "opt_compat_netbsd32.h"
163: #include "opt_compat_ibcs2.h"
1.32 bouyer 164: #include "opt_xen.h"
1.12 drochner 165:
1.1 fvdl 166: #include "assym.h"
167: #include "lapic.h"
168: #include "ioapic.h"
1.2 fvdl 169: #include "ksyms.h"
1.1 fvdl 170:
171: #include <sys/errno.h>
172: #include <sys/syscall.h>
173:
174: #include <machine/pte.h>
175: #include <machine/segments.h>
176: #include <machine/specialreg.h>
177: #include <machine/trap.h>
178: #include <machine/bootinfo.h>
179: #include <machine/frameasm.h>
1.44 ad 180: #include <machine/cputypes.h>
1.1 fvdl 181:
182: #if NLAPIC > 0
183: #include <machine/i82489reg.h>
184: #endif
185:
186: /* XXX temporary kluge; these should not be here */
187: /* Get definitions for IOM_BEGIN, IOM_END, and IOM_SIZE */
188: #include <dev/isa/isareg.h>
189:
1.32 bouyer 190: #ifdef XEN
191:
192: /*
193: * Xen Guest Loader Info
194: */
195:
196: .section __xen_guest
197:
198: .ascii "GUEST_OS=NetBSD,GUEST_VER=4.99"
199: .ascii ",XEN_VER=xen-3.0"
200: .ascii ",LOADER=generic"
201: .ascii ",VIRT_BASE=0xffffffff80000000"
202: .ascii ",ELF_PADDR_OFFSET=0xffffffff80000000"
203: .ascii ",VIRT_ENTRY=0xffffffff80100000"
204: .ascii ",HYPERCALL_PAGE=0x00000101" /* (???+HYPERCALL_PAGE_OFFSET)/PAGE_SIZE) */
1.49 ad 205: #if NKSYMS > 0 || defined(DDB) || defined(MODULAR)
1.33 bouyer 206: .ascii ",BSD_SYMTAB=yes"
1.32 bouyer 207: #endif
208: .byte 0
209:
210: #endif /* XEN */
1.1 fvdl 211:
212: /*
213: * Initialization
214: */
215: .data
216:
217: #if NLAPIC > 0
218: .align NBPG
219: .globl _C_LABEL(local_apic), _C_LABEL(lapic_id), _C_LABEL(lapic_tpr)
220: _C_LABEL(local_apic):
221: .space LAPIC_ID
222: _C_LABEL(lapic_id):
223: .long 0x00000000
224: .space LAPIC_TPRI-(LAPIC_ID+4)
225: _C_LABEL(lapic_tpr):
226: .space LAPIC_PPRI-LAPIC_TPRI
227: _C_LABEL(lapic_ppr):
228: .space LAPIC_ISR-LAPIC_PPRI
229: _C_LABEL(lapic_isr):
230: .space NBPG-LAPIC_ISR
231: #endif
232:
1.30 ad 233: .globl _C_LABEL(cpu_id),_C_LABEL(cpu_vendorname), _C_LABEL(cpu_brand_id)
1.57 jym 234: .globl _C_LABEL(cpuid_level)
1.43 ad 235: .globl _C_LABEL(esym),_C_LABEL(eblob),_C_LABEL(boothowto)
1.1 fvdl 236: .globl _C_LABEL(bootinfo),_C_LABEL(atdevbase)
1.54 matt 237: .globl _C_LABEL(PDPpaddr)
1.1 fvdl 238: .globl _C_LABEL(biosbasemem),_C_LABEL(biosextmem)
1.44 ad 239: .globl _C_LABEL(gdtstore),_C_LABEL(cpu)
1.43 ad 240:
1.1 fvdl 241: _C_LABEL(cpu): .long 0 # are we 386, 386sx, or 486,
242: # or Pentium, or..
243: _C_LABEL(cpu_id): .long 0 # saved from `cpuid' instruction
244: _C_LABEL(cpuid_level): .long -1 # max. level accepted by 'cpuid'
245: # instruction
1.30 ad 246: _C_LABEL(cpu_vendorname): .space 16 # vendor string returned by `cpuid'
1.1 fvdl 247: # instruction
248: _C_LABEL(cpu_brand_id): .long 0 # brand ID from 'cpuid' instruction
249: _C_LABEL(esym): .quad 0 # ptr to end of syms
1.43 ad 250: _C_LABEL(eblob): .quad 0 # ptr to end of modules
1.1 fvdl 251: _C_LABEL(atdevbase): .quad 0 # location of start of iomem in virtual
1.29 yamt 252: _C_LABEL(PDPpaddr): .quad 0 # paddr of PTD, for libkvm
1.1 fvdl 253: #ifndef REALBASEMEM
254: _C_LABEL(biosbasemem): .long 0 # base memory reported by BIOS
255: #else
256: _C_LABEL(biosbasemem): .long REALBASEMEM
257: #endif
258: #ifndef REALEXTMEM
259: _C_LABEL(biosextmem): .long 0 # extended memory reported by BIOS
260: #else
261: _C_LABEL(biosextmem): .long REALEXTMEM
262: #endif
263:
264: #define _RELOC(x) ((x) - KERNBASE)
265: #define RELOC(x) _RELOC(_C_LABEL(x))
266:
1.32 bouyer 267: #ifndef XEN
1.56 jym 268: .globl gdt64_lo
269: .globl gdt64_hi
1.1 fvdl 270:
1.56 jym 271: #define GDT64_LIMIT gdt64_end-gdt64_start-1
272:
273: /* Temporary gdt64, with base address in low memory */
274: gdt64_lo:
275: .word GDT64_LIMIT
1.1 fvdl 276: .quad _RELOC(gdt64_start)
277: .align 64
278:
1.56 jym 279: /* Temporary gdt64, with base address in high memory */
280: gdt64_hi:
281: .word GDT64_LIMIT
282: .quad gdt64_start
283: .align 64
284:
285: #undef GDT64_LIMIT
286:
1.1 fvdl 287: gdt64_start:
288: .quad 0x0000000000000000 /* always empty */
289: .quad 0x00af9a000000ffff /* kernel CS */
290: .quad 0x00cf92000000ffff /* kernel DS */
291: gdt64_end:
292:
293: farjmp64:
1.57 jym 294: .long _RELOC(longmode)
1.1 fvdl 295: .word GSEL(GCODE_SEL, SEL_KPL)
296:
1.32 bouyer 297: #endif /* !XEN */
1.1 fvdl 298: .space 512
299: tmpstk:
300:
301: .globl _C_LABEL(cpu_private)
302: .comm _C_LABEL(cpu_private),NBPG,NBPG
303:
304: /*
305: * Some hackage to deal with 64bit symbols in 32 bit mode.
306: * This may not be needed it things are cleaned up a little.
307: */
308:
309:
310: .text
311: .globl _C_LABEL(kernel_text)
312: .set _C_LABEL(kernel_text),KERNTEXTOFF
313:
1.32 bouyer 314: #ifndef XEN
1.1 fvdl 315: .code32
316:
317: .globl start
318: start: movw $0x1234,0x472 # warm boot
319:
320: /*
321: * Load parameters from stack
322: * (howto, [bootdev], bootinfo, esym, basemem, extmem).
323: */
324: movl 4(%esp),%eax
325: movl %eax,RELOC(boothowto)
326: movl 12(%esp),%eax
327: testl %eax, %eax
328: jz 1f
329: movl (%eax), %ebx /* number of entries */
330: movl $RELOC(bootinfo),%ebp
331: movl %ebp, %edx
332: addl $BOOTINFO_MAXSIZE,%ebp
333: movl %ebx, (%edx)
334: addl $4, %edx
335: 2:
336: testl %ebx, %ebx
337: jz 1f
338: addl $4, %eax
339: movl (%eax), %ecx /* address of entry */
340: pushl %edi
341: pushl %esi
342: pushl %eax
343:
344: movl (%ecx),%eax /* len */
345: movl %edx,%edi
346: addl (%ecx), %edx /* update dest pointer */
347: cmpl %ebp, %edx
348: jg 2f
349: movl %ecx,%esi
350: movl %eax,%ecx
1.43 ad 351: /*
352: * If any modules were loaded, record where they
353: * end. We'll need to skip over them.
354: */
355: cmpl $BTINFO_MODULELIST, 4(%esi)
356: jne 0f
357: pushl 12(%esi) /* endpa */
358: popl RELOC(eblob)
359: addl $KERNBASE_LO, RELOC(eblob)
360: adcl $KERNBASE_HI, RELOC(eblob)+4
361: 0:
1.1 fvdl 362: rep
363: movsb
364: popl %eax
365: popl %esi
366: popl %edi
367: subl $1, %ebx
368: jmp 2b
369: 2: /* cleanup for overflow case */
370: popl %eax
371: popl %esi
372: popl %edi
373: movl $RELOC(bootinfo),%ebp
374: movl %ebp, %edx
375: subl %ebx, (%edx) /* correct number of entries */
376: 1:
377:
378: movl 16(%esp),%eax
379: testl %eax,%eax
380: jz 1f
381: addl $KERNBASE_LO,%eax
382: 1: movl $RELOC(esym),%ebp
383: movl %eax,(%ebp)
384: movl $KERNBASE_HI,4(%ebp)
385:
386: movl $RELOC(biosextmem),%ebp
387: movl (%ebp),%eax
388: testl %eax,%eax
389: jnz 1f
390: movl 20(%esp),%eax
391: movl %eax,(%ebp)
392: 1:
393: movl $RELOC(biosbasemem),%ebp
394: movl (%ebp),%eax
395: testl %eax,%eax
396: jnz 1f
397: movl 24(%esp),%eax
398: movl %eax,(%ebp)
399: 1:
400:
401: /* First, reset the PSL. */
402: pushl $PSL_MBO
403: popfl
404:
405: xorl %eax,%eax
406: cpuid
407: movl %eax,RELOC(cpuid_level)
1.30 ad 408: movl $RELOC(cpu_vendorname),%ebp
1.1 fvdl 409: movl %ebx,(%ebp)
410: movl %edx,4(%ebp)
411: movl %ecx,8(%ebp)
412: movl $0, 12(%ebp)
413:
414: movl $1,%eax
415: cpuid
416: movl %eax,RELOC(cpu_id)
417:
418: /* Brand ID is bits 0-7 of %ebx */
419: andl $255,%ebx
420: movl %ebx,RELOC(cpu_brand_id)
421:
422: /*
423: * Finished with old stack; load new %esp now instead of later so we
424: * can trace this code without having to worry about the trace trap
425: * clobbering the memory test or the zeroing of the bss+bootstrap page
426: * tables.
427: *
428: * The boot program should check:
429: * text+data <= &stack_variable - more_space_for_stack
430: * text+data+bss+pad+space_for_page_tables <= end_of_memory
431: * Oops, the gdt is in the carcass of the boot program so clearing
432: * the rest of memory is still not possible.
433: */
434: movl $RELOC(tmpstk),%esp
435:
436: /*
437: * Virtual address space of kernel:
438: *
439: * text | data | bss | [syms] | page dir | proc0 kstack | L1 ptp | L2 ptp | L3
440: * 0 1 2 3
441: */
442:
443: #if L2_SLOT_KERNBASE > 0
444: #define TABLE_L2_ENTRIES (2 * (NKL2_KIMG_ENTRIES + 1))
445: #else
446: #define TABLE_L2_ENTRIES (NKL2_KIMG_ENTRIES + 1)
447: #endif
448:
449: #if L3_SLOT_KERNBASE > 0
450: #define TABLE_L3_ENTRIES (2 * NKL3_KIMG_ENTRIES)
451: #else
452: #define TABLE_L3_ENTRIES NKL3_KIMG_ENTRIES
453: #endif
454:
455:
456: #define PROC0_PML4_OFF 0
457: #define PROC0_STK_OFF (PROC0_PML4_OFF + NBPG)
458: #define PROC0_PTP3_OFF (PROC0_STK_OFF + UPAGES * NBPG)
459: #define PROC0_PTP2_OFF (PROC0_PTP3_OFF + NKL4_KIMG_ENTRIES * NBPG)
460: #define PROC0_PTP1_OFF (PROC0_PTP2_OFF + TABLE_L3_ENTRIES * NBPG)
461: #define TABLESIZE \
462: ((NKL4_KIMG_ENTRIES + TABLE_L3_ENTRIES + TABLE_L2_ENTRIES + 1 + UPAGES) \
463: * NBPG)
464:
465: #define fillkpt \
466: 1: movl %eax,(%ebx) ; /* store phys addr */ \
467: movl $0,4(%ebx) ; /* upper 32 bits 0 */ \
468: addl $8,%ebx ; /* next pte/pde */ \
469: addl $NBPG,%eax ; /* next phys page */ \
470: loop 1b ; \
471:
472:
473: /* Find end of kernel image. */
474: movl $RELOC(end),%edi
1.49 ad 475: #if (NKSYMS || defined(DDB) || defined(MODULAR)) && !defined(SYMTAB_SPACE)
1.1 fvdl 476: /* Save the symbols (if loaded). */
477: movl RELOC(esym),%eax
478: testl %eax,%eax
479: jz 1f
480: subl $KERNBASE_LO,%eax /* XXX */
481: movl %eax,%edi
482: 1:
483: #endif
1.43 ad 484: /* Skip over any modules/blobs. */
485: movl RELOC(eblob),%eax
486: testl %eax,%eax
487: jz 1f
488: subl $KERNBASE_LO,%eax /* XXX */
489: movl %eax,%edi
490: 1:
1.1 fvdl 491: /* Clear tables */
492: movl %edi,%esi
493: addl $PGOFSET,%esi
494: andl $~PGOFSET,%esi
495:
496: movl %esi,%edi
497: xorl %eax,%eax
498: cld
499: movl $TABLESIZE,%ecx
500: shrl $2,%ecx
501: rep
502: stosl
503:
504: leal (PROC0_PTP1_OFF)(%esi), %ebx
505:
506: /*
1.29 yamt 507: * Compute &__data_start - KERNBASE. This can't be > 4G,
508: * or we can't deal with it anyway, since we can't load it in
509: * 32 bit mode. So use the bottom 32 bits.
1.1 fvdl 510: */
1.29 yamt 511: movl $RELOC(__data_start),%edx
1.1 fvdl 512: andl $~PGOFSET,%edx
513:
514: /*
515: * Skip the first MB.
516: */
517: movl $(KERNTEXTOFF_LO - KERNBASE_LO),%eax
518: movl %eax,%ecx
519: shrl $(PGSHIFT-3),%ecx /* ((n >> PGSHIFT) << 3) for # pdes */
520: addl %ecx,%ebx
521:
522: /* Map kernel text read-only */
523: movl %edx,%ecx
524: subl %eax,%ecx
525: shrl $PGSHIFT,%ecx
526: orl $(PG_V|PG_KR),%eax
527: fillkpt
528:
529: /* Map the data, BSS, and bootstrap tables read-write. */
530: leal (PG_V|PG_KW)(%edx),%eax
531: movl $TABLESIZE,%ecx
532: addl %esi,%ecx /* %ecx = &end[TABLESIZE] */
533: subl %edx,%ecx /* %ecx = %ecx - etext */
534: shrl $PGSHIFT,%ecx
535: fillkpt
536:
537: /* Map ISA I/O mem (later atdevbase) */
538: movl $(IOM_BEGIN|PG_V|PG_KW/*|PG_N*/),%eax
539: movl $(IOM_SIZE>>PGSHIFT),%ecx
540: fillkpt
541:
542: /* Set up level 2 pages */
543: leal (PROC0_PTP2_OFF)(%esi),%ebx
544: leal (PROC0_PTP1_OFF)(%esi),%eax
545: orl $(PG_V|PG_KW), %eax
546: movl $(NKL2_KIMG_ENTRIES+1),%ecx
547: fillkpt
548:
549: #if L2_SLOT_KERNBASE > 0
550: /* If needed, set up level 2 entries for actual kernel mapping */
551: leal (PROC0_PTP2_OFF+ L2_SLOT_KERNBASE*8)(%esi),%ebx
552: leal (PROC0_PTP1_OFF)(%esi),%eax
553: orl $(PG_V|PG_KW), %eax
554: movl $(NKL2_KIMG_ENTRIES+1),%ecx
555: fillkpt
556: #endif
557:
558: /* Set up level 3 pages */
559: leal (PROC0_PTP3_OFF)(%esi),%ebx
560: leal (PROC0_PTP2_OFF)(%esi),%eax
561: orl $(PG_V|PG_KW), %eax
562: movl $NKL3_KIMG_ENTRIES,%ecx
563: fillkpt
564:
565: #if L3_SLOT_KERNBASE > 0
566: /* If needed, set up level 3 entries for actual kernel mapping */
567: leal (PROC0_PTP3_OFF+ L3_SLOT_KERNBASE*8)(%esi),%ebx
568: leal (PROC0_PTP2_OFF)(%esi),%eax
569: orl $(PG_V|PG_KW), %eax
570: movl $NKL3_KIMG_ENTRIES,%ecx
571: fillkpt
572: #endif
573:
574: /* Set up top level entries for identity mapping */
575: leal (PROC0_PML4_OFF)(%esi),%ebx
576: leal (PROC0_PTP3_OFF)(%esi),%eax
577: orl $(PG_V|PG_KW), %eax
578: movl $NKL4_KIMG_ENTRIES,%ecx
579: fillkpt
580:
581: /* Set up top level entries for actual kernel mapping */
582: leal (PROC0_PML4_OFF + L4_SLOT_KERNBASE*8)(%esi),%ebx
583: leal (PROC0_PTP3_OFF)(%esi),%eax
584: orl $(PG_V|PG_KW), %eax
585: movl $NKL4_KIMG_ENTRIES,%ecx
586: fillkpt
587:
588: /* Install recursive top level PDE */
589: leal (PROC0_PML4_OFF + PDIR_SLOT_PTE*8)(%esi),%ebx
590: leal (PROC0_PML4_OFF)(%esi),%eax
591: orl $(PG_V|PG_KW),%eax
592: movl %eax,(%ebx)
593: movl $0, 4(%ebx)
594:
595:
596: /* Save phys. addr of PTD, for libkvm. */
1.29 yamt 597: movl $RELOC(PDPpaddr),%ebp
1.1 fvdl 598: movl %esi,(%ebp)
599: movl $0,4(%ebp)
600:
601: /*
602: * Startup checklist:
603: * 1. Enable PAE (and SSE while here).
604: */
605: movl %cr4,%eax
606: orl $(CR4_PAE|CR4_OSFXSR|CR4_OSXMMEXCPT),%eax
607: movl %eax,%cr4
608:
609: /*
610: * 2. Set Long Mode Enable in EFER. Also enable the
611: * syscall extensions.
612: */
613: movl $MSR_EFER,%ecx
614: rdmsr
615: xorl %eax,%eax /* XXX */
616: orl $(EFER_LME|EFER_SCE),%eax
617: wrmsr
618:
619: /*
620: * 3. Load %cr3 with pointer to PML4.
621: */
622: movl %esi,%eax
623: movl %eax,%cr3
624:
625: /*
626: * 4. Enable paging and the rest of it.
627: */
628: movl %cr0,%eax
1.68 ! jym 629: orl $(CR0_PE|CR0_PG|CR0_NE|CR0_TS|CR0_MP|CR0_WP|CR0_AM),%eax
1.1 fvdl 630: movl %eax,%cr0
631: jmp compat
632: compat:
633:
634: /*
635: * 5.
636: * Not quite done yet, we're now in a compatibility segment,
637: * in legacy mode. We must jump to a long mode segment.
638: * Need to set up a temporary GDT with a long mode segment
639: * in it to do that.
640: */
641:
1.56 jym 642: movl $RELOC(gdt64_lo),%eax
1.1 fvdl 643: lgdt (%eax)
644: movl $RELOC(farjmp64),%eax
645: ljmp *(%eax)
646:
647: .code64
648: longmode:
649: /*
650: * 6.
651: * Finally, we're in long mode. However, we're still
652: * in the identity mapped area (could not jump out
653: * of that earlier because it would have been a > 32bit
654: * jump). We can do that now, so here we go.
655: */
656: movabsq $longmode_hi,%rax
657: jmp *%rax
1.56 jym 658:
1.1 fvdl 659: longmode_hi:
1.56 jym 660:
661: /*
662: * We left the identity mapped area. Base address of
663: * the temporary gdt64 should now be in high memory.
664: */
665: movq $RELOC(gdt64_hi),%rax
666: lgdt (%rax)
667:
1.1 fvdl 668: /*
669: * There's no need anymore for the identity mapping in low
670: * memory, remove it.
671: */
672: movq $KERNBASE,%r8
673:
674: #if L2_SLOT_KERNBASE > 0
675: movq $(NKL2_KIMG_ENTRIES+1),%rcx
676: leaq (PROC0_PTP2_OFF)(%rsi),%rbx
677: addq %r8, %rbx
678: 1: movq $0,(%rbx)
679: addq $8,%rbx
680: loop 1b
681: #endif
682:
683: #if L3_SLOT_KERNBASE > 0
684: movq $NKL3_KIMG_ENTRIES,%rcx
685: leaq (PROC0_PTP3_OFF)(%rsi),%rbx
686: addq %r8, %rbx
687: 1: movq $0,(%rbx)
688: addq $8,%rbx
689: loop 1b
690: #endif
691:
692: movq $NKL4_KIMG_ENTRIES,%rcx
693: leaq (PROC0_PML4_OFF)(%rsi),%rbx # old, phys address of PML4
1.8 wiz 694: addq %r8, %rbx # new, virtual address of PML4
1.1 fvdl 695: 1: movq $0,(%rbx)
696: addq $8,%rbx
697: loop 1b
698:
699: /* Relocate atdevbase. */
700: movq $(TABLESIZE+KERNBASE),%rdx
701: addq %rsi,%rdx
702: movq %rdx,_C_LABEL(atdevbase)(%rip)
703:
704: /* Set up bootstrap stack. */
705: leaq (PROC0_STK_OFF)(%rsi),%rax
706: addq %r8,%rax
1.55 rmind 707: movq %rax,(_C_LABEL(lwp0)+L_PCB)(%rip) /* XXX L_PCB != uarea */
1.1 fvdl 708: leaq (USPACE-FRAMESIZE)(%rax),%rsp
709: movq %rsi,PCB_CR3(%rax) # pcb->pcb_cr3
710: xorq %rbp,%rbp # mark end of frames
711:
712: xorw %ax,%ax
713: movw %ax,%gs
714: movw %ax,%fs
715:
716: /* XXX merge these */
1.46 ad 717: leaq (TABLESIZE+IOM_SIZE)(%rsi),%rdi
1.32 bouyer 718:
719: #else /* XEN */
720: .globl start
721: start:
722: /* First, reset the PSL. */
723: pushq $2
724: popfq
725:
726: cld
727:
728: /*
729: * Xen info:
730: * - %rsi -> start_info struct
731: * - %rsp -> stack, *theorically* the last used page
732: * by Xen bootstrap
733: */
734: movq %rsi, %rbx
735:
736: /* Clear BSS */
737: xorq %rax,%rax
738: movq $_C_LABEL(__bss_start),%rdi
739: movq $_C_LABEL(_end),%rcx
740: subq %rdi,%rcx
741: rep
742: stosb
743:
744: /* Copy start_info to a safe place */
745: movq %rbx,%rsi
746: movq $_C_LABEL(start_info_union),%rdi
747: movq $64,%rcx
748: rep
749: movsq
750:
751: /*
752: * Memory layout at start of the day:
753: * - Kernel image
754: * - Page frames list
755: * - start_info struct. we copied it, so it can be recycled.
756: * - xenstore
757: * - console
758: * - Xen bootstrap page tables
759: * - kernel stack. provided by Xen
760: * - guaranted 512kB padding
761: *
762: * As we want to rebuild our page tables and place our stack
763: * in proc0 struct, all data starting from after console can be
764: * discarded after we've done a little setup.
765: */
766:
767: /*
768: * We want our own page tables, let's rebuild them
769: * We will reclaim xen space afterward INCLUDING stack
770: * so let's change it to a temporary one
771: */
772:
773: movq $tmpstk, %rax
774: subq $8, %rax
775: movq %rax, %rsp
776:
1.45 bouyer 777: xorl %eax,%eax
778: cpuid
779: movl %eax,_C_LABEL(cpuid_level)
780:
1.67 cherry 781: movq $cpu_info_primary, %rdi
782: movq %rdi, CPU_INFO_SELF(%rdi) /* ci->ci_self = ci */
783: movq $1, %rsi
784: call cpu_init_msrs /* cpu_init_msrs(ci, true); */
785:
1.32 bouyer 786: call xen_pmap_bootstrap
787:
788: /*
789: * First avail returned by xen_pmap_bootstrap in %rax
790: */
791: movq %rax, %rsi
1.55 rmind 792: movq %rsi,(_C_LABEL(lwp0)+L_PCB) /* XXX L_PCB != uarea */
1.32 bouyer 793:
794: /*
795: * Set new stack and clear segments
796: */
797:
798: leaq (USPACE-FRAMESIZE)(%rsi),%rsp
799: xorq %rbp,%rbp
800:
1.63 jym 801: xorw %ax,%ax
1.32 bouyer 802: movw %ax,%gs
803: movw %ax,%fs
804:
805: /*
806: * Set first_avail after proc0
807: */
808:
809: movq %rsi,%rdi
810: addq $USPACE,%rdi
811: subq $KERNBASE,%rdi # init_x86_64 want a physical address
812:
813: #endif /* !XEN */
1.1 fvdl 814: call _C_LABEL(init_x86_64)
815:
816: call _C_LABEL(main)
817:
1.32 bouyer 818: #ifdef XEN
819: /* space for the hypercall call page */
820: #define HYPERCALL_PAGE_OFFSET 0x1000
821: .org HYPERCALL_PAGE_OFFSET
822: ENTRY(hypercall_page)
823: .skip 0x1000
824: #endif /* XEN */
1.1 fvdl 825:
826: /*
1.22 yamt 827: * int setjmp(label_t *)
828: *
829: * Used primarily by DDB.
830: */
1.1 fvdl 831: ENTRY(setjmp)
832: /*
833: * Only save registers that must be preserved across function
834: * calls according to the ABI (%rbx, %rsp, %rbp, %r12-%r15)
835: * and %rip.
836: */
837: movq %rdi,%rax
838: movq %rbx,(%rax)
839: movq %rsp,8(%rax)
840: movq %rbp,16(%rax)
841: movq %r12,24(%rax)
842: movq %r13,32(%rax)
843: movq %r14,40(%rax)
844: movq %r15,48(%rax)
845: movq (%rsp),%rdx
846: movq %rdx,56(%rax)
847: xorl %eax,%eax
848: ret
849:
1.22 yamt 850: /*
851: * int longjmp(label_t *)
852: *
853: * Used primarily by DDB.
854: */
1.1 fvdl 855: ENTRY(longjmp)
856: movq %rdi,%rax
857: movq (%rax),%rbx
858: movq 8(%rax),%rsp
859: movq 16(%rax),%rbp
860: movq 24(%rax),%r12
861: movq 32(%rax),%r13
862: movq 40(%rax),%r14
863: movq 48(%rax),%r15
864: movq 56(%rax),%rdx
865: movq %rdx,(%rsp)
1.22 yamt 866: movl $1,%eax
1.1 fvdl 867: ret
868:
1.25 yamt 869: ENTRY(dumpsys)
870: # mimic cpu_switchto() for postmortem debugging.
871:
872: # build a fake switch frame.
873: pushq %rbx
874: pushq %r12
875: pushq %r13
876: pushq %r14
877: pushq %r15
878:
879: # save a context.
880: movq $dumppcb, %rax
881: movq %rsp, PCB_RSP(%rax)
882: movq %rbp, PCB_RBP(%rax)
883:
884: call _C_LABEL(dodumpsys)
885:
886: addq $(5*8), %rsp # sizeof(switchframe) - sizeof(%rip)
887: ret
888:
1.1 fvdl 889: /*
1.58 chs 890: * struct lwp *cpu_switchto(struct lwp *oldlwp, struct lwp *newlwp,
1.35 ad 891: * bool returning)
1.22 yamt 892: *
893: * 1. if (oldlwp != NULL), save its context.
894: * 2. then, restore context of newlwp.
895: *
896: * Note that the stack frame layout is known to "struct switchframe" in
897: * <machine/frame.h> and to the code in cpu_lwp_fork() which initializes
898: * it for a new lwp.
1.1 fvdl 899: */
1.22 yamt 900: ENTRY(cpu_switchto)
1.1 fvdl 901: pushq %rbx
902: pushq %r12
903: pushq %r13
904: pushq %r14
905: pushq %r15
906:
1.22 yamt 907: movq %rdi,%r13 # oldlwp
908: movq %rsi,%r12 # newlwp
1.1 fvdl 909:
1.22 yamt 910: testq %r13,%r13
911: jz 1f
1.1 fvdl 912:
1.22 yamt 913: /* Save old context. */
1.55 rmind 914: movq L_PCB(%r13),%rax
1.22 yamt 915: movq %rsp,PCB_RSP(%rax)
916: movq %rbp,PCB_RBP(%rax)
917:
918: /* Switch to newlwp's stack. */
1.55 rmind 919: 1: movq L_PCB(%r12),%r14
1.32 bouyer 920: #ifdef XEN /* XXX debug code */
921: cmpq $0, PCB_RSP(%r14)
922: jne 999f
923: callq _C_LABEL(cpu_Debugger);
924: 999:
925: #endif
1.22 yamt 926: movq PCB_RSP(%r14),%rsp
927: movq PCB_RBP(%r14),%rbp
1.1 fvdl 928:
1.39 ad 929: /*
930: * Set curlwp. This must be globally visible in order to permit
931: * non-interlocked mutex release.
932: */
933: movq %r12,%rcx
934: xchgq %rcx,CPUVAR(CURLWP)
1.35 ad 935:
936: /* Skip the rest if returning to a pinned LWP. */
937: testb %dl,%dl
938: jnz 4f
939:
1.38 yamt 940: /* Switch ring0 stack */
941: #ifndef XEN
942: movq PCB_RSP0(%r14),%rax
943: movq %rax,CPUVAR(RSP0)
944: #else
1.32 bouyer 945: movq %r14, %rdi
946: callq _C_LABEL(x86_64_switch_context);
947: #endif
1.1 fvdl 948:
1.22 yamt 949: /* Don't bother with the rest if switching to a system process. */
950: testl $LW_SYSTEM,L_FLAG(%r12)
951: jnz 4f
1.1 fvdl 952:
1.22 yamt 953: /* Is this process using RAS (restartable atomic sequences)? */
954: movq L_PROC(%r12),%rdi
955: cmpq $0,P_RASLIST(%rdi)
956: jne 5f
1.1 fvdl 957:
958: /*
1.53 rmind 959: * Restore cr0 (including FPU state). Raise the IPL to IPL_HIGH.
1.22 yamt 960: * FPU IPIs can alter the LWP's saved cr0. Dropping the priority
961: * is deferred until mi_switch(), when cpu_switchto() returns.
1.1 fvdl 962: */
1.32 bouyer 963: 2:
964: #ifndef XEN
1.53 rmind 965: movl $IPL_HIGH,CPUVAR(ILEVEL)
1.48 ad 966: movl PCB_CR0(%r14),%ecx /* has CR0_TS clear */
1.22 yamt 967: movq %cr0,%rdx
1.1 fvdl 968:
969: /*
1.22 yamt 970: * If our floating point registers are on a different CPU,
971: * set CR0_TS so we'll trap rather than reuse bogus state.
1.1 fvdl 972: */
1.48 ad 973: cmpq CPUVAR(FPCURLWP),%r12
1.22 yamt 974: je 3f
975: orq $CR0_TS,%rcx
1.1 fvdl 976:
1.22 yamt 977: /* Reloading CR0 is very expensive - avoid if possible. */
978: 3: cmpq %rdx,%rcx
1.58 chs 979: je 6f
1.1 fvdl 980: movq %rcx,%cr0
1.58 chs 981:
982: 6: testl $PCB_COMPAT32, PCB_FLAGS(%r14)
983: jne 32f
984:
985: /* Zero out %fs/%gs registers and GDT descriptors. */
986: xorq %rax, %rax
987: movw %ax, %fs
988: CLI(cx)
989: swapgs
990: movw %ax, %gs
991: swapgs
992: STI(cx)
993:
994: movq CPUVAR(GDT),%rcx
995: movq %rax, (GUFS_SEL*8)(%rcx)
996: movq %rax, (GUGS_SEL*8)(%rcx)
997:
998: /* Reload 64-bit %fs/%gs MSRs. */
999: movl $MSR_FSBASE, %ecx
1000: movl PCB_FS(%r14), %eax
1001: movl 4+PCB_FS(%r14), %edx
1002: wrmsr
1003: movl $MSR_KERNELGSBASE, %ecx
1004: movl PCB_GS(%r14), %eax
1005: movl 4+PCB_GS(%r14), %edx
1006: wrmsr
1007: jmp 4f
1008:
1009: 32:
1010: /* Reload %fs/%gs GDT descriptors. */
1011: movq CPUVAR(GDT),%rcx
1012: movq PCB_FS(%r14), %rax
1013: movq %rax, (GUFS_SEL*8)(%rcx)
1014: movq PCB_GS(%r14), %rax
1015: movq %rax, (GUGS_SEL*8)(%rcx)
1016:
1017: /* Reload %fs and %gs */
1018: movq L_MD_REGS(%r12), %rbx
1019: movw TF_FS(%rbx), %fs
1020: CLI(ax)
1021: swapgs
1022: movw TF_GS(%rbx), %gs
1023: swapgs
1024: STI(ax)
1025:
1026: #else
1027: movq %r12,%rdi
1028: callq _C_LABEL(x86_64_tls_switch)
1.32 bouyer 1029: #endif
1.58 chs 1030:
1.22 yamt 1031: /* Return to the new LWP, returning 'oldlwp' in %rax. */
1032: 4: movq %r13,%rax
1.1 fvdl 1033: popq %r15
1034: popq %r14
1035: popq %r13
1036: popq %r12
1037: popq %rbx
1038: ret
1039:
1.22 yamt 1040: /* Check for restartable atomic sequences (RAS). */
1041: 5: movq L_MD_REGS(%r12),%rbx
1042: movq TF_RIP(%rbx),%rsi
1043: call _C_LABEL(ras_lookup)
1044: cmpq $-1,%rax
1045: je 2b
1046: movq %rax,TF_RIP(%rbx)
1047: jmp 2b
1.1 fvdl 1048:
1049: /*
1.22 yamt 1050: * void savectx(struct pcb *pcb);
1051: *
1.1 fvdl 1052: * Update pcb, saving current processor state.
1053: */
1054: ENTRY(savectx)
1055: /* Save stack pointers. */
1056: movq %rsp,PCB_RSP(%rdi)
1057: movq %rbp,PCB_RBP(%rdi)
1058: ret
1059:
1060: IDTVEC(syscall32)
1061: sysret /* go away please */
1062:
1063: /*
1.22 yamt 1064: * syscall()
1065: *
1.1 fvdl 1066: * syscall insn entry. This currently isn't much faster, but
1067: * it can be made faster in the future.
1068: */
1069: IDTVEC(syscall)
1.32 bouyer 1070: #ifndef XEN
1.1 fvdl 1071: swapgs
1072: movq %r15,CPUVAR(SCRATCH)
1.22 yamt 1073: movq CPUVAR(CURLWP),%r15
1.55 rmind 1074: movq L_PCB(%r15),%r15
1.1 fvdl 1075: movq PCB_RSP0(%r15),%r15
1076: xchgq %r15,%rsp
1077:
1078: /*
1079: * XXX don't need this whole frame, split of the
1080: * syscall frame and trapframe is needed.
1081: * First, leave some room for the trapno, error,
1082: * ss:rsp, etc, so that all GP registers can be
1083: * saved. Then, fill in the rest.
1084: */
1.37 dsl 1085: pushq $(LSEL(LUDATA_SEL, SEL_UPL)) /* Known to be user ss */
1086: pushq %r15 /* User space rsp */
1.1 fvdl 1087: movq CPUVAR(SCRATCH),%r15
1.59 yamt 1088: subq $TF_REGSIZE+(TF_RSP-TF_TRAPNO),%rsp
1089: movw %es,TF_ES(%rsp)
1.41 ad 1090: sti
1.1 fvdl 1091: INTR_SAVE_GPRS
1.24 fvdl 1092: movw %fs,TF_FS(%rsp)
1093: movw %gs,TF_GS(%rsp)
1.1 fvdl 1094: movw $(LSEL(LUDATA_SEL, SEL_UPL)),TF_DS(%rsp)
1095: movq %r11, TF_RFLAGS(%rsp) /* old rflags from syscall insn */
1096: movq $(LSEL(LUCODE_SEL, SEL_UPL)), TF_CS(%rsp)
1.37 dsl 1097: movq %rcx,TF_RIP(%rsp) /* syscall saves rip in rcx */
1.1 fvdl 1098: movq $2,TF_ERR(%rsp)
1099: movq $T_ASTFLT, TF_TRAPNO(%rsp)
1.32 bouyer 1100: #else
1101: /* Xen already switched to kernel stack */
1102: pushq %rsi
1.36 dsl 1103: STI(si)
1.32 bouyer 1104: popq %rsi
1.37 dsl 1105: addq $0x10,%rsp /* gap to match cs:rip */
1106: pushq $2 /* error code */
1.32 bouyer 1107: pushq $T_ASTFLT
1.37 dsl 1108: subq $TF_REGSIZE,%rsp
1.32 bouyer 1109: INTR_SAVE_GPRS
1110: movw %fs,TF_FS(%rsp)
1111: movw %gs,TF_GS(%rsp)
1112: movw %es,TF_ES(%rsp)
1113: movw $(LSEL(LUDATA_SEL, SEL_UPL)),TF_DS(%rsp)
1114: #endif
1.1 fvdl 1115:
1116: movq CPUVAR(CURLWP),%r14
1.64 matt 1117: incq CPUVAR(NSYSCALL) # count it atomically
1.1 fvdl 1118: movq %rsp,L_MD_REGS(%r14) # save pointer to frame
1119: movq L_PROC(%r14),%r15
1120: andl $~MDP_IRET,L_MD_FLAGS(%r14)
1.37 dsl 1121: movq %rsp,%rdi /* Pass frame as arg0 */
1.1 fvdl 1122: call *P_MD_SYSCALL(%r15)
1.29 yamt 1123: .Lsyscall_checkast:
1124: /* Check for ASTs on exit to user mode. */
1.36 dsl 1125: CLI(si)
1.41 ad 1126: movl L_MD_ASTPENDING(%r14), %eax
1127: orl CPUVAR(WANT_PMAPLOAD), %eax
1.29 yamt 1128: jnz 9f
1.1 fvdl 1129: testl $MDP_IRET, L_MD_FLAGS(%r14)
1130: jne iret_return;
1131: #ifdef DIAGNOSTIC
1132: cmpl $IPL_NONE,CPUVAR(ILEVEL)
1133: jne 3f
1134: #endif
1.58 chs 1135: movw TF_ES(%rsp),%es
1.32 bouyer 1136: #ifndef XEN
1.1 fvdl 1137: swapgs
1.32 bouyer 1138: #endif
1.1 fvdl 1139: INTR_RESTORE_GPRS
1.65 joerg 1140: movw $(LSEL(LUDATA_SEL, SEL_UPL)), %r11w
1141: movw %r11w,%ds
1.37 dsl 1142: addq $TF_REGSIZE+16,%rsp /* + T_xxx and error code */
1.32 bouyer 1143: #ifndef XEN
1.1 fvdl 1144: popq %rcx /* return rip */
1.37 dsl 1145: addq $8,%rsp /* discard cs */
1.1 fvdl 1146: popq %r11 /* flags as set by sysret insn */
1.11 drochner 1147: movq %ss:(%rsp),%rsp
1.1 fvdl 1148: sysretq
1.32 bouyer 1149: #else
1150: pushq $256 /* VGCF_IN_SYSCALL */
1151: jmp HYPERVISOR_iret
1152: #endif
1.1 fvdl 1153:
1154: #ifdef DIAGNOSTIC
1155: 3: movabsq $4f, %rdi
1156: movl TF_RAX(%rsp),%esi
1157: movl TF_RDI(%rsp),%edx
1158: movl %ebx,%ecx
1159: movl CPUVAR(ILEVEL),%r8d
1160: xorq %rax,%rax
1161: call _C_LABEL(printf)
1.35 ad 1162: movl $IPL_NONE,%edi
1163: call _C_LABEL(spllower)
1.29 yamt 1164: jmp .Lsyscall_checkast
1.1 fvdl 1165: 4: .asciz "WARNING: SPL NOT LOWERED ON SYSCALL %d %d EXIT %x %x\n"
1166: #endif
1.41 ad 1167: 9:
1168: cmpl $0, CPUVAR(WANT_PMAPLOAD)
1169: jz 10f
1170: STI(si)
1.29 yamt 1171: call _C_LABEL(do_pmap_load)
1172: jmp .Lsyscall_checkast /* re-check ASTs */
1.41 ad 1173: 10:
1174: CLEAR_ASTPENDING(%r14)
1175: STI(si)
1176: /* Pushed T_ASTFLT into tf_trapno on entry. */
1177: movq %rsp,%rdi
1178: call _C_LABEL(trap)
1179: jmp .Lsyscall_checkast /* re-check ASTs */
1.29 yamt 1180:
1.22 yamt 1181: /*
1182: * void lwp_trampoline(void);
1183: *
1184: * This is a trampoline function pushed run by newly created LWPs
1185: * in order to do additional setup in their context. 32-bit
1186: * binaries begin life here.
1187: */
1188: NENTRY(lwp_trampoline)
1189: movq %rbp,%rsi
1.41 ad 1190: movq %rbp,%r14 /* for .Losyscall_checkast */
1.22 yamt 1191: movq %rax,%rdi
1192: xorq %rbp,%rbp
1193: call _C_LABEL(lwp_startup)
1.1 fvdl 1194: movq %r13,%rdi
1195: call *%r12
1.41 ad 1196: jmp .Losyscall_checkast
1.1 fvdl 1197: /* NOTREACHED */
1198:
1.22 yamt 1199: /*
1200: * void child_trampoline(void);
1201: *
1202: * As per lwp_trampoline(), but 64-bit binaries start here.
1203: */
1.1 fvdl 1204: NENTRY(child_trampoline)
1.22 yamt 1205: movq %rbp,%rsi
1.41 ad 1206: movq %rbp,%r14 /* for .Lsyscall_checkast */
1.22 yamt 1207: movq %rax,%rdi
1208: xorq %rbp,%rbp
1209: call _C_LABEL(lwp_startup)
1.1 fvdl 1210: movq %r13,%rdi
1211: call *%r12
1.41 ad 1212: jmp .Lsyscall_checkast
1.1 fvdl 1213:
1214: .globl _C_LABEL(osyscall_return)
1215:
1216: /*
1.22 yamt 1217: * oosyscall()
1218: *
1.12 drochner 1219: * Old call gate entry for syscall. only needed if we're
1.34 dsl 1220: * going to support running old i386 NetBSD 1.0 or ibcs2 binaries, etc,
1.1 fvdl 1221: * on NetBSD/amd64.
1.34 dsl 1222: * The 64bit call gate can't request that arguments be copied from the
1223: * user stack (which the i386 code uses to get a gap for the flags).
1224: * push/pop are <read>:<modify_sp>:<write> cycles.
1.1 fvdl 1225: */
1226: IDTVEC(oosyscall)
1227: /* Set rflags in trap frame. */
1.34 dsl 1228: pushq (%rsp) # move user's %eip
1229: pushq 16(%rsp) # and %cs
1230: popq 8(%rsp)
1.1 fvdl 1231: pushfq
1.34 dsl 1232: popq 16(%rsp)
1.1 fvdl 1233: pushq $7 # size of instruction for restart
1234: jmp osyscall1
1235:
1236: /*
1.22 yamt 1237: * osyscall()
1238: *
1.1 fvdl 1239: * Trap gate entry for int $80 syscall, also used by sigreturn.
1240: */
1241: IDTVEC(osyscall)
1.32 bouyer 1242: #ifdef XEN
1243: movq (%rsp),%rcx
1244: movq 8(%rsp),%r11
1245: addq $0x10,%rsp
1246: #endif
1.1 fvdl 1247: pushq $2 # size of instruction for restart
1248: osyscall1:
1249: pushq $T_ASTFLT # trap # for doing ASTs
1250: INTRENTRY
1.36 dsl 1251: STI(si)
1.19 ad 1252: movq CPUVAR(CURLWP),%r14
1253: movq %rsp,L_MD_REGS(%r14) # save pointer to frame
1254: movq L_PROC(%r14),%rdx
1.6 fvdl 1255: movq %rsp,%rdi
1.1 fvdl 1256: call *P_MD_SYSCALL(%rdx)
1257: _C_LABEL(osyscall_return):
1.29 yamt 1258: .Losyscall_checkast:
1259: /* Check for ASTs on exit to user mode. */
1.36 dsl 1260: CLI(si)
1.62 yamt 1261: movl L_MD_ASTPENDING(%r14), %eax
1262: orl CPUVAR(WANT_PMAPLOAD), %eax
1.29 yamt 1263: jnz 9f
1.1 fvdl 1264: iret_return:
1.62 yamt 1265: #ifdef DIAGNOSTIC
1.29 yamt 1266: cmpl $IPL_NONE,CPUVAR(ILEVEL)
1.1 fvdl 1267: jne 3f
1.62 yamt 1268: #endif
1.1 fvdl 1269: INTRFASTEXIT
1.62 yamt 1270: #ifdef DIAGNOSTIC
1271: 3: movabsq $4f, %rdi
1272: movl TF_RAX(%rsp),%esi
1273: movl TF_RDI(%rsp),%edx
1274: movl %ebx,%ecx
1275: movl CPUVAR(ILEVEL),%r8d
1.1 fvdl 1276: xorq %rax,%rax
1277: call _C_LABEL(printf)
1.35 ad 1278: movl $IPL_NONE,%edi
1279: call _C_LABEL(spllower)
1.29 yamt 1280: jmp .Losyscall_checkast
1.62 yamt 1281: 4: .asciz "WARNING: SPL NOT LOWERED ON SYSCALL %d %d EXIT %x %x\n"
1282: #endif
1283: 9:
1284: cmpl $0, CPUVAR(WANT_PMAPLOAD)
1285: jz 10f
1286: STI(si)
1.29 yamt 1287: call _C_LABEL(do_pmap_load)
1288: jmp .Losyscall_checkast /* re-check ASTs */
1.62 yamt 1289: 10:
1290: CLEAR_ASTPENDING(%r14)
1291: STI(si)
1292: /* Pushed T_ASTFLT into tf_trapno on entry. */
1293: movq %rsp,%rdi
1294: call _C_LABEL(trap)
1295: jmp .Losyscall_checkast /* re-check ASTs */
1.26 ad 1296:
1297: /*
1.66 chs 1298: * bool sse2_idlezero_page(void *pg)
1.26 ad 1299: *
1.47 ad 1300: * Zero a page without polluting the cache. Preemption must be
1301: * disabled by the caller. Abort if a preemption is pending.
1.66 chs 1302: * Returns true if the page is zeroed, false if not.
1.26 ad 1303: */
1.47 ad 1304: ENTRY(sse2_idlezero_page)
1305: pushq %rbp
1306: movq %rsp,%rbp
1307: movl $(PAGE_SIZE/64), %ecx
1.26 ad 1308: xorq %rax, %rax
1309: .align 16
1310: 1:
1.50 ad 1311: testl $RESCHED_KPREEMPT, CPUVAR(RESCHED)
1.47 ad 1312: jnz 2f
1.26 ad 1313: movnti %rax, 0(%rdi)
1314: movnti %rax, 8(%rdi)
1315: movnti %rax, 16(%rdi)
1316: movnti %rax, 24(%rdi)
1317: movnti %rax, 32(%rdi)
1318: movnti %rax, 40(%rdi)
1319: movnti %rax, 48(%rdi)
1320: movnti %rax, 56(%rdi)
1.47 ad 1321: addq $64, %rdi
1322: decl %ecx
1.26 ad 1323: jnz 1b
1324: sfence
1.47 ad 1325: incl %eax
1326: popq %rbp
1.26 ad 1327: ret
1.47 ad 1328: 2:
1.26 ad 1329: sfence
1.47 ad 1330: popq %rbp
1.26 ad 1331: ret
1.66 chs 1332:
1333: /*
1334: * void pagezero(vaddr_t va)
1335: *
1336: * Zero a page without polluting the cache.
1337: */
1338:
1339: ENTRY(pagezero)
1340: movq $-PAGE_SIZE,%rdx
1341: subq %rdx,%rdi
1342: xorq %rax,%rax
1343: 1:
1344: movnti %rax,(%rdi,%rdx)
1345: movnti %rax,8(%rdi,%rdx)
1346: movnti %rax,16(%rdi,%rdx)
1347: movnti %rax,24(%rdi,%rdx)
1348: movnti %rax,32(%rdi,%rdx)
1349: movnti %rax,40(%rdi,%rdx)
1350: movnti %rax,48(%rdi,%rdx)
1351: movnti %rax,56(%rdi,%rdx)
1352: addq $64,%rdx
1353: jne 1b
1354: sfence
1355: ret
CVSweb <webmaster@jp.NetBSD.org>
![[BACK]](/icons/back.gif){kind=icon}
Return to locore.S CVS log ![[TXT]](/icons/text.gif){kind=icon}
![[DIR]](/icons/dir.gif){kind=icon}
Up to [cvs.NetBSD.org] / src / sys / arch / amd64 / amd64