Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.

===================================================================
RCS file: /ftp/cvs/cvsroot/src/sys/arch/i386/i386/locore.S,v
rcsdiff: /ftp/cvs/cvsroot/src/sys/arch/i386/i386/locore.S,v: warning: Unknown phrases like `commitid ...;' are present.
retrieving revision 1.111
retrieving revision 1.124
diff -u -p -r1.111 -r1.124
--- src/sys/arch/i386/i386/locore.S	2014/02/02 22:41:20	1.111
+++ src/sys/arch/i386/i386/locore.S	2016/05/15 07:17:53	1.124
@@ -1,4 +1,4 @@
-/*	$NetBSD: locore.S,v 1.111 2014/02/02 22:41:20 dsl Exp $	*/
+/*	$NetBSD: locore.S,v 1.124 2016/05/15 07:17:53 maxv Exp $	*/
 
 /*
  * Copyright-o-rama!
@@ -64,7 +64,6 @@
  * POSSIBILITY OF SUCH DAMAGE.
  */
 
-
 /*-
  * Copyright (c) 1998, 2000, 2004, 2006, 2007, 2009 The NetBSD Foundation, Inc.
  * All rights reserved.
@@ -129,9 +128,10 @@
  */
 
 #include <machine/asm.h>
-__KERNEL_RCSID(0, "$NetBSD: locore.S,v 1.111 2014/02/02 22:41:20 dsl Exp $");
+__KERNEL_RCSID(0, "$NetBSD: locore.S,v 1.124 2016/05/15 07:17:53 maxv Exp $");
 
 #include "opt_compat_oldboot.h"
+#include "opt_copy_symtab.h"
 #include "opt_ddb.h"
 #include "opt_modular.h"
 #include "opt_multiboot.h"
@@ -158,7 +158,6 @@ __KERNEL_RCSID(0, "$NetBSD: locore.S,v 1
 #include <machine/multiboot.h>
 #endif
 
-/* XXX temporary kluge; these should not be here */
 /* Get definitions for IOM_BEGIN, IOM_END, and IOM_SIZE */
 #include <dev/isa/isareg.h>
 
@@ -169,6 +168,61 @@ __KERNEL_RCSID(0, "$NetBSD: locore.S,v 1
 #endif /* XEN */
 #define	RELOC(x)	_RELOC(_C_LABEL(x))
 
+/* 32bit version of PG_NX */
+#define PG_NX32	0x80000000
+
+#ifndef PAE
+#define	PROC0_PDIR_OFF	0
+#else
+#define PROC0_L3_OFF	0
+#define PROC0_PDIR_OFF	1 * PAGE_SIZE
+#endif
+
+#define	PROC0_STK_OFF	(PROC0_PDIR_OFF + PDP_SIZE * PAGE_SIZE)
+#define	PROC0_PTP1_OFF	(PROC0_STK_OFF + UPAGES * PAGE_SIZE)
+
+/*
+ * fillkpt - Fill in a kernel page table
+ *	eax = pte (page frame | control | status)
+ *	ebx = page table address
+ *	ecx = number of pages to map
+ *
+ * For PAE, each entry is 8 bytes long: we must set the 4 upper bytes to 0.
+ * This is done by the first instruction of fillkpt. In the non-PAE case, this
+ * instruction just clears the page table entry.
+ */
+#define fillkpt	\
+1:	movl	$0,(PDE_SIZE-4)(%ebx)	;	/* upper 32 bits: 0 */	\
+	movl	%eax,(%ebx)		;	/* store phys addr */	\
+	addl	$PDE_SIZE,%ebx		;	/* next PTE/PDE */	\
+	addl	$PAGE_SIZE,%eax		;	/* next phys page */	\
+	loop	1b			;
+
+/*
+ * fillkpt_nox - Same as fillkpt, but sets the NX/XD bit.
+ */
+#define fillkpt_nox \
+	pushl	%ebp			; \
+	movl	RELOC(nox_flag),%ebp	; \
+1:	movl	%ebp,(PDE_SIZE-4)(%ebx)	;	/* upper 32 bits: NX */ \
+	movl	%eax,(%ebx)		;	/* store phys addr */	\
+	addl	$PDE_SIZE,%ebx		;	/* next PTE/PDE */	\
+	addl	$PAGE_SIZE,%eax		;	/* next phys page */	\
+	loop	1b			; \
+	popl	%ebp			;
+
+/*
+ * killkpt - Destroy a kernel page table
+ *	ebx = page table address
+ *	ecx = number of pages to destroy
+ */
+#define killkpt \
+1:	movl	$0,(PDE_SIZE-4)(%ebx)	;	/* upper bits (for PAE) */ \
+	movl	$0,(%ebx)		; \
+	addl	$PDE_SIZE,%ebx		; \
+	loop	1b			;
+
+
 #ifdef XEN
 /*
  * Xen guest identifier and loader selection
@@ -184,7 +238,7 @@ __KERNEL_RCSID(0, "$NetBSD: locore.S,v 1
 	.ascii	",PAE=yes[extended-cr3]"
 #endif
 	.ascii	",LOADER=generic"
-#if (NKSYMS || defined(DDB) || defined(MODULAR)) && !defined(SYMTAB_SPACE)
+#if (NKSYMS || defined(DDB) || defined(MODULAR)) && !defined(makeoptions_COPY_SYMTAB)
 	.ascii	",BSD_SYMTAB=yes"
 #endif
 	.byte	0
@@ -195,6 +249,7 @@ __KERNEL_RCSID(0, "$NetBSD: locore.S,v 1
  */
 	.data
 
+	.globl	_C_LABEL(nox_flag)
 	.globl	_C_LABEL(cputype)
 	.globl	_C_LABEL(cpuid_level)
 	.globl	_C_LABEL(esym)
@@ -240,31 +295,37 @@ LABEL(lapic_tpr)
 	.long 0
 END(lapic_tpr)
 #endif
+
+	.type	_C_LABEL(nox_flag), @object
+LABEL(nox_flag)		.long	0	/* 32bit NOX flag, set if supported */
+END(nox_flag)
 	.type	_C_LABEL(cputype), @object
-LABEL(cputype)		.long	0	# are we 80486, Pentium, or..
+LABEL(cputype)		.long	0	/* are we 80486, Pentium, or.. */
 END(cputype)
 	.type	_C_LABEL(cpuid_level), @object
 LABEL(cpuid_level)	.long	0
 END(cpuid_level)
 	.type	_C_LABEL(atdevbase), @object
-LABEL(atdevbase)	.long	0	# location of start of iomem in virtual
+LABEL(atdevbase)	.long	0	/* location of start of iomem in virt */
 END(atdevbase)
 	.type	_C_LABEL(lwp0uarea), @object
 LABEL(lwp0uarea)	.long	0
 END(lwp0uarea)
 	.type	_C_LABEL(PDPpaddr), @object
-LABEL(PDPpaddr)		.long	0	# paddr of PDP, for libkvm
+LABEL(PDPpaddr)		.long	0	/* paddr of PDP, for libkvm */
 END(PDPpaddr)
 	.type	_C_LABEL(tablesize), @object
 _C_LABEL(tablesize):	.long	0
 END(tablesize)
+
+	/* Space for the temporary stack */
 	.size	tmpstk, tmpstk - .
-	.space 512
+	.space	512
 tmpstk:
 #ifdef XEN
-	.align 		PAGE_SIZE, 0x0	# Align on page boundary
+	.align 		PAGE_SIZE, 0x0	/* Align on page boundary */
 LABEL(tmpgdt)
-	.space 		PAGE_SIZE	# Xen expects a page
+	.space 		PAGE_SIZE	/* Xen expects a page */
 END(tmpgdt)
 #endif /* XEN */
 
@@ -274,7 +335,10 @@ END(tmpgdt)
 
 ENTRY(start)
 #ifndef XEN
-	movw	$0x1234,0x472			# warm boot
+
+	/* Warm boot */
+	movw	$0x1234,0x472
+
 #if defined(MULTIBOOT)
 	jmp	1f
 
@@ -298,7 +362,7 @@ _C_LABEL(Multiboot_Header):
 	 * space to process it later -- after we are relocated.  It will
 	 * be safer to run complex C code than doing it at this point.
 	 */
-	pushl	%ebx		# Address of Multiboot information
+	pushl	%ebx		/* Address of Multiboot information */
 	call	_C_LABEL(multiboot_pre_reloc)
 	addl	$4,%esp
 	jmp	2f
@@ -307,11 +371,14 @@ _C_LABEL(Multiboot_Header):
 1:
 	/*
 	 * At this point, we know that a NetBSD-specific boot loader
-	 * booted this kernel.  The stack carries the following parameters:
-	 * (boothowto, [bootdev], bootinfo, esym, biosextmem, biosbasemem),
-	 * 4 bytes each.
+	 * booted this kernel.
+	 *
+	 * Load parameters from the stack (32 bits):
+	 *     boothowto, [bootdev], bootinfo, esym, biosextmem, biosbasemem
+	 * We are not interested in 'bootdev'.
 	 */
-	addl	$4,%esp		# Discard return address to boot loader
+
+	addl	$4,%esp		/* Discard return address to boot loader */
 	call	_C_LABEL(native_loader)
 	addl	$24,%esp
 
@@ -403,30 +470,30 @@ is486:	movl	$CPU_486,RELOC(cputype)
 	popfl
 	jmp 2f
 trycyrix486:
-	movl	$CPU_6x86,RELOC(cputype)	# set CPU type
+	movl	$CPU_6x86,RELOC(cputype)	/* set CPU type */
 	/*
 	 * Check for Cyrix 486 CPU by seeing if the flags change during a
 	 * divide. This is documented in the Cx486SLC/e SMM Programmer's
 	 * Guide.
 	 */
 	xorl	%edx,%edx
-	cmpl	%edx,%edx		# set flags to known state
+	cmpl	%edx,%edx		/* set flags to known state */
 	pushfl
-	popl	%ecx			# store flags in ecx
+	popl	%ecx			/* store flags in ecx */
 	movl	$-1,%eax
 	movl	$4,%ebx
-	divl	%ebx			# do a long division
+	divl	%ebx			/* do a long division */
 	pushfl
 	popl	%eax
-	xorl	%ecx,%eax		# are the flags different?
-	testl	$0x8d5,%eax		# only check C|PF|AF|Z|N|V
-	jne	2f			# yes; must be Cyrix 6x86 CPU
-	movl	$CPU_486DLC,RELOC(cputype) 	# set CPU type
+	xorl	%ecx,%eax		/* are the flags different? */
+	testl	$0x8d5,%eax		/* only check C|PF|AF|Z|N|V */
+	jne	2f			/* yes; must be Cyrix 6x86 CPU */
+	movl	$CPU_486DLC,RELOC(cputype) 	/* set CPU type */
 
 #ifndef CYRIX_CACHE_WORKS
 	/* Disable caching of the ISA hole only. */
 	invd
-	movb	$CCR0,%al		# Configuration Register index (CCR0)
+	movb	$CCR0,%al	/* Configuration Register index (CCR0) */
 	outb	%al,$0x22
 	inb	$0x23,%al
 	orb	$(CCR0_NC1|CCR0_BARB),%al
@@ -438,8 +505,8 @@ trycyrix486:
 	invd
 #else /* CYRIX_CACHE_WORKS */
 	/* Set cache parameters */
-	invd				# Start with guaranteed clean cache
-	movb	$CCR0,%al		# Configuration Register index (CCR0)
+	invd			/* Start with guaranteed clean cache */
+	movb	$CCR0,%al	/* Configuration Register index (CCR0) */
 	outb	%al,$0x22
 	inb	$0x23,%al
 	andb	$~CCR0_NC0,%al
@@ -487,6 +554,16 @@ try586:	/* Use the `cpuid' instruction. 
 	cpuid
 	movl	%eax,RELOC(cpuid_level)
 
+	/*
+	 * Retrieve the NX/XD flag. We use the 32bit version of PG_NX.
+	 */
+	movl	$0x80000001,%eax
+	cpuid
+	andl	$CPUID_NOX,%edx
+	jz	no_NOX
+	movl	$PG_NX32,RELOC(nox_flag)
+no_NOX:
+
 2:
 	/*
 	 * Finished with old stack; load new %esp now instead of later so we
@@ -497,55 +574,53 @@ try586:	/* Use the `cpuid' instruction. 
 	 * The boot program should check:
 	 *	text+data <= &stack_variable - more_space_for_stack
 	 *	text+data+bss+pad+space_for_page_tables <= end_of_memory
-	 * Oops, the gdt is in the carcass of the boot program so clearing
+	 *
+	 * XXX: the gdt is in the carcass of the boot program so clearing
 	 * the rest of memory is still not possible.
 	 */
-	movl	$_RELOC(tmpstk),%esp	# bootstrap stack end location
+	movl	$_RELOC(tmpstk),%esp
 
 /*
- * Virtual address space of kernel, without PAE. The page dir is 1 page long.
+ * There are two different layouts possible, depending on whether PAE is
+ * enabled or not.
  *
- * text | data | bss | [syms] | [blobs] | page dir | proc0 kstack | L1 ptp
- *			                0          1       2      3
+ * If PAE is not enabled, there are two levels of pages: PD -> PT. They will
+ * be referred to as: L2 -> L1. L2 is 1 page long. The BOOTSTRAP TABLES have
+ * the following layout:
+ * 	+-----+------------+----+
+ * 	| L2 -> PROC0 STK -> L1 |
+ * 	+-----+------------+----+
  *
- * Virtual address space of kernel, with PAE. We need 4 pages for the page dir
- * and 1 page for the L3.
- * text | data | bss | [syms] | [blobs] | L3 | page dir | proc0 kstack | L1 ptp
- * 					0    1          5       6      7
- */
-#ifndef PAE
-#define	PROC0_PDIR_OFF	0
-#else
-#define PROC0_L3_OFF	0
-#define PROC0_PDIR_OFF	1 * PAGE_SIZE
-#endif
-
-#define	PROC0_STK_OFF	(PROC0_PDIR_OFF + PDP_SIZE * PAGE_SIZE)
-#define	PROC0_PTP1_OFF	(PROC0_STK_OFF + UPAGES * PAGE_SIZE)
-
-/*
- * fillkpt - Fill in a kernel page table
- *	eax = pte (page frame | control | status)
- *	ebx = page table address
- *	ecx = number of pages to map
+ * If PAE is enabled, there are three levels of pages: PDP -> PD -> PT. They
+ * will be referred to as: L3 -> L2 -> L1. L3 is 1 page long, L2 is 4 page
+ * long. The BOOTSTRAP TABLES have the following layout:
+ * 	+-----+-----+------------+----+
+ * 	| L3 -> L2 -> PROC0 STK -> L1 |
+ * 	+-----+-----+------------+----+
  *
- * For PAE, each entry is 8 bytes long: we must set the 4 upper bytes to 0.
- * This is done by the first instruction of fillkpt. In the non-PAE case, this
- * instruction just clears the page table entry.
+ * Virtual address space of the kernel in both cases:
+ * +------+--------+------+-----+--------+---------------------+-----------
+ * | TEXT | RODATA | DATA | BSS | [SYMS] | [PRELOADED MODULES] | BOOTSTRAP
+ * +------+--------+------+-----+--------+---------------------+-----------
+ *                             (1)      (2)                   (3)
+ *
+ * -------+------------+
+ * TABLES | ISA IO MEM |
+ * -------+------------+
+ *       (4)
+ *
+ * PROC0 STK is obviously not linked as a page level. It just happens to be
+ * caught between L2 and L1.
+ *
+ * Important note: the kernel segments are properly 4k-aligned
+ * (see kern.ldscript), so there's no need to enforce alignment.
  */
 
-#define fillkpt	\
-1:	movl	$0,(PDE_SIZE-4)(%ebx)	;	/* clear bits */	\
-	movl	%eax,(%ebx)		;	/* store phys addr */	\
-	addl	$PDE_SIZE,%ebx		;	/* next pte/pde */	\
-	addl	$PAGE_SIZE,%eax		;	/* next phys page */	\
-	loop	1b			;
-
-	/* Find end of kernel image. */
+	/* Find end of kernel image; brings us on (1). */
 	movl	$RELOC(end),%edi
 
-#if (NKSYMS || defined(DDB) || defined(MODULAR)) && !defined(SYMTAB_SPACE)
-	/* Save the symbols (if loaded). */
+#if (NKSYMS || defined(DDB) || defined(MODULAR)) && !defined(makeoptions_COPY_SYMTAB)
+	/* Save the symbols (if loaded); brinds us on (2). */
 	movl	RELOC(esym),%eax
 	testl	%eax,%eax
 	jz	1f
@@ -554,36 +629,37 @@ try586:	/* Use the `cpuid' instruction. 
 1:
 #endif
 
-	/* Skip over any modules/blobs. */
+	/* Skip over any modules/blobs; brings us on (3). */
 	movl	RELOC(eblob),%eax
 	testl	%eax,%eax
 	jz	1f
 	subl	$KERNBASE,%eax
 	movl	%eax,%edi
 1:
-	/* Compute sizes */
+
+	/* We are on (3). Align up for BOOTSTRAP TABLES. */
 	movl	%edi,%esi
-	addl	$PGOFSET,%esi			# page align up
+	addl	$PGOFSET,%esi
 	andl	$~PGOFSET,%esi
 
 	/* nkptp[1] = (esi + ~L2_FRAME) >> L2_SHIFT + 1; */
 	movl	%esi,%eax
 	addl	$~L2_FRAME,%eax
 	shrl	$L2_SHIFT,%eax
-	incl	%eax		/* one more ptp for VAs stolen by bootstrap */
+	incl	%eax		/* one more PTP for VAs stolen by bootstrap */
 1:	movl	%eax,RELOC(nkptp)+1*4
 
 	/* tablesize = (PDP_SIZE + UPAGES + nkptp) << PGSHIFT; */
 	addl	$(PDP_SIZE+UPAGES),%eax
 #ifdef PAE
-	incl	%eax 		/* one more page for the L3 PD */
+	incl	%eax 		/* one more page for L3 */
 	shll	$PGSHIFT+1,%eax	/* PTP tables are twice larger with PAE */
 #else
 	shll	$PGSHIFT,%eax
 #endif
 	movl	%eax,RELOC(tablesize)
 
-	/* ensure that nkptp covers bootstrap tables */
+	/* Ensure that nkptp covers BOOTSTRAP TABLES. */
 	addl	%esi,%eax
 	addl	$~L2_FRAME,%eax
 	shrl	$L2_SHIFT,%eax
@@ -591,85 +667,104 @@ try586:	/* Use the `cpuid' instruction. 
 	cmpl	%eax,RELOC(nkptp)+1*4
 	jnz	1b
 
-	/* Clear tables */
+	/* Now, zero out the BOOTSTRAP TABLES (before filling them in). */
 	movl	%esi,%edi
 	xorl	%eax,%eax
 	cld
 	movl	RELOC(tablesize),%ecx
 	shrl	$2,%ecx
 	rep
-	stosl
-
-	leal	(PROC0_PTP1_OFF)(%esi), %ebx
+	stosl				/* copy eax -> edi */
 
 /*
- * Build initial page tables.
+ * Build the page tables and levels. We go from L1 to L2/L3, and link the levels
+ * together. Note: RELOC computes &addr - KERNBASE in 32 bits; the value can't
+ * be > 4G, or we can't deal with it anyway, since we are in 32bit mode.
  */
 	/*
-	 * Compute &__data_start - KERNBASE. This can't be > 4G,
-	 * or we can't deal with it anyway, since we can't load it in
-	 * 32 bit mode. So use the bottom 32 bits.
+	 * Build L1.
 	 */
-	movl	$RELOC(__data_start),%edx
-	andl	$~PGOFSET,%edx
+	leal	(PROC0_PTP1_OFF)(%esi),%ebx
 
-	/*
-	 * Skip the first MB.
-	 */
-	movl	$_RELOC(KERNTEXTOFF),%eax
+	/* Skip the first MB. */
+	movl	$(KERNTEXTOFF - KERNBASE),%eax
 	movl	%eax,%ecx
-	shrl	$(PGSHIFT-2),%ecx	/* ((n >> PGSHIFT) << 2) for # pdes */
+	shrl	$(PGSHIFT-2),%ecx	/* ((n >> PGSHIFT) << 2) for # PDEs */
 #ifdef PAE
-	shll	$1,%ecx			/* pdes are twice larger with PAE */
+	shll	$1,%ecx			/* PDEs are twice larger with PAE */
 #endif
 	addl	%ecx,%ebx
 
-	/* Map the kernel text read-only. */
-	movl	%edx,%ecx
+	/* Map the kernel text RX. */
+	movl	$RELOC(__rodata_start),%ecx
 	subl	%eax,%ecx
 	shrl	$PGSHIFT,%ecx
 	orl	$(PG_V|PG_KR),%eax
 	fillkpt
 
-	/* Map the data, BSS, and bootstrap tables read-write. */
-	leal	(PG_V|PG_KW)(%edx),%eax
-	movl	RELOC(tablesize),%ecx
-	addl	%esi,%ecx				# end of tables
-	subl	%edx,%ecx				# subtract end of text
+	/* Map the kernel rodata R. */
+	movl	$RELOC(__rodata_start),%eax
+	movl	$RELOC(__data_start),%ecx
+	subl	%eax,%ecx
 	shrl	$PGSHIFT,%ecx
+	orl	$(PG_V|PG_KR),%eax
+	fillkpt_nox
+
+	/* Map the kernel data+bss RW. */
+	movl	$RELOC(__data_start),%eax
+	movl	$RELOC(__kernel_end),%ecx
+	subl	%eax,%ecx
+	shrl	$PGSHIFT,%ecx
+	orl	$(PG_V|PG_KW),%eax
+	fillkpt_nox
+
+	/* Map [SYMS]+[PRELOADED MODULES] RWX. */
+	movl	$RELOC(__kernel_end),%eax
+	movl	%esi,%ecx		/* start of BOOTSTRAP TABLES */
+	subl	%eax,%ecx
+	shrl	$PGSHIFT,%ecx
+	orl	$(PG_V|PG_KW),%eax
 	fillkpt
 
-	/* Map ISA I/O mem (later atdevbase) */
-	movl	$(IOM_BEGIN|PG_V|PG_KW/*|PG_N*/),%eax	# having these bits set
-	movl	$(IOM_SIZE>>PGSHIFT),%ecx		# for this many pte s,
+	/* Map the BOOTSTRAP TABLES RW. */
+	movl	%esi,%eax		/* start of BOOTSTRAP TABLES */
+	movl	RELOC(tablesize),%ecx	/* length of BOOTSTRAP TABLES */
+	shrl	$PGSHIFT,%ecx
+	orl	$(PG_V|PG_KW),%eax
+	fillkpt_nox
+
+	/* We are on (4). Map ISA I/O mem (later atdevbase) RWX. */
+	movl	$(IOM_BEGIN|PG_V|PG_KW/*|PG_N*/),%eax
+	movl	$(IOM_SIZE>>PGSHIFT),%ecx
 	fillkpt
 
-/*
- * Construct a page table directory.
- */
-	/* Set up top level entries for identity mapping */
+	/*
+	 * Build L2 for identity mapping. Linked to L1.
+	 */
 	leal	(PROC0_PDIR_OFF)(%esi),%ebx
 	leal	(PROC0_PTP1_OFF)(%esi),%eax
-	orl	$(PG_V|PG_KW), %eax
+	orl	$(PG_V|PG_KW),%eax
 	movl	RELOC(nkptp)+1*4,%ecx
 	fillkpt
 
-	/* Set up top level entries for actual kernel mapping */
-	leal	(PROC0_PDIR_OFF + L2_SLOT_KERNBASE*PDE_SIZE)(%esi),%ebx
+	/* Set up L2 entries for actual kernel mapping */
+	leal	(PROC0_PDIR_OFF + L2_SLOT_KERNBASE * PDE_SIZE)(%esi),%ebx
 	leal	(PROC0_PTP1_OFF)(%esi),%eax
-	orl	$(PG_V|PG_KW), %eax
+	orl	$(PG_V|PG_KW),%eax
 	movl	RELOC(nkptp)+1*4,%ecx
 	fillkpt
 
 	/* Install a PDE recursively mapping page directory as a page table! */
-	leal	(PROC0_PDIR_OFF + PDIR_SLOT_PTE*PDE_SIZE)(%esi),%ebx
+	leal	(PROC0_PDIR_OFF + PDIR_SLOT_PTE * PDE_SIZE)(%esi),%ebx
 	leal	(PROC0_PDIR_OFF)(%esi),%eax
 	orl	$(PG_V|PG_KW),%eax
 	movl	$PDP_SIZE,%ecx
 	fillkpt
 
 #ifdef PAE
-	/* Fill in proc0 L3 page with entries pointing to the page dirs */
+	/*
+	 * Build L3. Linked to L2.
+	 */
 	leal	(PROC0_L3_OFF)(%esi),%ebx
 	leal	(PROC0_PDIR_OFF)(%esi),%eax
 	orl	$(PG_V),%eax
@@ -682,41 +777,49 @@ try586:	/* Use the `cpuid' instruction. 
 	movl	%eax,%cr4
 #endif
 
-	/* Save phys. addr of PDP, for libkvm. */
+	/* Save physical address of L2. */
 	leal	(PROC0_PDIR_OFF)(%esi),%eax
 	movl	%eax,RELOC(PDPpaddr)
 
 	/*
 	 * Startup checklist:
-	 * 1. Load %cr3 with pointer to PDIR (or L3 PD page for PAE).
+	 * 1. Load %cr3 with pointer to L2 (or L3 for PAE).
 	 */
-	movl	%esi,%eax		# phys address of ptd in proc 0
-	movl	%eax,%cr3		# load ptd addr into mmu
+	movl	%esi,%eax
+	movl	%eax,%cr3
 
 	/*
-	 * 2. Enable paging and the rest of it.
+	 * 2. Set NOX in EFER, if available.
 	 */
-	movl	%cr0,%eax		# get control word
-					# enable paging & NPX
+	movl	RELOC(nox_flag),%ebx
+	cmpl	$0,%ebx
+	je 	skip_NOX
+	movl	$MSR_EFER,%ecx
+	rdmsr
+	xorl	%eax,%eax
+	orl	$(EFER_NXE),%eax
+	wrmsr
+skip_NOX:
+
+	/*
+	 * 3. Enable paging and the rest of it.
+	 */
+	movl	%cr0,%eax
 	orl	$(CR0_PE|CR0_PG|CR0_NE|CR0_TS|CR0_MP|CR0_WP|CR0_AM),%eax
-	movl	%eax,%cr0		# and page NOW!
+	movl	%eax,%cr0
 
-	pushl	$begin			# jump to high mem
+	pushl	$begin			/* jump to high mem */
 	ret
 
 begin:
 	/*
-	 * We have arrived.
-	 * There's no need anymore for the identity mapping in low
-	 * memory, remove it.
+	 * We have arrived. There's no need anymore for the identity mapping in
+	 * low memory, remove it.
 	 */
 	movl	_C_LABEL(nkptp)+1*4,%ecx
-	leal	(PROC0_PDIR_OFF)(%esi),%ebx	# old, phys address of PDIR
-	addl	$(KERNBASE), %ebx		# new, virtual address of PDIR
-1:	movl	$0,(PDE_SIZE-4)(%ebx)		# Upper bits (for PAE)
-	movl	$0,(%ebx)
-	addl	$PDE_SIZE,%ebx
-	loop	1b
+	leal	(PROC0_PDIR_OFF)(%esi),%ebx	/* old, phys address of PDIR */
+	addl	$(KERNBASE), %ebx		/* new, virt address of PDIR */
+	killkpt
 
 	/* Relocate atdevbase. */
 	movl	$KERNBASE,%edx
@@ -727,9 +830,9 @@ begin:
 	/* Set up bootstrap stack. */
 	leal	(PROC0_STK_OFF+KERNBASE)(%esi),%eax
 	movl	%eax,_C_LABEL(lwp0uarea)
-	leal	(KSTACK_SIZE-FRAMESIZE)(%eax),%esp
-	movl	%esi,(KSTACK_SIZE+PCB_CR3)(%eax)	# pcb->pcb_cr3
-	xorl	%ebp,%ebp		# mark end of frames
+	leal	(USPACE-FRAMESIZE)(%eax),%esp
+	movl	%esi,PCB_CR3(%eax)	/* pcb->pcb_cr3 */
+	xorl	%ebp,%ebp		/* mark end of frames */
 
 #if defined(MULTIBOOT)
 	/* It is now safe to parse the Multiboot information structure
@@ -739,21 +842,21 @@ begin:
 	call	_C_LABEL(multiboot_post_reloc)
 #endif
 
-	subl	$NGDT*8, %esp		# space for temporary gdt
+	subl	$NGDT*8, %esp		/* space for temporary gdt */
 	pushl	%esp
 	call	_C_LABEL(initgdt)
 	addl	$4,%esp
 
 	movl	_C_LABEL(tablesize),%eax
-	addl	%esi,%eax		# skip past stack and page tables
+	addl	%esi,%eax		/* skip past stack and page tables */
 
 #ifdef PAE
-	pushl	$0	# init386() expects a 64 bits paddr_t with PAE
+	pushl	$0	/* init386() expects a 64 bits paddr_t with PAE */
 #endif
 	pushl	%eax
-	call	_C_LABEL(init386)	# wire 386 chip for unix operation
-	addl	$PDE_SIZE,%esp		# pop paddr_t
-	addl	$NGDT*8,%esp		# pop temporary gdt
+	call	_C_LABEL(init386)	/* wire 386 chip for unix operation */
+	addl	$PDE_SIZE,%esp		/* pop paddr_t */
+	addl	$NGDT*8,%esp		/* pop temporary gdt */
 
 #ifdef SAFARI_FIFO_HACK
 	movb	$5,%al
@@ -780,8 +883,8 @@ begin:
 	popfl
 
 	cld
-	movl	%esp, %ebx		# save start of available space
-	movl	$_RELOC(tmpstk),%esp	# bootstrap stack end location
+	movl	%esp, %ebx		/* save start of available space */
+	movl	$_RELOC(tmpstk),%esp	/* bootstrap stack end location */
 
 	/* Clear BSS. */
 	xorl	%eax,%eax
@@ -812,7 +915,7 @@ begin:
 	 * done using it.
 	 */
 	movl	$RELOC(tmpgdt), %eax
-	pushl	%eax		# start of temporary gdt
+	pushl	%eax		/* start of temporary gdt */
 	call	_C_LABEL(initgdt)
 	addl	$4,%esp
 
@@ -825,18 +928,18 @@ begin:
 	movl	%esi, _C_LABEL(lwp0uarea)
 
 	/* Set up bootstrap stack. */
-	leal	(KSTACK_SIZE-FRAMESIZE)(%eax),%esp
-	xorl	%ebp,%ebp		# mark end of frames
+	leal	(USPACE-FRAMESIZE)(%eax),%esp
+	xorl	%ebp,%ebp		/* mark end of frames */
 
 	addl	$USPACE, %esi
-	subl	$KERNBASE, %esi		#init386 want a physical address
+	subl	$KERNBASE, %esi		/* init386 wants a physical address */
 
 #ifdef PAE
-	pushl	$0	# init386() expects a 64 bits paddr_t with PAE
+	pushl	$0	/* init386() expects a 64 bits paddr_t with PAE */
 #endif
 	pushl	%esi
-	call	_C_LABEL(init386)	# wire 386 chip for unix operation
-	addl	$PDE_SIZE,%esp		# pop paddr_t
+	call	_C_LABEL(init386)	/* wire 386 chip for unix operation */
+	addl	$PDE_SIZE,%esp		/* pop paddr_t */
 	call 	_C_LABEL(main)
 #endif /* XEN */
 END(start)
@@ -910,13 +1013,13 @@ NENTRY(sigcode)
 	 * Handler has returned here as if we called it.  The sigcontext
 	 * is on the stack after the 3 args "we" pushed.
 	 */
-	leal	12(%esp),%eax		# get pointer to sigcontext
-	movl	%eax,4(%esp)		# put it in the argument slot
-					# fake return address already there
+	leal	12(%esp),%eax		/* get pointer to sigcontext */
+	movl	%eax,4(%esp)		/* put it in the argument slot */
+					/* fake return address already there */
 	movl	$SYS_compat_16___sigreturn14,%eax
-	int	$0x80	 		# enter kernel with args on stack
+	int	$0x80	 		/* enter kernel with args on stack */
 	movl	$SYS_exit,%eax
-	int	$0x80			# exit if sigreturn fails
+	int	$0x80			/* exit if sigreturn fails */
 	.globl	_C_LABEL(esigcode)
 _C_LABEL(esigcode):
 END(sigcode)
@@ -928,14 +1031,14 @@ END(sigcode)
  */
 ENTRY(setjmp)
 	movl	4(%esp),%eax
-	movl	%ebx,(%eax)		# save ebx
-	movl	%esp,4(%eax)		# save esp
-	movl	%ebp,8(%eax)		# save ebp
-	movl	%esi,12(%eax)		# save esi
-	movl	%edi,16(%eax)		# save edi
-	movl	(%esp),%edx		# get rta
-	movl	%edx,20(%eax)		# save eip
-	xorl	%eax,%eax		# return 0
+	movl	%ebx,(%eax)		/* save ebx */
+	movl	%esp,4(%eax)		/* save esp */
+	movl	%ebp,8(%eax)		/* save ebp */
+	movl	%esi,12(%eax)		/* save esi */
+	movl	%edi,16(%eax)		/* save edi */
+	movl	(%esp),%edx		/* get rta */
+	movl	%edx,20(%eax)		/* save eip */
+	xorl	%eax,%eax		/* return 0 */
 	ret
 END(setjmp)
 
@@ -946,14 +1049,14 @@ END(setjmp)
  */
 ENTRY(longjmp)
 	movl	4(%esp),%eax
-	movl	(%eax),%ebx		# restore ebx
-	movl	4(%eax),%esp		# restore esp
-	movl	8(%eax),%ebp		# restore ebp
-	movl	12(%eax),%esi		# restore esi
-	movl	16(%eax),%edi		# restore edi
-	movl	20(%eax),%edx		# get rta
-	movl	%edx,(%esp)		# put in return frame
-	movl	$1,%eax			# return 1
+	movl	(%eax),%ebx		/* restore ebx */
+	movl	4(%eax),%esp		/* restore esp */
+	movl	8(%eax),%ebp		/* restore ebp */
+	movl	12(%eax),%esi		/* restore esi */
+	movl	16(%eax),%edi		/* restore edi */
+	movl	20(%eax),%edx		/* get rta */
+	movl	%edx,(%esp)		/* put in return frame */
+	movl	$1,%eax			/* return 1 */
 	ret
 END(longjmp)
 
@@ -963,13 +1066,13 @@ END(longjmp)
  * Mimic cpu_switchto() for postmortem debugging.
  */
 ENTRY(dumpsys)
-	pushl	%ebx			# set up fake switchframe
-	pushl	%esi			#   and save context
-	pushl	%edi			
+	pushl	%ebx			/* set up fake switchframe */
+	pushl	%esi			/* and save context */
+	pushl	%edi
 	movl	%esp,_C_LABEL(dumppcb)+PCB_ESP
 	movl	%ebp,_C_LABEL(dumppcb)+PCB_EBP
-	call	_C_LABEL(dodumpsys)	# dump!
-	addl	$(3*4), %esp		# unwind switchframe
+	call	_C_LABEL(dodumpsys)	/* dump! */
+	addl	$(3*4), %esp		/* unwind switchframe */
 	ret
 END(dumpsys)
 
@@ -1000,9 +1103,9 @@ ENTRY(cpu_switchto)
 0:
 #endif
 
-	movl	16(%esp),%esi		# oldlwp
-	movl	20(%esp),%edi		# newlwp
-	movl	24(%esp),%edx		# returning
+	movl	16(%esp),%esi		/* oldlwp */
+	movl	20(%esp),%edi		/* newlwp */
+	movl	24(%esp),%edx		/* returning */
 	testl	%esi,%esi
 	jz	1f
 
@@ -1137,7 +1240,7 @@ END(cpu_switchto)
  * Update pcb, saving current processor state.
  */
 ENTRY(savectx)
-	movl	4(%esp),%edx		# edx = pcb
+	movl	4(%esp),%edx		/* edx = pcb */
 	movl	%esp,PCB_ESP(%edx)
 	movl	%ebp,PCB_EBP(%edx)
 	ret
@@ -1151,12 +1254,12 @@ END(savectx)
 IDTVEC(osyscall)
 #ifndef XEN
 	/* XXX we are in trouble! interrupts be off here. */
-	cli			# must be first instruction
+	cli			/* must be first instruction */
 #endif
-	pushfl			# set eflags in trap frame
+	pushfl			/* set eflags in trap frame */
 	popl	8(%esp)
-	orl	$PSL_I,8(%esp)	# re-enable ints on return to user
-	pushl	$7		# size of instruction for restart
+	orl	$PSL_I,8(%esp)	/* re-enable ints on return to user */
+	pushl	$7		/* size of instruction for restart */
 	jmp	syscall1
 IDTVEC_END(osyscall)
 
@@ -1166,9 +1269,9 @@ IDTVEC_END(osyscall)
  * Trap gate entry for syscall
  */
 IDTVEC(syscall)
-	pushl	$2		# size of instruction for restart
+	pushl	$2		/* size of instruction for restart */
 syscall1:
-	pushl	$T_ASTFLT	# trap # for doing ASTs
+	pushl	$T_ASTFLT	/* trap # for doing ASTs */
 	INTRENTRY
 	STI(%eax)
 #ifdef DIAGNOSTIC
@@ -1183,13 +1286,13 @@ syscall1:
 #endif
 1:
 #endif /* DIAGNOSTIC */
-	addl	$1,CPUVAR(NSYSCALL)	# count it atomically
-	adcl	$0,CPUVAR(NSYSCALL)+4	# count it atomically
+	addl	$1,CPUVAR(NSYSCALL)	/* count it atomically */
+	adcl	$0,CPUVAR(NSYSCALL)+4	/* count it atomically */
 	movl	CPUVAR(CURLWP),%edi
 	movl	L_PROC(%edi),%edx
-	movl	%esp,L_MD_REGS(%edi)	# save pointer to frame
+	movl	%esp,L_MD_REGS(%edi)	/* save pointer to frame */
 	pushl	%esp
-	call	*P_MD_SYSCALL(%edx)	# get pointer to syscall() function
+	call	*P_MD_SYSCALL(%edx)	/* get pointer to syscall() function */
 	addl	$4,%esp
 .Lsyscall_checkast:
 	/* Check for ASTs on exit to user mode. */
@@ -1206,17 +1309,17 @@ syscall1:
 	/* process pending interrupts */
 	CLI(%eax)
 	movl	CPUVAR(ILEVEL), %ebx
-	movl	$.Lsyscall_resume, %esi # address to resume loop at
+	movl	$.Lsyscall_resume, %esi /* address to resume loop at */
 .Lsyscall_resume:
-	movl	%ebx,%eax	# get cpl
+	movl	%ebx,%eax		/* get cpl */
 	movl	CPUVAR(IUNMASK)(,%eax,4),%eax
-	andl	CPUVAR(IPENDING),%eax	# any non-masked bits left?
+	andl	CPUVAR(IPENDING),%eax	/* any non-masked bits left? */
 	jz	17f
 	bsrl	%eax,%eax
 	btrl	%eax,CPUVAR(IPENDING)
 	movl	CPUVAR(ISOURCES)(,%eax,4),%eax
 	jmp	*IS_RESUME(%eax)
-17:	movl	%ebx, CPUVAR(ILEVEL)	#restore cpl
+17:	movl	%ebx, CPUVAR(ILEVEL)	/* restore cpl  */
 	jmp	.Lsyscall_checkast
 14:
 #endif /* XEN */
@@ -1255,8 +1358,8 @@ syscall1:
 IDTVEC_END(syscall)
 
 IDTVEC(svr4_fasttrap)
-	pushl	$2		# size of instruction for restart
-	pushl	$T_ASTFLT	# trap # for doing ASTs
+	pushl	$2		/* size of instruction for restart */
+	pushl	$T_ASTFLT	/* trap # for doing ASTs */
 	INTRENTRY
 	STI(%eax)
 	pushl	$RW_READER
@@ -1269,7 +1372,7 @@ IDTVEC(svr4_fasttrap)
 	addl	$4,%esp
 2:	/* Check for ASTs on exit to user mode. */
 	cli
-	CHECK_ASTPENDING(%eax)		
+	CHECK_ASTPENDING(%eax)
 	je	1f
 	/* Always returning to user mode here. */
 	CLEAR_ASTPENDING(%eax)
@@ -1293,12 +1396,12 @@ IDTVEC(svr4_fasttrap)
  * Anything 120MHz or above is fine.
  */
 NENTRY(npx586bug1)
-	fildl	4(%esp)		# x
-	fildl	8(%esp)		# y
+	fildl	4(%esp)		/* x */
+	fildl	8(%esp)		/* y */
 	fld	%st(1)
-	fdiv	%st(1),%st	# x/y
-	fmulp	%st,%st(1)	# (x/y)*y
-	fsubrp	%st,%st(1)	# x-(x/y)*y
+	fdiv	%st(1),%st	/* x/y */
+	fmulp	%st,%st(1)	/* (x/y)*y */
+	fsubrp	%st,%st(1)	/* x-(x/y)*y */
 	pushl	$0
 	fistpl	(%esp)
 	popl	%eax