Finish part C.

GNUmakefile

@@ -88,6 +88,7 @@ CFLAGS := $(CFLAGS) $(DEFS) $(LABDEFS) -O1 -fno-builtin -I$(TOP) -MD
 CFLAGS += -fno-omit-frame-pointer
 CFLAGS += -std=gnu99
 CFLAGS += -static
+CFLAGS += -fno-pie
 CFLAGS += -Wall -Wno-format -Wno-unused -Werror -gstabs -m32
 # -fno-tree-ch prevented gcc from sometimes reordering read_ebp() before
 # mon_backtrace()'s function prologue on gcc version: (Debian 4.7.2-5) 4.7.2
@@ -142,9 +143,12 @@ include lib/Makefrag
 include user/Makefrag
 
 
+CPUS ?= 1
+
 QEMUOPTS = -drive file=$(OBJDIR)/kern/kernel.img,index=0,media=disk,format=raw -serial mon:stdio -gdb tcp::$(GDBPORT)
 QEMUOPTS += $(shell if $(QEMU) -nographic -help | grep -q '^-D '; then echo '-D qemu.log'; fi)
 IMAGES = $(OBJDIR)/kern/kernel.img
+QEMUOPTS += -smp $(CPUS)
 QEMUOPTS += $(QEMUEXTRA)
 
 .gdbinit: .gdbinit.tmpl

answers-lab3.txt

@@ -1,12 +0,0 @@
-1. Some interrupts receive a different "shape" of stack frame - the kernel pushes
-an error code for some, but not for the others. We thus need individual handlers
-that would make sure that what we have on the stack is consistent, and
-then call one centralized function for that. The handlers additionally
-are able to pass in a trap number to that centrallized function. This way, a lot
-of code can be re-used.
-
-Conceptually, each trap has a different semantic meaning. Thus, it makes no sense
-to perform the same action for both traps - they don't mean the same thing.
-
-2. The general protection fault occurs when a process can access memory, but not do something else with it. We do not want page faults to be trigger-able by users (this is the case for many other traps). Thus, when the user tries to invoke int 14,
-it's not allowed to execute it from userspace, so it is thrown a general protection fault, which it then prints.

boot/main.c

@@ -39,6 +39,7 @@ void
 bootmain(void)
 {
 	struct Proghdr *ph, *eph;
+	int i;
 
 	// read 1st page off disk
 	readseg((uint32_t) ELFHDR, SECTSIZE*8, 0);
@@ -50,10 +51,14 @@ bootmain(void)
 	// load each program segment (ignores ph flags)
 	ph = (struct Proghdr *) ((uint8_t *) ELFHDR + ELFHDR->e_phoff);
 	eph = ph + ELFHDR->e_phnum;
-	for (; ph < eph; ph++)
+	for (; ph < eph; ph++) {
 		// p_pa is the load address of this segment (as well
 		// as the physical address)
 		readseg(ph->p_pa, ph->p_memsz, ph->p_offset);
+		for (i = 0; i < ph->p_memsz - ph->p_filesz; i++) {
+			*((char *) ph->p_pa + ph->p_filesz + i) = 0;
+		}
+	}
 
 	// call the entry point from the ELF header
 	// note: does not return!

conf/lab.mk

@@ -1,2 +1,2 @@
-LAB=3
+LAB=4
-PACKAGEDATE=Tue Sep 25 12:21:10 EDT 2018
+PACKAGEDATE=Mon Oct 8 21:31:51 PDT 2018

grade-lab4

@@ -0,0 +1,190 @@
+#!/usr/bin/env python
+
+import re
+from gradelib import *
+
+r = Runner(save("jos.out"),
+           stop_breakpoint("readline"))
+
+def E(s, trim=False):
+    """Expand $En in s to the environment ID of the n'th user
+    environment."""
+
+    tmpl = "%x" if trim else "%08x"
+    return re.sub(r"\$E([0-9]+)",
+                  lambda m: tmpl % (0x1000 + int(m.group(1))-1), s)
+
+@test(5)
+def test_dumbfork():
+    r.user_test("dumbfork")
+    r.match(E(".00000000. new env $E1"),
+            E(".$E1. new env $E2"),
+            "0: I am the parent.",
+            "9: I am the parent.",
+            "0: I am the child.",
+            "9: I am the child.",
+            "19: I am the child.",
+            E(".$E1. exiting gracefully"),
+            E(".$E1. free env $E1"),
+            E(".$E2. exiting gracefully"),
+            E(".$E2. free env $E2"))
+
+end_part("A")
+
+@test(5)
+def test_faultread():
+    r.user_test("faultread")
+    r.match(E(".$E1. user fault va 00000000 ip 008....."),
+            "TRAP frame at 0xf....... from CPU .",
+            "  trap 0x0000000e Page Fault",
+            "  err  0x00000004.*",
+            E(".$E1. free env $E1"),
+            no=["I read ........ from location 0."])
+
+@test(5)
+def test_faultwrite():
+    r.user_test("faultwrite")
+    r.match(E(".$E1. user fault va 00000000 ip 008....."),
+            "TRAP frame at 0xf....... from CPU .",
+            "  trap 0x0000000e Page Fault",
+            "  err  0x00000006.*",
+            E(".$E1. free env $E1"))
+
+@test(5)
+def test_faultdie():
+    r.user_test("faultdie")
+    r.match("i faulted at va deadbeef, err 6",
+            E(".$E1. exiting gracefully"),
+            E(".$E1. free env $E1"))
+
+@test(5)
+def test_faultregs():
+    r.user_test("faultregs")
+    r.match("Registers in UTrapframe OK",
+            "Registers after page-fault OK",
+            no=["Registers in UTrapframe MISMATCH",
+                "Registers after page-fault MISMATCH"])
+
+@test(5)
+def test_faultalloc():
+    r.user_test("faultalloc")
+    r.match("fault deadbeef",
+            "this string was faulted in at deadbeef",
+            "fault cafebffe",
+            "fault cafec000",
+            "this string was faulted in at cafebffe",
+            E(".$E1. exiting gracefully"),
+            E(".$E1. free env $E1"))
+
+@test(5)
+def test_faultallocbad():
+    r.user_test("faultallocbad")
+    r.match(E(".$E1. user_mem_check assertion failure for va deadbeef"),
+            E(".$E1. free env $E1"))
+
+@test(5)
+def test_faultnostack():
+    r.user_test("faultnostack")
+    r.match(E(".$E1. user_mem_check assertion failure for va eebfff.."),
+            E(".$E1. free env $E1"))
+
+@test(5)
+def test_faultbadhandler():
+    r.user_test("faultbadhandler")
+    r.match(E(".$E1. user_mem_check assertion failure for va (deadb|eebfe)..."),
+            E(".$E1. free env $E1"))
+
+@test(5)
+def test_faultevilhandler():
+    r.user_test("faultevilhandler")
+    r.match(E(".$E1. user_mem_check assertion failure for va (f0100|eebfe)..."),
+            E(".$E1. free env $E1"))
+
+@test(5)
+def test_forktree():
+    r.user_test("forktree")
+    r.match("....: I am .0.",
+            "....: I am .1.",
+            "....: I am .000.",
+            "....: I am .100.",
+            "....: I am .110.",
+            "....: I am .111.",
+            "....: I am .011.",
+            "....: I am .001.",
+            E(".$E1. exiting gracefully"),
+            E(".$E2. exiting gracefully"),
+            ".0000200.. exiting gracefully",
+            ".0000200.. free env 0000200.")
+
+end_part("B")
+
+@test(5)
+def test_spin():
+    r.user_test("spin")
+    r.match(E(".00000000. new env $E1"),
+            "I am the parent.  Forking the child...",
+            E(".$E1. new env $E2"),
+            "I am the parent.  Running the child...",
+            "I am the child.  Spinning...",
+            "I am the parent.  Killing the child...",
+            E(".$E1. destroying $E2"),
+            E(".$E1. free env $E2"),
+            E(".$E1. exiting gracefully"),
+            E(".$E1. free env $E1"))
+
+@test(5)
+def test_stresssched():
+    r.user_test("stresssched", make_args=["CPUS=4"])
+    r.match(".000010... stresssched on CPU 0",
+            ".000010... stresssched on CPU 1",
+            ".000010... stresssched on CPU 2",
+            ".000010... stresssched on CPU 3",
+            no=[".*ran on two CPUs at once"])
+
+@test(5)
+def test_sendpage():
+    r.user_test("sendpage", make_args=["CPUS=2"])
+    r.match(".00000000. new env 00001000",
+            E(".00000000. new env $E1"),
+            E(".$E1. new env $E2"),
+            E("$E1 got message: hello child environment! how are you?", trim=True),
+            E("child received correct message", trim=True),
+            E("$E2 got message: hello parent environment! I'm good", trim=True),
+            E("parent received correct message", trim=True),
+            E(".$E1. exiting gracefully"),
+            E(".$E1. free env $E1"),
+            E(".$E2. exiting gracefully"),
+            E(".$E2. free env $E2"))
+
+@test(5)
+def test_pingpong():
+    r.user_test("pingpong", make_args=["CPUS=4"])
+    r.match(E(".00000000. new env $E1"),
+            E(".$E1. new env $E2"),
+            E("send 0 from $E1 to $E2", trim=True),
+            E("$E2 got 0 from $E1", trim=True),
+            E("$E1 got 1 from $E2", trim=True),
+            E("$E2 got 8 from $E1", trim=True),
+            E("$E1 got 9 from $E2", trim=True),
+            E("$E2 got 10 from $E1", trim=True),
+            E(".$E1. exiting gracefully"),
+            E(".$E1. free env $E1"),
+            E(".$E2. exiting gracefully"),
+            E(".$E2. free env $E2"))
+
+@test(5)
+def test_primes():
+    r.user_test("primes", stop_on_line("CPU .: 1877"), stop_on_line(".*panic"),
+                make_args=["CPUS=4"], timeout=60)
+    r.match(E(".00000000. new env $E1"),
+            E(".$E1. new env $E2"),
+            E("CPU .: 2 .$E2. new env $E3"),
+            E("CPU .: 3 .$E3. new env $E4"),
+            E("CPU .: 5 .$E4. new env $E5"),
+            E("CPU .: 7 .$E5. new env $E6"),
+            E("CPU .: 11 .$E6. new env $E7"),
+            E("CPU .: 1877 .$E289. new env $E290"))
+
+end_part("C")
+
+run_tests()

inc/env.h

@@ -51,9 +51,20 @@ struct Env {
 	enum EnvType env_type;		// Indicates special system environments
 	unsigned env_status;		// Status of the environment
 	uint32_t env_runs;		// Number of times environment has run
+	int env_cpunum;			// The CPU that the env is running on
 
 	// Address space
 	pde_t *env_pgdir;		// Kernel virtual address of page dir
+
+	// Exception handling
+	void *env_pgfault_upcall;	// Page fault upcall entry point
+
+	// Lab 4 IPC
+	bool env_ipc_recving;		// Env is blocked receiving
+	void *env_ipc_dstva;		// VA at which to map received page
+	uint32_t env_ipc_value;		// Data value sent to us
+	envid_t env_ipc_from;		// envid of the sender
+	int env_ipc_perm;		// Perm of page mapping received
 };
 
 #endif // !JOS_INC_ENV_H

inc/error.h

@@ -14,6 +14,9 @@ enum {
 				// the maximum allowed
 	E_FAULT		,	// Memory fault
 
+	E_IPC_NOT_RECV	,	// Attempt to send to env that is not recving
+	E_EOF		,	// Unexpected end of file
+
 	MAXERROR
 };
 

inc/lib.h

@@ -16,6 +16,7 @@
 #include <inc/env.h>
 #include <inc/memlayout.h>
 #include <inc/syscall.h>
+#include <inc/trap.h>
 
 #define USED(x)		(void)(x)
 
@@ -31,6 +32,9 @@ extern const volatile struct PageInfo pages[];
 // exit.c
 void	exit(void);
 
+// pgfault.c
+void	set_pgfault_handler(void (*handler)(struct UTrapframe *utf));
+
 // readline.c
 char*	readline(const char *buf);
 
@@ -39,6 +43,37 @@ void	sys_cputs(const char *string, size_t len);
 int	sys_cgetc(void);
 envid_t	sys_getenvid(void);
 int	sys_env_destroy(envid_t);
+void	sys_yield(void);
+static envid_t sys_exofork(void);
+int	sys_env_set_status(envid_t env, int status);
+int	sys_env_set_pgfault_upcall(envid_t env, void *upcall);
+int	sys_page_alloc(envid_t env, void *pg, int perm);
+int	sys_page_map(envid_t src_env, void *src_pg,
+		     envid_t dst_env, void *dst_pg, int perm);
+int	sys_page_unmap(envid_t env, void *pg);
+int	sys_ipc_try_send(envid_t to_env, uint32_t value, void *pg, int perm);
+int	sys_ipc_recv(void *rcv_pg);
+
+// This must be inlined.  Exercise for reader: why?
+static inline envid_t __attribute__((always_inline))
+sys_exofork(void)
+{
+	envid_t ret;
+	asm volatile("int %2"
+		     : "=a" (ret)
+		     : "a" (SYS_exofork), "i" (T_SYSCALL));
+	return ret;
+}
+
+// ipc.c
+void	ipc_send(envid_t to_env, uint32_t value, void *pg, int perm);
+int32_t ipc_recv(envid_t *from_env_store, void *pg, int *perm_store);
+envid_t	ipc_find_env(enum EnvType type);
+
+// fork.c
+#define	PTE_SHARE	0x400
+envid_t	fork(void);
+envid_t	sfork(void);	// Challenge!
 
 
 

inc/memlayout.h

@@ -138,6 +138,9 @@
 // The location of the user-level STABS data structure
 #define USTABDATA	(PTSIZE / 2)
 
+// Physical address of startup code for non-boot CPUs (APs)
+#define MPENTRY_PADDR	0x7000
+
 #ifndef __ASSEMBLER__
 
 typedef uint32_t pte_t;

inc/syscall.h

@@ -7,6 +7,15 @@ enum {
 	SYS_cgetc,
 	SYS_getenvid,
 	SYS_env_destroy,
+	SYS_page_alloc,
+	SYS_page_map,
+	SYS_page_unmap,
+	SYS_exofork,
+	SYS_env_set_status,
+	SYS_env_set_pgfault_upcall,
+	SYS_yield,
+	SYS_ipc_try_send,
+	SYS_ipc_recv,
 	NSYSCALLS
 };
 

inc/trap.h

@@ -74,6 +74,17 @@ struct Trapframe {
 	uint16_t tf_padding4;
 } __attribute__((packed));
 
+struct UTrapframe {
+	/* information about the fault */
+	uint32_t utf_fault_va;	/* va for T_PGFLT, 0 otherwise */
+	uint32_t utf_err;
+	/* trap-time return state */
+	struct PushRegs utf_regs;
+	uintptr_t utf_eip;
+	uint32_t utf_eflags;
+	/* the trap-time stack to return to */
+	uintptr_t utf_esp;
+} __attribute__((packed));
 
 #endif /* !__ASSEMBLER__ */
 

kern/Makefrag

@@ -33,6 +33,12 @@ KERN_SRCFILES :=	kern/entry.S \
 			lib/readline.c \
 			lib/string.c
 
+# Source files for LAB4
+KERN_SRCFILES +=	kern/mpentry.S \
+			kern/mpconfig.c \
+			kern/lapic.c \
+			kern/spinlock.c
+
 # Only build files if they exist.
 KERN_SRCFILES := $(wildcard $(KERN_SRCFILES))
 
@@ -53,6 +59,25 @@ KERN_BINFILES :=	user/hello \
 			user/faultwritekernel \
 			user/getc
 
+# Binary files for LAB4
+KERN_BINFILES +=	user/idle \
+			user/yield \
+			user/dumbfork \
+			user/stresssched \
+			user/faultdie \
+			user/faultregs \
+			user/faultalloc \
+			user/faultallocbad \
+			user/faultnostack \
+			user/faultbadhandler \
+			user/faultevilhandler \
+			user/forktree \
+			user/sendpage \
+			user/spin \
+			user/fairness \
+			user/pingpong \
+			user/pingpongs \
+			user/primes
 KERN_OBJFILES := $(patsubst %.c, $(OBJDIR)/%.o, $(KERN_SRCFILES))
 KERN_OBJFILES := $(patsubst %.S, $(OBJDIR)/%.o, $(KERN_OBJFILES))
 KERN_OBJFILES := $(patsubst $(OBJDIR)/lib/%, $(OBJDIR)/kern/%, $(KERN_OBJFILES))

kern/console.c

@@ -7,6 +7,8 @@
 #include <inc/assert.h>
 
 #include <kern/console.h>
+#include <kern/trap.h>
+#include <kern/picirq.h>
 #include <kern/ansi.h>
 
 static void cons_intr(int (*proc)(void));
@@ -384,6 +386,9 @@ kbd_intr(void)
 static void
 kbd_init(void)
 {
+	// Drain the kbd buffer so that QEMU generates interrupts.
+	kbd_intr();
+	irq_setmask_8259A(irq_mask_8259A & ~(1<<IRQ_KBD));
 }
 
 

kern/cpu.h

@@ -0,0 +1,46 @@
+
+#ifndef JOS_INC_CPU_H
+#define JOS_INC_CPU_H
+
+#include <inc/types.h>
+#include <inc/memlayout.h>
+#include <inc/mmu.h>
+#include <inc/env.h>
+
+// Maximum number of CPUs
+#define NCPU  8
+
+// Values of status in struct Cpu
+enum {
+	CPU_UNUSED = 0,
+	CPU_STARTED,
+	CPU_HALTED,
+};
+
+// Per-CPU state
+struct CpuInfo {
+	uint8_t cpu_id;                 // Local APIC ID; index into cpus[] below
+	volatile unsigned cpu_status;   // The status of the CPU
+	struct Env *cpu_env;            // The currently-running environment.
+	struct Taskstate cpu_ts;        // Used by x86 to find stack for interrupt
+};
+
+// Initialized in mpconfig.c
+extern struct CpuInfo cpus[NCPU];
+extern int ncpu;                    // Total number of CPUs in the system
+extern struct CpuInfo *bootcpu;     // The boot-strap processor (BSP)
+extern physaddr_t lapicaddr;        // Physical MMIO address of the local APIC
+
+// Per-CPU kernel stacks
+extern unsigned char percpu_kstacks[NCPU][KSTKSIZE];
+
+int cpunum(void);
+#define thiscpu (&cpus[cpunum()])
+
+void mp_init(void);
+void lapic_init(void);
+void lapic_startap(uint8_t apicid, uint32_t addr);
+void lapic_eoi(void);
+void lapic_ipi(int vector);
+
+#endif

kern/env.c

@@ -11,9 +11,11 @@
 #include <kern/pmap.h>
 #include <kern/trap.h>
 #include <kern/monitor.h>
+#include <kern/sched.h>
+#include <kern/cpu.h>
+#include <kern/spinlock.h>
 
 struct Env *envs = NULL;		// All environments
-struct Env *curenv = NULL;		// The current env
 static struct Env *env_free_list;	// Free environment list
 					// (linked by Env->env_link)
 
@@ -34,7 +36,7 @@ static struct Env *env_free_list;	// Free environment list
 // definition of gdt specifies the Descriptor Privilege Level (DPL)
 // of that descriptor: 0 for kernel and 3 for user.
 //
-struct Segdesc gdt[] =
+struct Segdesc gdt[NCPU + 5] =
 {
 	// 0x0 - unused (always faults -- for trapping NULL far pointers)
 	SEG_NULL,
@@ -51,7 +53,8 @@ struct Segdesc gdt[] =
 	// 0x20 - user data segment
 	[GD_UD >> 3] = SEG(STA_W, 0x0, 0xffffffff, 3),
 
-	// 0x28 - tss, initialized in trap_init_percpu()
+	// Per-CPU TSS descriptors (starting from GD_TSS0) are initialized
+	// in trap_init_percpu()
 	[GD_TSS0 >> 3] = SEG_NULL
 };
 
@@ -250,6 +253,16 @@ env_alloc(struct Env **newenv_store, envid_t parent_id)
 	e->env_tf.tf_cs = GD_UT | 3;
 	// You will set e->env_tf.tf_eip later.
 
+	// Enable interrupts while in user mode.
+	// LAB 4: Your code here.
+	e->env_tf.tf_eflags |= FL_IF;
+
+	// Clear the page fault handler until user installs one.
+	e->env_pgfault_upcall = 0;
+
+	// Also clear the IPC receiving flag.
+	e->env_ipc_recving = 0;
+
 	// commit the allocation
 	env_free_list = e->env_link;
 	*newenv_store = e;
@@ -344,6 +357,7 @@ load_icode(struct Env *e, uint8_t *binary)
 	// LAB 3: Your code here.
 
 	// TODO validate the headers 
+	lcr3(PADDR(e->env_pgdir));
 	struct Elf* elf = (struct Elf*) binary;
 	struct Proghdr* ph = (struct Proghdr*) (binary + elf->e_phoff);
 	struct Proghdr* phend = ph + elf->e_phnum;
@@ -351,11 +365,10 @@ load_icode(struct Env *e, uint8_t *binary)
 		if(ph->p_type != ELF_PROG_LOAD) continue;
 
 		region_alloc(e, (void*) ph->p_va, ph->p_memsz);
-		lcr3(PADDR(e->env_pgdir));
 		memcpy((void*) ph->p_va, binary + ph->p_offset, ph->p_filesz);
 		memset((void*) ph->p_va + ph->p_filesz, 0, ph->p_memsz - ph->p_filesz);
-		lcr3(PADDR(kern_pgdir));
 	}
+	lcr3(PADDR(kern_pgdir));
 	e->env_tf.tf_eip = elf->e_entry;
 
 	// Now map one page for the program's initial stack
@@ -436,15 +449,26 @@ env_free(struct Env *e)
 
 //
 // Frees environment e.
+// If e was the current env, then runs a new environment (and does not return
+// to the caller).
 //
 void
 env_destroy(struct Env *e)
 {
+	// If e is currently running on other CPUs, we change its state to
+	// ENV_DYING. A zombie environment will be freed the next time
+	// it traps to the kernel.
+	if (e->env_status == ENV_RUNNING && curenv != e) {
+		e->env_status = ENV_DYING;
+		return;
+	}
+
 	env_free(e);
 
-	cprintf("Destroyed the only environment - nothing more to do!\n");
+	if (curenv == e) {
-	while (1)
+		curenv = NULL;
-		monitor(NULL);
+		sched_yield();
+	}
 }
 
 
@@ -457,6 +481,9 @@ env_destroy(struct Env *e)
 void
 env_pop_tf(struct Trapframe *tf)
 {
+	// Record the CPU we are running on for user-space debugging
+	curenv->env_cpunum = cpunum();
+
 	asm volatile(
 		"\tmovl %0,%%esp\n"
 		"\tpopal\n"
@@ -502,6 +529,7 @@ env_run(struct Env *e)
 	e->env_status = ENV_RUNNING;
 	e->env_runs++;
 	lcr3(PADDR(e->env_pgdir));
+	unlock_kernel();
 	env_pop_tf(&e->env_tf);
 }
 

kern/env.h

@@ -4,9 +4,10 @@
 #define JOS_KERN_ENV_H
 
 #include <inc/env.h>
+#include <kern/cpu.h>
 
 extern struct Env *envs;		// All environments
-extern struct Env *curenv;		// Current environment
+#define curenv (thiscpu->cpu_env)		// Current environment
 extern struct Segdesc gdt[];
 
 void	env_init(void);

kern/init.c

@@ -11,19 +11,18 @@
 #include <kern/kclock.h>
 #include <kern/env.h>
 #include <kern/trap.h>
+#include <kern/sched.h>
+#include <kern/picirq.h>
+#include <kern/cpu.h>
+#include <kern/spinlock.h>
+
+static void boot_aps(void);
 
 void sysenter_handler();
 
 void
 i386_init(void)
 {
-	extern char edata[], end[];
-
-	// Before doing anything else, complete the ELF loading process.
-	// Clear the uninitialized global data (BSS) section of our program.
-	// This ensures that all static/global variables start out zero.
-	memset(edata, 0, end - edata);
-
 	// Initialize the console.
 	// Can't call cprintf until after we do this!
 	cons_init();
@@ -47,18 +46,91 @@ i386_init(void)
 	env_init();
 	trap_init();
 
+	// Lab 4 multiprocessor initialization functions
+	mp_init();
+	lapic_init();
+
+	// Lab 4 multitasking initialization functions
+	pic_init();
+
+	// Acquire the big kernel lock before waking up APs
+	// Your code here:
+	lock_kernel();
+
+	// Starting non-boot CPUs
+	boot_aps();
+
 #if defined(TEST)
 	// Don't touch -- used by grading script!
 	ENV_CREATE(TEST, ENV_TYPE_USER);
 #else
 	// Touch all you want.
-	ENV_CREATE(user_hello, ENV_TYPE_USER);
+	ENV_CREATE(user_yield, ENV_TYPE_USER);
+	ENV_CREATE(user_yield, ENV_TYPE_USER);
+	ENV_CREATE(user_yield, ENV_TYPE_USER);
+	ENV_CREATE(user_yield, ENV_TYPE_USER);
 #endif // TEST*
 
-	// We only have one user environment for now, so just run it.
+	// Schedule and run the first user environment!
-	env_run(&envs[0]);
+	sched_yield();
 }
 
+// While boot_aps is booting a given CPU, it communicates the per-core
+// stack pointer that should be loaded by mpentry.S to that CPU in
+// this variable.
+void *mpentry_kstack;
+
+// Start the non-boot (AP) processors.
+static void
+boot_aps(void)
+{
+	extern unsigned char mpentry_start[], mpentry_end[];
+	void *code;
+	struct CpuInfo *c;
+
+	// Write entry code to unused memory at MPENTRY_PADDR
+	code = KADDR(MPENTRY_PADDR);
+	memmove(code, mpentry_start, mpentry_end - mpentry_start);
+
+	// Boot each AP one at a time
+	for (c = cpus; c < cpus + ncpu; c++) {
+		if (c == cpus + cpunum())  // We've started already.
+			continue;
+
+		// Tell mpentry.S what stack to use 
+		mpentry_kstack = percpu_kstacks[c - cpus] + KSTKSIZE;
+		// Start the CPU at mpentry_start
+		lapic_startap(c->cpu_id, PADDR(code));
+		// Wait for the CPU to finish some basic setup in mp_main()
+		while(c->cpu_status != CPU_STARTED)
+			;
+	}
+}
+
+// Setup code for APs
+void
+mp_main(void)
+{
+	// We are in high EIP now, safe to switch to kern_pgdir 
+	lcr3(PADDR(kern_pgdir));
+	cprintf("SMP: CPU %d starting\n", cpunum());
+
+	lapic_init();
+	env_init_percpu();
+	trap_init_percpu();
+	xchg(&thiscpu->cpu_status, CPU_STARTED); // tell boot_aps() we're up
+
+	// Now that we have finished some basic setup, call sched_yield()
+	// to start running processes on this CPU.  But make sure that
+	// only one CPU can enter the scheduler at a time!
+	//
+	// Your code here:
+	lock_kernel();
+	sched_yield();
+
+	// Remove this after you finish Exercise 6
+	for (;;);
+}
 
 /*
  * Variable panicstr contains argument to first call to panic; used as flag
@@ -83,7 +155,7 @@ _panic(const char *file, int line, const char *fmt,...)
 	asm volatile("cli; cld");
 
 	va_start(ap, fmt);
-	cprintf("kernel panic at %s:%d: ", file, line);
+	cprintf("kernel panic on CPU %d at %s:%d: ", cpunum(), file, line);
 	vcprintf(fmt, ap);
 	cprintf("\n");
 	va_end(ap);

kern/kernel.ld

@@ -44,18 +44,19 @@ SECTIONS
 
 	/* The data segment */
 	.data : {
-		*(.data)
+		*(.data .data.*)
 	}
 
 	.bss : {
 		PROVIDE(edata = .);
-		*(.bss)
+		*(.dynbss)
+		*(.bss .bss.*)
+		*(COMMON)
 		PROVIDE(end = .);
-		BYTE(0)
 	}
 
 
 	/DISCARD/ : {
-		*(.eh_frame .note.GNU-stack)
+		*(.eh_frame .note.GNU-stack .comment .note)
 	}
 }

kern/lapic.c

@@ -0,0 +1,182 @@
+// The local APIC manages internal (non-I/O) interrupts.
+// See Chapter 8 & Appendix C of Intel processor manual volume 3.
+
+#include <inc/types.h>
+#include <inc/memlayout.h>
+#include <inc/trap.h>
+#include <inc/mmu.h>
+#include <inc/stdio.h>
+#include <inc/x86.h>
+#include <kern/pmap.h>
+#include <kern/cpu.h>
+
+// Local APIC registers, divided by 4 for use as uint32_t[] indices.
+#define ID      (0x0020/4)   // ID
+#define VER     (0x0030/4)   // Version
+#define TPR     (0x0080/4)   // Task Priority
+#define EOI     (0x00B0/4)   // EOI
+#define SVR     (0x00F0/4)   // Spurious Interrupt Vector
+	#define ENABLE     0x00000100   // Unit Enable
+#define ESR     (0x0280/4)   // Error Status
+#define ICRLO   (0x0300/4)   // Interrupt Command
+	#define INIT       0x00000500   // INIT/RESET
+	#define STARTUP    0x00000600   // Startup IPI
+	#define DELIVS     0x00001000   // Delivery status
+	#define ASSERT     0x00004000   // Assert interrupt (vs deassert)
+	#define DEASSERT   0x00000000
+	#define LEVEL      0x00008000   // Level triggered
+	#define BCAST      0x00080000   // Send to all APICs, including self.
+	#define OTHERS     0x000C0000   // Send to all APICs, excluding self.
+	#define BUSY       0x00001000
+	#define FIXED      0x00000000
+#define ICRHI   (0x0310/4)   // Interrupt Command [63:32]
+#define TIMER   (0x0320/4)   // Local Vector Table 0 (TIMER)
+	#define X1         0x0000000B   // divide counts by 1
+	#define PERIODIC   0x00020000   // Periodic
+#define PCINT   (0x0340/4)   // Performance Counter LVT
+#define LINT0   (0x0350/4)   // Local Vector Table 1 (LINT0)
+#define LINT1   (0x0360/4)   // Local Vector Table 2 (LINT1)
+#define ERROR   (0x0370/4)   // Local Vector Table 3 (ERROR)
+	#define MASKED     0x00010000   // Interrupt masked
+#define TICR    (0x0380/4)   // Timer Initial Count
+#define TCCR    (0x0390/4)   // Timer Current Count
+#define TDCR    (0x03E0/4)   // Timer Divide Configuration
+
+physaddr_t lapicaddr;        // Initialized in mpconfig.c
+volatile uint32_t *lapic;
+
+static void
+lapicw(int index, int value)
+{
+	lapic[index] = value;
+	lapic[ID];  // wait for write to finish, by reading
+}
+
+void
+lapic_init(void)
+{
+	if (!lapicaddr)
+		return;
+
+	// lapicaddr is the physical address of the LAPIC's 4K MMIO
+	// region.  Map it in to virtual memory so we can access it.
+	lapic = mmio_map_region(lapicaddr, 4096);
+
+	// Enable local APIC; set spurious interrupt vector.
+	lapicw(SVR, ENABLE | (IRQ_OFFSET + IRQ_SPURIOUS));
+
+	// The timer repeatedly counts down at bus frequency
+	// from lapic[TICR] and then issues an interrupt.  
+	// If we cared more about precise timekeeping,
+	// TICR would be calibrated using an external time source.
+	lapicw(TDCR, X1);
+	lapicw(TIMER, PERIODIC | (IRQ_OFFSET + IRQ_TIMER));
+	lapicw(TICR, 10000000); 
+
+	// Leave LINT0 of the BSP enabled so that it can get
+	// interrupts from the 8259A chip.
+	//
+	// According to Intel MP Specification, the BIOS should initialize
+	// BSP's local APIC in Virtual Wire Mode, in which 8259A's
+	// INTR is virtually connected to BSP's LINTIN0. In this mode,
+	// we do not need to program the IOAPIC.
+	if (thiscpu != bootcpu)
+		lapicw(LINT0, MASKED);
+
+	// Disable NMI (LINT1) on all CPUs
+	lapicw(LINT1, MASKED);
+
+	// Disable performance counter overflow interrupts
+	// on machines that provide that interrupt entry.
+	if (((lapic[VER]>>16) & 0xFF) >= 4)
+		lapicw(PCINT, MASKED);
+
+	// Map error interrupt to IRQ_ERROR.
+	lapicw(ERROR, IRQ_OFFSET + IRQ_ERROR);
+
+	// Clear error status register (requires back-to-back writes).
+	lapicw(ESR, 0);
+	lapicw(ESR, 0);
+
+	// Ack any outstanding interrupts.
+	lapicw(EOI, 0);
+
+	// Send an Init Level De-Assert to synchronize arbitration ID's.
+	lapicw(ICRHI, 0);
+	lapicw(ICRLO, BCAST | INIT | LEVEL);
+	while(lapic[ICRLO] & DELIVS)
+		;
+
+	// Enable interrupts on the APIC (but not on the processor).
+	lapicw(TPR, 0);
+}
+
+int
+cpunum(void)
+{
+	if (lapic)
+		return lapic[ID] >> 24;
+	return 0;
+}
+
+// Acknowledge interrupt.
+void
+lapic_eoi(void)
+{
+	if (lapic)
+		lapicw(EOI, 0);
+}
+
+// Spin for a given number of microseconds.
+// On real hardware would want to tune this dynamically.
+static void
+microdelay(int us)
+{
+}
+
+#define IO_RTC  0x70
+
+// Start additional processor running entry code at addr.
+// See Appendix B of MultiProcessor Specification.
+void
+lapic_startap(uint8_t apicid, uint32_t addr)
+{
+	int i;
+	uint16_t *wrv;
+
+	// "The BSP must initialize CMOS shutdown code to 0AH
+	// and the warm reset vector (DWORD based at 40:67) to point at
+	// the AP startup code prior to the [universal startup algorithm]."
+	outb(IO_RTC, 0xF);  // offset 0xF is shutdown code
+	outb(IO_RTC+1, 0x0A);
+	wrv = (uint16_t *)KADDR((0x40 << 4 | 0x67));  // Warm reset vector
+	wrv[0] = 0;
+	wrv[1] = addr >> 4;
+
+	// "Universal startup algorithm."
+	// Send INIT (level-triggered) interrupt to reset other CPU.
+	lapicw(ICRHI, apicid << 24);
+	lapicw(ICRLO, INIT | LEVEL | ASSERT);
+	microdelay(200);
+	lapicw(ICRLO, INIT | LEVEL);
+	microdelay(100);    // should be 10ms, but too slow in Bochs!
+
+	// Send startup IPI (twice!) to enter code.
+	// Regular hardware is supposed to only accept a STARTUP
+	// when it is in the halted state due to an INIT.  So the second
+	// should be ignored, but it is part of the official Intel algorithm.
+	// Bochs complains about the second one.  Too bad for Bochs.
+	for (i = 0; i < 2; i++) {
+		lapicw(ICRHI, apicid << 24);
+		lapicw(ICRLO, STARTUP | (addr >> 12));
+		microdelay(200);
+	}
+}
+
+void
+lapic_ipi(int vector)
+{
+	lapicw(ICRLO, OTHERS | FIXED | vector);
+	while (lapic[ICRLO] & DELIVS)
+		;
+}

kern/mpconfig.c

@@ -0,0 +1,225 @@
+// Search for and parse the multiprocessor configuration table
+// See http://developer.intel.com/design/pentium/datashts/24201606.pdf
+
+#include <inc/types.h>
+#include <inc/string.h>
+#include <inc/memlayout.h>
+#include <inc/x86.h>
+#include <inc/mmu.h>
+#include <inc/env.h>
+#include <kern/cpu.h>
+#include <kern/pmap.h>
+
+struct CpuInfo cpus[NCPU];
+struct CpuInfo *bootcpu;
+int ismp;
+int ncpu;
+
+// Per-CPU kernel stacks
+unsigned char percpu_kstacks[NCPU][KSTKSIZE]
+__attribute__ ((aligned(PGSIZE)));
+
+
+// See MultiProcessor Specification Version 1.[14]
+
+struct mp {             // floating pointer [MP 4.1]
+	uint8_t signature[4];           // "_MP_"
+	physaddr_t physaddr;            // phys addr of MP config table
+	uint8_t length;                 // 1
+	uint8_t specrev;                // [14]
+	uint8_t checksum;               // all bytes must add up to 0
+	uint8_t type;                   // MP system config type
+	uint8_t imcrp;
+	uint8_t reserved[3];
+} __attribute__((__packed__));
+
+struct mpconf {         // configuration table header [MP 4.2]
+	uint8_t signature[4];           // "PCMP"
+	uint16_t length;                // total table length
+	uint8_t version;                // [14]
+	uint8_t checksum;               // all bytes must add up to 0
+	uint8_t product[20];            // product id
+	physaddr_t oemtable;            // OEM table pointer
+	uint16_t oemlength;             // OEM table length
+	uint16_t entry;                 // entry count
+	physaddr_t lapicaddr;           // address of local APIC
+	uint16_t xlength;               // extended table length
+	uint8_t xchecksum;              // extended table checksum
+	uint8_t reserved;
+	uint8_t entries[0];             // table entries
+} __attribute__((__packed__));
+
+struct mpproc {         // processor table entry [MP 4.3.1]
+	uint8_t type;                   // entry type (0)
+	uint8_t apicid;                 // local APIC id
+	uint8_t version;                // local APIC version
+	uint8_t flags;                  // CPU flags
+	uint8_t signature[4];           // CPU signature
+	uint32_t feature;               // feature flags from CPUID instruction
+	uint8_t reserved[8];
+} __attribute__((__packed__));
+
+// mpproc flags
+#define MPPROC_BOOT 0x02                // This mpproc is the bootstrap processor
+
+// Table entry types
+#define MPPROC    0x00  // One per processor
+#define MPBUS     0x01  // One per bus
+#define MPIOAPIC  0x02  // One per I/O APIC
+#define MPIOINTR  0x03  // One per bus interrupt source
+#define MPLINTR   0x04  // One per system interrupt source
+
+static uint8_t
+sum(void *addr, int len)
+{
+	int i, sum;
+
+	sum = 0;
+	for (i = 0; i < len; i++)
+		sum += ((uint8_t *)addr)[i];
+	return sum;
+}
+
+// Look for an MP structure in the len bytes at physical address addr.
+static struct mp *
+mpsearch1(physaddr_t a, int len)
+{
+	struct mp *mp = KADDR(a), *end = KADDR(a + len);
+
+	for (; mp < end; mp++)
+		if (memcmp(mp->signature, "_MP_", 4) == 0 &&
+		    sum(mp, sizeof(*mp)) == 0)
+			return mp;
+	return NULL;
+}
+
+// Search for the MP Floating Pointer Structure, which according to
+// [MP 4] is in one of the following three locations:
+// 1) in the first KB of the EBDA;
+// 2) if there is no EBDA, in the last KB of system base memory;
+// 3) in the BIOS ROM between 0xE0000 and 0xFFFFF.
+static struct mp *
+mpsearch(void)
+{
+	uint8_t *bda;
+	uint32_t p;
+	struct mp *mp;
+
+	static_assert(sizeof(*mp) == 16);
+
+	// The BIOS data area lives in 16-bit segment 0x40.
+	bda = (uint8_t *) KADDR(0x40 << 4);
+
+	// [MP 4] The 16-bit segment of the EBDA is in the two bytes
+	// starting at byte 0x0E of the BDA.  0 if not present.
+	if ((p = *(uint16_t *) (bda + 0x0E))) {
+		p <<= 4;	// Translate from segment to PA
+		if ((mp = mpsearch1(p, 1024)))
+			return mp;
+	} else {
+		// The size of base memory, in KB is in the two bytes
+		// starting at 0x13 of the BDA.
+		p = *(uint16_t *) (bda + 0x13) * 1024;
+		if ((mp = mpsearch1(p - 1024, 1024)))
+			return mp;
+	}
+	return mpsearch1(0xF0000, 0x10000);
+}
+
+// Search for an MP configuration table.  For now, don't accept the
+// default configurations (physaddr == 0).
+// Check for the correct signature, checksum, and version.
+static struct mpconf *
+mpconfig(struct mp **pmp)
+{
+	struct mpconf *conf;
+	struct mp *mp;
+
+	if ((mp = mpsearch()) == 0)
+		return NULL;
+	if (mp->physaddr == 0 || mp->type != 0) {
+		cprintf("SMP: Default configurations not implemented\n");
+		return NULL;
+	}
+	conf = (struct mpconf *) KADDR(mp->physaddr);
+	if (memcmp(conf, "PCMP", 4) != 0) {
+		cprintf("SMP: Incorrect MP configuration table signature\n");
+		return NULL;
+	}
+	if (sum(conf, conf->length) != 0) {
+		cprintf("SMP: Bad MP configuration checksum\n");
+		return NULL;
+	}
+	if (conf->version != 1 && conf->version != 4) {
+		cprintf("SMP: Unsupported MP version %d\n", conf->version);
+		return NULL;
+	}
+	if ((sum((uint8_t *)conf + conf->length, conf->xlength) + conf->xchecksum) & 0xff) {
+		cprintf("SMP: Bad MP configuration extended checksum\n");
+		return NULL;
+	}
+	*pmp = mp;
+	return conf;
+}
+
+void
+mp_init(void)
+{
+	struct mp *mp;
+	struct mpconf *conf;
+	struct mpproc *proc;
+	uint8_t *p;
+	unsigned int i;
+
+	bootcpu = &cpus[0];
+	if ((conf = mpconfig(&mp)) == 0)
+		return;
+	ismp = 1;
+	lapicaddr = conf->lapicaddr;
+
+	for (p = conf->entries, i = 0; i < conf->entry; i++) {
+		switch (*p) {
+		case MPPROC:
+			proc = (struct mpproc *)p;
+			if (proc->flags & MPPROC_BOOT)
+				bootcpu = &cpus[ncpu];
+			if (ncpu < NCPU) {
+				cpus[ncpu].cpu_id = ncpu;
+				ncpu++;
+			} else {
+				cprintf("SMP: too many CPUs, CPU %d disabled\n",
+					proc->apicid);
+			}
+			p += sizeof(struct mpproc);
+			continue;
+		case MPBUS:
+		case MPIOAPIC:
+		case MPIOINTR:
+		case MPLINTR:
+			p += 8;
+			continue;
+		default:
+			cprintf("mpinit: unknown config type %x\n", *p);
+			ismp = 0;
+			i = conf->entry;
+		}
+	}
+
+	bootcpu->cpu_status = CPU_STARTED;
+	if (!ismp) {
+		// Didn't like what we found; fall back to no MP.
+		ncpu = 1;
+		lapicaddr = 0;
+		cprintf("SMP: configuration not found, SMP disabled\n");
+		return;
+	}
+	cprintf("SMP: CPU %d found %d CPU(s)\n", bootcpu->cpu_id,  ncpu);
+
+	if (mp->imcrp) {
+		// [MP 3.2.6.1] If the hardware implements PIC mode,
+		// switch to getting interrupts from the LAPIC.
+		cprintf("SMP: Setting IMCR to switch from PIC mode to symmetric I/O mode\n");
+		outb(0x22, 0x70);   // Select IMCR
+		outb(0x23, inb(0x23) | 1);  // Mask external interrupts.
+	}
+}

kern/mpentry.S

@@ -0,0 +1,97 @@
+/* See COPYRIGHT for copyright information. */
+
+#include <inc/mmu.h>
+#include <inc/memlayout.h>
+
+###################################################################
+# entry point for APs
+###################################################################
+
+# Each non-boot CPU ("AP") is started up in response to a STARTUP
+# IPI from the boot CPU.  Section B.4.2 of the Multi-Processor
+# Specification says that the AP will start in real mode with CS:IP
+# set to XY00:0000, where XY is an 8-bit value sent with the
+# STARTUP. Thus this code must start at a 4096-byte boundary.
+#
+# Because this code sets DS to zero, it must run from an address in
+# the low 2^16 bytes of physical memory.
+#
+# boot_aps() (in init.c) copies this code to MPENTRY_PADDR (which
+# satisfies the above restrictions).  Then, for each AP, it stores the
+# address of the pre-allocated per-core stack in mpentry_kstack, sends
+# the STARTUP IPI, and waits for this code to acknowledge that it has
+# started (which happens in mp_main in init.c).
+#
+# This code is similar to boot/boot.S except that
+#    - it does not need to enable A20
+#    - it uses MPBOOTPHYS to calculate absolute addresses of its
+#      symbols, rather than relying on the linker to fill them
+
+#define RELOC(x) ((x) - KERNBASE)
+#define MPBOOTPHYS(s) ((s) - mpentry_start + MPENTRY_PADDR)
+
+.set PROT_MODE_CSEG, 0x8	# kernel code segment selector
+.set PROT_MODE_DSEG, 0x10	# kernel data segment selector
+
+.code16           
+.globl mpentry_start
+mpentry_start:
+	cli            
+
+	xorw    %ax, %ax
+	movw    %ax, %ds
+	movw    %ax, %es
+	movw    %ax, %ss
+
+	lgdt    MPBOOTPHYS(gdtdesc)
+	movl    %cr0, %eax
+	orl     $CR0_PE, %eax
+	movl    %eax, %cr0
+
+	ljmpl   $(PROT_MODE_CSEG), $(MPBOOTPHYS(start32))
+
+.code32
+start32:
+	movw    $(PROT_MODE_DSEG), %ax
+	movw    %ax, %ds
+	movw    %ax, %es
+	movw    %ax, %ss
+	movw    $0, %ax
+	movw    %ax, %fs
+	movw    %ax, %gs
+
+	# Set up initial page table. We cannot use kern_pgdir yet because
+	# we are still running at a low EIP.
+	movl    $(RELOC(entry_pgdir)), %eax
+	movl    %eax, %cr3
+	# Turn on paging.
+	movl    %cr0, %eax
+	orl     $(CR0_PE|CR0_PG|CR0_WP), %eax
+	movl    %eax, %cr0
+
+	# Switch to the per-cpu stack allocated in boot_aps()
+	movl    mpentry_kstack, %esp
+	movl    $0x0, %ebp       # nuke frame pointer
+
+	# Call mp_main().  (Exercise for the reader: why the indirect call?)
+	movl    $mp_main, %eax
+	call    *%eax
+
+	# If mp_main returns (it shouldn't), loop.
+spin:
+	jmp     spin
+
+# Bootstrap GDT
+.p2align 2					# force 4 byte alignment
+gdt:
+	SEG_NULL				# null seg
+	SEG(STA_X|STA_R, 0x0, 0xffffffff)	# code seg
+	SEG(STA_W, 0x0, 0xffffffff)		# data seg
+
+gdtdesc:
+	.word   0x17				# sizeof(gdt) - 1
+	.long   MPBOOTPHYS(gdt)			# address gdt
+
+.globl mpentry_end
+mpentry_end:
+	nop

kern/picirq.c

@@ -0,0 +1,86 @@
+/* See COPYRIGHT for copyright information. */
+
+#include <inc/assert.h>
+#include <inc/trap.h>
+
+#include <kern/picirq.h>
+
+
+// Current IRQ mask.
+// Initial IRQ mask has interrupt 2 enabled (for slave 8259A).
+uint16_t irq_mask_8259A = 0xFFFF & ~(1<<IRQ_SLAVE);
+static bool didinit;
+
+/* Initialize the 8259A interrupt controllers. */
+void
+pic_init(void)
+{
+	didinit = 1;
+
+	// mask all interrupts
+	outb(IO_PIC1+1, 0xFF);
+	outb(IO_PIC2+1, 0xFF);
+
+	// Set up master (8259A-1)
+
+	// ICW1:  0001g0hi
+	//    g:  0 = edge triggering, 1 = level triggering
+	//    h:  0 = cascaded PICs, 1 = master only
+	//    i:  0 = no ICW4, 1 = ICW4 required
+	outb(IO_PIC1, 0x11);
+
+	// ICW2:  Vector offset
+	outb(IO_PIC1+1, IRQ_OFFSET);
+
+	// ICW3:  bit mask of IR lines connected to slave PICs (master PIC),
+	//        3-bit No of IR line at which slave connects to master(slave PIC).
+	outb(IO_PIC1+1, 1<<IRQ_SLAVE);
+
+	// ICW4:  000nbmap
+	//    n:  1 = special fully nested mode
+	//    b:  1 = buffered mode
+	//    m:  0 = slave PIC, 1 = master PIC
+	//	  (ignored when b is 0, as the master/slave role
+	//	  can be hardwired).
+	//    a:  1 = Automatic EOI mode
+	//    p:  0 = MCS-80/85 mode, 1 = intel x86 mode
+	outb(IO_PIC1+1, 0x3);
+
+	// Set up slave (8259A-2)
+	outb(IO_PIC2, 0x11);			// ICW1
+	outb(IO_PIC2+1, IRQ_OFFSET + 8);	// ICW2
+	outb(IO_PIC2+1, IRQ_SLAVE);		// ICW3
+	// NB Automatic EOI mode doesn't tend to work on the slave.
+	// Linux source code says it's "to be investigated".
+	outb(IO_PIC2+1, 0x01);			// ICW4
+
+	// OCW3:  0ef01prs
+	//   ef:  0x = NOP, 10 = clear specific mask, 11 = set specific mask
+	//    p:  0 = no polling, 1 = polling mode
+	//   rs:  0x = NOP, 10 = read IRR, 11 = read ISR
+	outb(IO_PIC1, 0x68);             /* clear specific mask */
+	outb(IO_PIC1, 0x0a);             /* read IRR by default */
+
+	outb(IO_PIC2, 0x68);               /* OCW3 */
+	outb(IO_PIC2, 0x0a);               /* OCW3 */
+
+	if (irq_mask_8259A != 0xFFFF)
+		irq_setmask_8259A(irq_mask_8259A);
+}
+
+void
+irq_setmask_8259A(uint16_t mask)
+{
+	int i;
+	irq_mask_8259A = mask;
+	if (!didinit)
+		return;
+	outb(IO_PIC1+1, (char)mask);
+	outb(IO_PIC2+1, (char)(mask >> 8));
+	cprintf("enabled interrupts:");
+	for (i = 0; i < 16; i++)
+		if (~mask & (1<<i))
+			cprintf(" %d", i);
+	cprintf("\n");
+}
+

kern/picirq.h

@@ -0,0 +1,28 @@
+/* See COPYRIGHT for copyright information. */
+
+#ifndef JOS_KERN_PICIRQ_H
+#define JOS_KERN_PICIRQ_H
+#ifndef JOS_KERNEL
+# error "This is a JOS kernel header; user programs should not #include it"
+#endif
+
+#define MAX_IRQS	16	// Number of IRQs
+
+// I/O Addresses of the two 8259A programmable interrupt controllers
+#define IO_PIC1		0x20	// Master (IRQs 0-7)
+#define IO_PIC2		0xA0	// Slave (IRQs 8-15)
+
+#define IRQ_SLAVE	2	// IRQ at which slave connects to master
+
+
+#ifndef __ASSEMBLER__
+
+#include <inc/types.h>
+#include <inc/x86.h>
+
+extern uint16_t irq_mask_8259A;
+void pic_init(void);
+void irq_setmask_8259A(uint16_t mask);
+#endif // !__ASSEMBLER__
+
+#endif // !JOS_KERN_PICIRQ_H

kern/pmap.c

@@ -10,6 +10,7 @@
 #include <kern/pmap.h>
 #include <kern/kclock.h>
 #include <kern/env.h>
+#include <kern/cpu.h>
 
 // These variables are set by i386_detect_memory()
 size_t npages;			// Amount of physical memory (in pages)
@@ -63,6 +64,7 @@ i386_detect_memory(void)
 // Set up memory mappings above UTOP.
 // --------------------------------------------------------------
 
+static void mem_init_mp(void);
 static void boot_map_region(pde_t *pgdir, uintptr_t va, size_t size, physaddr_t pa, int perm);
 static void check_page_free_list(bool only_low_memory);
 static void check_page_alloc(void);
@@ -193,6 +195,7 @@ mem_init(void)
 	//    - the new image at UENVS  -- kernel R, user R
 	//    - envs itself -- kernel RW, user NONE
 	// LAB 3: Your code here.
+	cprintf("Mapping envs from %p to %p\n", UENVS, ROUNDUP(envs_size, PGSIZE));
 	boot_map_region(kern_pgdir,
 		UENVS, ROUNDUP(envs_size, PGSIZE),
 		PADDR(envs), PTE_U);
@@ -224,6 +227,9 @@ mem_init(void)
 			KERNBASE, 0x100000000 - KERNBASE,
 			0, PTE_W);
 
+	// Initialize the SMP-related parts of the memory map
+	mem_init_mp();
+
 	// Check that the initial page directory has been set up correctly.
 	check_kern_pgdir();
 
@@ -249,6 +255,36 @@ mem_init(void)
 	check_page_installed_pgdir();
 }
 
+// Modify mappings in kern_pgdir to support SMP
+//   - Map the per-CPU stacks in the region [KSTACKTOP-PTSIZE, KSTACKTOP)
+//
+static void
+mem_init_mp(void)
+{
+	// Map per-CPU stacks starting at KSTACKTOP, for up to 'NCPU' CPUs.
+	//
+	// For CPU i, use the physical memory that 'percpu_kstacks[i]' refers
+	// to as its kernel stack. CPU i's kernel stack grows down from virtual
+	// address kstacktop_i = KSTACKTOP - i * (KSTKSIZE + KSTKGAP), and is
+	// divided into two pieces, just like the single stack you set up in
+	// mem_init:
+	//     * [kstacktop_i - KSTKSIZE, kstacktop_i)
+	//          -- backed by physical memory
+	//     * [kstacktop_i - (KSTKSIZE + KSTKGAP), kstacktop_i - KSTKSIZE)
+	//          -- not backed; so if the kernel overflows its stack,
+	//             it will fault rather than overwrite another CPU's stack.
+	//             Known as a "guard page".
+	//     Permissions: kernel RW, user NONE
+	//
+	// LAB 4: Your code here:
+	for(int i = 0; i < NCPU; i++) {
+		uintptr_t kstacktop = KSTACKTOP - i * (KSTKSIZE + KSTKGAP);
+		boot_map_region(kern_pgdir, kstacktop - KSTKSIZE,
+				KSTKSIZE, PADDR(percpu_kstacks[i]), PTE_W);
+	}
+
+}
+
 // --------------------------------------------------------------
 // Tracking of physical pages.
 // The 'pages' array has one 'struct PageInfo' entry per physical page.
@@ -259,6 +295,7 @@ is_reserved(size_t pagenum) {
 	if(pagenum == 0) return true;
 	if(pagenum >= PGNUM(IOPHYSMEM) &&
 		pagenum < PGNUM(PADDR(boot_alloc(0)))) return true;
+	if(pagenum == PGNUM(MPENTRY_PADDR)) return true;
 
 	return false;
 }
@@ -272,6 +309,10 @@ is_reserved(size_t pagenum) {
 void
 page_init(void)
 {
+	// LAB 4:
+	// Change your code to mark the physical page at MPENTRY_PADDR
+	// as in use
+
 	// The example code here marks all physical pages as free.
 	// However this is not truly the case.  What memory is free?
 	//  1) Mark physical page 0 as in use.
@@ -526,10 +567,53 @@ void
 tlb_invalidate(pde_t *pgdir, void *va)
 {
 	// Flush the entry only if we're modifying the current address space.
-	// For now, there is only one address space, so always invalidate.
+	if (!curenv || curenv->env_pgdir == pgdir)
 		invlpg(va);
 }
 
+//
+// Reserve size bytes in the MMIO region and map [pa,pa+size) at this
+// location.  Return the base of the reserved region.  size does *not*
+// have to be multiple of PGSIZE.
+//
+void *
+mmio_map_region(physaddr_t pa, size_t size)
+{
+	// Where to start the next region.  Initially, this is the
+	// beginning of the MMIO region.  Because this is static, its
+	// value will be preserved between calls to mmio_map_region
+	// (just like nextfree in boot_alloc).
+	static uintptr_t base = MMIOBASE;
+
+	// Reserve size bytes of virtual memory starting at base and
+	// map physical pages [pa,pa+size) to virtual addresses
+	// [base,base+size).  Since this is device memory and not
+	// regular DRAM, you'll have to tell the CPU that it isn't
+	// safe to cache access to this memory.  Luckily, the page
+	// tables provide bits for this purpose; simply create the
+	// mapping with PTE_PCD|PTE_PWT (cache-disable and
+	// write-through) in addition to PTE_W.  (If you're interested
+	// in more details on this, see section 10.5 of IA32 volume
+	// 3A.)
+	//
+	// Be sure to round size up to a multiple of PGSIZE and to
+	// handle if this reservation would overflow MMIOLIM (it's
+	// okay to simply panic if this happens).
+	//
+	// Hint: The staff solution uses boot_map_region.
+	//
+	// Your code here:
+	size = ROUNDUP(size, PGSIZE);
+	if((base + size) > MMIOLIM)
+		panic("Not enough memory-mapped IO space!");
+
+	boot_map_region(kern_pgdir, base, size, pa, PTE_PCD | PTE_PWT | PTE_W);
+	uintptr_t to_return = base;
+	base += size;
+
+	return (void*) to_return;
+}
+
 static uintptr_t user_mem_check_addr;
 
 //
@@ -645,6 +729,8 @@ check_page_free_list(bool only_low_memory)
 		assert(page2pa(pp) != EXTPHYSMEM - PGSIZE);
 		assert(page2pa(pp) != EXTPHYSMEM);
 		assert(page2pa(pp) < EXTPHYSMEM || (char *) page2kva(pp) >= first_free_page);
+		// (new test for lab 4)
+		assert(page2pa(pp) != MPENTRY_PADDR);
 
 		if (page2pa(pp) < EXTPHYSMEM)
 			++nfree_basemem;
@@ -767,9 +853,15 @@ check_kern_pgdir(void)
 		assert(check_va2pa(pgdir, KERNBASE + i) == i);
 
 	// check kernel stack
+	// (updated in lab 4 to check per-CPU kernel stacks)
+	for (n = 0; n < NCPU; n++) {
+		uint32_t base = KSTACKTOP - (KSTKSIZE + KSTKGAP) * (n + 1);
 		for (i = 0; i < KSTKSIZE; i += PGSIZE)
-		assert(check_va2pa(pgdir, KSTACKTOP - KSTKSIZE + i) == PADDR(bootstack) + i);
+			assert(check_va2pa(pgdir, base + KSTKGAP + i)
-	assert(check_va2pa(pgdir, KSTACKTOP - PTSIZE) == ~0);
+				== PADDR(percpu_kstacks[n]) + i);
+		for (i = 0; i < KSTKGAP; i += PGSIZE)
+			assert(check_va2pa(pgdir, base + i) == ~0);
+	}
 
 	// check PDE permissions
 	for (i = 0; i < NPDENTRIES; i++) {
@@ -778,6 +870,7 @@ check_kern_pgdir(void)
 		case PDX(KSTACKTOP-1):
 		case PDX(UPAGES):
 		case PDX(UENVS):
+		case PDX(MMIOBASE):
 			assert(pgdir[i] & PTE_P);
 			break;
 		default:
@@ -820,6 +913,7 @@ check_page(void)
 	struct PageInfo *fl;
 	pte_t *ptep, *ptep1;
 	void *va;
+	uintptr_t mm1, mm2;
 	int i;
 	extern pde_t entry_pgdir[];
 
@@ -962,6 +1056,29 @@ check_page(void)
 	page_free(pp1);
 	page_free(pp2);
 
+	// test mmio_map_region
+	mm1 = (uintptr_t) mmio_map_region(0, 4097);
+	mm2 = (uintptr_t) mmio_map_region(0, 4096);
+	// check that they're in the right region
+	assert(mm1 >= MMIOBASE && mm1 + 8192 < MMIOLIM);
+	assert(mm2 >= MMIOBASE && mm2 + 8192 < MMIOLIM);
+	// check that they're page-aligned
+	assert(mm1 % PGSIZE == 0 && mm2 % PGSIZE == 0);
+	// check that they don't overlap
+	assert(mm1 + 8192 <= mm2);
+	// check page mappings
+	assert(check_va2pa(kern_pgdir, mm1) == 0);
+	assert(check_va2pa(kern_pgdir, mm1+PGSIZE) == PGSIZE);
+	assert(check_va2pa(kern_pgdir, mm2) == 0);
+	assert(check_va2pa(kern_pgdir, mm2+PGSIZE) == ~0);
+	// check permissions
+	assert(*pgdir_walk(kern_pgdir, (void*) mm1, 0) & (PTE_W|PTE_PWT|PTE_PCD));
+	assert(!(*pgdir_walk(kern_pgdir, (void*) mm1, 0) & PTE_U));
+	// clear the mappings
+	*pgdir_walk(kern_pgdir, (void*) mm1, 0) = 0;
+	*pgdir_walk(kern_pgdir, (void*) mm1 + PGSIZE, 0) = 0;
+	*pgdir_walk(kern_pgdir, (void*) mm2, 0) = 0;
+
 	cprintf("check_page() succeeded!\n");
 }
 

kern/pmap.h

@@ -63,6 +63,8 @@ void	page_decref(struct PageInfo *pp);
 
 void	tlb_invalidate(pde_t *pgdir, void *va);
 
+void *	mmio_map_region(physaddr_t pa, size_t size);
+
 int	user_mem_check(struct Env *env, const void *va, size_t len, int perm);
 void	user_mem_assert(struct Env *env, const void *va, size_t len, int perm);
 

kern/sched.c

@@ -0,0 +1,102 @@
+#include <inc/assert.h>
+#include <inc/x86.h>
+#include <kern/spinlock.h>
+#include <kern/env.h>
+#include <kern/pmap.h>
+#include <kern/monitor.h>
+
+void sched_halt(void);
+
+// Choose a user environment to run and run it.
+void
+sched_yield(void)
+{
+	struct Env *idle;
+
+	// Implement simple round-robin scheduling.
+	//
+	// Search through 'envs' for an ENV_RUNNABLE environment in
+	// circular fashion starting just after the env this CPU was
+	// last running.  Switch to the first such environment found.
+	//
+	// If no envs are runnable, but the environment previously
+	// running on this CPU is still ENV_RUNNING, it's okay to
+	// choose that environment.
+	//
+	// Never choose an environment that's currently running on
+	// another CPU (env_status == ENV_RUNNING). If there are
+	// no runnable environments, simply drop through to the code
+	// below to halt the cpu.
+
+	// LAB 4: Your code here.
+	struct Env* next_env = curenv ? curenv + 1 : envs;
+	struct Env* end_env = envs + NENV;
+	struct Env* to_run = NULL;
+
+	for(int i = 0; i < NENV; i++, next_env++) {
+		if(next_env == end_env) next_env = envs;
+		if(next_env->env_status == ENV_RUNNABLE) {
+			to_run = next_env;
+			break;
+		}
+	}
+
+	if(!to_run && curenv && curenv->env_status == ENV_RUNNING) {
+		to_run = curenv;
+	}
+
+	if(to_run) env_run(to_run);
+
+	// sched_halt never returns
+	sched_halt();
+}
+
+// Halt this CPU when there is nothing to do. Wait until the
+// timer interrupt wakes it up. This function never returns.
+//
+void
+sched_halt(void)
+{
+	int i;
+
+	// For debugging and testing purposes, if there are no runnable
+	// environments in the system, then drop into the kernel monitor.
+	for (i = 0; i < NENV; i++) {
+		if ((envs[i].env_status == ENV_RUNNABLE ||
+		     envs[i].env_status == ENV_RUNNING ||
+		     envs[i].env_status == ENV_DYING))
+			break;
+	}
+	if (i == NENV) {
+		cprintf("No runnable environments in the system!\n");
+		while (1)
+			monitor(NULL);
+	}
+
+	// Mark that no environment is running on this CPU
+	curenv = NULL;
+	lcr3(PADDR(kern_pgdir));
+
+	// Mark that this CPU is in the HALT state, so that when
+	// timer interupts come in, we know we should re-acquire the
+	// big kernel lock
+	xchg(&thiscpu->cpu_status, CPU_HALTED);
+
+	// Release the big kernel lock as if we were "leaving" the kernel
+	unlock_kernel();
+
+	// Reset stack pointer, enable interrupts and then halt.
+	asm volatile (
+		"movl $0, %%ebp\n"
+		"movl %0, %%esp\n"
+		"pushl $0\n"
+		"pushl $0\n"
+        // LAB 4:
+		// Uncomment the following line after completing exercise 13
+		"sti\n"
+		"1:\n"
+		"hlt\n"
+		"jmp 1b\n"
+	: : "a" (thiscpu->cpu_ts.ts_esp0));
+}
+

kern/sched.h

@@ -0,0 +1,12 @@
+/* See COPYRIGHT for copyright information. */
+
+#ifndef JOS_KERN_SCHED_H
+#define JOS_KERN_SCHED_H
+#ifndef JOS_KERNEL
+# error "This is a JOS kernel header; user programs should not #include it"
+#endif
+
+// This function does not return.
+void sched_yield(void) __attribute__((noreturn));
+
+#endif	// !JOS_KERN_SCHED_H

kern/spinlock.c

@@ -0,0 +1,116 @@
+// Mutual exclusion spin locks.
+
+#include <inc/types.h>
+#include <inc/assert.h>
+#include <inc/x86.h>
+#include <inc/memlayout.h>
+#include <inc/string.h>
+#include <kern/cpu.h>
+#include <kern/spinlock.h>
+#include <kern/kdebug.h>
+
+// The big kernel lock
+struct spinlock kernel_lock = {
+#ifdef DEBUG_SPINLOCK
+	.name = "kernel_lock"
+#endif
+};
+
+#ifdef DEBUG_SPINLOCK
+// Record the current call stack in pcs[] by following the %ebp chain.
+static void
+get_caller_pcs(uint32_t pcs[])
+{
+	uint32_t *ebp;
+	int i;
+
+	ebp = (uint32_t *)read_ebp();
+	for (i = 0; i < 10; i++){
+		if (ebp == 0 || ebp < (uint32_t *)ULIM)
+			break;
+		pcs[i] = ebp[1];          // saved %eip
+		ebp = (uint32_t *)ebp[0]; // saved %ebp
+	}
+	for (; i < 10; i++)
+		pcs[i] = 0;
+}
+
+// Check whether this CPU is holding the lock.
+static int
+holding(struct spinlock *lock)
+{
+	return lock->locked && lock->cpu == thiscpu;
+}
+#endif
+
+void
+__spin_initlock(struct spinlock *lk, char *name)
+{
+	lk->locked = 0;
+#ifdef DEBUG_SPINLOCK
+	lk->name = name;
+	lk->cpu = 0;
+#endif
+}
+
+// Acquire the lock.
+// Loops (spins) until the lock is acquired.
+// Holding a lock for a long time may cause
+// other CPUs to waste time spinning to acquire it.
+void
+spin_lock(struct spinlock *lk)
+{
+#ifdef DEBUG_SPINLOCK
+	if (holding(lk))
+		panic("CPU %d cannot acquire %s: already holding", cpunum(), lk->name);
+#endif
+
+	// The xchg is atomic.
+	// It also serializes, so that reads after acquire are not
+	// reordered before it. 
+	while (xchg(&lk->locked, 1) != 0)
+		asm volatile ("pause");
+
+	// Record info about lock acquisition for debugging.
+#ifdef DEBUG_SPINLOCK
+	lk->cpu = thiscpu;
+	get_caller_pcs(lk->pcs);
+#endif
+}
+
+// Release the lock.
+void
+spin_unlock(struct spinlock *lk)
+{
+#ifdef DEBUG_SPINLOCK
+	if (!holding(lk)) {
+		int i;
+		uint32_t pcs[10];
+		// Nab the acquiring EIP chain before it gets released
+		memmove(pcs, lk->pcs, sizeof pcs);
+		cprintf("CPU %d cannot release %s: held by CPU %d\nAcquired at:", 
+			cpunum(), lk->name, lk->cpu->cpu_id);
+		for (i = 0; i < 10 && pcs[i]; i++) {
+			struct Eipdebuginfo info;
+			if (debuginfo_eip(pcs[i], &info) >= 0)
+				cprintf("  %08x %s:%d: %.*s+%x\n", pcs[i],
+					info.eip_file, info.eip_line,
+					info.eip_fn_namelen, info.eip_fn_name,
+					pcs[i] - info.eip_fn_addr);
+			else
+				cprintf("  %08x\n", pcs[i]);
+		}
+		panic("spin_unlock");
+	}
+
+	lk->pcs[0] = 0;
+	lk->cpu = 0;
+#endif
+
+	// The xchg instruction is atomic (i.e. uses the "lock" prefix) with
+	// respect to any other instruction which references the same memory.
+	// x86 CPUs will not reorder loads/stores across locked instructions
+	// (vol 3, 8.2.2). Because xchg() is implemented using asm volatile,
+	// gcc will not reorder C statements across the xchg.
+	xchg(&lk->locked, 0);
+}

kern/spinlock.h

@@ -0,0 +1,48 @@
+#ifndef JOS_INC_SPINLOCK_H
+#define JOS_INC_SPINLOCK_H
+
+#include <inc/types.h>
+
+// Comment this to disable spinlock debugging
+#define DEBUG_SPINLOCK
+
+// Mutual exclusion lock.
+struct spinlock {
+	unsigned locked;       // Is the lock held?
+
+#ifdef DEBUG_SPINLOCK
+	// For debugging:
+	char *name;            // Name of lock.
+	struct CpuInfo *cpu;   // The CPU holding the lock.
+	uintptr_t pcs[10];     // The call stack (an array of program counters)
+	                       // that locked the lock.
+#endif
+};
+
+void __spin_initlock(struct spinlock *lk, char *name);
+void spin_lock(struct spinlock *lk);
+void spin_unlock(struct spinlock *lk);
+
+#define spin_initlock(lock)   __spin_initlock(lock, #lock)
+
+extern struct spinlock kernel_lock;
+
+static inline void
+lock_kernel(void)
+{
+	spin_lock(&kernel_lock);
+}
+
+static inline void
+unlock_kernel(void)
+{
+	spin_unlock(&kernel_lock);
+
+	// Normally we wouldn't need to do this, but QEMU only runs
+	// one CPU at a time and has a long time-slice.  Without the
+	// pause, this CPU is likely to reacquire the lock before
+	// another CPU has even been given a chance to acquire it.
+	asm volatile("pause");
+}
+
+#endif

kern/syscall.c

@@ -10,6 +10,7 @@
 #include <kern/trap.h>
 #include <kern/syscall.h>
 #include <kern/console.h>
+#include <kern/sched.h>
 
 // Print a string to the system console.
 // The string is exactly 'len' characters long.
@@ -60,6 +61,304 @@ sys_env_destroy(envid_t envid)
 	return 0;
 }
 
+// Deschedule current environment and pick a different one to run.
+static void
+sys_yield(void)
+{
+	sched_yield();
+}
+
+// Allocate a new environment.
+// Returns envid of new environment, or < 0 on error.  Errors are:
+//	-E_NO_FREE_ENV if no free environment is available.
+//	-E_NO_MEM on memory exhaustion.
+static envid_t
+sys_exofork(void)
+{
+	// Create the new environment with env_alloc(), from kern/env.c.
+	// It should be left as env_alloc created it, except that
+	// status is set to ENV_NOT_RUNNABLE, and the register set is copied
+	// from the current environment -- but tweaked so sys_exofork
+	// will appear to return 0.
+	struct Env* new_env;
+	int error_code;
+	
+	error_code = env_alloc(&new_env, curenv->env_id);
+	if(error_code < 0) return error_code;
+
+	new_env->env_tf = curenv->env_tf;
+	new_env->env_tf.tf_regs.reg_eax = 0;
+	new_env->env_status = ENV_NOT_RUNNABLE;
+	return new_env->env_id;
+}
+
+// Set envid's env_status to status, which must be ENV_RUNNABLE
+// or ENV_NOT_RUNNABLE.
+//
+// Returns 0 on success, < 0 on error.  Errors are:
+//	-E_BAD_ENV if environment envid doesn't currently exist,
+//		or the caller doesn't have permission to change envid.
+//	-E_INVAL if status is not a valid status for an environment.
+static int
+sys_env_set_status(envid_t envid, int status)
+{
+	// Hint: Use the 'envid2env' function from kern/env.c to translate an
+	// envid to a struct Env.
+	// You should set envid2env's third argument to 1, which will
+	// check whether the current environment has permission to set
+	// envid's status.
+	struct Env* env;
+	int error_code;
+
+	error_code = envid2env(envid, &env, 1);
+	if(error_code < 0) return error_code;
+
+	if(status != ENV_RUNNABLE && status != ENV_NOT_RUNNABLE)
+		return -E_INVAL;
+
+	env->env_status = status;
+	return 0;
+}
+
+// Set the page fault upcall for 'envid' by modifying the corresponding struct
+// Env's 'env_pgfault_upcall' field.  When 'envid' causes a page fault, the
+// kernel will push a fault record onto the exception stack, then branch to
+// 'func'.
+//
+// Returns 0 on success, < 0 on error.  Errors are:
+//	-E_BAD_ENV if environment envid doesn't currently exist,
+//		or the caller doesn't have permission to change envid.
+static int
+sys_env_set_pgfault_upcall(envid_t envid, void *func)
+{
+	struct Env* env;
+	int return_code;
+	if((return_code = envid2env(envid, &env, 1)) < 0) return return_code;
+	env->env_pgfault_upcall = func;
+	return 0;
+}
+
+#define SYS_CHECKPERMS(perm) \
+	((((perm) & (PTE_P | PTE_U)) == (PTE_P | PTE_U)) && \
+	 (((perm) & ~(PTE_P | PTE_U | PTE_W | PTE_AVAIL)) == 0))
+#define SYS_CHECKADDR(addr) (((uintptr_t) (addr) < UTOP) && ((uintptr_t) (addr) % PGSIZE == 0))
+
+// Allocate a page of memory and map it at 'va' with permission
+// 'perm' in the address space of 'envid'.
+// The page's contents are set to 0.
+// If a page is already mapped at 'va', that page is unmapped as a
+// side effect.
+//
+// perm -- PTE_U | PTE_P must be set, PTE_AVAIL | PTE_W may or may not be set,
+//         but no other bits may be set.  See PTE_SYSCALL in inc/mmu.h.
+//
+// Return 0 on success, < 0 on error.  Errors are:
+//	-E_BAD_ENV if environment envid doesn't currently exist,
+//		or the caller doesn't have permission to change envid.
+//	-E_INVAL if va >= UTOP, or va is not page-aligned.
+//	-E_INVAL if perm is inappropriate (see above).
+//	-E_NO_MEM if there's no memory to allocate the new page,
+//		or to allocate any necessary page tables.
+static int
+sys_page_alloc(envid_t envid, void *va, int perm)
+{
+	// Hint: This function is a wrapper around page_alloc() and
+	//   page_insert() from kern/pmap.c.
+	//   Most of the new code you write should be to check the
+	//   parameters for correctness.
+	//   If page_insert() fails, remember to free the page you
+	//   allocated!
+	struct Env* env;
+	int return_code;
+	if((return_code = envid2env(envid, &env, 1)) < 0) return return_code;
+	
+	if(!SYS_CHECKPERMS(perm)) return -E_INVAL;
+	if(!SYS_CHECKADDR(va)) return -E_INVAL;
+
+	struct PageInfo* page = page_alloc(1);
+	if(!page) return -E_NO_MEM;
+
+	if((return_code = page_insert(env->env_pgdir, page, va, perm)) < 0) {
+		page_free(page);
+		return return_code;
+	}
+
+	return 0;
+}
+
+// Map the page of memory at 'srcva' in srcenvid's address space
+// at 'dstva' in dstenvid's address space with permission 'perm'.
+// Perm has the same restrictions as in sys_page_alloc, except
+// that it also must not grant write access to a read-only
+// page.
+//
+// Return 0 on success, < 0 on error.  Errors are:
+//	-E_BAD_ENV if srcenvid and/or dstenvid doesn't currently exist,
+//		or the caller doesn't have permission to change one of them.
+//	-E_INVAL if srcva >= UTOP or srcva is not page-aligned,
+//		or dstva >= UTOP or dstva is not page-aligned.
+//	-E_INVAL is srcva is not mapped in srcenvid's address space.
+//	-E_INVAL if perm is inappropriate (see sys_page_alloc).
+//	-E_INVAL if (perm & PTE_W), but srcva is read-only in srcenvid's
+//		address space.
+//	-E_NO_MEM if there's no memory to allocate any necessary page tables.
+static int
+sys_page_map(envid_t srcenvid, void *srcva,
+	     envid_t dstenvid, void *dstva, int perm)
+{
+	// Hint: This function is a wrapper around page_lookup() and
+	//   page_insert() from kern/pmap.c.
+	//   Again, most of the new code you write should be to check the
+	//   parameters for correctness.
+	//   Use the third argument to page_lookup() to
+	//   check the current permissions on the page.
+	struct Env *srcenv, *dstenv;
+	pte_t* srcpte;
+	int return_code;
+
+	if((return_code = envid2env(srcenvid, &srcenv, 1)) < 0) return return_code;
+	if((return_code = envid2env(dstenvid, &dstenv, 1)) < 0) return return_code;
+
+	if(!SYS_CHECKADDR(srcva)) return -E_INVAL;
+	if(!SYS_CHECKADDR(dstva)) return -E_INVAL;
+	if(!SYS_CHECKPERMS(perm)) return -E_INVAL;
+
+	struct PageInfo* page = page_lookup(srcenv->env_pgdir, srcva, &srcpte);
+	if(page == NULL) return -E_INVAL;
+	if(perm & PTE_W && !(*srcpte & PTE_W)) return -E_INVAL;
+
+	if((return_code = page_insert(dstenv->env_pgdir, page, dstva, perm)) < 0)
+		return return_code;
+	return 0;
+}
+
+// Unmap the page of memory at 'va' in the address space of 'envid'.
+// If no page is mapped, the function silently succeeds.
+//
+// Return 0 on success, < 0 on error.  Errors are:
+//	-E_BAD_ENV if environment envid doesn't currently exist,
+//		or the caller doesn't have permission to change envid.
+//	-E_INVAL if va >= UTOP, or va is not page-aligned.
+static int
+sys_page_unmap(envid_t envid, void *va)
+{
+	// Hint: This function is a wrapper around page_remove().
+	struct Env* env;
+	int return_code;
+
+	if((return_code = envid2env(envid, &env, 1)) < 0) return return_code;
+	if(!SYS_CHECKADDR(va)) return -E_INVAL;
+
+	page_remove(env->env_pgdir, va);
+	return 0;
+}
+
+// Try to send 'value' to the target env 'envid'.
+// If srcva < UTOP, then also send page currently mapped at 'srcva',
+// so that receiver gets a duplicate mapping of the same page.
+//
+// The send fails with a return value of -E_IPC_NOT_RECV if the
+// target is not blocked, waiting for an IPC.
+//
+// The send also can fail for the other reasons listed below.
+//
+// Otherwise, the send succeeds, and the target's ipc fields are
+// updated as follows:
+//    env_ipc_recving is set to 0 to block future sends;
+//    env_ipc_from is set to the sending envid;
+//    env_ipc_value is set to the 'value' parameter;
+//    env_ipc_perm is set to 'perm' if a page was transferred, 0 otherwise.
+// The target environment is marked runnable again, returning 0
+// from the paused sys_ipc_recv system call.  (Hint: does the
+// sys_ipc_recv function ever actually return?)
+//
+// If the sender wants to send a page but the receiver isn't asking for one,
+// then no page mapping is transferred, but no error occurs.
+// The ipc only happens when no errors occur.
+//
+// Returns 0 on success, < 0 on error.
+// Errors are:
+//	-E_BAD_ENV if environment envid doesn't currently exist.
+//		(No need to check permissions.)
+//	-E_IPC_NOT_RECV if envid is not currently blocked in sys_ipc_recv,
+//		or another environment managed to send first.
+//	-E_INVAL if srcva < UTOP but srcva is not page-aligned.
+//	-E_INVAL if srcva < UTOP and perm is inappropriate
+//		(see sys_page_alloc).
+//	-E_INVAL if srcva < UTOP but srcva is not mapped in the caller's
+//		address space.
+//	-E_INVAL if (perm & PTE_W), but srcva is read-only in the
+//		current environment's address space.
+//	-E_NO_MEM if there's not enough memory to map srcva in envid's
+//		address space.
+static int
+sys_ipc_try_send(envid_t envid, uint32_t value, void *srcva, unsigned perm)
+{
+	struct Env* dest_env;
+	struct Env* src_env;
+	int return_code;
+
+	if((return_code = envid2env(0, &src_env, 0)) < 0)
+		return return_code;
+	if((return_code = envid2env(envid, &dest_env, 0)) < 0)
+		return return_code;
+
+	if(!dest_env->env_ipc_recving)
+		return -E_IPC_NOT_RECV;
+
+	if((uintptr_t) srcva < UTOP && dest_env->env_ipc_dstva) {
+		if(!SYS_CHECKADDR(srcva)) return -E_INVAL;
+		if(!SYS_CHECKPERMS(perm)) return -E_INVAL;
+
+		pte_t* srcpte;
+		struct PageInfo* page = page_lookup(src_env->env_pgdir, srcva, &srcpte);
+		if(page == NULL) return -E_INVAL;
+		if(perm & PTE_W && !(*srcpte & PTE_W)) return -E_INVAL;
+
+		page_insert(dest_env->env_pgdir, page, dest_env->env_ipc_dstva, perm);
+		dest_env->env_ipc_perm = perm;
+	}
+
+	dest_env->env_ipc_from = src_env->env_id;
+	dest_env->env_ipc_value = value;
+	dest_env->env_ipc_recving = false;
+	if(dest_env->env_status == ENV_NOT_RUNNABLE)
+		dest_env->env_status = ENV_RUNNABLE;
+
+	return 0;
+}
+
+// Block until a value is ready.  Record that you want to receive
+// using the env_ipc_recving and env_ipc_dstva fields of struct Env,
+// mark yourself not runnable, and then give up the CPU.
+//
+// If 'dstva' is < UTOP, then you are willing to receive a page of data.
+// 'dstva' is the virtual address at which the sent page should be mapped.
+//
+// This function only returns on error, but the system call will eventually
+// return 0 on success.
+// Return < 0 on error.  Errors are:
+//	-E_INVAL if dstva < UTOP but dstva is not page-aligned.
+static int
+sys_ipc_recv(void *dstva)
+{
+	struct Env* env;
+	int return_code;
+
+	if((return_code = envid2env(0, &env, 1)) < 0)
+		return return_code;
+
+	// LAB 4: Your code here.
+	if((uintptr_t) dstva < UTOP) {
+		if(!SYS_CHECKADDR(dstva)) return -E_INVAL;
+		env->env_ipc_dstva = dstva;
+	}
+	env->env_ipc_recving = true;
+	env->env_status = ENV_NOT_RUNNABLE;
+
+	return 0;
+}
+
 // Dispatches to the correct kernel function, passing the arguments.
 int32_t
 syscall(uint32_t syscallno, uint32_t a1, uint32_t a2, uint32_t a3, uint32_t a4, uint32_t a5)
@@ -78,6 +377,25 @@ syscall(uint32_t syscallno, uint32_t a1, uint32_t a2, uint32_t a3, uint32_t a4,
 		return sys_getenvid();
 	case SYS_env_destroy:
 		return sys_env_destroy(a1);
+	case SYS_yield:
+		sys_yield();
+		return 0;
+	case SYS_exofork:
+		return sys_exofork();
+	case SYS_env_set_status:
+		return sys_env_set_status(a1, a2);
+	case SYS_env_set_pgfault_upcall:
+		return sys_env_set_pgfault_upcall(a1, (void*) a2);
+	case SYS_page_alloc:
+		return sys_page_alloc(a1, (void*) a2, a3);
+	case SYS_page_map:
+		return sys_page_map(a1, (void*) a2, a3, (void*) a4, a5);
+	case SYS_page_unmap:
+		return sys_page_unmap(a1, (void*) a2);
+	case SYS_ipc_try_send:
+		return sys_ipc_try_send(a1, a2, (void*) a3, a4);
+	case SYS_ipc_recv:
+		return sys_ipc_recv((void*) a1);
 	default:
 		return -E_INVAL;
 	}

kern/trap.c

@@ -8,6 +8,11 @@
 #include <kern/monitor.h>
 #include <kern/env.h>
 #include <kern/syscall.h>
+#include <kern/sched.h>
+#include <kern/kclock.h>
+#include <kern/picirq.h>
+#include <kern/cpu.h>
+#include <kern/spinlock.h>
 
 static struct Taskstate ts;
 
@@ -55,6 +60,8 @@ static const char *trapname(int trapno)
 		return excnames[trapno];
 	if (trapno == T_SYSCALL)
 		return "System call";
+	if (trapno >= IRQ_OFFSET && trapno < IRQ_OFFSET + 16)
+		return "Hardware Interrupt";
 	return "(unknown trap)";
 }
 
@@ -82,6 +89,13 @@ void t_simderr();
 void t_syscall();
 void t_default();
 
+void irq_timer();
+void irq_kbd();
+void irq_serial();
+void irq_spurious();
+void irq_ide();
+void irq_error();
+
 void
 trap_init(void)
 {
@@ -118,6 +132,13 @@ trap_init(void)
 	SETGATE(idt[T_SYSCALL], 0, GD_KT, t_syscall, 3);
 	SETGATE(idt[T_DEFAULT], 0, GD_KT, t_default, 0);
 
+	SETGATE(idt[IRQ_OFFSET + IRQ_TIMER], 0, GD_KT, irq_timer, 0);
+	SETGATE(idt[IRQ_OFFSET + IRQ_KBD], 0, GD_KT, irq_kbd, 0);
+	SETGATE(idt[IRQ_OFFSET + IRQ_SERIAL], 0, GD_KT, irq_serial, 0);
+	SETGATE(idt[IRQ_OFFSET + IRQ_SPURIOUS], 0, GD_KT, irq_spurious, 0);
+	SETGATE(idt[IRQ_OFFSET + IRQ_IDE], 0, GD_KT, irq_ide, 0);
+	SETGATE(idt[IRQ_OFFSET + IRQ_ERROR], 0, GD_KT, irq_error, 0);
+
 	// Per-CPU setup
 	trap_init_percpu();
 }
@@ -126,20 +147,45 @@ trap_init(void)
 void
 trap_init_percpu(void)
 {
+	// The example code here sets up the Task State Segment (TSS) and
+	// the TSS descriptor for CPU 0. But it is incorrect if we are
+	// running on other CPUs because each CPU has its own kernel stack.
+	// Fix the code so that it works for all CPUs.
+	//
+	// Hints:
+	//   - The macro "thiscpu" always refers to the current CPU's
+	//     struct CpuInfo;
+	//   - The ID of the current CPU is given by cpunum() or
+	//     thiscpu->cpu_id;
+	//   - Use "thiscpu->cpu_ts" as the TSS for the current CPU,
+	//     rather than the global "ts" variable;
+	//   - Use gdt[(GD_TSS0 >> 3) + i] for CPU i's TSS descriptor;
+	//   - You mapped the per-CPU kernel stacks in mem_init_mp()
+	//   - Initialize cpu_ts.ts_iomb to prevent unauthorized environments
+	//     from doing IO (0 is not the correct value!)
+	//
+	// ltr sets a 'busy' flag in the TSS selector, so if you
+	// accidentally load the same TSS on more than one CPU, you'll
+	// get a triple fault.  If you set up an individual CPU's TSS
+	// wrong, you may not get a fault until you try to return from
+	// user space on that CPU.
+	//
+	// LAB 4: Your code here:
+
 	// Setup a TSS so that we get the right stack
 	// when we trap to the kernel.
-	ts.ts_esp0 = KSTACKTOP;
+	thiscpu->cpu_ts.ts_esp0 = KSTACKTOP - thiscpu->cpu_id * (KSTKSIZE + KSTKGAP);
-	ts.ts_ss0 = GD_KD;
+	thiscpu->cpu_ts.ts_ss0 = GD_KD;
-	ts.ts_iomb = sizeof(struct Taskstate);
+	thiscpu->cpu_ts.ts_iomb = sizeof(struct Taskstate);
 
 	// Initialize the TSS slot of the gdt.
-	gdt[GD_TSS0 >> 3] = SEG16(STS_T32A, (uint32_t) (&ts),
+	gdt[(GD_TSS0 >> 3) + cpunum()] = SEG16(STS_T32A, (uint32_t) (&thiscpu->cpu_ts),
 					sizeof(struct Taskstate) - 1, 0);
-	gdt[GD_TSS0 >> 3].sd_s = 0;
+	gdt[(GD_TSS0 >> 3) + cpunum()].sd_s = 0;
 
 	// Load the TSS selector (like other segment selectors, the
 	// bottom three bits are special; we leave them 0)
-	ltr(GD_TSS0);
+	ltr(GD_TSS0 + (cpunum() << 3));
 
 	// Load the IDT
 	lidt(&idt_pd);
@@ -148,7 +194,7 @@ trap_init_percpu(void)
 void
 print_trapframe(struct Trapframe *tf)
 {
-	cprintf("TRAP frame at %p\n", tf);
+	cprintf("TRAP frame at %p from CPU %d\n", tf, cpunum());
 	print_regs(&tf->tf_regs);
 	cprintf("  es   0x----%04x\n", tf->tf_es);
 	cprintf("  ds   0x----%04x\n", tf->tf_ds);
@@ -211,8 +257,24 @@ trap_dispatch(struct Trapframe *tf)
 				tf->tf_regs.reg_esi);
 		tf->tf_regs.reg_eax = returned;
 		return;
+	} else if (tf->tf_trapno == IRQ_OFFSET + IRQ_TIMER) {
+		lapic_eoi();
+		sched_yield();
 	}
 
+	// Handle spurious interrupts
+	// The hardware sometimes raises these because of noise on the
+	// IRQ line or other reasons. We don't care.
+	if (tf->tf_trapno == IRQ_OFFSET + IRQ_SPURIOUS) {
+		cprintf("Spurious interrupt on irq 7\n");
+		print_trapframe(tf);
+		return;
+	}
+
+	// Handle clock interrupts. Don't forget to acknowledge the
+	// interrupt using lapic_eoi() before calling the scheduler!
+	// LAB 4: Your code here.
+
 	// Unexpected trap: The user process or the kernel has a bug.
 	print_trapframe(tf);
 	if (tf->tf_cs == GD_KT)
@@ -230,17 +292,35 @@ trap(struct Trapframe *tf)
 	// of GCC rely on DF being clear
 	asm volatile("cld" ::: "cc");
 
+	// Halt the CPU if some other CPU has called panic()
+	extern char *panicstr;
+	if (panicstr)
+		asm volatile("hlt");
+
+	// Re-acqurie the big kernel lock if we were halted in
+	// sched_yield()
+	if (xchg(&thiscpu->cpu_status, CPU_STARTED) == CPU_HALTED)
+		lock_kernel();
 	// Check that interrupts are disabled.  If this assertion
 	// fails, DO NOT be tempted to fix it by inserting a "cli" in
 	// the interrupt path.
 	assert(!(read_eflags() & FL_IF));
 
-	cprintf("Incoming TRAP frame at %p\n", tf);
-
 	if ((tf->tf_cs & 3) == 3) {
 		// Trapped from user mode.
+		// Acquire the big kernel lock before doing any
+		// serious kernel work.
+		// LAB 4: Your code here.
+		lock_kernel();
 		assert(curenv);
 
+		// Garbage collect if current enviroment is a zombie
+		if (curenv->env_status == ENV_DYING) {
+			env_free(curenv);
+			curenv = NULL;
+			sched_yield();
+		}
+
 		// Copy trap frame (which is currently on the stack)
 		// into 'curenv->env_tf', so that running the environment
 		// will restart at the trap point.
@@ -256,9 +336,13 @@ trap(struct Trapframe *tf)
 	// Dispatch based on what type of trap occurred
 	trap_dispatch(tf);
 
-	// Return to the current environment, which should be running.
+	// If we made it to this point, then no other environment was
-	assert(curenv && curenv->env_status == ENV_RUNNING);
+	// scheduled, so we should return to the current environment
+	// if doing so makes sense.
+	if (curenv && curenv->env_status == ENV_RUNNING)
 		env_run(curenv);
+	else
+		sched_yield();
 }
 
 
@@ -277,10 +361,69 @@ page_fault_handler(struct Trapframe *tf)
 	// We've already handled kernel-mode exceptions, so if we get here,
 	// the page fault happened in user mode.
 
+	// Call the environment's page fault upcall, if one exists.  Set up a
+	// page fault stack frame on the user exception stack (below
+	// UXSTACKTOP), then branch to curenv->env_pgfault_upcall.
+	//
+	// The page fault upcall might cause another page fault, in which case
+	// we branch to the page fault upcall recursively, pushing another
+	// page fault stack frame on top of the user exception stack.
+	//
+	// It is convenient for our code which returns from a page fault
+	// (lib/pfentry.S) to have one word of scratch space at the top of the
+	// trap-time stack; it allows us to more easily restore the eip/esp. In
+	// the non-recursive case, we don't have to worry about this because
+	// the top of the regular user stack is free.  In the recursive case,
+	// this means we have to leave an extra word between the current top of
+	// the exception stack and the new stack frame because the exception
+	// stack _is_ the trap-time stack.
+	//
+	// If there's no page fault upcall, the environment didn't allocate a
+	// page for its exception stack or can't write to it, or the exception
+	// stack overflows, then destroy the environment that caused the fault.
+	// Note that the grade script assumes you will first check for the page
+	// fault upcall and print the "user fault va" message below if there is
+	// none.  The remaining three checks can be combined into a single test.
+	//
+	// Hints:
+	//   user_mem_assert() and env_run() are useful here.
+	//   To change what the user environment runs, modify 'curenv->env_tf'
+	//   (the 'tf' variable points at 'curenv->env_tf').
+
+	// LAB 4: Your code here.
+	if(!curenv->env_pgfault_upcall) {
 		// Destroy the environment that caused the fault.
 		cprintf("[%08x] user fault va %08x ip %08x\n",
 				curenv->env_id, fault_va, tf->tf_eip);
 		print_trapframe(tf);
 		env_destroy(curenv);
+	}
+	user_mem_assert(curenv, curenv->env_pgfault_upcall, 1, PTE_U | PTE_P);
+	user_mem_assert(curenv, (void*) UXSTACKTOP - 1, 1, PTE_U | PTE_P | PTE_W);
+
+	uintptr_t top_addr = UXSTACKTOP;
+	if(tf->tf_esp <= UXSTACKTOP && tf->tf_esp >= (UXSTACKTOP - PGSIZE)) {
+		top_addr = tf->tf_esp - 4;
+	}
+
+	struct UTrapframe utf;
+	utf.utf_eflags = tf->tf_eflags;
+	utf.utf_eip = tf->tf_eip;
+	utf.utf_err = tf->tf_err;
+	utf.utf_esp = tf->tf_esp;
+	utf.utf_fault_va = fault_va;
+	utf.utf_regs = tf->tf_regs;
+
+	struct UTrapframe* push_to = (struct UTrapframe*) top_addr - 1;
+	if((uintptr_t) push_to < USTACKTOP - PGSIZE) {
+		cprintf("[%08x] stack overflow in page fault handler\n",
+				curenv->env_id);
+		env_destroy(curenv);
+	}
+
+	*push_to = utf;
+	curenv->env_tf.tf_eip = (uintptr_t) curenv->env_pgfault_upcall;
+	curenv->env_tf.tf_esp = (uintptr_t) push_to;
+	env_run(curenv);
 }
 

kern/trapentry.S

@@ -4,6 +4,7 @@
 #include <inc/memlayout.h>
 #include <inc/trap.h>
 
+#include <kern/picirq.h>
 
 
 ###################################################################
@@ -81,6 +82,12 @@ TRAPHANDLER_NOEC(t_mchk, T_MCHK);
 TRAPHANDLER_NOEC(t_simderr, T_SIMDERR);
 TRAPHANDLER_NOEC(t_syscall, T_SYSCALL);
 TRAPHANDLER(t_default, T_DEFAULT);
+TRAPHANDLER_NOEC(irq_timer, IRQ_OFFSET + IRQ_TIMER);
+TRAPHANDLER_NOEC(irq_kbd, IRQ_OFFSET + IRQ_KBD);
+TRAPHANDLER_NOEC(irq_serial, IRQ_OFFSET + IRQ_SERIAL);
+TRAPHANDLER_NOEC(irq_spurious, IRQ_OFFSET + IRQ_SPURIOUS);
+TRAPHANDLER_NOEC(irq_ide, IRQ_OFFSET + IRQ_IDE);
+TRAPHANDLER_NOEC(irq_error, IRQ_OFFSET + IRQ_ERROR);
 
 // HINT 1 : TRAPHANDLER_NOEC(t_divide, T_DIVIDE);
 //          Do something like this if there is no error code for the trap

lib/Makefrag

@@ -10,6 +10,11 @@ LIB_SRCFILES :=		lib/console.c \
 			lib/string.c \
 			lib/syscall.c
 
+LIB_SRCFILES :=		$(LIB_SRCFILES) \
+			lib/pgfault.c \
+			lib/pfentry.S \
+			lib/fork.c \
+			lib/ipc.c
 
 
 

lib/console.c

@@ -18,7 +18,7 @@ getchar(void)
 	int r;
 	// sys_cgetc does not block, but getchar should.
 	while ((r = sys_cgetc()) == 0)
-		;
+		sys_yield();
 	return r;
 }
 

lib/fork.c

@@ -0,0 +1,147 @@
+// implement fork from user space
+
+#include <inc/string.h>
+#include <inc/lib.h>
+
+// PTE_COW marks copy-on-write page table entries.
+// It is one of the bits explicitly allocated to user processes (PTE_AVAIL).
+#define PTE_COW		0x800
+
+//
+// Custom page fault handler - if faulting page is copy-on-write,
+// map in our own private writable copy.
+//
+static void
+pgfault(struct UTrapframe *utf)
+{
+	void *addr = (void *) utf->utf_fault_va;
+	uint32_t err = utf->utf_err;
+	int r;
+
+	// Check that the faulting access was (1) a write, and (2) to a
+	// copy-on-write page.  If not, panic.
+	// Hint:
+	//   Use the read-only page table mappings at uvpt
+	//   (see <inc/memlayout.h>).
+	if(!((err & FEC_WR) && (uvpt[(uintptr_t) addr >> PGSHIFT] & PTE_COW)))
+		panic("page fault (addr %p)! %c", addr, (err & FEC_WR) ? 'w' : 'r');
+
+	// Allocate a new page, map it at a temporary location (PFTEMP),
+	// copy the data from the old page to the new page, then move the new
+	// page to the old page's address.
+	// Hint:
+	//   You should make three system calls.
+	void* temp_addr = (void*) PFTEMP;
+	void* fault_addr = ROUNDDOWN(addr, PGSIZE);
+	if(sys_page_alloc(0, temp_addr, PTE_P | PTE_W | PTE_U) < 0)
+		panic("failed to allocate new page");
+
+	memcpy(temp_addr, fault_addr, PGSIZE);
+	sys_page_map(0, temp_addr, 0, fault_addr, PTE_P | PTE_U | PTE_W);
+	sys_page_unmap(0, temp_addr);
+}
+
+//
+// Map our virtual page pn (address pn*PGSIZE) into the target envid
+// at the same virtual address.  If the page is writable or copy-on-write,
+// the new mapping must be created copy-on-write, and then our mapping must be
+// marked copy-on-write as well.  (Exercise: Why do we need to mark ours
+// copy-on-write again if it was already copy-on-write at the beginning of
+// this function?)
+//
+// Returns: 0 on success, < 0 on error.
+// It is also OK to panic on error.
+//
+static int
+duppage(envid_t envid, unsigned pn)
+{
+	int r;
+	bool change_own = false;
+	pte_t new_pte = uvpt[pn];
+	pte_t perms = new_pte & (PTE_P | PTE_U | PTE_W | PTE_AVAIL);
+	void* addr = (void*) (pn * PGSIZE);
+
+	// If we're writable, remove write permission
+	if((new_pte & PTE_W) || (new_pte & PTE_COW)) {
+		perms = (perms & ~PTE_W) | PTE_COW;
+		change_own = true;
+	}
+
+	// Map either with the same permissions or with COW.
+	if((r = sys_page_map(0, addr, envid, addr, perms)) < 0)
+		return r;
+
+	// Update our own permissions if necessary
+	if(change_own) {
+		if((r = sys_page_map(0, addr, 0, addr, perms)) < 0)
+			return r;
+	}
+
+	return 0;
+}
+
+//
+// User-level fork with copy-on-write.
+// Set up our page fault handler appropriately.
+// Create a child.
+// Copy our address space and page fault handler setup to the child.
+// Then mark the child as runnable and return.
+//
+// Returns: child's envid to the parent, 0 to the child, < 0 on error.
+// It is also OK to panic on error.
+//
+// Hint:
+//   Use uvpd, uvpt, and duppage.
+//   Remember to fix "thisenv" in the child process.
+//   Neither user exception stack should ever be marked copy-on-write,
+//   so you must allocate a new page for the child's user exception stack.
+//
+envid_t
+fork(void)
+{
+	set_pgfault_handler(pgfault);
+
+	int return_code;
+	envid_t forked;
+
+	forked = sys_exofork();
+	if(forked < 0) return forked;
+	if(forked == 0) { thisenv = &envs[ENVX(sys_getenvid())]; return 0; }
+
+	// Map all accessible page directory entries
+	for(int pde_i = 0; pde_i < PDX(UTOP); pde_i++) {
+		pde_t pde = uvpd[pde_i];
+		if(!(pde & PTE_P)) continue;
+
+		// For each PDE, map all the underlying PTEs
+		for(int pte_i = 0; pte_i < NPTENTRIES; pte_i++) {
+			int pn = pde_i * NPTENTRIES + pte_i;
+			pte_t pte = uvpt[pn];
+			if(!(pte & PTE_P)) continue;
+
+			// Do not map user exception stack, though
+			if(pn == ((UXSTACKTOP - PGSIZE) >> PGSHIFT)) continue;
+
+			if((return_code = duppage(forked, pn)) < 0) return return_code;
+		}
+	}
+
+	// Allocate new page for the exception stack
+	return_code = sys_page_alloc(forked, (void*) UXSTACKTOP - PGSIZE,
+			PTE_P | PTE_U | PTE_W);
+	if(return_code < 0) return return_code;
+
+	// Set the upcall entry point
+	sys_env_set_pgfault_upcall(forked, thisenv->env_pgfault_upcall);
+	sys_env_set_status(forked, ENV_RUNNABLE);
+
+	return forked;
+}
+
+// Challenge!
+int
+sfork(void)
+{
+	panic("sfork not implemented");
+	return -E_INVAL;
+}

lib/ipc.c

@@ -0,0 +1,65 @@
+// User-level IPC library routines
+
+#include <inc/lib.h>
+
+// Receive a value via IPC and return it.
+// If 'pg' is nonnull, then any page sent by the sender will be mapped at
+//	that address.
+// If 'from_env_store' is nonnull, then store the IPC sender's envid in
+//	*from_env_store.
+// If 'perm_store' is nonnull, then store the IPC sender's page permission
+//	in *perm_store (this is nonzero iff a page was successfully
+//	transferred to 'pg').
+// If the system call fails, then store 0 in *fromenv and *perm (if
+//	they're nonnull) and return the error.
+// Otherwise, return the value sent by the sender
+//
+// Hint:
+//   Use 'thisenv' to discover the value and who sent it.
+//   If 'pg' is null, pass sys_ipc_recv a value that it will understand
+//   as meaning "no page".  (Zero is not the right value, since that's
+//   a perfectly valid place to map a page.)
+int32_t
+ipc_recv(envid_t *from_env_store, void *pg, int *perm_store)
+{
+	int return_code;
+	if((return_code = sys_ipc_recv(pg ? pg : (void*) ~0)) < 0)
+		return return_code;
+
+	if(from_env_store) *from_env_store = thisenv->env_ipc_from;
+	if(perm_store) *perm_store = thisenv->env_ipc_perm;
+
+	return thisenv->env_ipc_value;
+}
+
+// Send 'val' (and 'pg' with 'perm', if 'pg' is nonnull) to 'toenv'.
+// This function keeps trying until it succeeds.
+// It should panic() on any error other than -E_IPC_NOT_RECV.
+//
+// Hint:
+//   Use sys_yield() to be CPU-friendly.
+//   If 'pg' is null, pass sys_ipc_try_send a value that it will understand
+//   as meaning "no page".  (Zero is not the right value.)
+void
+ipc_send(envid_t to_env, uint32_t val, void *pg, int perm)
+{
+	int return_code = -E_IPC_NOT_RECV;
+	while(return_code == -E_IPC_NOT_RECV) {
+		return_code = sys_ipc_try_send(to_env, val, pg ? pg : (void*) ~0, perm);
+		sys_yield();
+	}
+	if(return_code != 0) panic("failed to send\n");
+}
+
+// Find the first environment of the given type.  We'll use this to
+// find special environments.
+// Returns 0 if no such environment exists.
+envid_t
+ipc_find_env(enum EnvType type)
+{
+	int i;
+	for (i = 0; i < NENV; i++)
+		if (envs[i].env_type == type)
+			return envs[i].env_id;
+	return 0;
+}

lib/pfentry.S

@@ -0,0 +1,93 @@
+#include <inc/mmu.h>
+#include <inc/memlayout.h>
+
+// Page fault upcall entrypoint.
+
+// This is where we ask the kernel to redirect us to whenever we cause
+// a page fault in user space (see the call to sys_set_pgfault_handler
+// in pgfault.c).
+//
+// When a page fault actually occurs, the kernel switches our ESP to
+// point to the user exception stack if we're not already on the user
+// exception stack, and then it pushes a UTrapframe onto our user
+// exception stack:
+//
+//	trap-time esp
+//	trap-time eflags
+//	trap-time eip
+//	utf_regs.reg_eax
+//	...
+//	utf_regs.reg_esi
+//	utf_regs.reg_edi
+//	utf_err (error code)
+//	utf_fault_va            <-- %esp
+//
+// If this is a recursive fault, the kernel will reserve for us a
+// blank word above the trap-time esp for scratch work when we unwind
+// the recursive call.
+//
+// We then have call up to the appropriate page fault handler in C
+// code, pointed to by the global variable '_pgfault_handler'.
+
+.text
+.globl _pgfault_upcall
+_pgfault_upcall:
+	// Call the C page fault handler.
+	pushl %esp			// function argument: pointer to UTF
+	movl _pgfault_handler, %eax
+	call *%eax
+	addl $4, %esp			// pop function argument
+	
+	// Now the C page fault handler has returned and you must return
+	// to the trap time state.
+	// Push trap-time %eip onto the trap-time stack.
+	//
+	// Explanation:
+	//   We must prepare the trap-time stack for our eventual return to
+	//   re-execute the instruction that faulted.
+	//   Unfortunately, we can't return directly from the exception stack:
+	//   We can't call 'jmp', since that requires that we load the address
+	//   into a register, and all registers must have their trap-time
+	//   values after the return.
+	//   We can't call 'ret' from the exception stack either, since if we
+	//   did, %esp would have the wrong value.
+	//   So instead, we push the trap-time %eip onto the *trap-time* stack!
+	//   Below we'll switch to that stack and call 'ret', which will
+	//   restore %eip to its pre-fault value.
+	//
+	//   In the case of a recursive fault on the exception stack,
+	//   note that the word we're pushing now will fit in the
+	//   blank word that the kernel reserved for us.
+	//
+	// Throughout the remaining code, think carefully about what
+	// registers are available for intermediate calculations.  You
+	// may find that you have to rearrange your code in non-obvious
+	// ways as registers become unavailable as scratch space.
+	//
+	// LAB 4: Your code here.
+	mov 40(%esp), %eax // Take the EIP from memory
+	mov 48(%esp), %ebp // Take the ESP from memory
+	sub $4, %ebp // Push onto trap-time ESP
+	mov %eax, (%ebp)
+	mov %ebp, 48(%esp) // Put ESP back
+
+	// Restore the trap-time registers.  After you do this, you
+	// can no longer modify any general-purpose registers.
+	// LAB 4: Your code here.
+	add $0x8, %esp
+	popal
+
+	// Restore eflags from the stack.  After you do this, you can
+	// no longer use arithmetic operations or anything else that
+	// modifies eflags.
+	// LAB 4: Your code here.
+	add $0x4, %esp
+	popfl
+
+	// Switch back to the adjusted trap-time stack.
+	// LAB 4: Your code here.
+	pop %esp
+
+	// Return to re-execute the instruction that faulted.
+	// LAB 4: Your code here.
+	ret

lib/pgfault.c

@@ -0,0 +1,37 @@
+// User-level page fault handler support.
+// Rather than register the C page fault handler directly with the
+// kernel as the page fault handler, we register the assembly language
+// wrapper in pfentry.S, which in turns calls the registered C
+// function.
+
+#include <inc/lib.h>
+
+
+// Assembly language pgfault entrypoint defined in lib/pfentry.S.
+extern void _pgfault_upcall(void);
+
+// Pointer to currently installed C-language pgfault handler.
+void (*_pgfault_handler)(struct UTrapframe *utf);
+
+//
+// Set the page fault handler function.
+// If there isn't one yet, _pgfault_handler will be 0.
+// The first time we register a handler, we need to
+// allocate an exception stack (one page of memory with its top
+// at UXSTACKTOP), and tell the kernel to call the assembly-language
+// _pgfault_upcall routine when a page fault occurs.
+//
+void
+set_pgfault_handler(void (*handler)(struct UTrapframe *utf))
+{
+	int r;
+
+	if (_pgfault_handler == 0) {
+		if(sys_page_alloc(0, (void*) UXSTACKTOP - PGSIZE, PTE_U | PTE_P | PTE_W) < 0)
+			panic("set_pgfault_handler");
+		sys_env_set_pgfault_upcall(0, _pgfault_upcall);
+	}
+
+	// Save handler pointer for assembly to call.
+	_pgfault_handler = handler;
+}

lib/printfmt.c

@@ -26,6 +26,8 @@ static const char * const error_string[MAXERROR] =
 	[E_NO_MEM]	= "out of memory",
 	[E_NO_FREE_ENV]	= "out of environments",
 	[E_FAULT]	= "segmentation fault",
+	[E_IPC_NOT_RECV]= "env is not recving",
+	[E_EOF]		= "unexpected end of file",
 };
 
 /*

lib/syscall.c

@@ -55,13 +55,13 @@ fast_syscall(int num, uint32_t a1, uint32_t a2, uint32_t a3, uint32_t a4) {
 void
 sys_cputs(const char *s, size_t len)
 {
-	fast_syscall(SYS_cputs, (uint32_t)s, len, 0, 0);
+	syscall(SYS_cputs, 0, (uint32_t)s, len, 0, 0, 0);
 }
 
 int
 sys_cgetc(void)
 {
-	return fast_syscall(SYS_cgetc, 0, 0, 0, 0);
+	return syscall(SYS_cgetc, 0, 0, 0, 0, 0, 0);
 }
 
 int
@@ -76,3 +76,53 @@ sys_getenvid(void)
 	 return syscall(SYS_getenvid, 0, 0, 0, 0, 0, 0);
 }
 
+void
+sys_yield(void)
+{
+	syscall(SYS_yield, 0, 0, 0, 0, 0, 0);
+}
+
+int
+sys_page_alloc(envid_t envid, void *va, int perm)
+{
+	return syscall(SYS_page_alloc, 1, envid, (uint32_t) va, perm, 0, 0);
+}
+
+int
+sys_page_map(envid_t srcenv, void *srcva, envid_t dstenv, void *dstva, int perm)
+{
+	return syscall(SYS_page_map, 1, srcenv, (uint32_t) srcva, dstenv, (uint32_t) dstva, perm);
+}
+
+int
+sys_page_unmap(envid_t envid, void *va)
+{
+	return syscall(SYS_page_unmap, 1, envid, (uint32_t) va, 0, 0, 0);
+}
+
+// sys_exofork is inlined in lib.h
+
+int
+sys_env_set_status(envid_t envid, int status)
+{
+	return syscall(SYS_env_set_status, 1, envid, status, 0, 0, 0);
+}
+
+int
+sys_env_set_pgfault_upcall(envid_t envid, void *upcall)
+{
+	return syscall(SYS_env_set_pgfault_upcall, 1, envid, (uint32_t) upcall, 0, 0, 0);
+}
+
+int
+sys_ipc_try_send(envid_t envid, uint32_t value, void *srcva, int perm)
+{
+	return syscall(SYS_ipc_try_send, 0, envid, value, (uint32_t) srcva, perm, 0);
+}
+
+int
+sys_ipc_recv(void *dstva)
+{
+	return syscall(SYS_ipc_recv, 1, (uint32_t)dstva, 0, 0, 0, 0);
+}
+

user/dumbfork.c

@@ -0,0 +1,80 @@
+// Ping-pong a counter between two processes.
+// Only need to start one of these -- splits into two, crudely.
+
+#include <inc/string.h>
+#include <inc/lib.h>
+
+envid_t dumbfork(void);
+
+void
+umain(int argc, char **argv)
+{
+	envid_t who;
+	int i;
+
+	// fork a child process
+	who = dumbfork();
+
+	// print a message and yield to the other a few times
+	for (i = 0; i < (who ? 10 : 20); i++) {
+		cprintf("%d: I am the %s!\n", i, who ? "parent" : "child");
+		sys_yield();
+	}
+}
+
+void
+duppage(envid_t dstenv, void *addr)
+{
+	int r;
+
+	// This is NOT what you should do in your fork.
+	if ((r = sys_page_alloc(dstenv, addr, PTE_P|PTE_U|PTE_W)) < 0)
+		panic("sys_page_alloc: %e", r);
+	if ((r = sys_page_map(dstenv, addr, 0, UTEMP, PTE_P|PTE_U|PTE_W)) < 0)
+		panic("sys_page_map: %e", r);
+	memmove(UTEMP, addr, PGSIZE);
+	if ((r = sys_page_unmap(0, UTEMP)) < 0)
+		panic("sys_page_unmap: %e", r);
+}
+
+envid_t
+dumbfork(void)
+{
+	envid_t envid;
+	uint8_t *addr;
+	int r;
+	extern unsigned char end[];
+
+	// Allocate a new child environment.
+	// The kernel will initialize it with a copy of our register state,
+	// so that the child will appear to have called sys_exofork() too -
+	// except that in the child, this "fake" call to sys_exofork()
+	// will return 0 instead of the envid of the child.
+	envid = sys_exofork();
+	if (envid < 0)
+		panic("sys_exofork: %e", envid);
+	if (envid == 0) {
+		// We're the child.
+		// The copied value of the global variable 'thisenv'
+		// is no longer valid (it refers to the parent!).
+		// Fix it and return 0.
+		thisenv = &envs[ENVX(sys_getenvid())];
+		return 0;
+	}
+
+	// We're the parent.
+	// Eagerly copy our entire address space into the child.
+	// This is NOT what you should do in your fork implementation.
+	for (addr = (uint8_t*) UTEXT; addr < end; addr += PGSIZE)
+		duppage(envid, addr);
+
+	// Also copy the stack we are currently running on.
+	duppage(envid, ROUNDDOWN(&addr, PGSIZE));
+
+	// Start the child environment running
+	if ((r = sys_env_set_status(envid, ENV_RUNNABLE)) < 0)
+		panic("sys_env_set_status: %e", r);
+
+	return envid;
+}
+

user/fairness.c

@@ -0,0 +1,25 @@
+// Demonstrate lack of fairness in IPC.
+// Start three instances of this program as envs 1, 2, and 3.
+// (user/idle is env 0).
+
+#include <inc/lib.h>
+
+void
+umain(int argc, char **argv)
+{
+	envid_t who, id;
+
+	id = sys_getenvid();
+
+	if (thisenv == &envs[1]) {
+		while (1) {
+			ipc_recv(&who, 0, 0);
+			cprintf("%x recv from %x\n", id, who);
+		}
+	} else {
+		cprintf("%x loop sending to %x\n", id, envs[1].env_id);
+		while (1)
+			ipc_send(envs[1].env_id, 0, 0, 0);
+	}
+}
+

user/faultalloc.c

@@ -0,0 +1,24 @@
+// test user-level fault handler -- alloc pages to fix faults
+
+#include <inc/lib.h>
+
+void
+handler(struct UTrapframe *utf)
+{
+	int r;
+	void *addr = (void*)utf->utf_fault_va;
+
+	cprintf("fault %x\n", addr);
+	if ((r = sys_page_alloc(0, ROUNDDOWN(addr, PGSIZE),
+				PTE_P|PTE_U|PTE_W)) < 0)
+		panic("allocating at %x in page fault handler: %e", addr, r);
+	snprintf((char*) addr, 100, "this string was faulted in at %x", addr);
+}
+
+void
+umain(int argc, char **argv)
+{
+	set_pgfault_handler(handler);
+	cprintf("%s\n", (char*)0xDeadBeef);
+	cprintf("%s\n", (char*)0xCafeBffe);
+}

user/faultallocbad.c

@@ -0,0 +1,24 @@
+// test user-level fault handler -- alloc pages to fix faults
+// doesn't work because we sys_cputs instead of cprintf (exercise: why?)
+
+#include <inc/lib.h>
+
+void
+handler(struct UTrapframe *utf)
+{
+	int r;
+	void *addr = (void*)utf->utf_fault_va;
+
+	cprintf("fault %x\n", addr);
+	if ((r = sys_page_alloc(0, ROUNDDOWN(addr, PGSIZE),
+				PTE_P|PTE_U|PTE_W)) < 0)
+		panic("allocating at %x in page fault handler: %e", addr, r);
+	snprintf((char*) addr, 100, "this string was faulted in at %x", addr);
+}
+
+void
+umain(int argc, char **argv)
+{
+	set_pgfault_handler(handler);
+	sys_cputs((char*)0xDEADBEEF, 4);
+}

user/faultbadhandler.c

@@ -0,0 +1,14 @@
+// test bad pointer for user-level fault handler
+// this is going to fault in the fault handler accessing eip (always!)
+// so eventually the kernel kills it (PFM_KILL) because
+// we outrun the stack with invocations of the user-level handler
+
+#include <inc/lib.h>
+
+void
+umain(int argc, char **argv)
+{
+	sys_page_alloc(0, (void*) (UXSTACKTOP - PGSIZE), PTE_P|PTE_U|PTE_W);
+	sys_env_set_pgfault_upcall(0, (void*) 0xDeadBeef);
+	*(int*)0 = 0;
+}

user/faultdie.c

@@ -0,0 +1,19 @@
+// test user-level fault handler -- just exit when we fault
+
+#include <inc/lib.h>
+
+void
+handler(struct UTrapframe *utf)
+{
+	void *addr = (void*)utf->utf_fault_va;
+	uint32_t err = utf->utf_err;
+	cprintf("i faulted at va %x, err %x\n", addr, err & 7);
+	sys_env_destroy(sys_getenvid());
+}
+
+void
+umain(int argc, char **argv)
+{
+	set_pgfault_handler(handler);
+	*(int*)0xDeadBeef = 0;
+}

user/faultevilhandler.c

@@ -0,0 +1,11 @@
+// test evil pointer for user-level fault handler
+
+#include <inc/lib.h>
+
+void
+umain(int argc, char **argv)
+{
+	sys_page_alloc(0, (void*) (UXSTACKTOP - PGSIZE), PTE_P|PTE_U|PTE_W);
+	sys_env_set_pgfault_upcall(0, (void*) 0xF0100020);
+	*(int*)0 = 0;
+}

user/faultnostack.c

@@ -0,0 +1,12 @@
+// test user fault handler being called with no exception stack mapped
+
+#include <inc/lib.h>
+
+void _pgfault_upcall();
+
+void
+umain(int argc, char **argv)
+{
+	sys_env_set_pgfault_upcall(0, (void*) _pgfault_upcall);
+	*(int*)0 = 0;
+}

user/faultregs.c

@@ -0,0 +1,146 @@
+// test register restore on user-level page fault return
+
+#include <inc/lib.h>
+
+struct regs
+{
+	struct PushRegs regs;
+	uintptr_t eip;
+	uint32_t eflags;
+	uintptr_t esp;
+};
+
+#define SAVE_REGS(base) \
+	"\tmovl %%edi, "base"+0x00\n" \
+	"\tmovl %%esi, "base"+0x04\n" \
+	"\tmovl %%ebp, "base"+0x08\n" \
+	"\tmovl %%ebx, "base"+0x10\n" \
+	"\tmovl %%edx, "base"+0x14\n" \
+	"\tmovl %%ecx, "base"+0x18\n" \
+	"\tmovl %%eax, "base"+0x1c\n" \
+	"\tmovl %%esp, "base"+0x28\n"
+
+#define LOAD_REGS(base) \
+	"\tmovl "base"+0x00, %%edi\n" \
+	"\tmovl "base"+0x04, %%esi\n" \
+	"\tmovl "base"+0x08, %%ebp\n" \
+	"\tmovl "base"+0x10, %%ebx\n" \
+	"\tmovl "base"+0x14, %%edx\n" \
+	"\tmovl "base"+0x18, %%ecx\n" \
+	"\tmovl "base"+0x1c, %%eax\n" \
+	"\tmovl "base"+0x28, %%esp\n"
+
+static struct regs before, during, after;
+
+static void
+check_regs(struct regs* a, const char *an, struct regs* b, const char *bn,
+	   const char *testname)
+{
+	int mismatch = 0;
+
+	cprintf("%-6s %-8s %-8s\n", "", an, bn);
+
+#define CHECK(name, field)						\
+	do {								\
+		cprintf("%-6s %08x %08x ", #name, a->field, b->field);	\
+		if (a->field == b->field)				\
+			cprintf("OK\n");				\
+		else {							\
+			cprintf("MISMATCH\n");				\
+			mismatch = 1;					\
+		}							\
+	} while (0)
+
+	CHECK(edi, regs.reg_edi);
+	CHECK(esi, regs.reg_esi);
+	CHECK(ebp, regs.reg_ebp);
+	CHECK(ebx, regs.reg_ebx);
+	CHECK(edx, regs.reg_edx);
+	CHECK(ecx, regs.reg_ecx);
+	CHECK(eax, regs.reg_eax);
+	CHECK(eip, eip);
+	CHECK(eflags, eflags);
+	CHECK(esp, esp);
+
+#undef CHECK
+
+	cprintf("Registers %s ", testname);
+	if (!mismatch)
+		cprintf("OK\n");
+	else
+		cprintf("MISMATCH\n");
+}
+
+static void
+pgfault(struct UTrapframe *utf)
+{
+	int r;
+
+	if (utf->utf_fault_va != (uint32_t)UTEMP)
+		panic("pgfault expected at UTEMP, got 0x%08x (eip %08x)",
+		      utf->utf_fault_va, utf->utf_eip);
+
+	// Check registers in UTrapframe
+	during.regs = utf->utf_regs;
+	during.eip = utf->utf_eip;
+	during.eflags = utf->utf_eflags & ~FL_RF;
+	during.esp = utf->utf_esp;
+	check_regs(&before, "before", &during, "during", "in UTrapframe");
+
+	// Map UTEMP so the write succeeds
+	if ((r = sys_page_alloc(0, UTEMP, PTE_U|PTE_P|PTE_W)) < 0)
+		panic("sys_page_alloc: %e", r);
+}
+
+void
+umain(int argc, char **argv)
+{
+	set_pgfault_handler(pgfault);
+
+	asm volatile(
+		// Light up eflags to catch more errors
+		"\tpushl %%eax\n"
+		"\tpushfl\n"
+		"\tpopl %%eax\n"
+		"\torl $0x8d5, %%eax\n"
+		"\tpushl %%eax\n"
+		"\tpopfl\n"
+
+		// Save before registers
+		// eflags
+		"\tmov %%eax, %0+0x24\n"
+		// eip
+		"\tleal 0f, %%eax\n"
+		"\tmovl %%eax, %0+0x20\n"
+		"\tpopl %%eax\n"
+		// others
+		SAVE_REGS("%0")
+
+		// Fault at UTEMP
+		"\t0: movl $42, 0x400000\n"
+
+		// Save after registers (except eip and eflags)
+		SAVE_REGS("%1")
+		// Restore registers (except eip and eflags).  This
+		// way, the test will run even if EIP is the *only*
+		// thing restored correctly.
+		LOAD_REGS("%0")
+		// Save after eflags (now that stack is back); note
+		// that we were very careful not to modify eflags in
+		// since we saved it
+		"\tpushl %%eax\n"
+		"\tpushfl\n"
+		"\tpopl %%eax\n"
+		"\tmov %%eax, %1+0x24\n"
+		"\tpopl %%eax\n"
+		: : "m" (before), "m" (after) : "memory", "cc");
+
+	// Check UTEMP to roughly determine that EIP was restored
+	// correctly (of course, we probably wouldn't get this far if
+	// it weren't)
+	if (*(int*)UTEMP != 42)
+		cprintf("EIP after page-fault MISMATCH\n");
+	after.eip = before.eip;
+
+	check_regs(&before, "before", &after, "after", "after page-fault");
+}

user/forktree.c

@@ -0,0 +1,38 @@
+// Fork a binary tree of processes and display their structure.
+
+#include <inc/lib.h>
+
+#define DEPTH 3
+
+void forktree(const char *cur);
+
+void
+forkchild(const char *cur, char branch)
+{
+	char nxt[DEPTH+1];
+
+	if (strlen(cur) >= DEPTH)
+		return;
+
+	snprintf(nxt, DEPTH+1, "%s%c", cur, branch);
+	if (fork() == 0) {
+		forktree(nxt);
+		exit();
+	}
+}
+
+void
+forktree(const char *cur)
+{
+	cprintf("%04x: I am '%s'\n", sys_getenvid(), cur);
+
+	forkchild(cur, '0');
+	forkchild(cur, '1');
+}
+
+void
+umain(int argc, char **argv)
+{
+	forktree("");
+}
+

user/idle.c

@@ -0,0 +1,20 @@
+// idle loop
+
+#include <inc/x86.h>
+#include <inc/lib.h>
+
+void
+umain(int argc, char **argv)
+{
+	binaryname = "idle";
+
+	// Loop forever, simply trying to yield to a different environment.
+	// Instead of busy-waiting like this,
+	// a better way would be to use the processor's HLT instruction
+	// to cause the processor to stop executing until the next interrupt -
+	// doing so allows the processor to conserve power more effectively.
+	while (1) {
+		sys_yield();
+	}
+}
+

user/pingpong.c

@@ -0,0 +1,29 @@
+// Ping-pong a counter between two processes.
+// Only need to start one of these -- splits into two with fork.
+
+#include <inc/lib.h>
+
+void
+umain(int argc, char **argv)
+{
+	envid_t who;
+
+	if ((who = fork()) != 0) {
+		// get the ball rolling
+		cprintf("send 0 from %x to %x\n", sys_getenvid(), who);
+		ipc_send(who, 0, 0, 0);
+	}
+
+	while (1) {
+		uint32_t i = ipc_recv(&who, 0, 0);
+		cprintf("%x got %d from %x\n", sys_getenvid(), i, who);
+		if (i == 10)
+			return;
+		i++;
+		ipc_send(who, i, 0, 0);
+		if (i == 10)
+			return;
+	}
+
+}
+

user/pingpongs.c

@@ -0,0 +1,33 @@
+// Ping-pong a counter between two shared-memory processes.
+// Only need to start one of these -- splits into two with sfork.
+
+#include <inc/lib.h>
+
+uint32_t val;
+
+void
+umain(int argc, char **argv)
+{
+	envid_t who;
+	uint32_t i;
+
+	i = 0;
+	if ((who = sfork()) != 0) {
+		cprintf("i am %08x; thisenv is %p\n", sys_getenvid(), thisenv);
+		// get the ball rolling
+		cprintf("send 0 from %x to %x\n", sys_getenvid(), who);
+		ipc_send(who, 0, 0, 0);
+	}
+
+	while (1) {
+		ipc_recv(&who, 0, 0);
+		cprintf("%x got %d from %x (thisenv is %p %x)\n", sys_getenvid(), val, who, thisenv, thisenv->env_id);
+		if (val == 10)
+			return;
+		++val;
+		ipc_send(who, 0, 0, 0);
+		if (val == 10)
+			return;
+	}
+
+}

user/primes.c

@@ -0,0 +1,53 @@
+// Concurrent version of prime sieve of Eratosthenes.
+// Invented by Doug McIlroy, inventor of Unix pipes.
+// See http://swtch.com/~rsc/thread/.
+// The picture halfway down the page and the text surrounding it
+// explain what's going on here.
+//
+// Since NENV is 1024, we can print 1022 primes before running out.
+// The remaining two environments are the integer generator at the bottom
+// of main and user/idle.
+
+#include <inc/lib.h>
+
+unsigned
+primeproc(void)
+{
+	int i, id, p;
+	envid_t envid;
+
+	// fetch a prime from our left neighbor
+top:
+	p = ipc_recv(&envid, 0, 0);
+	cprintf("CPU %d: %d ", thisenv->env_cpunum, p);
+
+	// fork a right neighbor to continue the chain
+	if ((id = fork()) < 0)
+		panic("fork: %e", id);
+	if (id == 0)
+		goto top;
+
+	// filter out multiples of our prime
+	while (1) {
+		i = ipc_recv(&envid, 0, 0);
+		if (i % p)
+			ipc_send(id, i, 0, 0);
+	}
+}
+
+void
+umain(int argc, char **argv)
+{
+	int i, id;
+
+	// fork the first prime process in the chain
+	if ((id = fork()) < 0)
+		panic("fork: %e", id);
+	if (id == 0)
+		primeproc();
+
+	// feed all the integers through
+	for (i = 2; ; i++)
+		ipc_send(id, i, 0, 0);
+}
+

user/spin.c

@@ -0,0 +1,31 @@
+// Test preemption by forking off a child process that just spins forever.
+// Let it run for a couple time slices, then kill it.
+
+#include <inc/lib.h>
+
+void
+umain(int argc, char **argv)
+{
+	envid_t env;
+
+	cprintf("I am the parent.  Forking the child...\n");
+	if ((env = fork()) == 0) {
+		cprintf("I am the child.  Spinning...\n");
+		while (1)
+			/* do nothing */;
+	}
+
+	cprintf("I am the parent.  Running the child...\n");
+	sys_yield();
+	sys_yield();
+	sys_yield();
+	sys_yield();
+	sys_yield();
+	sys_yield();
+	sys_yield();
+	sys_yield();
+
+	cprintf("I am the parent.  Killing the child...\n");
+	sys_env_destroy(env);
+}
+

user/stresssched.c

@@ -0,0 +1,39 @@
+#include <inc/lib.h>
+
+volatile int counter;
+
+void
+umain(int argc, char **argv)
+{
+	int i, j;
+	int seen;
+	envid_t parent = sys_getenvid();
+
+	// Fork several environments
+	for (i = 0; i < 20; i++)
+		if (fork() == 0)
+			break;
+	if (i == 20) {
+		sys_yield();
+		return;
+	}
+
+	// Wait for the parent to finish forking
+	while (envs[ENVX(parent)].env_status != ENV_FREE)
+		asm volatile("pause");
+
+	// Check that one environment doesn't run on two CPUs at once
+	for (i = 0; i < 10; i++) {
+		sys_yield();
+		for (j = 0; j < 10000; j++)
+			counter++;
+	}
+
+	if (counter != 10*10000)
+		panic("ran on two CPUs at once (counter is %d)", counter);
+
+	// Check that we see environments running on different CPUs
+	cprintf("[%08x] stresssched on CPU %d\n", thisenv->env_id, thisenv->env_cpunum);
+
+}
+

user/yield.c

@@ -0,0 +1,17 @@
+// yield the processor to other environments
+
+#include <inc/lib.h>
+
+void
+umain(int argc, char **argv)
+{
+	int i;
+
+	cprintf("Hello, I am environment %08x.\n", thisenv->env_id);
+	for (i = 0; i < 5; i++) {
+		sys_yield();
+		cprintf("Back in environment %08x, iteration %d.\n",
+			thisenv->env_id, i);
+	}
+	cprintf("All done in environment %08x.\n", thisenv->env_id);
+}