Lab 4

kern/Makefrag

@@ -32,6 +32,12 @@ KERN_SRCFILES :=	kern/entry.S \
 			lib/readline.c \
 			lib/string.c
 
+# Source files for LAB4
+KERN_SRCFILES +=	kern/mpentry.S \
+			kern/mpconfig.c \
+			kern/lapic.c \
+			kern/spinlock.c
+
 # Only build files if they exist.
 KERN_SRCFILES := $(wildcard $(KERN_SRCFILES))
 
@@ -51,6 +57,25 @@ KERN_BINFILES :=	user/hello \
 			user/faultwrite \
 			user/faultwritekernel
 
+# Binary files for LAB4
+KERN_BINFILES +=	user/idle \
+			user/yield \
+			user/dumbfork \
+			user/stresssched \
+			user/faultdie \
+			user/faultregs \
+			user/faultalloc \
+			user/faultallocbad \
+			user/faultnostack \
+			user/faultbadhandler \
+			user/faultevilhandler \
+			user/forktree \
+			user/sendpage \
+			user/spin \
+			user/fairness \
+			user/pingpong \
+			user/pingpongs \
+			user/primes
 KERN_OBJFILES := $(patsubst %.c, $(OBJDIR)/%.o, $(KERN_SRCFILES))
 KERN_OBJFILES := $(patsubst %.S, $(OBJDIR)/%.o, $(KERN_OBJFILES))
 KERN_OBJFILES := $(patsubst $(OBJDIR)/lib/%, $(OBJDIR)/kern/%, $(KERN_OBJFILES))

kern/console.c

@@ -7,6 +7,8 @@
 #include <inc/assert.h>
 
 #include <kern/console.h>
+#include <kern/trap.h>
+#include <kern/picirq.h>
 
 static void cons_intr(int (*proc)(void));
 static void cons_putc(int c);
@@ -373,6 +375,9 @@ kbd_intr(void)
 static void
 kbd_init(void)
 {
+	// Drain the kbd buffer so that QEMU generates interrupts.
+	kbd_intr();
+	irq_setmask_8259A(irq_mask_8259A & ~(1<<IRQ_KBD));
 }
 
 

kern/cpu.h

@@ -0,0 +1,46 @@
+
+#ifndef JOS_INC_CPU_H
+#define JOS_INC_CPU_H
+
+#include <inc/types.h>
+#include <inc/memlayout.h>
+#include <inc/mmu.h>
+#include <inc/env.h>
+
+// Maximum number of CPUs
+#define NCPU  8
+
+// Values of status in struct Cpu
+enum {
+	CPU_UNUSED = 0,
+	CPU_STARTED,
+	CPU_HALTED,
+};
+
+// Per-CPU state
+struct CpuInfo {
+	uint8_t cpu_id;                 // Local APIC ID; index into cpus[] below
+	volatile unsigned cpu_status;   // The status of the CPU
+	struct Env *cpu_env;            // The currently-running environment.
+	struct Taskstate cpu_ts;        // Used by x86 to find stack for interrupt
+};
+
+// Initialized in mpconfig.c
+extern struct CpuInfo cpus[NCPU];
+extern int ncpu;                    // Total number of CPUs in the system
+extern struct CpuInfo *bootcpu;     // The boot-strap processor (BSP)
+extern physaddr_t lapicaddr;        // Physical MMIO address of the local APIC
+
+// Per-CPU kernel stacks
+extern unsigned char percpu_kstacks[NCPU][KSTKSIZE];
+
+int cpunum(void);
+#define thiscpu (&cpus[cpunum()])
+
+void mp_init(void);
+void lapic_init(void);
+void lapic_startap(uint8_t apicid, uint32_t addr);
+void lapic_eoi(void);
+void lapic_ipi(int vector);
+
+#endif

kern/env.c

@@ -11,9 +11,11 @@
 #include <kern/pmap.h>
 #include <kern/trap.h>
 #include <kern/monitor.h>
+#include <kern/sched.h>
+#include <kern/cpu.h>
+#include <kern/spinlock.h>
 
 struct Env *envs = NULL;		// All environments
-struct Env *curenv = NULL;		// The current env
 static struct Env *env_free_list;	// Free environment list
 					// (linked by Env->env_link)
 
@@ -34,7 +36,7 @@ static struct Env *env_free_list;	// Free environment list
 // definition of gdt specifies the Descriptor Privilege Level (DPL)
 // of that descriptor: 0 for kernel and 3 for user.
 //
-struct Segdesc gdt[] =
+struct Segdesc gdt[NCPU + 5] =
 {
 	// 0x0 - unused (always faults -- for trapping NULL far pointers)
 	SEG_NULL,
@@ -51,7 +53,8 @@ struct Segdesc gdt[] =
 	// 0x20 - user data segment
 	[GD_UD >> 3] = SEG(STA_W, 0x0, 0xffffffff, 3),
 
-	// 0x28 - tss, initialized in trap_init_percpu()
+	// Per-CPU TSS descriptors (starting from GD_TSS0) are initialized
+	// in trap_init_percpu()
 	[GD_TSS0 >> 3] = SEG_NULL
 };
 
@@ -242,6 +245,15 @@ env_alloc(struct Env **newenv_store, envid_t parent_id)
 	e->env_tf.tf_cs = GD_UT | 3;
 	// You will set e->env_tf.tf_eip later.
 
+	// Enable interrupts while in user mode.
+	// LAB 4: Your code here.
+
+	// Clear the page fault handler until user installs one.
+	e->env_pgfault_upcall = 0;
+
+	// Also clear the IPC receiving flag.
+	e->env_ipc_recving = 0;
+
 	// commit the allocation
 	env_free_list = e->env_link;
 	*newenv_store = e;
@@ -398,15 +410,26 @@ env_free(struct Env *e)
 
 //
 // Frees environment e.
+// If e was the current env, then runs a new environment (and does not return
+// to the caller).
 //
 void
 env_destroy(struct Env *e)
 {
+	// If e is currently running on other CPUs, we change its state to
+	// ENV_DYING. A zombie environment will be freed the next time
+	// it traps to the kernel.
+	if (e->env_status == ENV_RUNNING && curenv != e) {
+		e->env_status = ENV_DYING;
+		return;
+	}
+
 	env_free(e);
 
-	cprintf("Destroyed the only environment - nothing more to do!\n");
-	while (1)
-		monitor(NULL);
+	if (curenv == e) {
+		curenv = NULL;
+		sched_yield();
+	}
 }
 
 
@@ -419,6 +442,9 @@ env_destroy(struct Env *e)
 void
 env_pop_tf(struct Trapframe *tf)
 {
+	// Record the CPU we are running on for user-space debugging
+	curenv->env_cpunum = cpunum();
+
 	asm volatile(
 		"\tmovl %0,%%esp\n"
 		"\tpopal\n"

kern/env.h

@@ -4,9 +4,10 @@
 #define JOS_KERN_ENV_H
 
 #include <inc/env.h>
+#include <kern/cpu.h>
 
 extern struct Env *envs;		// All environments
-extern struct Env *curenv;		// Current environment
+#define curenv (thiscpu->cpu_env)		// Current environment
 extern struct Segdesc gdt[];
 
 void	env_init(void);

kern/init.c

@@ -10,18 +10,17 @@
 #include <kern/kclock.h>
 #include <kern/env.h>
 #include <kern/trap.h>
+#include <kern/sched.h>
+#include <kern/picirq.h>
+#include <kern/cpu.h>
+#include <kern/spinlock.h>
+
+static void boot_aps(void);
 
 
 void
 i386_init(void)
 {
-	extern char edata[], end[];
-
-	// Before doing anything else, complete the ELF loading process.
-	// Clear the uninitialized global data (BSS) section of our program.
-	// This ensures that all static/global variables start out zero.
-	memset(edata, 0, end - edata);
-
 	// Initialize the console.
 	// Can't call cprintf until after we do this!
 	cons_init();
@@ -35,18 +34,85 @@ i386_init(void)
 	env_init();
 	trap_init();
 
+	// Lab 4 multiprocessor initialization functions
+	mp_init();
+	lapic_init();
+
+	// Lab 4 multitasking initialization functions
+	pic_init();
+
+	// Acquire the big kernel lock before waking up APs
+	// Your code here:
+
+	// Starting non-boot CPUs
+	boot_aps();
+
 #if defined(TEST)
 	// Don't touch -- used by grading script!
 	ENV_CREATE(TEST, ENV_TYPE_USER);
 #else
 	// Touch all you want.
-	ENV_CREATE(user_hello, ENV_TYPE_USER);
+	ENV_CREATE(user_primes, ENV_TYPE_USER);
 #endif // TEST*
 
-	// We only have one user environment for now, so just run it.
-	env_run(&envs[0]);
+	// Schedule and run the first user environment!
+	sched_yield();
 }
 
+// While boot_aps is booting a given CPU, it communicates the per-core
+// stack pointer that should be loaded by mpentry.S to that CPU in
+// this variable.
+void *mpentry_kstack;
+
+// Start the non-boot (AP) processors.
+static void
+boot_aps(void)
+{
+	extern unsigned char mpentry_start[], mpentry_end[];
+	void *code;
+	struct CpuInfo *c;
+
+	// Write entry code to unused memory at MPENTRY_PADDR
+	code = KADDR(MPENTRY_PADDR);
+	memmove(code, mpentry_start, mpentry_end - mpentry_start);
+
+	// Boot each AP one at a time
+	for (c = cpus; c < cpus + ncpu; c++) {
+		if (c == cpus + cpunum())  // We've started already.
+			continue;
+
+		// Tell mpentry.S what stack to use 
+		mpentry_kstack = percpu_kstacks[c - cpus] + KSTKSIZE;
+		// Start the CPU at mpentry_start
+		lapic_startap(c->cpu_id, PADDR(code));
+		// Wait for the CPU to finish some basic setup in mp_main()
+		while(c->cpu_status != CPU_STARTED)
+			;
+	}
+}
+
+// Setup code for APs
+void
+mp_main(void)
+{
+	// We are in high EIP now, safe to switch to kern_pgdir 
+	lcr3(PADDR(kern_pgdir));
+	cprintf("SMP: CPU %d starting\n", cpunum());
+
+	lapic_init();
+	env_init_percpu();
+	trap_init_percpu();
+	xchg(&thiscpu->cpu_status, CPU_STARTED); // tell boot_aps() we're up
+
+	// Now that we have finished some basic setup, call sched_yield()
+	// to start running processes on this CPU.  But make sure that
+	// only one CPU can enter the scheduler at a time!
+	//
+	// Your code here:
+
+	// Remove this after you finish Exercise 6
+	for (;;);
+}
 
 /*
  * Variable panicstr contains argument to first call to panic; used as flag
@@ -71,7 +137,7 @@ _panic(const char *file, int line, const char *fmt,...)
 	asm volatile("cli; cld");
 
 	va_start(ap, fmt);
-	cprintf("kernel panic at %s:%d: ", file, line);
+	cprintf("kernel panic on CPU %d at %s:%d: ", cpunum(), file, line);
 	vcprintf(fmt, ap);
 	cprintf("\n");
 	va_end(ap);

kern/kernel.ld

@@ -44,18 +44,19 @@ SECTIONS
 
 	/* The data segment */
 	.data : {
-		*(.data)
+		*(.data .data.*)
 	}
 
 	.bss : {
 		PROVIDE(edata = .);
-		*(.bss)
+		*(.dynbss)
+		*(.bss .bss.*)
+		*(COMMON)
 		PROVIDE(end = .);
-		BYTE(0)
 	}
 
 
 	/DISCARD/ : {
-		*(.eh_frame .note.GNU-stack)
+		*(.eh_frame .note.GNU-stack .comment .note)
 	}
 }

kern/lapic.c

@@ -0,0 +1,182 @@
+// The local APIC manages internal (non-I/O) interrupts.
+// See Chapter 8 & Appendix C of Intel processor manual volume 3.
+
+#include <inc/types.h>
+#include <inc/memlayout.h>
+#include <inc/trap.h>
+#include <inc/mmu.h>
+#include <inc/stdio.h>
+#include <inc/x86.h>
+#include <kern/pmap.h>
+#include <kern/cpu.h>
+
+// Local APIC registers, divided by 4 for use as uint32_t[] indices.
+#define ID      (0x0020/4)   // ID
+#define VER     (0x0030/4)   // Version
+#define TPR     (0x0080/4)   // Task Priority
+#define EOI     (0x00B0/4)   // EOI
+#define SVR     (0x00F0/4)   // Spurious Interrupt Vector
+	#define ENABLE     0x00000100   // Unit Enable
+#define ESR     (0x0280/4)   // Error Status
+#define ICRLO   (0x0300/4)   // Interrupt Command
+	#define INIT       0x00000500   // INIT/RESET
+	#define STARTUP    0x00000600   // Startup IPI
+	#define DELIVS     0x00001000   // Delivery status
+	#define ASSERT     0x00004000   // Assert interrupt (vs deassert)
+	#define DEASSERT   0x00000000
+	#define LEVEL      0x00008000   // Level triggered
+	#define BCAST      0x00080000   // Send to all APICs, including self.
+	#define OTHERS     0x000C0000   // Send to all APICs, excluding self.
+	#define BUSY       0x00001000
+	#define FIXED      0x00000000
+#define ICRHI   (0x0310/4)   // Interrupt Command [63:32]
+#define TIMER   (0x0320/4)   // Local Vector Table 0 (TIMER)
+	#define X1         0x0000000B   // divide counts by 1
+	#define PERIODIC   0x00020000   // Periodic
+#define PCINT   (0x0340/4)   // Performance Counter LVT
+#define LINT0   (0x0350/4)   // Local Vector Table 1 (LINT0)
+#define LINT1   (0x0360/4)   // Local Vector Table 2 (LINT1)
+#define ERROR   (0x0370/4)   // Local Vector Table 3 (ERROR)
+	#define MASKED     0x00010000   // Interrupt masked
+#define TICR    (0x0380/4)   // Timer Initial Count
+#define TCCR    (0x0390/4)   // Timer Current Count
+#define TDCR    (0x03E0/4)   // Timer Divide Configuration
+
+physaddr_t lapicaddr;        // Initialized in mpconfig.c
+volatile uint32_t *lapic;
+
+static void
+lapicw(int index, int value)
+{
+	lapic[index] = value;
+	lapic[ID];  // wait for write to finish, by reading
+}
+
+void
+lapic_init(void)
+{
+	if (!lapicaddr)
+		return;
+
+	// lapicaddr is the physical address of the LAPIC's 4K MMIO
+	// region.  Map it in to virtual memory so we can access it.
+	lapic = mmio_map_region(lapicaddr, 4096);
+
+	// Enable local APIC; set spurious interrupt vector.
+	lapicw(SVR, ENABLE | (IRQ_OFFSET + IRQ_SPURIOUS));
+
+	// The timer repeatedly counts down at bus frequency
+	// from lapic[TICR] and then issues an interrupt.  
+	// If we cared more about precise timekeeping,
+	// TICR would be calibrated using an external time source.
+	lapicw(TDCR, X1);
+	lapicw(TIMER, PERIODIC | (IRQ_OFFSET + IRQ_TIMER));
+	lapicw(TICR, 10000000); 
+
+	// Leave LINT0 of the BSP enabled so that it can get
+	// interrupts from the 8259A chip.
+	//
+	// According to Intel MP Specification, the BIOS should initialize
+	// BSP's local APIC in Virtual Wire Mode, in which 8259A's
+	// INTR is virtually connected to BSP's LINTIN0. In this mode,
+	// we do not need to program the IOAPIC.
+	if (thiscpu != bootcpu)
+		lapicw(LINT0, MASKED);
+
+	// Disable NMI (LINT1) on all CPUs
+	lapicw(LINT1, MASKED);
+
+	// Disable performance counter overflow interrupts
+	// on machines that provide that interrupt entry.
+	if (((lapic[VER]>>16) & 0xFF) >= 4)
+		lapicw(PCINT, MASKED);
+
+	// Map error interrupt to IRQ_ERROR.
+	lapicw(ERROR, IRQ_OFFSET + IRQ_ERROR);
+
+	// Clear error status register (requires back-to-back writes).
+	lapicw(ESR, 0);
+	lapicw(ESR, 0);
+
+	// Ack any outstanding interrupts.
+	lapicw(EOI, 0);
+
+	// Send an Init Level De-Assert to synchronize arbitration ID's.
+	lapicw(ICRHI, 0);
+	lapicw(ICRLO, BCAST | INIT | LEVEL);
+	while(lapic[ICRLO] & DELIVS)
+		;
+
+	// Enable interrupts on the APIC (but not on the processor).
+	lapicw(TPR, 0);
+}
+
+int
+cpunum(void)
+{
+	if (lapic)
+		return lapic[ID] >> 24;
+	return 0;
+}
+
+// Acknowledge interrupt.
+void
+lapic_eoi(void)
+{
+	if (lapic)
+		lapicw(EOI, 0);
+}
+
+// Spin for a given number of microseconds.
+// On real hardware would want to tune this dynamically.
+static void
+microdelay(int us)
+{
+}
+
+#define IO_RTC  0x70
+
+// Start additional processor running entry code at addr.
+// See Appendix B of MultiProcessor Specification.
+void
+lapic_startap(uint8_t apicid, uint32_t addr)
+{
+	int i;
+	uint16_t *wrv;
+
+	// "The BSP must initialize CMOS shutdown code to 0AH
+	// and the warm reset vector (DWORD based at 40:67) to point at
+	// the AP startup code prior to the [universal startup algorithm]."
+	outb(IO_RTC, 0xF);  // offset 0xF is shutdown code
+	outb(IO_RTC+1, 0x0A);
+	wrv = (uint16_t *)KADDR((0x40 << 4 | 0x67));  // Warm reset vector
+	wrv[0] = 0;
+	wrv[1] = addr >> 4;
+
+	// "Universal startup algorithm."
+	// Send INIT (level-triggered) interrupt to reset other CPU.
+	lapicw(ICRHI, apicid << 24);
+	lapicw(ICRLO, INIT | LEVEL | ASSERT);
+	microdelay(200);
+	lapicw(ICRLO, INIT | LEVEL);
+	microdelay(100);    // should be 10ms, but too slow in Bochs!
+
+	// Send startup IPI (twice!) to enter code.
+	// Regular hardware is supposed to only accept a STARTUP
+	// when it is in the halted state due to an INIT.  So the second
+	// should be ignored, but it is part of the official Intel algorithm.
+	// Bochs complains about the second one.  Too bad for Bochs.
+	for (i = 0; i < 2; i++) {
+		lapicw(ICRHI, apicid << 24);
+		lapicw(ICRLO, STARTUP | (addr >> 12));
+		microdelay(200);
+	}
+}
+
+void
+lapic_ipi(int vector)
+{
+	lapicw(ICRLO, OTHERS | FIXED | vector);
+	while (lapic[ICRLO] & DELIVS)
+		;
+}

kern/mpconfig.c

@@ -0,0 +1,225 @@
+// Search for and parse the multiprocessor configuration table
+// See http://developer.intel.com/design/pentium/datashts/24201606.pdf
+
+#include <inc/types.h>
+#include <inc/string.h>
+#include <inc/memlayout.h>
+#include <inc/x86.h>
+#include <inc/mmu.h>
+#include <inc/env.h>
+#include <kern/cpu.h>
+#include <kern/pmap.h>
+
+struct CpuInfo cpus[NCPU];
+struct CpuInfo *bootcpu;
+int ismp;
+int ncpu;
+
+// Per-CPU kernel stacks
+unsigned char percpu_kstacks[NCPU][KSTKSIZE]
+__attribute__ ((aligned(PGSIZE)));
+
+
+// See MultiProcessor Specification Version 1.[14]
+
+struct mp {             // floating pointer [MP 4.1]
+	uint8_t signature[4];           // "_MP_"
+	physaddr_t physaddr;            // phys addr of MP config table
+	uint8_t length;                 // 1
+	uint8_t specrev;                // [14]
+	uint8_t checksum;               // all bytes must add up to 0
+	uint8_t type;                   // MP system config type
+	uint8_t imcrp;
+	uint8_t reserved[3];
+} __attribute__((__packed__));
+
+struct mpconf {         // configuration table header [MP 4.2]
+	uint8_t signature[4];           // "PCMP"
+	uint16_t length;                // total table length
+	uint8_t version;                // [14]
+	uint8_t checksum;               // all bytes must add up to 0
+	uint8_t product[20];            // product id
+	physaddr_t oemtable;            // OEM table pointer
+	uint16_t oemlength;             // OEM table length
+	uint16_t entry;                 // entry count
+	physaddr_t lapicaddr;           // address of local APIC
+	uint16_t xlength;               // extended table length
+	uint8_t xchecksum;              // extended table checksum
+	uint8_t reserved;
+	uint8_t entries[0];             // table entries
+} __attribute__((__packed__));
+
+struct mpproc {         // processor table entry [MP 4.3.1]
+	uint8_t type;                   // entry type (0)
+	uint8_t apicid;                 // local APIC id
+	uint8_t version;                // local APIC version
+	uint8_t flags;                  // CPU flags
+	uint8_t signature[4];           // CPU signature
+	uint32_t feature;               // feature flags from CPUID instruction
+	uint8_t reserved[8];
+} __attribute__((__packed__));
+
+// mpproc flags
+#define MPPROC_BOOT 0x02                // This mpproc is the bootstrap processor
+
+// Table entry types
+#define MPPROC    0x00  // One per processor
+#define MPBUS     0x01  // One per bus
+#define MPIOAPIC  0x02  // One per I/O APIC
+#define MPIOINTR  0x03  // One per bus interrupt source
+#define MPLINTR   0x04  // One per system interrupt source
+
+static uint8_t
+sum(void *addr, int len)
+{
+	int i, sum;
+
+	sum = 0;
+	for (i = 0; i < len; i++)
+		sum += ((uint8_t *)addr)[i];
+	return sum;
+}
+
+// Look for an MP structure in the len bytes at physical address addr.
+static struct mp *
+mpsearch1(physaddr_t a, int len)
+{
+	struct mp *mp = KADDR(a), *end = KADDR(a + len);
+
+	for (; mp < end; mp++)
+		if (memcmp(mp->signature, "_MP_", 4) == 0 &&
+		    sum(mp, sizeof(*mp)) == 0)
+			return mp;
+	return NULL;
+}
+
+// Search for the MP Floating Pointer Structure, which according to
+// [MP 4] is in one of the following three locations:
+// 1) in the first KB of the EBDA;
+// 2) if there is no EBDA, in the last KB of system base memory;
+// 3) in the BIOS ROM between 0xE0000 and 0xFFFFF.
+static struct mp *
+mpsearch(void)
+{
+	uint8_t *bda;
+	uint32_t p;
+	struct mp *mp;
+
+	static_assert(sizeof(*mp) == 16);
+
+	// The BIOS data area lives in 16-bit segment 0x40.
+	bda = (uint8_t *) KADDR(0x40 << 4);
+
+	// [MP 4] The 16-bit segment of the EBDA is in the two bytes
+	// starting at byte 0x0E of the BDA.  0 if not present.
+	if ((p = *(uint16_t *) (bda + 0x0E))) {
+		p <<= 4;	// Translate from segment to PA
+		if ((mp = mpsearch1(p, 1024)))
+			return mp;
+	} else {
+		// The size of base memory, in KB is in the two bytes
+		// starting at 0x13 of the BDA.
+		p = *(uint16_t *) (bda + 0x13) * 1024;
+		if ((mp = mpsearch1(p - 1024, 1024)))
+			return mp;
+	}
+	return mpsearch1(0xF0000, 0x10000);
+}
+
+// Search for an MP configuration table.  For now, don't accept the
+// default configurations (physaddr == 0).
+// Check for the correct signature, checksum, and version.
+static struct mpconf *
+mpconfig(struct mp **pmp)
+{
+	struct mpconf *conf;
+	struct mp *mp;
+
+	if ((mp = mpsearch()) == 0)
+		return NULL;
+	if (mp->physaddr == 0 || mp->type != 0) {
+		cprintf("SMP: Default configurations not implemented\n");
+		return NULL;
+	}
+	conf = (struct mpconf *) KADDR(mp->physaddr);
+	if (memcmp(conf, "PCMP", 4) != 0) {
+		cprintf("SMP: Incorrect MP configuration table signature\n");
+		return NULL;
+	}
+	if (sum(conf, conf->length) != 0) {
+		cprintf("SMP: Bad MP configuration checksum\n");
+		return NULL;
+	}
+	if (conf->version != 1 && conf->version != 4) {
+		cprintf("SMP: Unsupported MP version %d\n", conf->version);
+		return NULL;
+	}
+	if ((sum((uint8_t *)conf + conf->length, conf->xlength) + conf->xchecksum) & 0xff) {
+		cprintf("SMP: Bad MP configuration extended checksum\n");
+		return NULL;
+	}
+	*pmp = mp;
+	return conf;
+}
+
+void
+mp_init(void)
+{
+	struct mp *mp;
+	struct mpconf *conf;
+	struct mpproc *proc;
+	uint8_t *p;
+	unsigned int i;
+
+	bootcpu = &cpus[0];
+	if ((conf = mpconfig(&mp)) == 0)
+		return;
+	ismp = 1;
+	lapicaddr = conf->lapicaddr;
+
+	for (p = conf->entries, i = 0; i < conf->entry; i++) {
+		switch (*p) {
+		case MPPROC:
+			proc = (struct mpproc *)p;
+			if (proc->flags & MPPROC_BOOT)
+				bootcpu = &cpus[ncpu];
+			if (ncpu < NCPU) {
+				cpus[ncpu].cpu_id = ncpu;
+				ncpu++;
+			} else {
+				cprintf("SMP: too many CPUs, CPU %d disabled\n",
+					proc->apicid);
+			}
+			p += sizeof(struct mpproc);
+			continue;
+		case MPBUS:
+		case MPIOAPIC:
+		case MPIOINTR:
+		case MPLINTR:
+			p += 8;
+			continue;
+		default:
+			cprintf("mpinit: unknown config type %x\n", *p);
+			ismp = 0;
+			i = conf->entry;
+		}
+	}
+
+	bootcpu->cpu_status = CPU_STARTED;
+	if (!ismp) {
+		// Didn't like what we found; fall back to no MP.
+		ncpu = 1;
+		lapicaddr = 0;
+		cprintf("SMP: configuration not found, SMP disabled\n");
+		return;
+	}
+	cprintf("SMP: CPU %d found %d CPU(s)\n", bootcpu->cpu_id,  ncpu);
+
+	if (mp->imcrp) {
+		// [MP 3.2.6.1] If the hardware implements PIC mode,
+		// switch to getting interrupts from the LAPIC.
+		cprintf("SMP: Setting IMCR to switch from PIC mode to symmetric I/O mode\n");
+		outb(0x22, 0x70);   // Select IMCR
+		outb(0x23, inb(0x23) | 1);  // Mask external interrupts.
+	}
+}

kern/mpentry.S

@@ -0,0 +1,97 @@
+/* See COPYRIGHT for copyright information. */
+
+#include <inc/mmu.h>
+#include <inc/memlayout.h>
+
+###################################################################
+# entry point for APs
+###################################################################
+
+# Each non-boot CPU ("AP") is started up in response to a STARTUP
+# IPI from the boot CPU.  Section B.4.2 of the Multi-Processor
+# Specification says that the AP will start in real mode with CS:IP
+# set to XY00:0000, where XY is an 8-bit value sent with the
+# STARTUP. Thus this code must start at a 4096-byte boundary.
+#
+# Because this code sets DS to zero, it must run from an address in
+# the low 2^16 bytes of physical memory.
+#
+# boot_aps() (in init.c) copies this code to MPENTRY_PADDR (which
+# satisfies the above restrictions).  Then, for each AP, it stores the
+# address of the pre-allocated per-core stack in mpentry_kstack, sends
+# the STARTUP IPI, and waits for this code to acknowledge that it has
+# started (which happens in mp_main in init.c).
+#
+# This code is similar to boot/boot.S except that
+#    - it does not need to enable A20
+#    - it uses MPBOOTPHYS to calculate absolute addresses of its
+#      symbols, rather than relying on the linker to fill them
+
+#define RELOC(x) ((x) - KERNBASE)
+#define MPBOOTPHYS(s) ((s) - mpentry_start + MPENTRY_PADDR)
+
+.set PROT_MODE_CSEG, 0x8	# kernel code segment selector
+.set PROT_MODE_DSEG, 0x10	# kernel data segment selector
+
+.code16           
+.globl mpentry_start
+mpentry_start:
+	cli            
+
+	xorw    %ax, %ax
+	movw    %ax, %ds
+	movw    %ax, %es
+	movw    %ax, %ss
+
+	lgdt    MPBOOTPHYS(gdtdesc)
+	movl    %cr0, %eax
+	orl     $CR0_PE, %eax
+	movl    %eax, %cr0
+
+	ljmpl   $(PROT_MODE_CSEG), $(MPBOOTPHYS(start32))
+
+.code32
+start32:
+	movw    $(PROT_MODE_DSEG), %ax
+	movw    %ax, %ds
+	movw    %ax, %es
+	movw    %ax, %ss
+	movw    $0, %ax
+	movw    %ax, %fs
+	movw    %ax, %gs
+
+	# Set up initial page table. We cannot use kern_pgdir yet because
+	# we are still running at a low EIP.
+	movl    $(RELOC(entry_pgdir)), %eax
+	movl    %eax, %cr3
+	# Turn on paging.
+	movl    %cr0, %eax
+	orl     $(CR0_PE|CR0_PG|CR0_WP), %eax
+	movl    %eax, %cr0
+
+	# Switch to the per-cpu stack allocated in boot_aps()
+	movl    mpentry_kstack, %esp
+	movl    $0x0, %ebp       # nuke frame pointer
+
+	# Call mp_main().  (Exercise for the reader: why the indirect call?)
+	movl    $mp_main, %eax
+	call    *%eax
+
+	# If mp_main returns (it shouldn't), loop.
+spin:
+	jmp     spin
+
+# Bootstrap GDT
+.p2align 2					# force 4 byte alignment
+gdt:
+	SEG_NULL				# null seg
+	SEG(STA_X|STA_R, 0x0, 0xffffffff)	# code seg
+	SEG(STA_W, 0x0, 0xffffffff)		# data seg
+
+gdtdesc:
+	.word   0x17				# sizeof(gdt) - 1
+	.long   MPBOOTPHYS(gdt)			# address gdt
+
+.globl mpentry_end
+mpentry_end:
+	nop

kern/picirq.c

@@ -0,0 +1,86 @@
+/* See COPYRIGHT for copyright information. */
+
+#include <inc/assert.h>
+#include <inc/trap.h>
+
+#include <kern/picirq.h>
+
+
+// Current IRQ mask.
+// Initial IRQ mask has interrupt 2 enabled (for slave 8259A).
+uint16_t irq_mask_8259A = 0xFFFF & ~(1<<IRQ_SLAVE);
+static bool didinit;
+
+/* Initialize the 8259A interrupt controllers. */
+void
+pic_init(void)
+{
+	didinit = 1;
+
+	// mask all interrupts
+	outb(IO_PIC1+1, 0xFF);
+	outb(IO_PIC2+1, 0xFF);
+
+	// Set up master (8259A-1)
+
+	// ICW1:  0001g0hi
+	//    g:  0 = edge triggering, 1 = level triggering
+	//    h:  0 = cascaded PICs, 1 = master only
+	//    i:  0 = no ICW4, 1 = ICW4 required
+	outb(IO_PIC1, 0x11);
+
+	// ICW2:  Vector offset
+	outb(IO_PIC1+1, IRQ_OFFSET);
+
+	// ICW3:  bit mask of IR lines connected to slave PICs (master PIC),
+	//        3-bit No of IR line at which slave connects to master(slave PIC).
+	outb(IO_PIC1+1, 1<<IRQ_SLAVE);
+
+	// ICW4:  000nbmap
+	//    n:  1 = special fully nested mode
+	//    b:  1 = buffered mode
+	//    m:  0 = slave PIC, 1 = master PIC
+	//	  (ignored when b is 0, as the master/slave role
+	//	  can be hardwired).
+	//    a:  1 = Automatic EOI mode
+	//    p:  0 = MCS-80/85 mode, 1 = intel x86 mode
+	outb(IO_PIC1+1, 0x3);
+
+	// Set up slave (8259A-2)
+	outb(IO_PIC2, 0x11);			// ICW1
+	outb(IO_PIC2+1, IRQ_OFFSET + 8);	// ICW2
+	outb(IO_PIC2+1, IRQ_SLAVE);		// ICW3
+	// NB Automatic EOI mode doesn't tend to work on the slave.
+	// Linux source code says it's "to be investigated".
+	outb(IO_PIC2+1, 0x01);			// ICW4
+
+	// OCW3:  0ef01prs
+	//   ef:  0x = NOP, 10 = clear specific mask, 11 = set specific mask
+	//    p:  0 = no polling, 1 = polling mode
+	//   rs:  0x = NOP, 10 = read IRR, 11 = read ISR
+	outb(IO_PIC1, 0x68);             /* clear specific mask */
+	outb(IO_PIC1, 0x0a);             /* read IRR by default */
+
+	outb(IO_PIC2, 0x68);               /* OCW3 */
+	outb(IO_PIC2, 0x0a);               /* OCW3 */
+
+	if (irq_mask_8259A != 0xFFFF)
+		irq_setmask_8259A(irq_mask_8259A);
+}
+
+void
+irq_setmask_8259A(uint16_t mask)
+{
+	int i;
+	irq_mask_8259A = mask;
+	if (!didinit)
+		return;
+	outb(IO_PIC1+1, (char)mask);
+	outb(IO_PIC2+1, (char)(mask >> 8));
+	cprintf("enabled interrupts:");
+	for (i = 0; i < 16; i++)
+		if (~mask & (1<<i))
+			cprintf(" %d", i);
+	cprintf("\n");
+}
+

kern/picirq.h

@@ -0,0 +1,28 @@
+/* See COPYRIGHT for copyright information. */
+
+#ifndef JOS_KERN_PICIRQ_H
+#define JOS_KERN_PICIRQ_H
+#ifndef JOS_KERNEL
+# error "This is a JOS kernel header; user programs should not #include it"
+#endif
+
+#define MAX_IRQS	16	// Number of IRQs
+
+// I/O Addresses of the two 8259A programmable interrupt controllers
+#define IO_PIC1		0x20	// Master (IRQs 0-7)
+#define IO_PIC2		0xA0	// Slave (IRQs 8-15)
+
+#define IRQ_SLAVE	2	// IRQ at which slave connects to master
+
+
+#ifndef __ASSEMBLER__
+
+#include <inc/types.h>
+#include <inc/x86.h>
+
+extern uint16_t irq_mask_8259A;
+void pic_init(void);
+void irq_setmask_8259A(uint16_t mask);
+#endif // !__ASSEMBLER__
+
+#endif // !JOS_KERN_PICIRQ_H

kern/pmap.c

@@ -9,6 +9,7 @@
 #include <kern/pmap.h>
 #include <kern/kclock.h>
 #include <kern/env.h>
+#include <kern/cpu.h>
 
 // These variables are set by i386_detect_memory()
 size_t npages;			// Amount of physical memory (in pages)
@@ -62,6 +63,7 @@ i386_detect_memory(void)
 // Set up memory mappings above UTOP.
 // --------------------------------------------------------------
 
+static void mem_init_mp(void);
 static void boot_map_region(pde_t *pgdir, uintptr_t va, size_t size, physaddr_t pa, int perm);
 static void check_page_free_list(bool only_low_memory);
 static void check_page_alloc(void);
@@ -207,6 +209,9 @@ mem_init(void)
 	// Permissions: kernel RW, user NONE
 	// Your code goes here:
 
+	// Initialize the SMP-related parts of the memory map
+	mem_init_mp();
+
 	// Check that the initial page directory has been set up correctly.
 	check_kern_pgdir();
 
@@ -232,6 +237,31 @@ mem_init(void)
 	check_page_installed_pgdir();
 }
 
+// Modify mappings in kern_pgdir to support SMP
+//   - Map the per-CPU stacks in the region [KSTACKTOP-PTSIZE, KSTACKTOP)
+//
+static void
+mem_init_mp(void)
+{
+	// Map per-CPU stacks starting at KSTACKTOP, for up to 'NCPU' CPUs.
+	//
+	// For CPU i, use the physical memory that 'percpu_kstacks[i]' refers
+	// to as its kernel stack. CPU i's kernel stack grows down from virtual
+	// address kstacktop_i = KSTACKTOP - i * (KSTKSIZE + KSTKGAP), and is
+	// divided into two pieces, just like the single stack you set up in
+	// mem_init:
+	//     * [kstacktop_i - KSTKSIZE, kstacktop_i)
+	//          -- backed by physical memory
+	//     * [kstacktop_i - (KSTKSIZE + KSTKGAP), kstacktop_i - KSTKSIZE)
+	//          -- not backed; so if the kernel overflows its stack,
+	//             it will fault rather than overwrite another CPU's stack.
+	//             Known as a "guard page".
+	//     Permissions: kernel RW, user NONE
+	//
+	// LAB 4: Your code here:
+
+}
+
 // --------------------------------------------------------------
 // Tracking of physical pages.
 // The 'pages' array has one 'struct PageInfo' entry per physical page.
@@ -247,6 +277,10 @@ mem_init(void)
 void
 page_init(void)
 {
+	// LAB 4:
+	// Change your code to mark the physical page at MPENTRY_PADDR
+	// as in use
+
 	// The example code here marks all physical pages as free.
 	// However this is not truly the case.  What memory is free?
 	//  1) Mark physical page 0 as in use.
@@ -439,8 +473,43 @@ void
 tlb_invalidate(pde_t *pgdir, void *va)
 {
 	// Flush the entry only if we're modifying the current address space.
-	// For now, there is only one address space, so always invalidate.
-	invlpg(va);
+	if (!curenv || curenv->env_pgdir == pgdir)
+		invlpg(va);
+}
+
+//
+// Reserve size bytes in the MMIO region and map [pa,pa+size) at this
+// location.  Return the base of the reserved region.  size does *not*
+// have to be multiple of PGSIZE.
+//
+void *
+mmio_map_region(physaddr_t pa, size_t size)
+{
+	// Where to start the next region.  Initially, this is the
+	// beginning of the MMIO region.  Because this is static, its
+	// value will be preserved between calls to mmio_map_region
+	// (just like nextfree in boot_alloc).
+	static uintptr_t base = MMIOBASE;
+
+	// Reserve size bytes of virtual memory starting at base and
+	// map physical pages [pa,pa+size) to virtual addresses
+	// [base,base+size).  Since this is device memory and not
+	// regular DRAM, you'll have to tell the CPU that it isn't
+	// safe to cache access to this memory.  Luckily, the page
+	// tables provide bits for this purpose; simply create the
+	// mapping with PTE_PCD|PTE_PWT (cache-disable and
+	// write-through) in addition to PTE_W.  (If you're interested
+	// in more details on this, see section 10.5 of IA32 volume
+	// 3A.)
+	//
+	// Be sure to round size up to a multiple of PGSIZE and to
+	// handle if this reservation would overflow MMIOLIM (it's
+	// okay to simply panic if this happens).
+	//
+	// Hint: The staff solution uses boot_map_region.
+	//
+	// Your code here:
+	panic("mmio_map_region not implemented");
 }
 
 static uintptr_t user_mem_check_addr;
@@ -541,6 +610,8 @@ check_page_free_list(bool only_low_memory)
 		assert(page2pa(pp) != EXTPHYSMEM - PGSIZE);
 		assert(page2pa(pp) != EXTPHYSMEM);
 		assert(page2pa(pp) < EXTPHYSMEM || (char *) page2kva(pp) >= first_free_page);
+		// (new test for lab 4)
+		assert(page2pa(pp) != MPENTRY_PADDR);
 
 		if (page2pa(pp) < EXTPHYSMEM)
 			++nfree_basemem;
@@ -663,9 +734,15 @@ check_kern_pgdir(void)
 		assert(check_va2pa(pgdir, KERNBASE + i) == i);
 
 	// check kernel stack
-	for (i = 0; i < KSTKSIZE; i += PGSIZE)
-		assert(check_va2pa(pgdir, KSTACKTOP - KSTKSIZE + i) == PADDR(bootstack) + i);
-	assert(check_va2pa(pgdir, KSTACKTOP - PTSIZE) == ~0);
+	// (updated in lab 4 to check per-CPU kernel stacks)
+	for (n = 0; n < NCPU; n++) {
+		uint32_t base = KSTACKTOP - (KSTKSIZE + KSTKGAP) * (n + 1);
+		for (i = 0; i < KSTKSIZE; i += PGSIZE)
+			assert(check_va2pa(pgdir, base + KSTKGAP + i)
+				== PADDR(percpu_kstacks[n]) + i);
+		for (i = 0; i < KSTKGAP; i += PGSIZE)
+			assert(check_va2pa(pgdir, base + i) == ~0);
+	}
 
 	// check PDE permissions
 	for (i = 0; i < NPDENTRIES; i++) {
@@ -674,6 +751,7 @@ check_kern_pgdir(void)
 		case PDX(KSTACKTOP-1):
 		case PDX(UPAGES):
 		case PDX(UENVS):
+		case PDX(MMIOBASE):
 			assert(pgdir[i] & PTE_P);
 			break;
 		default:
@@ -716,6 +794,7 @@ check_page(void)
 	struct PageInfo *fl;
 	pte_t *ptep, *ptep1;
 	void *va;
+	uintptr_t mm1, mm2;
 	int i;
 	extern pde_t entry_pgdir[];
 
@@ -858,6 +937,29 @@ check_page(void)
 	page_free(pp1);
 	page_free(pp2);
 
+	// test mmio_map_region
+	mm1 = (uintptr_t) mmio_map_region(0, 4097);
+	mm2 = (uintptr_t) mmio_map_region(0, 4096);
+	// check that they're in the right region
+	assert(mm1 >= MMIOBASE && mm1 + 8192 < MMIOLIM);
+	assert(mm2 >= MMIOBASE && mm2 + 8192 < MMIOLIM);
+	// check that they're page-aligned
+	assert(mm1 % PGSIZE == 0 && mm2 % PGSIZE == 0);
+	// check that they don't overlap
+	assert(mm1 + 8192 <= mm2);
+	// check page mappings
+	assert(check_va2pa(kern_pgdir, mm1) == 0);
+	assert(check_va2pa(kern_pgdir, mm1+PGSIZE) == PGSIZE);
+	assert(check_va2pa(kern_pgdir, mm2) == 0);
+	assert(check_va2pa(kern_pgdir, mm2+PGSIZE) == ~0);
+	// check permissions
+	assert(*pgdir_walk(kern_pgdir, (void*) mm1, 0) & (PTE_W|PTE_PWT|PTE_PCD));
+	assert(!(*pgdir_walk(kern_pgdir, (void*) mm1, 0) & PTE_U));
+	// clear the mappings
+	*pgdir_walk(kern_pgdir, (void*) mm1, 0) = 0;
+	*pgdir_walk(kern_pgdir, (void*) mm1 + PGSIZE, 0) = 0;
+	*pgdir_walk(kern_pgdir, (void*) mm2, 0) = 0;
+
 	cprintf("check_page() succeeded!\n");
 }
 

kern/pmap.h

@@ -63,6 +63,8 @@ void	page_decref(struct PageInfo *pp);
 
 void	tlb_invalidate(pde_t *pgdir, void *va);
 
+void *	mmio_map_region(physaddr_t pa, size_t size);
+
 int	user_mem_check(struct Env *env, const void *va, size_t len, int perm);
 void	user_mem_assert(struct Env *env, const void *va, size_t len, int perm);
 

kern/sched.c

@@ -0,0 +1,84 @@
+#include <inc/assert.h>
+#include <inc/x86.h>
+#include <kern/spinlock.h>
+#include <kern/env.h>
+#include <kern/pmap.h>
+#include <kern/monitor.h>
+
+void sched_halt(void);
+
+// Choose a user environment to run and run it.
+void
+sched_yield(void)
+{
+	struct Env *idle;
+
+	// Implement simple round-robin scheduling.
+	//
+	// Search through 'envs' for an ENV_RUNNABLE environment in
+	// circular fashion starting just after the env this CPU was
+	// last running.  Switch to the first such environment found.
+	//
+	// If no envs are runnable, but the environment previously
+	// running on this CPU is still ENV_RUNNING, it's okay to
+	// choose that environment.
+	//
+	// Never choose an environment that's currently running on
+	// another CPU (env_status == ENV_RUNNING). If there are
+	// no runnable environments, simply drop through to the code
+	// below to halt the cpu.
+
+	// LAB 4: Your code here.
+
+	// sched_halt never returns
+	sched_halt();
+}
+
+// Halt this CPU when there is nothing to do. Wait until the
+// timer interrupt wakes it up. This function never returns.
+//
+void
+sched_halt(void)
+{
+	int i;
+
+	// For debugging and testing purposes, if there are no runnable
+	// environments in the system, then drop into the kernel monitor.
+	for (i = 0; i < NENV; i++) {
+		if ((envs[i].env_status == ENV_RUNNABLE ||
+		     envs[i].env_status == ENV_RUNNING ||
+		     envs[i].env_status == ENV_DYING))
+			break;
+	}
+	if (i == NENV) {
+		cprintf("No runnable environments in the system!\n");
+		while (1)
+			monitor(NULL);
+	}
+
+	// Mark that no environment is running on this CPU
+	curenv = NULL;
+	lcr3(PADDR(kern_pgdir));
+
+	// Mark that this CPU is in the HALT state, so that when
+	// timer interupts come in, we know we should re-acquire the
+	// big kernel lock
+	xchg(&thiscpu->cpu_status, CPU_HALTED);
+
+	// Release the big kernel lock as if we were "leaving" the kernel
+	unlock_kernel();
+
+	// Reset stack pointer, enable interrupts and then halt.
+	asm volatile (
+		"movl $0, %%ebp\n"
+		"movl %0, %%esp\n"
+		"pushl $0\n"
+		"pushl $0\n"
+		// Uncomment the following line after completing exercise 13
+		//"sti\n"
+		"1:\n"
+		"hlt\n"
+		"jmp 1b\n"
+	: : "a" (thiscpu->cpu_ts.ts_esp0));
+}
+

kern/sched.h

@@ -0,0 +1,12 @@
+/* See COPYRIGHT for copyright information. */
+
+#ifndef JOS_KERN_SCHED_H
+#define JOS_KERN_SCHED_H
+#ifndef JOS_KERNEL
+# error "This is a JOS kernel header; user programs should not #include it"
+#endif
+
+// This function does not return.
+void sched_yield(void) __attribute__((noreturn));
+
+#endif	// !JOS_KERN_SCHED_H

kern/spinlock.c

@@ -0,0 +1,116 @@
+// Mutual exclusion spin locks.
+
+#include <inc/types.h>
+#include <inc/assert.h>
+#include <inc/x86.h>
+#include <inc/memlayout.h>
+#include <inc/string.h>
+#include <kern/cpu.h>
+#include <kern/spinlock.h>
+#include <kern/kdebug.h>
+
+// The big kernel lock
+struct spinlock kernel_lock = {
+#ifdef DEBUG_SPINLOCK
+	.name = "kernel_lock"
+#endif
+};
+
+#ifdef DEBUG_SPINLOCK
+// Record the current call stack in pcs[] by following the %ebp chain.
+static void
+get_caller_pcs(uint32_t pcs[])
+{
+	uint32_t *ebp;
+	int i;
+
+	ebp = (uint32_t *)read_ebp();
+	for (i = 0; i < 10; i++){
+		if (ebp == 0 || ebp < (uint32_t *)ULIM)
+			break;
+		pcs[i] = ebp[1];          // saved %eip
+		ebp = (uint32_t *)ebp[0]; // saved %ebp
+	}
+	for (; i < 10; i++)
+		pcs[i] = 0;
+}
+
+// Check whether this CPU is holding the lock.
+static int
+holding(struct spinlock *lock)
+{
+	return lock->locked && lock->cpu == thiscpu;
+}
+#endif
+
+void
+__spin_initlock(struct spinlock *lk, char *name)
+{
+	lk->locked = 0;
+#ifdef DEBUG_SPINLOCK
+	lk->name = name;
+	lk->cpu = 0;
+#endif
+}
+
+// Acquire the lock.
+// Loops (spins) until the lock is acquired.
+// Holding a lock for a long time may cause
+// other CPUs to waste time spinning to acquire it.
+void
+spin_lock(struct spinlock *lk)
+{
+#ifdef DEBUG_SPINLOCK
+	if (holding(lk))
+		panic("CPU %d cannot acquire %s: already holding", cpunum(), lk->name);
+#endif
+
+	// The xchg is atomic.
+	// It also serializes, so that reads after acquire are not
+	// reordered before it. 
+	while (xchg(&lk->locked, 1) != 0)
+		asm volatile ("pause");
+
+	// Record info about lock acquisition for debugging.
+#ifdef DEBUG_SPINLOCK
+	lk->cpu = thiscpu;
+	get_caller_pcs(lk->pcs);
+#endif
+}
+
+// Release the lock.
+void
+spin_unlock(struct spinlock *lk)
+{
+#ifdef DEBUG_SPINLOCK
+	if (!holding(lk)) {
+		int i;
+		uint32_t pcs[10];
+		// Nab the acquiring EIP chain before it gets released
+		memmove(pcs, lk->pcs, sizeof pcs);
+		cprintf("CPU %d cannot release %s: held by CPU %d\nAcquired at:", 
+			cpunum(), lk->name, lk->cpu->cpu_id);
+		for (i = 0; i < 10 && pcs[i]; i++) {
+			struct Eipdebuginfo info;
+			if (debuginfo_eip(pcs[i], &info) >= 0)
+				cprintf("  %08x %s:%d: %.*s+%x\n", pcs[i],
+					info.eip_file, info.eip_line,
+					info.eip_fn_namelen, info.eip_fn_name,
+					pcs[i] - info.eip_fn_addr);
+			else
+				cprintf("  %08x\n", pcs[i]);
+		}
+		panic("spin_unlock");
+	}
+
+	lk->pcs[0] = 0;
+	lk->cpu = 0;
+#endif
+
+	// The xchg instruction is atomic (i.e. uses the "lock" prefix) with
+	// respect to any other instruction which references the same memory.
+	// x86 CPUs will not reorder loads/stores across locked instructions
+	// (vol 3, 8.2.2). Because xchg() is implemented using asm volatile,
+	// gcc will not reorder C statements across the xchg.
+	xchg(&lk->locked, 0);
+}

kern/spinlock.h

@@ -0,0 +1,48 @@
+#ifndef JOS_INC_SPINLOCK_H
+#define JOS_INC_SPINLOCK_H
+
+#include <inc/types.h>
+
+// Comment this to disable spinlock debugging
+#define DEBUG_SPINLOCK
+
+// Mutual exclusion lock.
+struct spinlock {
+	unsigned locked;       // Is the lock held?
+
+#ifdef DEBUG_SPINLOCK
+	// For debugging:
+	char *name;            // Name of lock.
+	struct CpuInfo *cpu;   // The CPU holding the lock.
+	uintptr_t pcs[10];     // The call stack (an array of program counters)
+	                       // that locked the lock.
+#endif
+};
+
+void __spin_initlock(struct spinlock *lk, char *name);
+void spin_lock(struct spinlock *lk);
+void spin_unlock(struct spinlock *lk);
+
+#define spin_initlock(lock)   __spin_initlock(lock, #lock)
+
+extern struct spinlock kernel_lock;
+
+static inline void
+lock_kernel(void)
+{
+	spin_lock(&kernel_lock);
+}
+
+static inline void
+unlock_kernel(void)
+{
+	spin_unlock(&kernel_lock);
+
+	// Normally we wouldn't need to do this, but QEMU only runs
+	// one CPU at a time and has a long time-slice.  Without the
+	// pause, this CPU is likely to reacquire the lock before
+	// another CPU has even been given a chance to acquire it.
+	asm volatile("pause");
+}
+
+#endif

kern/syscall.c

@@ -10,6 +10,7 @@
 #include <kern/trap.h>
 #include <kern/syscall.h>
 #include <kern/console.h>
+#include <kern/sched.h>
 
 // Print a string to the system console.
 // The string is exactly 'len' characters long.
@@ -62,6 +63,206 @@ sys_env_destroy(envid_t envid)
 	return 0;
 }
 
+// Deschedule current environment and pick a different one to run.
+static void
+sys_yield(void)
+{
+	sched_yield();
+}
+
+// Allocate a new environment.
+// Returns envid of new environment, or < 0 on error.  Errors are:
+//	-E_NO_FREE_ENV if no free environment is available.
+//	-E_NO_MEM on memory exhaustion.
+static envid_t
+sys_exofork(void)
+{
+	// Create the new environment with env_alloc(), from kern/env.c.
+	// It should be left as env_alloc created it, except that
+	// status is set to ENV_NOT_RUNNABLE, and the register set is copied
+	// from the current environment -- but tweaked so sys_exofork
+	// will appear to return 0.
+
+	// LAB 4: Your code here.
+	panic("sys_exofork not implemented");
+}
+
+// Set envid's env_status to status, which must be ENV_RUNNABLE
+// or ENV_NOT_RUNNABLE.
+//
+// Returns 0 on success, < 0 on error.  Errors are:
+//	-E_BAD_ENV if environment envid doesn't currently exist,
+//		or the caller doesn't have permission to change envid.
+//	-E_INVAL if status is not a valid status for an environment.
+static int
+sys_env_set_status(envid_t envid, int status)
+{
+	// Hint: Use the 'envid2env' function from kern/env.c to translate an
+	// envid to a struct Env.
+	// You should set envid2env's third argument to 1, which will
+	// check whether the current environment has permission to set
+	// envid's status.
+
+	// LAB 4: Your code here.
+	panic("sys_env_set_status not implemented");
+}
+
+// Set the page fault upcall for 'envid' by modifying the corresponding struct
+// Env's 'env_pgfault_upcall' field.  When 'envid' causes a page fault, the
+// kernel will push a fault record onto the exception stack, then branch to
+// 'func'.
+//
+// Returns 0 on success, < 0 on error.  Errors are:
+//	-E_BAD_ENV if environment envid doesn't currently exist,
+//		or the caller doesn't have permission to change envid.
+static int
+sys_env_set_pgfault_upcall(envid_t envid, void *func)
+{
+	// LAB 4: Your code here.
+	panic("sys_env_set_pgfault_upcall not implemented");
+}
+
+// Allocate a page of memory and map it at 'va' with permission
+// 'perm' in the address space of 'envid'.
+// The page's contents are set to 0.
+// If a page is already mapped at 'va', that page is unmapped as a
+// side effect.
+//
+// perm -- PTE_U | PTE_P must be set, PTE_AVAIL | PTE_W may or may not be set,
+//         but no other bits may be set.  See PTE_SYSCALL in inc/mmu.h.
+//
+// Return 0 on success, < 0 on error.  Errors are:
+//	-E_BAD_ENV if environment envid doesn't currently exist,
+//		or the caller doesn't have permission to change envid.
+//	-E_INVAL if va >= UTOP, or va is not page-aligned.
+//	-E_INVAL if perm is inappropriate (see above).
+//	-E_NO_MEM if there's no memory to allocate the new page,
+//		or to allocate any necessary page tables.
+static int
+sys_page_alloc(envid_t envid, void *va, int perm)
+{
+	// Hint: This function is a wrapper around page_alloc() and
+	//   page_insert() from kern/pmap.c.
+	//   Most of the new code you write should be to check the
+	//   parameters for correctness.
+	//   If page_insert() fails, remember to free the page you
+	//   allocated!
+
+	// LAB 4: Your code here.
+	panic("sys_page_alloc not implemented");
+}
+
+// Map the page of memory at 'srcva' in srcenvid's address space
+// at 'dstva' in dstenvid's address space with permission 'perm'.
+// Perm has the same restrictions as in sys_page_alloc, except
+// that it also must not grant write access to a read-only
+// page.
+//
+// Return 0 on success, < 0 on error.  Errors are:
+//	-E_BAD_ENV if srcenvid and/or dstenvid doesn't currently exist,
+//		or the caller doesn't have permission to change one of them.
+//	-E_INVAL if srcva >= UTOP or srcva is not page-aligned,
+//		or dstva >= UTOP or dstva is not page-aligned.
+//	-E_INVAL is srcva is not mapped in srcenvid's address space.
+//	-E_INVAL if perm is inappropriate (see sys_page_alloc).
+//	-E_INVAL if (perm & PTE_W), but srcva is read-only in srcenvid's
+//		address space.
+//	-E_NO_MEM if there's no memory to allocate any necessary page tables.
+static int
+sys_page_map(envid_t srcenvid, void *srcva,
+	     envid_t dstenvid, void *dstva, int perm)
+{
+	// Hint: This function is a wrapper around page_lookup() and
+	//   page_insert() from kern/pmap.c.
+	//   Again, most of the new code you write should be to check the
+	//   parameters for correctness.
+	//   Use the third argument to page_lookup() to
+	//   check the current permissions on the page.
+
+	// LAB 4: Your code here.
+	panic("sys_page_map not implemented");
+}
+
+// Unmap the page of memory at 'va' in the address space of 'envid'.
+// If no page is mapped, the function silently succeeds.
+//
+// Return 0 on success, < 0 on error.  Errors are:
+//	-E_BAD_ENV if environment envid doesn't currently exist,
+//		or the caller doesn't have permission to change envid.
+//	-E_INVAL if va >= UTOP, or va is not page-aligned.
+static int
+sys_page_unmap(envid_t envid, void *va)
+{
+	// Hint: This function is a wrapper around page_remove().
+
+	// LAB 4: Your code here.
+	panic("sys_page_unmap not implemented");
+}
+
+// Try to send 'value' to the target env 'envid'.
+// If srcva < UTOP, then also send page currently mapped at 'srcva',
+// so that receiver gets a duplicate mapping of the same page.
+//
+// The send fails with a return value of -E_IPC_NOT_RECV if the
+// target is not blocked, waiting for an IPC.
+//
+// The send also can fail for the other reasons listed below.
+//
+// Otherwise, the send succeeds, and the target's ipc fields are
+// updated as follows:
+//    env_ipc_recving is set to 0 to block future sends;
+//    env_ipc_from is set to the sending envid;
+//    env_ipc_value is set to the 'value' parameter;
+//    env_ipc_perm is set to 'perm' if a page was transferred, 0 otherwise.
+// The target environment is marked runnable again, returning 0
+// from the paused sys_ipc_recv system call.  (Hint: does the
+// sys_ipc_recv function ever actually return?)
+//
+// If the sender wants to send a page but the receiver isn't asking for one,
+// then no page mapping is transferred, but no error occurs.
+// The ipc only happens when no errors occur.
+//
+// Returns 0 on success, < 0 on error.
+// Errors are:
+//	-E_BAD_ENV if environment envid doesn't currently exist.
+//		(No need to check permissions.)
+//	-E_IPC_NOT_RECV if envid is not currently blocked in sys_ipc_recv,
+//		or another environment managed to send first.
+//	-E_INVAL if srcva < UTOP but srcva is not page-aligned.
+//	-E_INVAL if srcva < UTOP and perm is inappropriate
+//		(see sys_page_alloc).
+//	-E_INVAL if srcva < UTOP but srcva is not mapped in the caller's
+//		address space.
+//	-E_INVAL if (perm & PTE_W), but srcva is read-only in the
+//		current environment's address space.
+//	-E_NO_MEM if there's not enough memory to map srcva in envid's
+//		address space.
+static int
+sys_ipc_try_send(envid_t envid, uint32_t value, void *srcva, unsigned perm)
+{
+	// LAB 4: Your code here.
+	panic("sys_ipc_try_send not implemented");
+}
+
+// Block until a value is ready.  Record that you want to receive
+// using the env_ipc_recving and env_ipc_dstva fields of struct Env,
+// mark yourself not runnable, and then give up the CPU.
+//
+// If 'dstva' is < UTOP, then you are willing to receive a page of data.
+// 'dstva' is the virtual address at which the sent page should be mapped.
+//
+// This function only returns on error, but the system call will eventually
+// return 0 on success.
+// Return < 0 on error.  Errors are:
+//	-E_INVAL if dstva < UTOP but dstva is not page-aligned.
+static int
+sys_ipc_recv(void *dstva)
+{
+	// LAB 4: Your code here.
+	panic("sys_ipc_recv not implemented");
+	return 0;
+}
+
 // Dispatches to the correct kernel function, passing the arguments.
 int32_t
 syscall(uint32_t syscallno, uint32_t a1, uint32_t a2, uint32_t a3, uint32_t a4, uint32_t a5)

kern/trap.c

@@ -8,6 +8,11 @@
 #include <kern/monitor.h>
 #include <kern/env.h>
 #include <kern/syscall.h>
+#include <kern/sched.h>
+#include <kern/kclock.h>
+#include <kern/picirq.h>
+#include <kern/cpu.h>
+#include <kern/spinlock.h>
 
 static struct Taskstate ts;
 
@@ -55,6 +60,8 @@ static const char *trapname(int trapno)
 		return excnames[trapno];
 	if (trapno == T_SYSCALL)
 		return "System call";
+	if (trapno >= IRQ_OFFSET && trapno < IRQ_OFFSET + 16)
+		return "Hardware Interrupt";
 	return "(unknown trap)";
 }
 
@@ -74,6 +81,31 @@ trap_init(void)
 void
 trap_init_percpu(void)
 {
+	// The example code here sets up the Task State Segment (TSS) and
+	// the TSS descriptor for CPU 0. But it is incorrect if we are
+	// running on other CPUs because each CPU has its own kernel stack.
+	// Fix the code so that it works for all CPUs.
+	//
+	// Hints:
+	//   - The macro "thiscpu" always refers to the current CPU's
+	//     struct CpuInfo;
+	//   - The ID of the current CPU is given by cpunum() or
+	//     thiscpu->cpu_id;
+	//   - Use "thiscpu->cpu_ts" as the TSS for the current CPU,
+	//     rather than the global "ts" variable;
+	//   - Use gdt[(GD_TSS0 >> 3) + i] for CPU i's TSS descriptor;
+	//   - You mapped the per-CPU kernel stacks in mem_init_mp()
+	//   - Initialize cpu_ts.ts_iomb to prevent unauthorized environments
+	//     from doing IO (0 is not the correct value!)
+	//
+	// ltr sets a 'busy' flag in the TSS selector, so if you
+	// accidentally load the same TSS on more than one CPU, you'll
+	// get a triple fault.  If you set up an individual CPU's TSS
+	// wrong, you may not get a fault until you try to return from
+	// user space on that CPU.
+	//
+	// LAB 4: Your code here:
+
 	// Setup a TSS so that we get the right stack
 	// when we trap to the kernel.
 	ts.ts_esp0 = KSTACKTOP;
@@ -96,7 +128,7 @@ trap_init_percpu(void)
 void
 print_trapframe(struct Trapframe *tf)
 {
-	cprintf("TRAP frame at %p\n", tf);
+	cprintf("TRAP frame at %p from CPU %d\n", tf, cpunum());
 	print_regs(&tf->tf_regs);
 	cprintf("  es   0x----%04x\n", tf->tf_es);
 	cprintf("  ds   0x----%04x\n", tf->tf_ds);
@@ -145,6 +177,19 @@ trap_dispatch(struct Trapframe *tf)
 	// Handle processor exceptions.
 	// LAB 3: Your code here.
 
+	// Handle spurious interrupts
+	// The hardware sometimes raises these because of noise on the
+	// IRQ line or other reasons. We don't care.
+	if (tf->tf_trapno == IRQ_OFFSET + IRQ_SPURIOUS) {
+		cprintf("Spurious interrupt on irq 7\n");
+		print_trapframe(tf);
+		return;
+	}
+
+	// Handle clock interrupts. Don't forget to acknowledge the
+	// interrupt using lapic_eoi() before calling the scheduler!
+	// LAB 4: Your code here.
+
 	// Unexpected trap: The user process or the kernel has a bug.
 	print_trapframe(tf);
 	if (tf->tf_cs == GD_KT)
@@ -162,17 +207,34 @@ trap(struct Trapframe *tf)
 	// of GCC rely on DF being clear
 	asm volatile("cld" ::: "cc");
 
+	// Halt the CPU if some other CPU has called panic()
+	extern char *panicstr;
+	if (panicstr)
+		asm volatile("hlt");
+
+	// Re-acqurie the big kernel lock if we were halted in
+	// sched_yield()
+	if (xchg(&thiscpu->cpu_status, CPU_STARTED) == CPU_HALTED)
+		lock_kernel();
 	// Check that interrupts are disabled.  If this assertion
 	// fails, DO NOT be tempted to fix it by inserting a "cli" in
 	// the interrupt path.
 	assert(!(read_eflags() & FL_IF));
 
-	cprintf("Incoming TRAP frame at %p\n", tf);
-
 	if ((tf->tf_cs & 3) == 3) {
 		// Trapped from user mode.
+		// Acquire the big kernel lock before doing any
+		// serious kernel work.
+		// LAB 4: Your code here.
 		assert(curenv);
 
+		// Garbage collect if current enviroment is a zombie
+		if (curenv->env_status == ENV_DYING) {
+			env_free(curenv);
+			curenv = NULL;
+			sched_yield();
+		}
+
 		// Copy trap frame (which is currently on the stack)
 		// into 'curenv->env_tf', so that running the environment
 		// will restart at the trap point.
@@ -188,9 +250,13 @@ trap(struct Trapframe *tf)
 	// Dispatch based on what type of trap occurred
 	trap_dispatch(tf);
 
-	// Return to the current environment, which should be running.
-	assert(curenv && curenv->env_status == ENV_RUNNING);
-	env_run(curenv);
+	// If we made it to this point, then no other environment was
+	// scheduled, so we should return to the current environment
+	// if doing so makes sense.
+	if (curenv && curenv->env_status == ENV_RUNNING)
+		env_run(curenv);
+	else
+		sched_yield();
 }
 
 
@@ -209,6 +275,37 @@ page_fault_handler(struct Trapframe *tf)
 	// We've already handled kernel-mode exceptions, so if we get here,
 	// the page fault happened in user mode.
 
+	// Call the environment's page fault upcall, if one exists.  Set up a
+	// page fault stack frame on the user exception stack (below
+	// UXSTACKTOP), then branch to curenv->env_pgfault_upcall.
+	//
+	// The page fault upcall might cause another page fault, in which case
+	// we branch to the page fault upcall recursively, pushing another
+	// page fault stack frame on top of the user exception stack.
+	//
+	// It is convenient for our code which returns from a page fault
+	// (lib/pfentry.S) to have one word of scratch space at the top of the
+	// trap-time stack; it allows us to more easily restore the eip/esp. In
+	// the non-recursive case, we don't have to worry about this because
+	// the top of the regular user stack is free.  In the recursive case,
+	// this means we have to leave an extra word between the current top of
+	// the exception stack and the new stack frame because the exception
+	// stack _is_ the trap-time stack.
+	//
+	// If there's no page fault upcall, the environment didn't allocate a
+	// page for its exception stack or can't write to it, or the exception
+	// stack overflows, then destroy the environment that caused the fault.
+	// Note that the grade script assumes you will first check for the page
+	// fault upcall and print the "user fault va" message below if there is
+	// none.  The remaining three checks can be combined into a single test.
+	//
+	// Hints:
+	//   user_mem_assert() and env_run() are useful here.
+	//   To change what the user environment runs, modify 'curenv->env_tf'
+	//   (the 'tf' variable points at 'curenv->env_tf').
+
+	// LAB 4: Your code here.
+
 	// Destroy the environment that caused the fault.
 	cprintf("[%08x] user fault va %08x ip %08x\n",
 		curenv->env_id, fault_va, tf->tf_eip);

kern/trapentry.S

@@ -4,6 +4,7 @@
 #include <inc/memlayout.h>
 #include <inc/trap.h>
 
+#include <kern/picirq.h>
 
 
 ###################################################################