[4.8] Merge GCC 4.8.2

Change-Id: I0f1fcf69c5076d8534c5c45562745e1a37adb197

1 files changed, 698 insertions, 166 deletions

diff --git a/gcc-4.8/libgo/runtime/mgc0.c b/gcc-4.8/libgo/runtime/mgc0.c
index ffbe2cefb..c3b32111c 100644
--- a/gcc-4.8/libgo/runtime/mgc0.c
+++ b/gcc-4.8/libgo/runtime/mgc0.c
@@ -21,8 +21,11 @@
 #define tab __methods
 // Eface aka __go_empty_interface.
 #define type __type_descriptor
+// Hmap aka __go_map
+typedef struct __go_map Hmap;
 // Type aka __go_type_descriptor
 #define kind __code
+#define string __reflection
 #define KindPtr GO_PTR
 #define KindNoPointers GO_NO_POINTERS
 // PtrType aka __go_ptr_type
@@ -41,6 +44,9 @@ extern void * __splitstack_find_context (void *context[10], size_t *, void **,
 enum {
 	Debug = 0,
 	DebugMark = 0,  // run second pass to check mark
+	CollectStats = 0,
+	ScanStackByFrames = 0,
+	IgnorePreciseGC = 0,
 
 	// Four bits per word (see #defines below).
 	wordsPerBitmapWord = sizeof(void*)*8/4,
@@ -120,7 +126,7 @@ struct Workbuf
 typedef struct Finalizer Finalizer;
 struct Finalizer
 {
-	void (*fn)(void*);
+	FuncVal *fn;
 	void *arg;
 	const struct __go_func_type *ft;
 };
@@ -147,6 +153,7 @@ static Workbuf* getempty(Workbuf*);
 static Workbuf* getfull(Workbuf*);
 static void	putempty(Workbuf*);
 static Workbuf* handoff(Workbuf*);
+static void	gchelperstart(void);
 
 static struct {
 	uint64	full;  // lock-free list of full blocks
@@ -170,11 +177,114 @@ static struct {
 } work;
 
 enum {
-	// TODO(atom): to be expanded in a next CL
 	GC_DEFAULT_PTR = GC_NUM_INSTR,
+	GC_MAP_NEXT,
+	GC_CHAN,
+
+	GC_NUM_INSTR2
 };
 
-// PtrTarget and BitTarget are structures used by intermediate buffers.
+static struct {
+	struct {
+		uint64 sum;
+		uint64 cnt;
+	} ptr;
+	uint64 nbytes;
+	struct {
+		uint64 sum;
+		uint64 cnt;
+		uint64 notype;
+		uint64 typelookup;
+	} obj;
+	uint64 rescan;
+	uint64 rescanbytes;
+	uint64 instr[GC_NUM_INSTR2];
+	uint64 putempty;
+	uint64 getfull;
+} gcstats;
+
+// markonly marks an object. It returns true if the object
+// has been marked by this function, false otherwise.
+// This function doesn't append the object to any buffer.
+static bool
+markonly(void *obj)
+{
+	byte *p;
+	uintptr *bitp, bits, shift, x, xbits, off;
+	MSpan *s;
+	PageID k;
+
+	// Words outside the arena cannot be pointers.
+	if((byte*)obj < runtime_mheap->arena_start || (byte*)obj >= runtime_mheap->arena_used)
+		return false;
+
+	// obj may be a pointer to a live object.
+	// Try to find the beginning of the object.
+
+	// Round down to word boundary.
+	obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1));
+
+	// Find bits for this word.
+	off = (uintptr*)obj - (uintptr*)runtime_mheap->arena_start;
+	bitp = (uintptr*)runtime_mheap->arena_start - off/wordsPerBitmapWord - 1;
+	shift = off % wordsPerBitmapWord;
+	xbits = *bitp;
+	bits = xbits >> shift;
+
+	// Pointing at the beginning of a block?
+	if((bits & (bitAllocated|bitBlockBoundary)) != 0)
+		goto found;
+
+	// Otherwise consult span table to find beginning.
+	// (Manually inlined copy of MHeap_LookupMaybe.)
+	k = (uintptr)obj>>PageShift;
+	x = k;
+	if(sizeof(void*) == 8)
+		x -= (uintptr)runtime_mheap->arena_start>>PageShift;
+	s = runtime_mheap->map[x];
+	if(s == nil || k < s->start || k - s->start >= s->npages || s->state != MSpanInUse)
+		return false;
+	p = (byte*)((uintptr)s->start<<PageShift);
+	if(s->sizeclass == 0) {
+		obj = p;
+	} else {
+		if((byte*)obj >= (byte*)s->limit)
+			return false;
+		uintptr size = s->elemsize;
+		int32 i = ((byte*)obj - p)/size;
+		obj = p+i*size;
+	}
+
+	// Now that we know the object header, reload bits.
+	off = (uintptr*)obj - (uintptr*)runtime_mheap->arena_start;
+	bitp = (uintptr*)runtime_mheap->arena_start - off/wordsPerBitmapWord - 1;
+	shift = off % wordsPerBitmapWord;
+	xbits = *bitp;
+	bits = xbits >> shift;
+
+found:
+	// Now we have bits, bitp, and shift correct for
+	// obj pointing at the base of the object.
+	// Only care about allocated and not marked.
+	if((bits & (bitAllocated|bitMarked)) != bitAllocated)
+		return false;
+	if(work.nproc == 1)
+		*bitp |= bitMarked<<shift;
+	else {
+		for(;;) {
+			x = *bitp;
+			if(x & (bitMarked<<shift))
+				return false;
+			if(runtime_casp((void**)bitp, (void*)x, (void*)(x|(bitMarked<<shift))))
+				break;
+		}
+	}
+
+	// The object is now marked
+	return true;
+}
+
+// PtrTarget is a structure used by intermediate buffers.
 // The intermediate buffers hold GC data before it
 // is moved/flushed to the work buffer (Workbuf).
 // The size of an intermediate buffer is very small,
@@ -186,24 +296,16 @@ struct PtrTarget
 	uintptr ti;
 };
 
-typedef struct BitTarget BitTarget;
-struct BitTarget
-{
-	void *p;
-	uintptr ti;
-	uintptr *bitp, shift;
-};
-
 typedef struct BufferList BufferList;
 struct BufferList
 {
 	PtrTarget ptrtarget[IntermediateBufferCapacity];
-	BitTarget bittarget[IntermediateBufferCapacity];
-	BufferList *next;
+	Obj obj[IntermediateBufferCapacity];
+	uint32 busy;
+	byte pad[CacheLineSize];
 };
-static BufferList *bufferList;
+static BufferList bufferList[MaxGcproc];
 
-static Lock lock;
 static Type *itabtype;
 
 static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj);
@@ -214,7 +316,6 @@ static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj);
 // and are prepared to be scanned by the garbage collector.
 //
 // _wp, _wbuf, _nobj are input/output parameters and are specifying the work buffer.
-// bitbuf holds temporary data generated by this function.
 //
 // A simplified drawing explaining how the todo-list moves from a structure to another:
 //
@@ -222,14 +323,12 @@ static void enqueue(Obj obj, Workbuf **_wbuf, Obj **_wp, uintptr *_nobj);
 //  (find pointers)
 //    Obj ------> PtrTarget (pointer targets)
 //     ↑          |
-//     |          | flushptrbuf (1st part,
-//     |          | find block start)
-//     |          ↓
-//     `--------- BitTarget (pointer targets and the corresponding locations in bitmap)
-//  flushptrbuf
-//  (2nd part, mark and enqueue)
+//     |          |
+//     `----------'
+//     flushptrbuf
+//  (find block start, mark and enqueue)
 static void
-flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj, BitTarget *bitbuf)
+flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj)
 {
 	byte *p, *arena_start, *obj;
 	uintptr size, *bitp, bits, shift, j, x, xbits, off, nobj, ti, n;
@@ -238,9 +337,8 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
 	Obj *wp;
 	Workbuf *wbuf;
 	PtrTarget *ptrbuf_end;
-	BitTarget *bitbufpos, *bt;
 
-	arena_start = runtime_mheap.arena_start;
+	arena_start = runtime_mheap->arena_start;
 
 	wp = *_wp;
 	wbuf = *_wbuf;
@@ -250,6 +348,11 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
 	n = ptrbuf_end - ptrbuf;
 	*ptrbufpos = ptrbuf;
 
+	if(CollectStats) {
+		runtime_xadd64(&gcstats.ptr.sum, n);
+		runtime_xadd64(&gcstats.ptr.cnt, 1);
+	}
+
 	// If buffer is nearly full, get a new one.
 	if(wbuf == nil || nobj+n >= nelem(wbuf->obj)) {
 		if(wbuf != nil)
@@ -267,8 +370,6 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
 	{
 		// Multi-threaded version.
 
-		bitbufpos = bitbuf;
-
 		while(ptrbuf < ptrbuf_end) {
 			obj = ptrbuf->p;
 			ti = ptrbuf->ti;
@@ -276,7 +377,7 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
 
 			// obj belongs to interval [mheap.arena_start, mheap.arena_used).
 			if(Debug > 1) {
-				if(obj < runtime_mheap.arena_start || obj >= runtime_mheap.arena_used)
+				if(obj < runtime_mheap->arena_start || obj >= runtime_mheap->arena_used)
 					runtime_throw("object is outside of mheap");
 			}
 
@@ -319,7 +420,7 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
 			x = k;
 			if(sizeof(void*) == 8)
 				x -= (uintptr)arena_start>>PageShift;
-			s = runtime_mheap.map[x];
+			s = runtime_mheap->map[x];
 			if(s == nil || k < s->start || k - s->start >= s->npages || s->state != MSpanInUse)
 				continue;
 			p = (byte*)((uintptr)s->start<<PageShift);
@@ -346,40 +447,36 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
 			// Only care about allocated and not marked.
 			if((bits & (bitAllocated|bitMarked)) != bitAllocated)
 				continue;
-
-			*bitbufpos++ = (BitTarget){obj, ti, bitp, shift};
-		}
-
-		runtime_lock(&lock);
-		for(bt=bitbuf; bt<bitbufpos; bt++){
-			xbits = *bt->bitp;
-			bits = xbits >> bt->shift;
-			if((bits & bitMarked) != 0)
-				continue;
-
-			// Mark the block
-			*bt->bitp = xbits | (bitMarked << bt->shift);
+			if(work.nproc == 1)
+				*bitp |= bitMarked<<shift;
+			else {
+				for(;;) {
+					x = *bitp;
+					if(x & (bitMarked<<shift))
+						goto continue_obj;
+					if(runtime_casp((void**)bitp, (void*)x, (void*)(x|(bitMarked<<shift))))
+						break;
+				}
+			}
 
 			// If object has no pointers, don't need to scan further.
 			if((bits & bitNoPointers) != 0)
 				continue;
 
-			obj = bt->p;
-
 			// Ask span about size class.
 			// (Manually inlined copy of MHeap_Lookup.)
 			x = (uintptr)obj >> PageShift;
 			if(sizeof(void*) == 8)
 				x -= (uintptr)arena_start>>PageShift;
-			s = runtime_mheap.map[x];
+			s = runtime_mheap->map[x];
 
 			PREFETCH(obj);
 
-			*wp = (Obj){obj, s->elemsize, bt->ti};
+			*wp = (Obj){obj, s->elemsize, ti};
 			wp++;
 			nobj++;
+		continue_obj:;
 		}
-		runtime_unlock(&lock);
 
 		// If another proc wants a pointer, give it some.
 		if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) {
@@ -395,9 +492,73 @@ flushptrbuf(PtrTarget *ptrbuf, PtrTarget **ptrbufpos, Obj **_wp, Workbuf **_wbuf
 	*_nobj = nobj;
 }
 
+static void
+flushobjbuf(Obj *objbuf, Obj **objbufpos, Obj **_wp, Workbuf **_wbuf, uintptr *_nobj)
+{
+	uintptr nobj, off;
+	Obj *wp, obj;
+	Workbuf *wbuf;
+	Obj *objbuf_end;
+
+	wp = *_wp;
+	wbuf = *_wbuf;
+	nobj = *_nobj;
+
+	objbuf_end = *objbufpos;
+	*objbufpos = objbuf;
+
+	while(objbuf < objbuf_end) {
+		obj = *objbuf++;
+
+		// Align obj.b to a word boundary.
+		off = (uintptr)obj.p & (PtrSize-1);
+		if(off != 0) {
+			obj.p += PtrSize - off;
+			obj.n -= PtrSize - off;
+			obj.ti = 0;
+		}
+
+		if(obj.p == nil || obj.n == 0)
+			continue;
+
+		// If buffer is full, get a new one.
+		if(wbuf == nil || nobj >= nelem(wbuf->obj)) {
+			if(wbuf != nil)
+				wbuf->nobj = nobj;
+			wbuf = getempty(wbuf);
+			wp = wbuf->obj;
+			nobj = 0;
+		}
+
+		*wp = obj;
+		wp++;
+		nobj++;
+	}
+
+	// If another proc wants a pointer, give it some.
+	if(work.nwait > 0 && nobj > handoffThreshold && work.full == 0) {
+		wbuf->nobj = nobj;
+		wbuf = handoff(wbuf);
+		nobj = wbuf->nobj;
+		wp = wbuf->obj + nobj;
+	}
+
+	*_wp = wp;
+	*_wbuf = wbuf;
+	*_nobj = nobj;
+}
+
 // Program that scans the whole block and treats every block element as a potential pointer
 static uintptr defaultProg[2] = {PtrSize, GC_DEFAULT_PTR};
 
+#if 0
+// Hashmap iterator program
+static uintptr mapProg[2] = {0, GC_MAP_NEXT};
+
+// Hchan program
+static uintptr chanProg[2] = {0, GC_CHAN};
+#endif
+
 // Local variables of a program fragment or loop
 typedef struct Frame Frame;
 struct Frame {
@@ -405,6 +566,61 @@ struct Frame {
 	uintptr *loop_or_ret;
 };
 
+// Sanity check for the derived type info objti.
+static void
+checkptr(void *obj, uintptr objti)
+{
+	uintptr type, tisize, i, x;
+	byte *objstart;
+	Type *t;
+	MSpan *s;
+
+	if(!Debug)
+		runtime_throw("checkptr is debug only");
+
+	if((byte*)obj < runtime_mheap->arena_start || (byte*)obj >= runtime_mheap->arena_used)
+		return;
+	type = runtime_gettype(obj);
+	t = (Type*)(type & ~(uintptr)(PtrSize-1));
+	if(t == nil)
+		return;
+	x = (uintptr)obj >> PageShift;
+	if(sizeof(void*) == 8)
+		x -= (uintptr)(runtime_mheap->arena_start)>>PageShift;
+	s = runtime_mheap->map[x];
+	objstart = (byte*)((uintptr)s->start<<PageShift);
+	if(s->sizeclass != 0) {
+		i = ((byte*)obj - objstart)/s->elemsize;
+		objstart += i*s->elemsize;
+	}
+	tisize = *(uintptr*)objti;
+	// Sanity check for object size: it should fit into the memory block.
+	if((byte*)obj + tisize > objstart + s->elemsize)
+		runtime_throw("invalid gc type info");
+	if(obj != objstart)
+		return;
+	// If obj points to the beginning of the memory block,
+	// check type info as well.
+	if(t->string == nil ||
+		// Gob allocates unsafe pointers for indirection.
+		(runtime_strcmp((const char *)t->string->str, (const char*)"unsafe.Pointer") &&
+		// Runtime and gc think differently about closures.
+		 runtime_strstr((const char *)t->string->str, (const char*)"struct { F uintptr") != (const char *)t->string->str)) {
+#if 0
+		pc1 = (uintptr*)objti;
+		pc2 = (uintptr*)t->gc;
+		// A simple best-effort check until first GC_END.
+		for(j = 1; pc1[j] != GC_END && pc2[j] != GC_END; j++) {
+			if(pc1[j] != pc2[j]) {
+				runtime_printf("invalid gc type info for '%s' at %p, type info %p, block info %p\n",
+					t->string ? (const int8*)t->string->str : (const int8*)"?", j, pc1[j], pc2[j]);
+				runtime_throw("invalid gc type info");
+			}
+		}
+#endif
+	}
+}					
+
 // scanblock scans a block of n bytes starting at pointer b for references
 // to other objects, scanning any it finds recursively until there are no
 // unscanned objects left.  Instead of using an explicit recursion, it keeps
@@ -419,49 +635,64 @@ static void
 scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
 {
 	byte *b, *arena_start, *arena_used;
-	uintptr n, i, end_b, elemsize, ti, objti, count /* , type */;
+	uintptr n, i, end_b, elemsize, size, ti, objti, count /* , type */;
 	uintptr *pc, precise_type, nominal_size;
+#if 0
+	uintptr *map_ret, mapkey_size, mapval_size, mapkey_ti, mapval_ti, *chan_ret, chancap;
+#endif
 	void *obj;
 	const Type *t;
 	Slice *sliceptr;
 	Frame *stack_ptr, stack_top, stack[GC_STACK_CAPACITY+4];
 	BufferList *scanbuffers;
 	PtrTarget *ptrbuf, *ptrbuf_end, *ptrbufpos;
-	BitTarget *bitbuf;
+	Obj *objbuf, *objbuf_end, *objbufpos;
 	Eface *eface;
 	Iface *iface;
+#if 0
+	Hmap *hmap;
+	MapType *maptype;
+	bool mapkey_kind, mapval_kind;
+	struct hash_gciter map_iter;
+	struct hash_gciter_data d;
+	Hchan *chan;
+	ChanType *chantype;
+#endif
 
 	if(sizeof(Workbuf) % PageSize != 0)
 		runtime_throw("scanblock: size of Workbuf is suboptimal");
 
 	// Memory arena parameters.
-	arena_start = runtime_mheap.arena_start;
-	arena_used = runtime_mheap.arena_used;
+	arena_start = runtime_mheap->arena_start;
+	arena_used = runtime_mheap->arena_used;
 
 	stack_ptr = stack+nelem(stack)-1;
 	
 	precise_type = false;
 	nominal_size = 0;
 
-	// Allocate ptrbuf, bitbuf
+	// Allocate ptrbuf
 	{
-		runtime_lock(&lock);
-
-		if(bufferList == nil) {
-			bufferList = runtime_SysAlloc(sizeof(*bufferList));
-			bufferList->next = nil;
-		}
-		scanbuffers = bufferList;
-		bufferList = bufferList->next;
-
+		scanbuffers = &bufferList[runtime_m()->helpgc];
 		ptrbuf = &scanbuffers->ptrtarget[0];
 		ptrbuf_end = &scanbuffers->ptrtarget[0] + nelem(scanbuffers->ptrtarget);
-		bitbuf = &scanbuffers->bittarget[0];
-
-		runtime_unlock(&lock);
+		objbuf = &scanbuffers->obj[0];
+		objbuf_end = &scanbuffers->obj[0] + nelem(scanbuffers->obj);
 	}
 
 	ptrbufpos = ptrbuf;
+	objbufpos = objbuf;
+
+	// (Silence the compiler)
+#if 0
+	map_ret = nil;
+	mapkey_size = mapval_size = 0;
+	mapkey_kind = mapval_kind = false;
+	mapkey_ti = mapval_ti = 0;
+	chan = nil;
+	chantype = nil;
+	chan_ret = nil;
+#endif
 
 	goto next_block;
 
@@ -472,7 +703,13 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
 			runtime_printf("scanblock %p %D\n", b, (int64)n);
 		}
 
-		if(ti != 0 && 0) {
+		if(CollectStats) {
+			runtime_xadd64(&gcstats.nbytes, n);
+			runtime_xadd64(&gcstats.obj.sum, nobj);
+			runtime_xadd64(&gcstats.obj.cnt, 1);
+		}
+
+		if(ti != 0 && false) {
 			pc = (uintptr*)(ti & ~(uintptr)PC_BITS);
 			precise_type = (ti & PRECISE);
 			stack_top.elemsize = pc[0];
@@ -484,10 +721,27 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
 			} else {
 				stack_top.count = 1;
 			}
-		} else if(UseSpanType && 0) {
+			if(Debug) {
+				// Simple sanity check for provided type info ti:
+				// The declared size of the object must be not larger than the actual size
+				// (it can be smaller due to inferior pointers).
+				// It's difficult to make a comprehensive check due to inferior pointers,
+				// reflection, gob, etc.
+				if(pc[0] > n) {
+					runtime_printf("invalid gc type info: type info size %p, block size %p\n", pc[0], n);
+					runtime_throw("invalid gc type info");
+				}
+			}
+		} else if(UseSpanType && false) {
+			if(CollectStats)
+				runtime_xadd64(&gcstats.obj.notype, 1);
+
 #if 0
 			type = runtime_gettype(b);
 			if(type != 0) {
+				if(CollectStats)
+					runtime_xadd64(&gcstats.obj.typelookup, 1);
+
 				t = (Type*)(type & ~(uintptr)(PtrSize-1));
 				switch(type & (PtrSize-1)) {
 				case TypeInfo_SingleObject:
@@ -506,8 +760,27 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
 					stack_top.loop_or_ret = pc+1;
 					break;
 				case TypeInfo_Map:
-					// TODO(atom): to be expanded in a next CL
-					pc = defaultProg;
+					hmap = (Hmap*)b;
+					maptype = (MapType*)t;
+					if(hash_gciter_init(hmap, &map_iter)) {
+						mapkey_size = maptype->key->size;
+						mapkey_kind = maptype->key->kind;
+						mapkey_ti   = (uintptr)maptype->key->gc | PRECISE;
+						mapval_size = maptype->elem->size;
+						mapval_kind = maptype->elem->kind;
+						mapval_ti   = (uintptr)maptype->elem->gc | PRECISE;
+
+						map_ret = nil;
+						pc = mapProg;
+					} else {
+						goto next_block;
+					}
+					break;
+				case TypeInfo_Chan:
+					chan = (Hchan*)b;
+					chantype = (ChanType*)t;
+					chan_ret = nil;
+					pc = chanProg;
 					break;
 				default:
 					runtime_throw("scanblock: invalid type");
@@ -521,12 +794,18 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
 			pc = defaultProg;
 		}
 
+		if(IgnorePreciseGC)
+			pc = defaultProg;
+
 		pc++;
 		stack_top.b = (uintptr)b;
 
 		end_b = (uintptr)b + n - PtrSize;
 
 	for(;;) {
+		if(CollectStats)
+			runtime_xadd64(&gcstats.instr[pc[0]], 1);
+
 		obj = nil;
 		objti = 0;
 		switch(pc[0]) {
@@ -534,13 +813,19 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
 			obj = *(void**)(stack_top.b + pc[1]);
 			objti = pc[2];
 			pc += 3;
+			if(Debug)
+				checkptr(obj, objti);
 			break;
 
 		case GC_SLICE:
 			sliceptr = (Slice*)(stack_top.b + pc[1]);
 			if(sliceptr->cap != 0) {
 				obj = sliceptr->array;
-				objti = pc[2] | PRECISE | LOOP;
+				// Can't use slice element type for scanning,
+				// because if it points to an array embedded
+				// in the beginning of a struct,
+				// we will scan the whole struct as the slice.
+				// So just obtain type info from heap.
 			}
 			pc += 3;
 			break;
@@ -552,17 +837,31 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
 
 		case GC_STRING:
 			obj = *(void**)(stack_top.b + pc[1]);
+			markonly(obj);
 			pc += 2;
-			break;
+			continue;
 
 		case GC_EFACE:
 			eface = (Eface*)(stack_top.b + pc[1]);
 			pc += 2;
-			if(eface->type != nil && ((byte*)eface->__object >= arena_start && (byte*)eface->__object < arena_used)) {
-				t = eface->type;
+			if(eface->type == nil)
+				continue;
+
+			// eface->type
+			t = eface->type;
+			if((const byte*)t >= arena_start && (const byte*)t < arena_used) {
+				union { const Type *tc; Type *tr; } u;
+				u.tc = t;
+				*ptrbufpos++ = (struct PtrTarget){(void*)u.tr, 0};
+				if(ptrbufpos == ptrbuf_end)
+					flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj);
+			}
+
+			// eface->__object
+			if((byte*)eface->__object >= arena_start && (byte*)eface->__object < arena_used) {
 				if(t->__size <= sizeof(void*)) {
 					if((t->kind & KindNoPointers))
-						break;
+						continue;
 
 					obj = eface->__object;
 					if((t->kind & ~KindNoPointers) == KindPtr)
@@ -580,14 +879,14 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
 			iface = (Iface*)(stack_top.b + pc[1]);
 			pc += 2;
 			if(iface->tab == nil)
-				break;
+				continue;
 			
 			// iface->tab
 			if((byte*)iface->tab >= arena_start && (byte*)iface->tab < arena_used) {
 				// *ptrbufpos++ = (struct PtrTarget){iface->tab, (uintptr)itabtype->gc};
 				*ptrbufpos++ = (struct PtrTarget){iface->tab, 0};
 				if(ptrbufpos == ptrbuf_end)
-					flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf);
+					flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj);
 			}
 
 			// iface->data
@@ -596,7 +895,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
 				t = nil;
 				if(t->__size <= sizeof(void*)) {
 					if((t->kind & KindNoPointers))
-						break;
+						continue;
 
 					obj = iface->__object;
 					if((t->kind & ~KindNoPointers) == KindPtr)
@@ -611,13 +910,13 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
 			break;
 
 		case GC_DEFAULT_PTR:
-			while((i = stack_top.b) <= end_b) {
+			while(stack_top.b <= end_b) {
+				obj = *(byte**)stack_top.b;
 				stack_top.b += PtrSize;
-				obj = *(byte**)i;
 				if((byte*)obj >= arena_start && (byte*)obj < arena_used) {
 					*ptrbufpos++ = (struct PtrTarget){obj, 0};
 					if(ptrbufpos == ptrbuf_end)
-						flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf);
+						flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj);
 				}
 			}
 			goto next_block;
@@ -625,9 +924,8 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
 		case GC_END:
 			if(--stack_top.count != 0) {
 				// Next iteration of a loop if possible.
-				elemsize = stack_top.elemsize;
-				stack_top.b += elemsize;
-				if(stack_top.b + elemsize <= end_b+PtrSize) {
+				stack_top.b += stack_top.elemsize;
+				if(stack_top.b + stack_top.elemsize <= end_b+PtrSize) {
 					pc = stack_top.loop_or_ret;
 					continue;
 				}
@@ -648,6 +946,10 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
 						// Found a value that may be a pointer.
 						// Do a rescan of the entire block.
 						enqueue((Obj){b, n, 0}, &wbuf, &wp, &nobj);
+						if(CollectStats) {
+							runtime_xadd64(&gcstats.rescan, 1);
+							runtime_xadd64(&gcstats.rescanbytes, n);
+						}
 						break;
 					}
 				}
@@ -680,20 +982,136 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
 			// Stack push.
 			*stack_ptr-- = stack_top;
 			stack_top = (Frame){1, 0, stack_top.b + pc[1], pc+3 /*return address*/};
-			pc = (uintptr*)pc[2];  // target of the CALL instruction
+			pc = (uintptr*)((byte*)pc + *(int32*)(pc+2));  // target of the CALL instruction
 			continue;
 
+#if 0
 		case GC_MAP_PTR:
-			// TODO(atom): to be expanded in a next CL. Same as GC_APTR for now.
-			obj = *(void**)(stack_top.b + pc[1]);
-			pc += 3;
-			break;
+			hmap = *(Hmap**)(stack_top.b + pc[1]);
+			if(hmap == nil) {
+				pc += 3;
+				continue;
+			}
+			if(markonly(hmap)) {
+				maptype = (MapType*)pc[2];
+				if(hash_gciter_init(hmap, &map_iter)) {
+					mapkey_size = maptype->key->size;
+					mapkey_kind = maptype->key->kind;
+					mapkey_ti   = (uintptr)maptype->key->gc | PRECISE;
+					mapval_size = maptype->elem->size;
+					mapval_kind = maptype->elem->kind;
+					mapval_ti   = (uintptr)maptype->elem->gc | PRECISE;
+
+					// Start mapProg.
+					map_ret = pc+3;
+					pc = mapProg+1;
+				} else {
+					pc += 3;
+				}
+			} else {
+				pc += 3;
+			}
+			continue;
+
+		case GC_MAP_NEXT:
+			// Add all keys and values to buffers, mark all subtables.
+			while(hash_gciter_next(&map_iter, &d)) {
+				// buffers: reserve space for 2 objects.
+				if(ptrbufpos+2 >= ptrbuf_end)
+					flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj);
+				if(objbufpos+2 >= objbuf_end)
+					flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj);
+
+				if(d.st != nil)
+					markonly(d.st);
+
+				if(d.key_data != nil) {
+					if(!(mapkey_kind & KindNoPointers) || d.indirectkey) {
+						if(!d.indirectkey)
+							*objbufpos++ = (Obj){d.key_data, mapkey_size, mapkey_ti};
+						else {
+							if(Debug) {
+								obj = *(void**)d.key_data;
+								if(!(arena_start <= obj && obj < arena_used))
+									runtime_throw("scanblock: inconsistent hashmap");
+							}
+							*ptrbufpos++ = (struct PtrTarget){*(void**)d.key_data, mapkey_ti};
+						}
+					}
+					if(!(mapval_kind & KindNoPointers) || d.indirectval) {
+						if(!d.indirectval)
+							*objbufpos++ = (Obj){d.val_data, mapval_size, mapval_ti};
+						else {
+							if(Debug) {
+								obj = *(void**)d.val_data;
+								if(!(arena_start <= obj && obj < arena_used))
+									runtime_throw("scanblock: inconsistent hashmap");
+							}
+							*ptrbufpos++ = (struct PtrTarget){*(void**)d.val_data, mapval_ti};
+						}
+					}
+				}
+			}
+			if(map_ret == nil)
+				goto next_block;
+			pc = map_ret;
+			continue;
+#endif
 
 		case GC_REGION:
-			// TODO(atom): to be expanded in a next CL. Same as GC_APTR for now.
 			obj = (void*)(stack_top.b + pc[1]);
+			size = pc[2];
+			objti = pc[3];
 			pc += 4;
-			break;
+
+			*objbufpos++ = (Obj){obj, size, objti};
+			if(objbufpos == objbuf_end)
+				flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj);
+			continue;
+
+#if 0
+		case GC_CHAN_PTR:
+			// Similar to GC_MAP_PTR
+			chan = *(Hchan**)(stack_top.b + pc[1]);
+			if(chan == nil) {
+				pc += 3;
+				continue;
+			}
+			if(markonly(chan)) {
+				chantype = (ChanType*)pc[2];
+				if(!(chantype->elem->kind & KindNoPointers)) {
+					// Start chanProg.
+					chan_ret = pc+3;
+					pc = chanProg+1;
+					continue;
+				}
+			}
+			pc += 3;
+			continue;
+
+		case GC_CHAN:
+			// There are no heap pointers in struct Hchan,
+			// so we can ignore the leading sizeof(Hchan) bytes.
+			if(!(chantype->elem->kind & KindNoPointers)) {
+				// Channel's buffer follows Hchan immediately in memory.
+				// Size of buffer (cap(c)) is second int in the chan struct.
+				chancap = ((uintgo*)chan)[1];
+				if(chancap > 0) {
+					// TODO(atom): split into two chunks so that only the
+					// in-use part of the circular buffer is scanned.
+					// (Channel routines zero the unused part, so the current
+					// code does not lead to leaks, it's just a little inefficient.)
+					*objbufpos++ = (Obj){(byte*)chan+runtime_Hchansize, chancap*chantype->elem->size,
+						(uintptr)chantype->elem->gc | PRECISE | LOOP};
+					if(objbufpos == objbuf_end)
+						flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj);
+				}
+			}
+			if(chan_ret == nil)
+				goto next_block;
+			pc = chan_ret;
+			continue;
+#endif
 
 		default:
 			runtime_throw("scanblock: invalid GC instruction");
@@ -701,9 +1119,9 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
 		}
 
 		if((byte*)obj >= arena_start && (byte*)obj < arena_used) {
-			*ptrbufpos++ = (PtrTarget){obj, objti};
+			*ptrbufpos++ = (struct PtrTarget){obj, objti};
 			if(ptrbufpos == ptrbuf_end)
-				flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf);
+				flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj);
 		}
 	}
 
@@ -712,7 +1130,8 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
 		// the loop by setting b, n, ti to the parameters for the next block.
 
 		if(nobj == 0) {
-			flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj, bitbuf);
+			flushptrbuf(ptrbuf, &ptrbufpos, &wp, &wbuf, &nobj);
+			flushobjbuf(objbuf, &objbufpos, &wp, &wbuf, &nobj);
 
 			if(nobj == 0) {
 				if(!keepworking) {
@@ -737,11 +1156,7 @@ scanblock(Workbuf *wbuf, Obj *wp, uintptr nobj, bool keepworking)
 		nobj--;
 	}
 
-endscan:
-	runtime_lock(&lock);
-	scanbuffers->next = bufferList;
-	bufferList = scanbuffers;
-	runtime_unlock(&lock);
+endscan:;
 }
 
 // debug_scanblock is the debug copy of scanblock.
@@ -776,14 +1191,14 @@ debug_scanblock(byte *b, uintptr n)
 		obj = (byte*)vp[i];
 
 		// Words outside the arena cannot be pointers.
-		if((byte*)obj < runtime_mheap.arena_start || (byte*)obj >= runtime_mheap.arena_used)
+		if((byte*)obj < runtime_mheap->arena_start || (byte*)obj >= runtime_mheap->arena_used)
 			continue;
 
 		// Round down to word boundary.
 		obj = (void*)((uintptr)obj & ~((uintptr)PtrSize-1));
 
 		// Consult span table to find beginning.
-		s = runtime_MHeap_LookupMaybe(&runtime_mheap, obj);
+		s = runtime_MHeap_LookupMaybe(runtime_mheap, obj);
 		if(s == nil)
 			continue;
 
@@ -799,8 +1214,8 @@ debug_scanblock(byte *b, uintptr n)
 		}
 
 		// Now that we know the object header, reload bits.
-		off = (uintptr*)obj - (uintptr*)runtime_mheap.arena_start;
-		bitp = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1;
+		off = (uintptr*)obj - (uintptr*)runtime_mheap->arena_start;
+		bitp = (uintptr*)runtime_mheap->arena_start - off/wordsPerBitmapWord - 1;
 		shift = off % wordsPerBitmapWord;
 		xbits = *bitp;
 		bits = xbits >> shift;
@@ -906,6 +1321,8 @@ getempty(Workbuf *b)
 		if(work.nchunk < sizeof *b) {
 			work.nchunk = 1<<20;
 			work.chunk = runtime_SysAlloc(work.nchunk);
+			if(work.chunk == nil)
+				runtime_throw("runtime: cannot allocate memory");
 		}
 		b = (Workbuf*)work.chunk;
 		work.chunk += sizeof *b;
@@ -919,6 +1336,9 @@ getempty(Workbuf *b)
 static void
 putempty(Workbuf *b)
 {
+	if(CollectStats)
+		runtime_xadd64(&gcstats.putempty, 1);
+
 	runtime_lfstackpush(&work.empty, &b->node);
 }
 
@@ -929,6 +1349,9 @@ getfull(Workbuf *b)
 	M *m;
 	int32 i;
 
+	if(CollectStats)
+		runtime_xadd64(&gcstats.getfull, 1);
+
 	if(b != nil)
 		runtime_lfstackpush(&work.empty, &b->node);
 	b = (Workbuf*)runtime_lfstackpop(&work.full);
@@ -994,6 +1417,8 @@ addroot(Obj obj)
 		if(cap < 2*work.rootcap)
 			cap = 2*work.rootcap;
 		new = (Obj*)runtime_SysAlloc(cap*sizeof(Obj));
+		if(new == nil)
+			runtime_throw("runtime: cannot allocate memory");
 		if(work.roots != nil) {
 			runtime_memmove(new, work.roots, work.rootcap*sizeof(Obj));
 			runtime_SysFree(work.roots, work.rootcap*sizeof(Obj));
@@ -1081,13 +1506,14 @@ static void
 addfinroots(void *v)
 {
 	uintptr size;
+	void *base;
 
 	size = 0;
-	if(!runtime_mlookup(v, (byte**)&v, &size, nil) || !runtime_blockspecial(v))
+	if(!runtime_mlookup(v, (byte**)&base, &size, nil) || !runtime_blockspecial(base))
 		runtime_throw("mark - finalizer inconsistency");
 
 	// do not mark the finalizer block itself.  just mark the things it points at.
-	addroot((Obj){v, size, 0});
+	addroot((Obj){base, size, 0});
 }
 
 static struct root_list* roots;
@@ -1128,23 +1554,27 @@ addroots(void)
 	addroot((Obj){(byte*)&runtime_g0, sizeof runtime_g0, 0});
 	addroot((Obj){(byte*)&runtime_allg, sizeof runtime_allg, 0});
 	addroot((Obj){(byte*)&runtime_allm, sizeof runtime_allm, 0});
+	addroot((Obj){(byte*)&runtime_allp, sizeof runtime_allp, 0});
+	runtime_proc_scan(addroot);
 	runtime_MProf_Mark(addroot);
 	runtime_time_scan(addroot);
-	runtime_trampoline_scan(addroot);
 
 	// MSpan.types
-	allspans = runtime_mheap.allspans;
-	for(spanidx=0; spanidx<runtime_mheap.nspan; spanidx++) {
+	allspans = runtime_mheap->allspans;
+	for(spanidx=0; spanidx<runtime_mheap->nspan; spanidx++) {
 		s = allspans[spanidx];
 		if(s->state == MSpanInUse) {
+			// The garbage collector ignores type pointers stored in MSpan.types:
+			//  - Compiler-generated types are stored outside of heap.
+			//  - The reflect package has runtime-generated types cached in its data structures.
+			//    The garbage collector relies on finding the references via that cache.
 			switch(s->types.compression) {
 			case MTypes_Empty:
 			case MTypes_Single:
 				break;
 			case MTypes_Words:
 			case MTypes_Bytes:
-				// TODO(atom): consider using defaultProg instead of 0
-				addroot((Obj){(byte*)&s->types.data, sizeof(void*), 0});
+				markonly((byte*)s->types.data);
 				break;
 			}
 		}
@@ -1182,7 +1612,7 @@ addroots(void)
 static bool
 handlespecial(byte *p, uintptr size)
 {
-	void (*fn)(void*);
+	FuncVal *fn;
 	const struct __go_func_type *ft;
 	FinBlock *block;
 	Finalizer *f;
@@ -1197,6 +1627,8 @@ handlespecial(byte *p, uintptr size)
 	if(finq == nil || finq->cnt == finq->cap) {
 		if(finc == nil) {
 			finc = runtime_SysAlloc(PageSize);
+			if(finc == nil)
+				runtime_throw("runtime: cannot allocate memory");
 			finc->cap = (PageSize - sizeof(FinBlock)) / sizeof(Finalizer) + 1;
 			finc->alllink = allfin;
 			allfin = finc;
@@ -1236,10 +1668,10 @@ sweepspan(ParFor *desc, uint32 idx)
 	m = runtime_m();
 
 	USED(&desc);
-	s = runtime_mheap.allspans[idx];
+	s = runtime_mheap->allspans[idx];
 	if(s->state != MSpanInUse)
 		return;
-	arena_start = runtime_mheap.arena_start;
+	arena_start = runtime_mheap->arena_start;
 	p = (byte*)(s->start << PageShift);
 	cl = s->sizeclass;
 	size = s->elemsize;
@@ -1302,8 +1734,8 @@ sweepspan(ParFor *desc, uint32 idx)
 		if(cl == 0) {
 			// Free large span.
 			runtime_unmarkspan(p, 1<<PageShift);
-			*(uintptr*)p = 1;	// needs zeroing
-			runtime_MHeap_Free(&runtime_mheap, s, 1);
+			*(uintptr*)p = (uintptr)0xdeaddeaddeaddeadll;	// needs zeroing
+			runtime_MHeap_Free(runtime_mheap, s, 1);
 			c->local_alloc -= size;
 			c->local_nfree++;
 		} else {
@@ -1317,7 +1749,7 @@ sweepspan(ParFor *desc, uint32 idx)
 				break;
 			}
 			if(size > sizeof(uintptr))
-				((uintptr*)p)[1] = 1;	// mark as "needs to be zeroed"
+				((uintptr*)p)[1] = (uintptr)0xdeaddeaddeaddeadll;	// mark as "needs to be zeroed"
 			
 			end->next = (MLink*)p;
 			end = (MLink*)p;
@@ -1331,7 +1763,7 @@ sweepspan(ParFor *desc, uint32 idx)
 		c->local_nfree += nfree;
 		c->local_cachealloc -= nfree * size;
 		c->local_objects -= nfree;
-		runtime_MCentral_FreeSpan(&runtime_mheap.central[cl], s, nfree, head.next, end);
+		runtime_MCentral_FreeSpan(&runtime_mheap->central[cl], s, nfree, head.next, end);
 	}
 }
 
@@ -1345,10 +1777,10 @@ dumpspan(uint32 idx)
 	MSpan *s;
 	bool allocated, special;
 
-	s = runtime_mheap.allspans[idx];
+	s = runtime_mheap->allspans[idx];
 	if(s->state != MSpanInUse)
 		return;
-	arena_start = runtime_mheap.arena_start;
+	arena_start = runtime_mheap->arena_start;
 	p = (byte*)(s->start << PageShift);
 	sizeclass = s->sizeclass;
 	size = s->elemsize;
@@ -1406,7 +1838,7 @@ runtime_memorydump(void)
 {
 	uint32 spanidx;
 
-	for(spanidx=0; spanidx<runtime_mheap.nspan; spanidx++) {
+	for(spanidx=0; spanidx<runtime_mheap->nspan; spanidx++) {
 		dumpspan(spanidx);
 	}
 }
@@ -1414,6 +1846,8 @@ runtime_memorydump(void)
 void
 runtime_gchelper(void)
 {
+	gchelperstart();
+
 	// parallel mark for over gc roots
 	runtime_parfordo(work.markfor);
 
@@ -1427,10 +1861,13 @@ runtime_gchelper(void)
 	}
 
 	runtime_parfordo(work.sweepfor);
+	bufferList[runtime_m()->helpgc].busy = 0;
 	if(runtime_xadd(&work.ndone, +1) == work.nproc-1)
 		runtime_notewakeup(&work.alldone);
 }
 
+#define GcpercentUnknown (-2)
+
 // Initialized from $GOGC.  GOGC=off means no gc.
 //
 // Next gc is after we've allocated an extra amount of
@@ -1440,22 +1877,14 @@ runtime_gchelper(void)
 // proportion to the allocation cost.  Adjusting gcpercent
 // just changes the linear constant (and also the amount of
 // extra memory used).
-static int32 gcpercent = -2;
-
-static void
-stealcache(void)
-{
-	M *mp;
-
-	for(mp=runtime_allm; mp; mp=mp->alllink)
-		runtime_MCache_ReleaseAll(mp->mcache);
-}
+static int32 gcpercent = GcpercentUnknown;
 
 static void
 cachestats(GCStats *stats)
 {
 	M *mp;
 	MCache *c;
+	P *p, **pp;
 	uint32 i;
 	uint64 stacks_inuse;
 	uint64 *src, *dst;
@@ -1464,9 +1893,7 @@ cachestats(GCStats *stats)
 		runtime_memclr((byte*)stats, sizeof(*stats));
 	stacks_inuse = 0;
 	for(mp=runtime_allm; mp; mp=mp->alllink) {
-		c = mp->mcache;
-		runtime_purgecachedstats(c);
-		// stacks_inuse += mp->stackinuse*FixedStack;
+		//stacks_inuse += mp->stackinuse*FixedStack;
 		if(stats) {
 			src = (uint64*)&mp->gcstats;
 			dst = (uint64*)stats;
@@ -1474,6 +1901,12 @@ cachestats(GCStats *stats)
 				dst[i] += src[i];
 			runtime_memclr((byte*)&mp->gcstats, sizeof(mp->gcstats));
 		}
+	}
+	for(pp=runtime_allp; (p=*pp) != nil; pp++) {
+		c = p->mcache;
+		if(c==nil)
+			continue;
+		runtime_purgecachedstats(c);
 		for(i=0; i<nelem(c->local_by_size); i++) {
 			mstats.by_size[i].nmalloc += c->local_by_size[i].nmalloc;
 			c->local_by_size[i].nmalloc = 0;
@@ -1493,6 +1926,19 @@ struct gc_args
 
 static void gc(struct gc_args *args);
 
+static int32
+readgogc(void)
+{
+	const byte *p;
+
+	p = runtime_getenv("GOGC");
+	if(p == nil || p[0] == '\0')
+		return 100;
+	if(runtime_strcmp((const char *)p, "off") == 0)
+		return -1;
+	return runtime_atoi(p);
+}
+
 void
 runtime_gc(int32 force)
 {
@@ -1505,6 +1951,8 @@ runtime_gc(int32 force)
 	// a problem in the past.
 	if((((uintptr)&work.empty) & 7) != 0)
 		runtime_throw("runtime: gc work buffer is misaligned");
+	if((((uintptr)&work.full) & 7) != 0)
+		runtime_throw("runtime: gc work buffer is misaligned");
 
 	// Make sure all registers are saved on stack so that
 	// scanstack sees them.
@@ -1522,14 +1970,8 @@ runtime_gc(int32 force)
 	if(!mstats.enablegc || m->locks > 0 || runtime_panicking)
 		return;
 
-	if(gcpercent == -2) {	// first time through
-		p = runtime_getenv("GOGC");
-		if(p == nil || p[0] == '\0')
-			gcpercent = 100;
-		else if(runtime_strcmp((const char*)p, "off") == 0)
-			gcpercent = -1;
-		else
-			gcpercent = runtime_atoi(p);
+	if(gcpercent == GcpercentUnknown) {	// first time through
+		gcpercent = readgogc();
 
 		p = runtime_getenv("GOGCTRACE");
 		if(p != nil)
@@ -1556,7 +1998,7 @@ gc(struct gc_args *args)
 {
 	M *m;
 	int64 t0, t1, t2, t3, t4;
-	uint64 heap0, heap1, obj0, obj1;
+	uint64 heap0, heap1, obj0, obj1, ninstr;
 	GCStats stats;
 	M *mp;
 	uint32 i;
@@ -1575,6 +2017,9 @@ gc(struct gc_args *args)
 	m->gcing = 1;
 	runtime_stoptheworld();
 
+	if(CollectStats)
+		runtime_memclr((byte*)&gcstats, sizeof(gcstats));
+
 	for(mp=runtime_allm; mp; mp=mp->alllink)
 		runtime_settype_flush(mp, false);
 
@@ -1605,7 +2050,7 @@ gc(struct gc_args *args)
 	work.nproc = runtime_gcprocs();
 	addroots();
 	runtime_parforsetup(work.markfor, work.nproc, work.nroot, nil, false, markroot);
-	runtime_parforsetup(work.sweepfor, work.nproc, runtime_mheap.nspan, nil, true, sweepspan);
+	runtime_parforsetup(work.sweepfor, work.nproc, runtime_mheap->nspan, nil, true, sweepspan);
 	if(work.nproc > 1) {
 		runtime_noteclear(&work.alldone);
 		runtime_helpgc(work.nproc);
@@ -1613,6 +2058,7 @@ gc(struct gc_args *args)
 
 	t1 = runtime_nanotime();
 
+	gchelperstart();
 	runtime_parfordo(work.markfor);
 	scanblock(nil, nil, 0, true);
 
@@ -1624,14 +2070,14 @@ gc(struct gc_args *args)
 	t2 = runtime_nanotime();
 
 	runtime_parfordo(work.sweepfor);
+	bufferList[m->helpgc].busy = 0;
 	t3 = runtime_nanotime();
 
-	stealcache();
-	cachestats(&stats);
-
 	if(work.nproc > 1)
 		runtime_notesleep(&work.alldone);
 
+	cachestats(&stats);
+
 	stats.nprocyield += work.sweepfor->nprocyield;
 	stats.nosyield += work.sweepfor->nosyield;
 	stats.nsleep += work.sweepfor->nsleep;
@@ -1671,6 +2117,27 @@ gc(struct gc_args *args)
 			stats.nhandoff, stats.nhandoffcnt,
 			work.sweepfor->nsteal, work.sweepfor->nstealcnt,
 			stats.nprocyield, stats.nosyield, stats.nsleep);
+		if(CollectStats) {
+			runtime_printf("scan: %D bytes, %D objects, %D untyped, %D types from MSpan\n",
+				gcstats.nbytes, gcstats.obj.cnt, gcstats.obj.notype, gcstats.obj.typelookup);
+			if(gcstats.ptr.cnt != 0)
+				runtime_printf("avg ptrbufsize: %D (%D/%D)\n",
+					gcstats.ptr.sum/gcstats.ptr.cnt, gcstats.ptr.sum, gcstats.ptr.cnt);
+			if(gcstats.obj.cnt != 0)
+				runtime_printf("avg nobj: %D (%D/%D)\n",
+					gcstats.obj.sum/gcstats.obj.cnt, gcstats.obj.sum, gcstats.obj.cnt);
+			runtime_printf("rescans: %D, %D bytes\n", gcstats.rescan, gcstats.rescanbytes);
+
+			runtime_printf("instruction counts:\n");
+			ninstr = 0;
+			for(i=0; i<nelem(gcstats.instr); i++) {
+				runtime_printf("\t%d:\t%D\n", i, gcstats.instr[i]);
+				ninstr += gcstats.instr[i];
+			}
+			runtime_printf("\ttotal:\t%D\n", ninstr);
+
+			runtime_printf("putempty: %D, getfull: %D\n", gcstats.putempty, gcstats.getfull);
+		}
 	}
 
 	runtime_MProf_GC();
@@ -1705,6 +2172,71 @@ runtime_ReadMemStats(MStats *stats)
 	runtime_starttheworld();
 }
 
+void runtime_debug_readGCStats(Slice*)
+  __asm__("runtime_debug.readGCStats");
+
+void
+runtime_debug_readGCStats(Slice *pauses)
+{
+	uint64 *p;
+	uint32 i, n;
+
+	// Calling code in runtime/debug should make the slice large enough.
+	if((size_t)pauses->cap < nelem(mstats.pause_ns)+3)
+		runtime_throw("runtime: short slice passed to readGCStats");
+
+	// Pass back: pauses, last gc (absolute time), number of gc, total pause ns.
+	p = (uint64*)pauses->array;
+	runtime_lock(runtime_mheap);
+	n = mstats.numgc;
+	if(n > nelem(mstats.pause_ns))
+		n = nelem(mstats.pause_ns);
+	
+	// The pause buffer is circular. The most recent pause is at
+	// pause_ns[(numgc-1)%nelem(pause_ns)], and then backward
+	// from there to go back farther in time. We deliver the times
+	// most recent first (in p[0]).
+	for(i=0; i<n; i++)
+		p[i] = mstats.pause_ns[(mstats.numgc-1-i)%nelem(mstats.pause_ns)];
+
+	p[n] = mstats.last_gc;
+	p[n+1] = mstats.numgc;
+	p[n+2] = mstats.pause_total_ns;	
+	runtime_unlock(runtime_mheap);
+	pauses->__count = n+3;
+}
+
+intgo runtime_debug_setGCPercent(intgo)
+  __asm__("runtime_debug.setGCPercent");
+
+intgo
+runtime_debug_setGCPercent(intgo in)
+{
+	intgo out;
+
+	runtime_lock(runtime_mheap);
+	if(gcpercent == GcpercentUnknown)
+		gcpercent = readgogc();
+	out = gcpercent;
+	if(in < 0)
+		in = -1;
+	gcpercent = in;
+	runtime_unlock(runtime_mheap);
+	return out;
+}
+
+static void
+gchelperstart(void)
+{
+	M *m;
+
+	m = runtime_m();
+	if(m->helpgc < 0 || m->helpgc >= MaxGcproc)
+		runtime_throw("gchelperstart: bad m->helpgc");
+	if(runtime_xchg(&bufferList[m->helpgc].busy, 1))
+		runtime_throw("gchelperstart: already busy");
+}
+
 static void
 runfinq(void* dummy __attribute__ ((unused)))
 {
@@ -1731,11 +2263,11 @@ runfinq(void* dummy __attribute__ ((unused)))
 		for(; fb; fb=next) {
 			next = fb->next;
 			for(i=0; i<(uint32)fb->cnt; i++) {
-				void *params[1];
+				void *param;
 
 				f = &fb->fin[i];
-				params[0] = &f->arg;
-				reflect_call(f->ft, (void*)f->fn, 0, 0, params, nil);
+				param = &f->arg;
+				reflect_call(f->ft, f->fn, 0, 0, &param, nil);
 				f->fn = nil;
 				f->arg = nil;
 			}
@@ -1757,11 +2289,11 @@ runtime_markallocated(void *v, uintptr n, bool noptr)
 	if(0)
 		runtime_printf("markallocated %p+%p\n", v, n);
 
-	if((byte*)v+n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start)
+	if((byte*)v+n > (byte*)runtime_mheap->arena_used || (byte*)v < runtime_mheap->arena_start)
 		runtime_throw("markallocated: bad pointer");
 
-	off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start;  // word offset
-	b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1;
+	off = (uintptr*)v - (uintptr*)runtime_mheap->arena_start;  // word offset
+	b = (uintptr*)runtime_mheap->arena_start - off/wordsPerBitmapWord - 1;
 	shift = off % wordsPerBitmapWord;
 
 	for(;;) {
@@ -1789,11 +2321,11 @@ runtime_markfreed(void *v, uintptr n)
 	if(0)
 		runtime_printf("markallocated %p+%p\n", v, n);
 
-	if((byte*)v+n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start)
+	if((byte*)v+n > (byte*)runtime_mheap->arena_used || (byte*)v < runtime_mheap->arena_start)
 		runtime_throw("markallocated: bad pointer");
 
-	off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start;  // word offset
-	b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1;
+	off = (uintptr*)v - (uintptr*)runtime_mheap->arena_start;  // word offset
+	b = (uintptr*)runtime_mheap->arena_start - off/wordsPerBitmapWord - 1;
 	shift = off % wordsPerBitmapWord;
 
 	for(;;) {
@@ -1819,11 +2351,11 @@ runtime_checkfreed(void *v, uintptr n)
 	if(!runtime_checking)
 		return;
 
-	if((byte*)v+n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start)
+	if((byte*)v+n > (byte*)runtime_mheap->arena_used || (byte*)v < runtime_mheap->arena_start)
 		return;	// not allocated, so okay
 
-	off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start;  // word offset
-	b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1;
+	off = (uintptr*)v - (uintptr*)runtime_mheap->arena_start;  // word offset
+	b = (uintptr*)runtime_mheap->arena_start - off/wordsPerBitmapWord - 1;
 	shift = off % wordsPerBitmapWord;
 
 	bits = *b>>shift;
@@ -1842,7 +2374,7 @@ runtime_markspan(void *v, uintptr size, uintptr n, bool leftover)
 	uintptr *b, off, shift;
 	byte *p;
 
-	if((byte*)v+size*n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start)
+	if((byte*)v+size*n > (byte*)runtime_mheap->arena_used || (byte*)v < runtime_mheap->arena_start)
 		runtime_throw("markspan: bad pointer");
 
 	p = v;
@@ -1853,8 +2385,8 @@ runtime_markspan(void *v, uintptr size, uintptr n, bool leftover)
 		// the entire span, and each bitmap word has bits for only
 		// one span, so no other goroutines are changing these
 		// bitmap words.
-		off = (uintptr*)p - (uintptr*)runtime_mheap.arena_start;  // word offset
-		b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1;
+		off = (uintptr*)p - (uintptr*)runtime_mheap->arena_start;  // word offset
+		b = (uintptr*)runtime_mheap->arena_start - off/wordsPerBitmapWord - 1;
 		shift = off % wordsPerBitmapWord;
 		*b = (*b & ~(bitMask<<shift)) | (bitBlockBoundary<<shift);
 	}
@@ -1866,14 +2398,14 @@ runtime_unmarkspan(void *v, uintptr n)
 {
 	uintptr *p, *b, off;
 
-	if((byte*)v+n > (byte*)runtime_mheap.arena_used || (byte*)v < runtime_mheap.arena_start)
+	if((byte*)v+n > (byte*)runtime_mheap->arena_used || (byte*)v < runtime_mheap->arena_start)
 		runtime_throw("markspan: bad pointer");
 
 	p = v;
-	off = p - (uintptr*)runtime_mheap.arena_start;  // word offset
+	off = p - (uintptr*)runtime_mheap->arena_start;  // word offset
 	if(off % wordsPerBitmapWord != 0)
 		runtime_throw("markspan: unaligned pointer");
-	b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1;
+	b = (uintptr*)runtime_mheap->arena_start - off/wordsPerBitmapWord - 1;
 	n /= PtrSize;
 	if(n%wordsPerBitmapWord != 0)
 		runtime_throw("unmarkspan: unaligned length");
@@ -1894,8 +2426,8 @@ runtime_blockspecial(void *v)
 	if(DebugMark)
 		return true;
 
-	off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start;
-	b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1;
+	off = (uintptr*)v - (uintptr*)runtime_mheap->arena_start;
+	b = (uintptr*)runtime_mheap->arena_start - off/wordsPerBitmapWord - 1;
 	shift = off % wordsPerBitmapWord;
 
 	return (*b & (bitSpecial<<shift)) != 0;
@@ -1909,8 +2441,8 @@ runtime_setblockspecial(void *v, bool s)
 	if(DebugMark)
 		return;
 
-	off = (uintptr*)v - (uintptr*)runtime_mheap.arena_start;
-	b = (uintptr*)runtime_mheap.arena_start - off/wordsPerBitmapWord - 1;
+	off = (uintptr*)v - (uintptr*)runtime_mheap->arena_start;
+	b = (uintptr*)runtime_mheap->arena_start - off/wordsPerBitmapWord - 1;
 	shift = off % wordsPerBitmapWord;
 
 	for(;;) {