Imported Upstream version 5.4.0.167

Former-commit-id: 5624ac747d633e885131e8349322922b6a59baaa
2017-08-21 15:34:15 +00:00
parent e49d6f06c0
commit 536cd135cc
12856 changed files with 563812 additions and 223249 deletions
--- a/mono/sgen/Makefile.in
+++ b/mono/sgen/Makefile.in
@@ -319,7 +319,6 @@ MANIFEST_TOOL = @MANIFEST_TOOL@
 MKDIR_P = @MKDIR_P@
 MONO_CORLIB_VERSION = @MONO_CORLIB_VERSION@
 MONO_DL_NEED_USCORE = @MONO_DL_NEED_USCORE@
-MONO_NACL_ALIGN_MASK_OFF = @MONO_NACL_ALIGN_MASK_OFF@
 MSGFMT = @MSGFMT@
 MSGFMT_015 = @MSGFMT_015@
 MSGMERGE = @MSGMERGE@
@@ -417,7 +416,6 @@ mkdir_p = @mkdir_p@
 mono_build_root = @mono_build_root@
 mono_cfg_dir = @mono_cfg_dir@
 mono_runtime = @mono_runtime@
-nacl_self_host = @nacl_self_host@
 oldincludedir = @oldincludedir@
 pdfdir = @pdfdir@
 prefix = @prefix@
--- a/mono/sgen/sgen-alloc.c
+++ b/mono/sgen/sgen-alloc.c
@@ -211,7 +211,7 @@ sgen_alloc_obj_nolock (GCVTable vtable, size_t size)
 			/* when running in degraded mode, we continue allocing that way
 			 * for a while, to decrease the number of useless nursery collections.
 			 */
-			if (degraded_mode && degraded_mode < DEFAULT_NURSERY_SIZE)
+			if (degraded_mode && degraded_mode < sgen_nursery_size)
 				return alloc_degraded (vtable, size, FALSE);

 			available_in_tlab = (int)(TLAB_REAL_END - TLAB_NEXT);//We'll never have tlabs > 2Gb
--- a/mono/sgen/sgen-archdep.h
+++ b/mono/sgen/sgen-archdep.h
@@ -39,13 +39,6 @@

 #define REDZONE_SIZE	224

-/* MS_BLOCK_SIZE must be a multiple of the system pagesize, which for some
-   architectures is 64k.  */
-#if defined(TARGET_POWERPC) || defined(TARGET_POWERPC64)
-#define ARCH_MIN_MS_BLOCK_SIZE	(64*1024)
-#define ARCH_MIN_MS_BLOCK_SIZE_SHIFT	16
-#endif
-
 #elif defined(TARGET_ARM)

 #define REDZONE_SIZE	0
--- a/mono/sgen/sgen-cardtable.c
+++ b/mono/sgen/sgen-cardtable.c
@@ -49,13 +49,6 @@ guint64 remarked_cards;
 static guint64 large_objects;
 static guint64 bloby_objects;
 #endif
-static guint64 major_card_scan_time;
-static guint64 los_card_scan_time;
-
-static guint64 last_major_scan_time;
-static guint64 last_los_scan_time;
-
-static void sgen_card_tables_collect_stats (gboolean begin);

 mword
 sgen_card_table_number_of_cards_in_range (mword address, mword size)
@@ -139,6 +132,36 @@ sgen_card_table_wbarrier_generic_nostore (gpointer ptr)
 	sgen_card_table_mark_address ((mword)ptr);	
 }

+static void
+sgen_card_table_wbarrier_range_copy (gpointer _dest, gpointer _src, int size)
+{
+	GCObject **dest = (GCObject **)_dest;
+	GCObject **src = (GCObject **)_src;
+
+	size_t nursery_bits = sgen_nursery_bits;
+	char *start = sgen_nursery_start;
+	G_GNUC_UNUSED char *end = sgen_nursery_end;
+
+	/*
+	 * It's cardtable theory time!
+	 * Our cardtable scanning code supports marking any card that an object/valuetype belongs to.
+	 * This function is supposed to be used to copy a range that fully belongs to a single type.
+	 * It must not be used, for example, to copy 2 adjacent VTs in an array.
+	*/
+	volatile guint8 *card_address = (volatile guint8 *)sgen_card_table_get_card_address ((mword)dest);
+	while (size) {
+		GCObject *value = *src;
+		*dest = value;
+		if (SGEN_PTR_IN_NURSERY (value, nursery_bits, start, end) || concurrent_collection_in_progress) {
+			*card_address = 1;
+			sgen_dummy_use (value);
+		}
+		++src;
+		++dest;
+		size -= SIZEOF_VOID_P;
+	}
+}
+
 #ifdef SGEN_HAVE_OVERLAPPING_CARDS

 guint8 *sgen_shadow_cardtable;
@@ -417,19 +440,8 @@ sgen_card_table_clear_cards (void)
 }

 static void
-sgen_card_table_finish_minor_collection (void)
+sgen_card_table_start_scan_remsets (void)
 {
-	sgen_card_tables_collect_stats (FALSE);
-}
-
-static void
-sgen_card_table_scan_remsets (ScanCopyContext ctx)
-{
-	SGEN_TV_DECLARE (atv);
-	SGEN_TV_DECLARE (btv);
-
-	sgen_card_tables_collect_stats (TRUE);
-
 #ifdef SGEN_HAVE_OVERLAPPING_CARDS
 	/*FIXME we should have a bit on each block/los object telling if the object have marked cards.*/
 	/*First we copy*/
@@ -440,17 +452,6 @@ sgen_card_table_scan_remsets (ScanCopyContext ctx)
 	/*Then we clear*/
 	sgen_card_table_clear_cards ();
 #endif
-	SGEN_TV_GETTIME (atv);
-	sgen_get_major_collector ()->scan_card_table (CARDTABLE_SCAN_GLOBAL, ctx, 0, 1);
-	SGEN_TV_GETTIME (btv);
-	last_major_scan_time = SGEN_TV_ELAPSED (atv, btv); 
-	major_card_scan_time += last_major_scan_time;
-	sgen_los_scan_card_table (CARDTABLE_SCAN_GLOBAL, ctx, 0, 1);
-	SGEN_TV_GETTIME (atv);
-	last_los_scan_time = SGEN_TV_ELAPSED (btv, atv);
-	los_card_scan_time += last_los_scan_time;
-
-	sgen_wbroots_scan_card_table (ctx);
 }

 guint8*
@@ -573,69 +574,6 @@ sgen_cardtable_scan_object (GCObject *obj, mword block_obj_size, guint8 *cards,
 	binary_protocol_card_scan (obj, sgen_safe_object_get_size (obj));
 }

-#ifdef CARDTABLE_STATS
-
-typedef struct {
-	int total, marked, remarked, gc_marked;	
-} card_stats;
-
-static card_stats major_stats, los_stats;
-static card_stats *cur_stats;
-
-static void
-count_marked_cards (mword start, mword size)
-{
-	mword end = start + size;
-	while (start <= end) {
-		guint8 card = *sgen_card_table_get_card_address (start);
-		++cur_stats->total;
-		if (card)
-			++cur_stats->marked;
-		if (card == 2)
-			++cur_stats->gc_marked;
-		start += CARD_SIZE_IN_BYTES;
-	}
-}
-
-static void
-count_remarked_cards (mword start, mword size)
-{
-	mword end = start + size;
-	while (start <= end) {
-		if (sgen_card_table_address_is_marked (start)) {
-			++cur_stats->remarked;
-			*sgen_card_table_get_card_address (start) = 2;
-		}
-		start += CARD_SIZE_IN_BYTES;
-	}
-}
-
-#endif
-
-static void
-sgen_card_tables_collect_stats (gboolean begin)
-{
-#ifdef CARDTABLE_STATS
-	if (begin) {
-		memset (&major_stats, 0, sizeof (card_stats));
-		memset (&los_stats, 0, sizeof (card_stats));
-		cur_stats = &major_stats;
-		sgen_major_collector_iterate_live_block_ranges (count_marked_cards);
-		cur_stats = &los_stats;
-		sgen_los_iterate_live_block_ranges (count_marked_cards);
-	} else {
-		cur_stats = &major_stats;
-		sgen_major_collector_iterate_live_block_ranges (count_remarked_cards);
-		cur_stats = &los_stats;
-		sgen_los_iterate_live_block_ranges (count_remarked_cards);
-		printf ("cards major (t %d m %d g %d r %d)  los (t %d m %d g %d r %d) major_scan %.2fms los_scan %.2fms\n", 
-			major_stats.total, major_stats.marked, major_stats.gc_marked, major_stats.remarked,
-			los_stats.total, los_stats.marked, los_stats.gc_marked, los_stats.remarked,
-			last_major_scan_time / 10000.0f, last_los_scan_time / 10000.0f);
-	}
-#endif
-}
-
 void
 sgen_card_table_init (SgenRememberedSet *remset)
 {
@@ -654,9 +592,6 @@ sgen_card_table_init (SgenRememberedSet *remset)
 	mono_counters_register ("cardtable large objects", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &large_objects);
 	mono_counters_register ("cardtable bloby objects", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &bloby_objects);
 #endif
-	mono_counters_register ("cardtable major scan time", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &major_card_scan_time);
-	mono_counters_register ("cardtable los scan time", MONO_COUNTER_GC | MONO_COUNTER_ULONG | MONO_COUNTER_TIME, &los_card_scan_time);
-

 	remset->wbarrier_set_field = sgen_card_table_wbarrier_set_field;
 	remset->wbarrier_arrayref_copy = sgen_card_table_wbarrier_arrayref_copy;
@@ -665,13 +600,13 @@ sgen_card_table_init (SgenRememberedSet *remset)
 	remset->wbarrier_generic_nostore = sgen_card_table_wbarrier_generic_nostore;
 	remset->record_pointer = sgen_card_table_record_pointer;

-	remset->scan_remsets = sgen_card_table_scan_remsets;
+	remset->start_scan_remsets = sgen_card_table_start_scan_remsets;

-	remset->finish_minor_collection = sgen_card_table_finish_minor_collection;
 	remset->clear_cards = sgen_card_table_clear_cards;

 	remset->find_address = sgen_card_table_find_address;
 	remset->find_address_with_cards = sgen_card_table_find_address_with_cards;
+	remset->wbarrier_range_copy = sgen_card_table_wbarrier_range_copy;

 	need_mod_union = sgen_get_major_collector ()->is_concurrent;
 }
--- a/mono/sgen/sgen-conf.h
+++ b/mono/sgen/sgen-conf.h
@@ -37,13 +37,6 @@ typedef mword SgenDescriptor;
 #define HEAVY_STAT(x)
 #endif

-/*
- * Define this to allow the user to change the nursery size by
- * specifying its value in the MONO_GC_PARAMS environmental
- * variable. See mono_gc_base_init for details.
- */
-#define USER_CONFIG 1
-
 /*
 * The binary protocol enables logging a lot of the GC ativity in a way that is not very
 * intrusive and produces a compact file that can be searched using a custom tool.  This
@@ -211,4 +204,19 @@ typedef mword SgenDescriptor;
 #define SGEN_CEMENT_HASH(hv)	(((hv) ^ ((hv) >> SGEN_CEMENT_HASH_SHIFT)) & (SGEN_CEMENT_HASH_SIZE - 1))
 #define SGEN_CEMENT_THRESHOLD	1000

+/*
+ * Default values for the nursery size
+ */
+#define SGEN_DEFAULT_NURSERY_MIN_SIZE	(1 << 19)
+#define SGEN_DEFAULT_NURSERY_SIZE	(1 << 22)
+#define SGEN_DEFAULT_NURSERY_MAX_SIZE	(1 << 25)
+
+/*
+ * We are trying to keep pauses lower than this (ms). We use it for dynamic nursery
+ * sizing heuristics. We are keeping leeway in order to be prepared for work-load
+ * variations.
+ */
+#define SGEN_MAX_PAUSE_TIME 30
+#define SGEN_MAX_PAUSE_MARGIN 0.66f
+
 #endif
--- a/mono/sgen/sgen-copy-object.h
+++ b/mono/sgen/sgen-copy-object.h
@@ -108,7 +108,7 @@ copy_object_no_checks_par (GCObject *obj, SgenGrayQueue *queue)
 		 */
 		gboolean has_references = SGEN_VTABLE_HAS_REFERENCES (vt);
 		mword objsize = SGEN_ALIGN_UP (sgen_client_par_object_get_size (vt, obj));
-		destination = major_collector.alloc_object_par (vt, objsize, has_references);
+		destination = COLLECTOR_PARALLEL_ALLOC_FOR_PROMOTION (vt, obj, objsize, has_references);

 		par_copy_object_no_checks ((char*)destination, vt, obj, objsize);

@@ -124,6 +124,12 @@ copy_object_no_checks_par (GCObject *obj, SgenGrayQueue *queue)
 				GRAY_OBJECT_ENQUEUE_PARALLEL (queue, (GCObject *)destination, sgen_vtable_get_descriptor (vt));
 			}
 		} else {
+			/*
+			 * Unlikely case. Clear the allocated object so it doesn't confuse nursery
+			 * card table scanning, since it can contain old invalid refs.
+			 * FIXME make sure it is not a problem if another threads scans it while we clear
+			 */
+			mono_gc_bzero_aligned (destination, objsize);
 			destination = final_destination;
 		}
 	}
@@ -133,5 +139,6 @@ copy_object_no_checks_par (GCObject *obj, SgenGrayQueue *queue)
 #endif

 #undef COLLECTOR_SERIAL_ALLOC_FOR_PROMOTION
+#undef COLLECTOR_PARALLEL_ALLOC_FOR_PROMOTION
 #undef collector_pin_object
 #undef COPY_OR_MARK_PARALLEL
--- a/mono/sgen/sgen-debug.c
+++ b/mono/sgen/sgen-debug.c
@@ -325,7 +325,7 @@ static void
 setup_valid_nursery_objects (void)
 {
 	if (!valid_nursery_objects)
-		valid_nursery_objects = (GCObject **)sgen_alloc_os_memory (DEFAULT_NURSERY_SIZE, (SgenAllocFlags)(SGEN_ALLOC_INTERNAL | SGEN_ALLOC_ACTIVATE), "debugging data", MONO_MEM_ACCOUNT_SGEN_DEBUGGING);
+		valid_nursery_objects = (GCObject **)sgen_alloc_os_memory (sgen_nursery_max_size, (SgenAllocFlags)(SGEN_ALLOC_INTERNAL | SGEN_ALLOC_ACTIVATE), "debugging data", MONO_MEM_ACCOUNT_SGEN_DEBUGGING);
 	valid_nursery_object_count = 0;
 	sgen_scan_area_with_callback (nursery_section->data, nursery_section->end_data, setup_mono_sgen_scan_area_with_callback, NULL, FALSE, FALSE);
 }
@@ -1065,10 +1065,10 @@ void
 sgen_dump_section (GCMemSection *section, const char *type)
 {
 	char *start = section->data;
-	char *end = section->data + section->size;
+	char *end = section->end_data;
 	char *occ_start = NULL;

-	fprintf (heap_dump_file, "<section type=\"%s\" size=\"%lu\">\n", type, (unsigned long)section->size);
+	fprintf (heap_dump_file, "<section type=\"%s\" size=\"%lu\">\n", type, (unsigned long)(section->end_data - section->data));

 	while (start < end) {
 		guint size;
@@ -1083,7 +1083,6 @@ sgen_dump_section (GCMemSection *section, const char *type)
 			start += sizeof (void*); /* should be ALLOC_ALIGN, really */
 			continue;
 		}
-		g_assert (start < section->next_data);

 		if (!occ_start)
 			occ_start = start;
--- a/mono/sgen/sgen-gc.c.REMOVED.git-id
+++ b/mono/sgen/sgen-gc.c.REMOVED.git-id
@@ -1 +1 @@
-ab46a371126ca76de0901d9cb657988d91409d4f
+66eb9ab194bb76293336041dfca14767efe3fb3a
--- a/mono/sgen/sgen-gc.h
+++ b/mono/sgen/sgen-gc.h
@@ -35,6 +35,7 @@ typedef struct _SgenThreadInfo SgenThreadInfo;
 #include "mono/sgen/sgen-hash-table.h"
 #include "mono/sgen/sgen-protocol.h"
 #include "mono/sgen/gc-internal-agnostic.h"
+#include "mono/sgen/sgen-thread-pool.h"

 /* The method used to clear the nursery */
 /* Clearing at nursery collections is the safest, but has bad interactions with caches.
@@ -59,9 +60,6 @@ NurseryClearPolicy sgen_get_nursery_clear_policy (void);
 typedef struct _GCMemSection GCMemSection;
 struct _GCMemSection {
 	char *data;
-	mword size;
-	/* pointer where more data could be allocated if it fits */
-	char *next_data;
 	char *end_data;
 	/*
 	 * scan starts is an array of pointers to objects equally spaced in the allocation area
@@ -185,29 +183,17 @@ sgen_aligned_addr_hash (gconstpointer ptr)

 #define SGEN_PTR_IN_NURSERY(p,bits,start,end)	(((mword)(p) & ~(((mword)1 << (bits)) - 1)) == (mword)(start))

-#ifdef USER_CONFIG
-
-/* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
-#define DEFAULT_NURSERY_SIZE (sgen_nursery_size)
 extern size_t sgen_nursery_size;
-/* The number of trailing 0 bits in DEFAULT_NURSERY_SIZE */
-#define DEFAULT_NURSERY_BITS (sgen_nursery_bits)
+extern size_t sgen_nursery_max_size;
 extern int sgen_nursery_bits;

-#else
-
-#define DEFAULT_NURSERY_SIZE (4*1024*1024)
-#define DEFAULT_NURSERY_BITS 22
-
-#endif
-
 extern char *sgen_nursery_start;
 extern char *sgen_nursery_end;

 static inline MONO_ALWAYS_INLINE gboolean
 sgen_ptr_in_nursery (void *p)
 {
-	return SGEN_PTR_IN_NURSERY ((p), DEFAULT_NURSERY_BITS, sgen_nursery_start, sgen_nursery_end);
+	return SGEN_PTR_IN_NURSERY ((p), sgen_nursery_bits, sgen_nursery_start, sgen_nursery_end);
 }

 static inline MONO_ALWAYS_INLINE char*
@@ -494,7 +480,6 @@ typedef struct {

 void sgen_fragment_allocator_add (SgenFragmentAllocator *allocator, char *start, char *end);
 void sgen_fragment_allocator_release (SgenFragmentAllocator *allocator);
-void* sgen_fragment_allocator_serial_alloc (SgenFragmentAllocator *allocator, size_t size);
 void* sgen_fragment_allocator_par_alloc (SgenFragmentAllocator *allocator, size_t size);
 void* sgen_fragment_allocator_serial_range_alloc (SgenFragmentAllocator *allocator, size_t desired_size, size_t minimum_size, size_t *out_alloc_size);
 void* sgen_fragment_allocator_par_range_alloc (SgenFragmentAllocator *allocator, size_t desired_size, size_t minimum_size, size_t *out_alloc_size);
@@ -559,11 +544,14 @@ sgen_nursery_is_object_alive (GCObject *obj)

 typedef struct {
 	gboolean is_split;
+	gboolean is_parallel;

 	GCObject* (*alloc_for_promotion) (GCVTable vtable, GCObject *obj, size_t objsize, gboolean has_references);
+	GCObject* (*alloc_for_promotion_par) (GCVTable vtable, GCObject *obj, size_t objsize, gboolean has_references);

 	SgenObjectOperations serial_ops;
 	SgenObjectOperations serial_ops_with_concurrent_major;
+	SgenObjectOperations parallel_ops;

 	void (*prepare_to_space) (char *to_space_bitmap, size_t space_bitmap_size);
 	void (*clear_fragments) (void);
@@ -578,7 +566,7 @@ typedef struct {

 extern SgenMinorCollector sgen_minor_collector;

-void sgen_simple_nursery_init (SgenMinorCollector *collector);
+void sgen_simple_nursery_init (SgenMinorCollector *collector, gboolean parallel);
 void sgen_split_nursery_init (SgenMinorCollector *collector);

 /* Updating references */
@@ -591,7 +579,7 @@ sgen_update_reference (GCObject **p, GCObject *o, gboolean allow_null)
 {
 	if (!allow_null)
 		SGEN_ASSERT (0, o, "Cannot update a reference with a NULL pointer");
-	SGEN_ASSERT (0, !sgen_thread_pool_is_thread_pool_thread (mono_native_thread_id_get ()), "Can't update a reference in the worker thread");
+	SGEN_ASSERT (0, !sgen_workers_is_worker_thread (mono_native_thread_id_get ()), "Can't update a reference in the worker thread");
 	*p = o;
 }

@@ -634,11 +622,10 @@ struct _SgenMajorCollector {
 	size_t section_size;
 	gboolean is_concurrent;
 	gboolean is_parallel;
-	gboolean needs_thread_pool;
 	gboolean supports_cardtable;
 	gboolean sweeps_lazily;

-	void* (*alloc_heap) (mword nursery_size, mword nursery_align, int nursery_bits);
+	void* (*alloc_heap) (mword nursery_size, mword nursery_align);
 	gboolean (*is_object_live) (GCObject *obj);
 	GCObject* (*alloc_small_pinned_obj) (GCVTable vtable, size_t size, gboolean has_references);
 	GCObject* (*alloc_degraded) (GCVTable vtable, size_t size);
@@ -691,6 +678,7 @@ struct _SgenMajorCollector {
 	guint8* (*get_cardtable_mod_union_for_reference) (char *object);
 	long long (*get_and_reset_num_major_objects_marked) (void);
 	void (*count_cards) (long long *num_total_cards, long long *num_marked_cards);
+	SgenThreadPool* (*get_sweep_pool) (void);

 	void (*worker_init_cb) (gpointer worker);
 };
@@ -701,6 +689,7 @@ void sgen_marksweep_init (SgenMajorCollector *collector);
 void sgen_marksweep_conc_init (SgenMajorCollector *collector);
 void sgen_marksweep_conc_par_init (SgenMajorCollector *collector);
 SgenMajorCollector* sgen_get_major_collector (void);
+SgenMinorCollector* sgen_get_minor_collector (void);


 typedef struct _SgenRememberedSet {
@@ -710,12 +699,12 @@ typedef struct _SgenRememberedSet {
 	void (*wbarrier_object_copy) (GCObject* obj, GCObject *src);
 	void (*wbarrier_generic_nostore) (gpointer ptr);
 	void (*record_pointer) (gpointer ptr);
+	void (*wbarrier_range_copy) (gpointer dest, gpointer src, int count);

-	void (*scan_remsets) (ScanCopyContext ctx);
+	void (*start_scan_remsets) (void);

 	void (*clear_cards) (void);

-	void (*finish_minor_collection) (void);
 	gboolean (*find_address) (char *addr);
 	gboolean (*find_address_with_cards) (char *cards_start, guint8 *cards, char *addr);
 } SgenRememberedSet;
@@ -731,7 +720,7 @@ void mono_gc_wbarrier_generic_nostore (gpointer ptr);
 void mono_gc_wbarrier_generic_store (gpointer ptr, GCObject* value);
 void mono_gc_wbarrier_generic_store_atomic (gpointer ptr, GCObject *value);

-void sgen_wbarrier_value_copy_bitmap (gpointer _dest, gpointer _src, int size, unsigned bitmap);
+void sgen_wbarrier_range_copy (gpointer _dest, gpointer _src, int size);

 static inline SgenDescriptor
 sgen_obj_get_descriptor (GCObject *obj)
@@ -908,12 +897,12 @@ void sgen_los_mark_mod_union_card (GCObject *mono_obj, void **ptr);

 void sgen_clear_nursery_fragments (void);
 void sgen_nursery_allocator_prepare_for_pinning (void);
-void sgen_nursery_allocator_set_nursery_bounds (char *nursery_start, char *nursery_end);
+void sgen_nursery_allocator_set_nursery_bounds (char *nursery_start, size_t min_size, size_t max_size);
+void sgen_resize_nursery (gboolean need_shrink);
 mword sgen_build_nursery_fragments (GCMemSection *nursery_section, SgenGrayQueue *unpin_queue);
 void sgen_init_nursery_allocator (void);
 void sgen_nursery_allocator_init_heavy_stats (void);
 void sgen_init_allocator (void);
-char* sgen_nursery_alloc_get_upper_alloc_bound (void);
 void* sgen_nursery_alloc (size_t size);
 void* sgen_nursery_alloc_range (size_t size, size_t min_size, size_t *out_alloc_size);
 gboolean sgen_can_alloc_size (size_t size);
@@ -1000,6 +989,7 @@ extern mword total_promoted_size;
 extern mword total_allocated_major;
 extern volatile gboolean sgen_suspend_finalizers;
 extern MonoCoopMutex gc_mutex;
+extern volatile gboolean concurrent_collection_in_progress;

 /* Nursery helpers. */

--- a/mono/sgen/sgen-gray.h
+++ b/mono/sgen/sgen-gray.h
@@ -47,7 +47,7 @@
 #define SGEN_GRAY_QUEUE_HEADER_SIZE	3
 #endif

-#define SGEN_GRAY_QUEUE_SECTION_SIZE	(128 - SGEN_GRAY_QUEUE_HEADER_SIZE)
+#define SGEN_GRAY_QUEUE_SECTION_SIZE	(512 - SGEN_GRAY_QUEUE_HEADER_SIZE)

 #ifdef SGEN_CHECK_GRAY_OBJECT_SECTIONS
 typedef enum {
--- a/mono/sgen/sgen-major-copy-object.h
+++ b/mono/sgen/sgen-major-copy-object.h
@@ -19,5 +19,6 @@
 } while (0)

 #define COLLECTOR_SERIAL_ALLOC_FOR_PROMOTION sgen_minor_collector.alloc_for_promotion
+#define COLLECTOR_PARALLEL_ALLOC_FOR_PROMOTION sgen_minor_collector.alloc_for_promotion_par

 #include "sgen-copy-object.h"
--- a/mono/sgen/sgen-marksweep.c
+++ b/mono/sgen/sgen-marksweep.c
@@ -33,15 +33,15 @@
 #include "mono/sgen/sgen-client.h"
 #include "mono/utils/mono-memory-model.h"

-#if defined(ARCH_MIN_MS_BLOCK_SIZE) && defined(ARCH_MIN_MS_BLOCK_SIZE_SHIFT)
-#define MS_BLOCK_SIZE	ARCH_MIN_MS_BLOCK_SIZE
-#define MS_BLOCK_SIZE_SHIFT	ARCH_MIN_MS_BLOCK_SIZE_SHIFT
-#else
-#define MS_BLOCK_SIZE_SHIFT     14      /* INT FASTENABLE */
-#define MS_BLOCK_SIZE           (1 << MS_BLOCK_SIZE_SHIFT)
-#endif
-#define MAJOR_SECTION_SIZE	MS_BLOCK_SIZE
-#define CARDS_PER_BLOCK (MS_BLOCK_SIZE / CARD_SIZE_IN_BYTES)
+static int ms_block_size;
+
+/*
+ * Blocks must be at least this size, meaning that if we detect a
+ * page size lower than this, we'll use this instead.
+ */
+#define MS_BLOCK_SIZE_MIN (1024 * 16)
+
+#define CARDS_PER_BLOCK (ms_block_size / CARD_SIZE_IN_BYTES)

 /*
 * Don't allocate single blocks, but alloc a contingent of this many
@@ -49,16 +49,22 @@
 */
 #define MS_BLOCK_ALLOC_NUM	32

+#define MS_NUM_MARK_WORDS	((ms_block_size / SGEN_ALLOC_ALIGN + sizeof (guint32) * 8 - 1) / (sizeof (guint32) * 8))
+
+/*
+ * Use this instead of sizeof (MSBlockInfo) since the mark_words
+ * array size depends on page size at runtime.
+ */
+#define SIZEOF_MS_BLOCK_INFO (sizeof (MSBlockInfo) + sizeof (guint32) * (MS_NUM_MARK_WORDS - MONO_ZERO_LEN_ARRAY))
+
 /*
 * Number of bytes before the first object in a block.  At the start
 * of a block is the MSBlockHeader, then opional padding, then come
- * the objects, so this must be >= sizeof (MSBlockHeader).
+ * the objects, so this must be >= SIZEOF_MS_BLOCK_INFO.
 */
-#define MS_BLOCK_SKIP	((sizeof (MSBlockHeader) + 15) & ~15)
+#define MS_BLOCK_SKIP	((SIZEOF_MS_BLOCK_INFO + 15) & ~15)

-#define MS_BLOCK_FREE	(MS_BLOCK_SIZE - MS_BLOCK_SKIP)
-
-#define MS_NUM_MARK_WORDS	(MS_BLOCK_SIZE / SGEN_ALLOC_ALIGN + sizeof (guint32) * 8 - 1) / (sizeof (guint32) * 8)
+#define MS_BLOCK_FREE	(ms_block_size - MS_BLOCK_SKIP)

 /*
 * Blocks progress from one state to the next:
@@ -108,14 +114,14 @@ struct _MSBlockInfo {
 	void ** volatile free_list;
 	MSBlockInfo * volatile next_free;
 	guint8 * volatile cardtable_mod_union;
-	guint32 mark_words [MS_NUM_MARK_WORDS];
+	guint32 mark_words [MONO_ZERO_LEN_ARRAY];
 };

 #define MS_BLOCK_FOR_BLOCK_INFO(b)	((char*)(b))

 #define MS_BLOCK_OBJ(b,i)		((GCObject *)(MS_BLOCK_FOR_BLOCK_INFO(b) + MS_BLOCK_SKIP + (b)->obj_size * (i)))
 #define MS_BLOCK_OBJ_FOR_SIZE(b,i,obj_size)		(MS_BLOCK_FOR_BLOCK_INFO(b) + MS_BLOCK_SKIP + (obj_size) * (i))
-#define MS_BLOCK_DATA_FOR_OBJ(o)	((char*)((mword)(o) & ~(mword)(MS_BLOCK_SIZE - 1)))
+#define MS_BLOCK_DATA_FOR_OBJ(o)	((char*)((mword)(o) & ~(mword)(ms_block_size - 1)))

 typedef struct {
 	MSBlockInfo info;
@@ -150,7 +156,7 @@ typedef struct {
 	} while (0)


-#define MS_OBJ_ALLOCED(o,b)	(*(void**)(o) && (*(char**)(o) < MS_BLOCK_FOR_BLOCK_INFO (b) || *(char**)(o) >= MS_BLOCK_FOR_BLOCK_INFO (b) + MS_BLOCK_SIZE))
+#define MS_OBJ_ALLOCED(o,b)	(*(void**)(o) && (*(char**)(o) < MS_BLOCK_FOR_BLOCK_INFO (b) || *(char**)(o) >= MS_BLOCK_FOR_BLOCK_INFO (b) + ms_block_size))

 #define MS_BLOCK_OBJ_SIZE_FACTOR	(pow (2.0, 1.0 / 3))

@@ -187,6 +193,9 @@ static volatile int sweep_state = SWEEP_STATE_SWEPT;
 static gboolean concurrent_mark;
 static gboolean concurrent_sweep = TRUE;

+SgenThreadPool sweep_pool_inst;
+SgenThreadPool *sweep_pool;
+
 #define BLOCK_IS_TAGGED_HAS_REFERENCES(bl)	SGEN_POINTER_IS_TAGGED_1 ((bl))
 #define BLOCK_TAG_HAS_REFERENCES(bl)		SGEN_POINTER_TAG_1 ((bl))

@@ -303,7 +312,7 @@ ms_find_block_obj_size_index (size_t size)
 	 ms_find_block_obj_size_index ((s)))

 static void*
-major_alloc_heap (mword nursery_size, mword nursery_align, int the_nursery_bits)
+major_alloc_heap (mword nursery_size, mword nursery_align)
 {
 	char *start;
 	if (nursery_align)
@@ -317,7 +326,7 @@ major_alloc_heap (mword nursery_size, mword nursery_align, int the_nursery_bits)
 static void
 update_heap_boundaries_for_block (MSBlockInfo *block)
 {
-	sgen_update_heap_boundaries ((mword)MS_BLOCK_FOR_BLOCK_INFO (block), (mword)MS_BLOCK_FOR_BLOCK_INFO (block) + MS_BLOCK_SIZE);
+	sgen_update_heap_boundaries ((mword)MS_BLOCK_FOR_BLOCK_INFO (block), (mword)MS_BLOCK_FOR_BLOCK_INFO (block) + ms_block_size);
 }

 /*
@@ -339,7 +348,7 @@ ms_get_empty_block (void)
 		 */
 		int alloc_num = MS_BLOCK_ALLOC_NUM;
 		for (;;) {
-			p = (char *)sgen_alloc_os_memory_aligned (MS_BLOCK_SIZE * alloc_num, MS_BLOCK_SIZE,
+			p = (char *)sgen_alloc_os_memory_aligned (ms_block_size * alloc_num, ms_block_size,
 				(SgenAllocFlags)(SGEN_ALLOC_HEAP | SGEN_ALLOC_ACTIVATE),
 				alloc_num == 1 ? "major heap section" : NULL, MONO_MEM_ACCOUNT_SGEN_MARKSWEEP);
 			if (p)
@@ -358,7 +367,7 @@ ms_get_empty_block (void)
 				empty = empty_blocks;
 				*(void**)block = empty;
 			} while (SGEN_CAS_PTR ((gpointer*)&empty_blocks, block, empty) != empty);
-			p += MS_BLOCK_SIZE;
+			p += ms_block_size;
 		}

 		SGEN_ATOMIC_ADD_P (num_empty_blocks, alloc_num);
@@ -382,7 +391,7 @@ ms_get_empty_block (void)

 	*(void**)block = NULL;

-	g_assert (!((mword)block & (MS_BLOCK_SIZE - 1)));
+	g_assert (!((mword)block & (ms_block_size - 1)));

 	return block;
 }
@@ -397,10 +406,10 @@ ms_free_block (MSBlockInfo *info)
 	void *empty;
 	char *block = MS_BLOCK_FOR_BLOCK_INFO (info);

-	sgen_memgov_release_space (MS_BLOCK_SIZE, SPACE_MAJOR);
+	sgen_memgov_release_space (ms_block_size, SPACE_MAJOR);
 	if (info->cardtable_mod_union)
-		sgen_card_table_free_mod_union (info->cardtable_mod_union, block, MS_BLOCK_SIZE);
-	memset (block, 0, MS_BLOCK_SIZE);
+		sgen_card_table_free_mod_union (info->cardtable_mod_union, block, ms_block_size);
+	memset (block, 0, ms_block_size);

 	do {
 		empty = empty_blocks;
@@ -409,7 +418,7 @@ ms_free_block (MSBlockInfo *info)

 	SGEN_ATOMIC_ADD_P (num_empty_blocks, 1);

-	binary_protocol_block_free (block, MS_BLOCK_SIZE);
+	binary_protocol_block_free (block, ms_block_size);
 }

 static gboolean
@@ -524,7 +533,7 @@ ms_alloc_block (int size_index, gboolean pinned, gboolean has_references)
 	char *obj_start;
 	int i;

-	if (!sgen_memgov_try_alloc_space (MS_BLOCK_SIZE, SPACE_MAJOR))
+	if (!sgen_memgov_try_alloc_space (ms_block_size, SPACE_MAJOR))
 		return FALSE;

 	info = (MSBlockInfo*)ms_get_empty_block ();
@@ -550,7 +559,7 @@ ms_alloc_block (int size_index, gboolean pinned, gboolean has_references)

 	update_heap_boundaries_for_block (info);

-	binary_protocol_block_alloc (info, MS_BLOCK_SIZE);
+	binary_protocol_block_alloc (info, ms_block_size);

 	/* build free list */
 	obj_start = MS_BLOCK_FOR_BLOCK_INFO (info) + MS_BLOCK_SKIP;
@@ -578,7 +587,7 @@ ptr_is_in_major_block (char *ptr, char **start, gboolean *pinned)
 	MSBlockInfo *block;

 	FOREACH_BLOCK_NO_LOCK (block) {
-		if (ptr >= MS_BLOCK_FOR_BLOCK_INFO (block) && ptr <= MS_BLOCK_FOR_BLOCK_INFO (block) + MS_BLOCK_SIZE) {
+		if (ptr >= MS_BLOCK_FOR_BLOCK_INFO (block) && ptr <= MS_BLOCK_FOR_BLOCK_INFO (block) + ms_block_size) {
 			int count = MS_BLOCK_FREE / block->obj_size;
 			int i;

@@ -727,7 +736,6 @@ get_block:
 			 */
 			if (SGEN_CAS_PTR ((volatile gpointer *)&free_blocks [size_index], next_free, block) != block)
 				goto get_block;
-			g_assert (block->free_list);
 			block->next_free = free_blocks_local [size_index];
 			free_blocks_local [size_index] = block;

@@ -919,7 +927,7 @@ major_finish_sweep_checking (void)
 wait:
 	job = sweep_job;
 	if (job)
-		sgen_thread_pool_job_wait (job);
+		sgen_thread_pool_job_wait (sweep_pool, job);
 	SGEN_ASSERT (0, !sweep_job, "Why did the sweep job not null itself?");
 	SGEN_ASSERT (0, sweep_state == SWEEP_STATE_SWEPT, "How is the sweep job done but we're not swept?");
 }
@@ -964,7 +972,7 @@ major_is_valid_object (char *object)
 		int idx;
 		char *obj;

-		if ((MS_BLOCK_FOR_BLOCK_INFO (block) > object) || ((MS_BLOCK_FOR_BLOCK_INFO (block) + MS_BLOCK_SIZE) <= object))
+		if ((MS_BLOCK_FOR_BLOCK_INFO (block) > object) || ((MS_BLOCK_FOR_BLOCK_INFO (block) + ms_block_size) <= object))
 			continue;

 		idx = MS_BLOCK_OBJ_INDEX (object, block);
@@ -991,7 +999,7 @@ major_describe_pointer (char *ptr)
 		int w, b;
 		gboolean marked;

-		if ((MS_BLOCK_FOR_BLOCK_INFO (block) > ptr) || ((MS_BLOCK_FOR_BLOCK_INFO (block) + MS_BLOCK_SIZE) <= ptr))
+		if ((MS_BLOCK_FOR_BLOCK_INFO (block) > ptr) || ((MS_BLOCK_FOR_BLOCK_INFO (block) + ms_block_size) <= ptr))
 			continue;

 		SGEN_LOG (0, "major-ptr (block %p sz %d pin %d ref %d)\n",
@@ -1092,13 +1100,13 @@ get_cardtable_mod_union_for_block (MSBlockInfo *block, gboolean allocate)
 		return mod_union;
 	else if (!allocate)
 		return NULL;
-	mod_union = sgen_card_table_alloc_mod_union (MS_BLOCK_FOR_BLOCK_INFO (block), MS_BLOCK_SIZE);
+	mod_union = sgen_card_table_alloc_mod_union (MS_BLOCK_FOR_BLOCK_INFO (block), ms_block_size);
 	other = (guint8 *)SGEN_CAS_PTR ((gpointer*)&block->cardtable_mod_union, mod_union, NULL);
 	if (!other) {
 		SGEN_ASSERT (0, block->cardtable_mod_union == mod_union, "Why did CAS not replace?");
 		return mod_union;
 	}
-	sgen_card_table_free_mod_union (mod_union, MS_BLOCK_FOR_BLOCK_INFO (block), MS_BLOCK_SIZE);
+	sgen_card_table_free_mod_union (mod_union, MS_BLOCK_FOR_BLOCK_INFO (block), ms_block_size);
 	return other;
 }

@@ -1401,10 +1409,10 @@ static inline void
 sweep_block_for_size (MSBlockInfo *block, int count, int obj_size)
 {
 	int obj_index;
+	void *obj = MS_BLOCK_OBJ_FOR_SIZE (block, 0, obj_size);

-	for (obj_index = 0; obj_index < count; ++obj_index) {
+	for (obj_index = 0; obj_index < count; ++obj_index, obj = (void*)((mword)obj + obj_size)) {
 		int word, bit;
-		void *obj = MS_BLOCK_OBJ_FOR_SIZE (block, obj_index, obj_size);

 		MS_CALC_MARK_BIT (word, bit, obj);
 		if (MS_MARK_BIT (block, word, bit)) {
@@ -1433,7 +1441,7 @@ try_set_block_state (MSBlockInfo *block, gint32 new_state, gint32 expected_state
 	gint32 old_state = SGEN_CAS (&block->state, new_state, expected_state);
 	gboolean success = old_state == expected_state;
 	if (success)
-		binary_protocol_block_set_state (block, MS_BLOCK_SIZE, old_state, new_state);
+		binary_protocol_block_set_state (block, ms_block_size, old_state, new_state);
 	return success;
 }

@@ -1442,7 +1450,7 @@ set_block_state (MSBlockInfo *block, gint32 new_state, gint32 expected_state)
 {
 	SGEN_ASSERT (6, block->state == expected_state, "Block state incorrect before set");
 	block->state = new_state;
-	binary_protocol_block_set_state (block, MS_BLOCK_SIZE, expected_state, new_state);
+	binary_protocol_block_set_state (block, ms_block_size, expected_state, new_state);
 }

 /*
@@ -1556,6 +1564,25 @@ sgen_worker_clear_free_block_lists (WorkerData *worker)
 	}
 }

+static void
+sgen_worker_clear_free_block_lists_evac (WorkerData *worker)
+{
+	int i, j;
+
+	if (!worker->free_block_lists)
+		return;
+
+	for (i = 0; i < MS_BLOCK_TYPE_MAX; i++) {
+		for (j = 0; j < num_block_obj_sizes; j++) {
+			if (((MSBlockInfo***) worker->free_block_lists) [i][j])
+				SGEN_ASSERT (0, !((MSBlockInfo***) worker->free_block_lists) [i][j]->next_free, "Why do we have linked free blocks on the workers");
+
+			if (evacuate_block_obj_sizes [j])
+				((MSBlockInfo***) worker->free_block_lists) [i][j] = NULL;
+		}
+	}
+}
+
 static void
 sweep_start (void)
 {
@@ -1788,7 +1815,7 @@ sweep_job_func (void *thread_data_untyped, SgenThreadPoolJob *job)
 	 */
 	if (concurrent_sweep && lazy_sweep) {
 		sweep_blocks_job = sgen_thread_pool_job_alloc ("sweep_blocks", sweep_blocks_job_func, sizeof (SgenThreadPoolJob));
-		sgen_thread_pool_job_enqueue (sweep_blocks_job);
+		sgen_thread_pool_job_enqueue (sweep_pool, sweep_blocks_job);
 	}

 	sweep_finish ();
@@ -1837,7 +1864,7 @@ major_sweep (void)
 	SGEN_ASSERT (0, !sweep_job, "We haven't finished the last sweep?");
 	if (concurrent_sweep) {
 		sweep_job = sgen_thread_pool_job_alloc ("sweep", sweep_job_func, sizeof (SgenThreadPoolJob));
-		sgen_thread_pool_job_enqueue (sweep_job);
+		sgen_thread_pool_job_enqueue (sweep_pool, sweep_job);
 	} else {
 		sweep_job_func (NULL, NULL);
 	}
@@ -2039,6 +2066,9 @@ major_start_major_collection (void)
 		sgen_evacuation_freelist_blocks (&free_block_lists [MS_BLOCK_FLAG_REFS][i], i);
 	}

+	/* We expect workers to have very few blocks on the freelist, just evacuate them */
+	sgen_workers_foreach (sgen_worker_clear_free_block_lists_evac);
+
 	if (lazy_sweep && concurrent_sweep) {
 		/*
 		 * sweep_blocks_job is created before sweep_finish, which we wait for above
@@ -2047,7 +2077,7 @@ major_start_major_collection (void)
 		 */
 		SgenThreadPoolJob *job = sweep_blocks_job;
 		if (job)
-			sgen_thread_pool_job_wait (job);
+			sgen_thread_pool_job_wait (sweep_pool, job);
 	}

 	if (lazy_sweep && !concurrent_sweep)
@@ -2087,6 +2117,12 @@ major_finish_major_collection (ScannedObjectCounts *counts)
 #endif
 }

+static SgenThreadPool*
+major_get_sweep_pool (void)
+{
+	return sweep_pool;
+}
+
 static int
 compare_pointers (const void *va, const void *vb) {
 	char *a = *(char**)va, *b = *(char**)vb;
@@ -2171,7 +2207,7 @@ major_free_swept_blocks (size_t section_reserve)

 				SGEN_ASSERT (6, first >= 0 && d > first, "algorithm is wrong");

-				if ((char*)block != ((char*)empty_block_arr [d-1]) + MS_BLOCK_SIZE) {
+				if ((char*)block != ((char*)empty_block_arr [d-1]) + ms_block_size) {
 					first = d;
 					continue;
 				}
@@ -2185,7 +2221,7 @@ major_free_swept_blocks (size_t section_reserve)
 					 * we're iterating.
 					 */
 					int j;
-					sgen_free_os_memory (empty_block_arr [first], MS_BLOCK_SIZE * num_blocks, SGEN_ALLOC_HEAP, MONO_MEM_ACCOUNT_SGEN_MARKSWEEP);
+					sgen_free_os_memory (empty_block_arr [first], ms_block_size * num_blocks, SGEN_ALLOC_HEAP, MONO_MEM_ACCOUNT_SGEN_MARKSWEEP);
 					for (j = first; j <= d; ++j)
 						empty_block_arr [j] = NULL;
 					dest = first;
@@ -2235,7 +2271,7 @@ major_free_swept_blocks (size_t section_reserve)

 	while (num_empty_blocks > section_reserve) {
 		void *next = *(void**)empty_blocks;
-		sgen_free_os_memory (empty_blocks, MS_BLOCK_SIZE, SGEN_ALLOC_HEAP, MONO_MEM_ACCOUNT_SGEN_MARKSWEEP);
+		sgen_free_os_memory (empty_blocks, ms_block_size, SGEN_ALLOC_HEAP, MONO_MEM_ACCOUNT_SGEN_MARKSWEEP);
 		empty_blocks = next;
 		/*
 		 * Needs not be atomic because this is running
@@ -2256,7 +2292,7 @@ major_pin_objects (SgenGrayQueue *queue)
 	FOREACH_BLOCK_NO_LOCK (block) {
 		size_t first_entry, last_entry;
 		SGEN_ASSERT (6, block_is_swept_or_marking (block), "All blocks must be swept when we're pinning.");
-		sgen_find_optimized_pin_queue_area (MS_BLOCK_FOR_BLOCK_INFO (block) + MS_BLOCK_SKIP, MS_BLOCK_FOR_BLOCK_INFO (block) + MS_BLOCK_SIZE,
+		sgen_find_optimized_pin_queue_area (MS_BLOCK_FOR_BLOCK_INFO (block) + MS_BLOCK_SKIP, MS_BLOCK_FOR_BLOCK_INFO (block) + ms_block_size,
 				&first_entry, &last_entry);
 		mark_pinned_objects_in_block (block, first_entry, last_entry, queue);
 	} END_FOREACH_BLOCK_NO_LOCK;
@@ -2312,7 +2348,7 @@ static size_t
 get_bytes_survived_last_sweep (void)
 {
 	SGEN_ASSERT (0, sweep_state == SWEEP_STATE_SWEPT, "Can only query unswept sections after sweep");
-	return (num_major_sections_before_sweep - num_major_sections_freed_in_sweep) * MS_BLOCK_SIZE;
+	return (num_major_sections_before_sweep - num_major_sections_freed_in_sweep) * ms_block_size;
 }

 static gboolean
@@ -2366,7 +2402,7 @@ major_iterate_block_ranges (sgen_cardtable_block_callback callback)

 	FOREACH_BLOCK_HAS_REFERENCES_NO_LOCK (block, has_references) {
 		if (has_references)
-			callback ((mword)MS_BLOCK_FOR_BLOCK_INFO (block), MS_BLOCK_SIZE);
+			callback ((mword)MS_BLOCK_FOR_BLOCK_INFO (block), ms_block_size);
 	} END_FOREACH_BLOCK_NO_LOCK;
 }

@@ -2379,7 +2415,7 @@ major_iterate_live_block_ranges (sgen_cardtable_block_callback callback)
 	major_finish_sweep_checking ();
 	FOREACH_BLOCK_HAS_REFERENCES_NO_LOCK (block, has_references) {
 		if (has_references)
-			callback ((mword)MS_BLOCK_FOR_BLOCK_INFO (block), MS_BLOCK_SIZE);
+			callback ((mword)MS_BLOCK_FOR_BLOCK_INFO (block), ms_block_size);
 	} END_FOREACH_BLOCK_NO_LOCK;
 }

@@ -2400,7 +2436,7 @@ static guint8*
 initial_skip_card (guint8 *card_data)
 {
 	mword *cards = (mword*)card_data;
-	mword card;
+	mword card = 0;
 	int i;
 	for (i = 0; i < CARD_WORDS_PER_BLOCK; ++i) {
 		card = cards [i];
@@ -2428,17 +2464,22 @@ initial_skip_card (guint8 *card_data)

 #define MS_BLOCK_OBJ_INDEX_FAST(o,b,os)	(((char*)(o) - ((b) + MS_BLOCK_SKIP)) / (os))
 #define MS_BLOCK_OBJ_FAST(b,os,i)			((b) + MS_BLOCK_SKIP + (os) * (i))
-#define MS_OBJ_ALLOCED_FAST(o,b)		(*(void**)(o) && (*(char**)(o) < (b) || *(char**)(o) >= (b) + MS_BLOCK_SIZE))
+#define MS_OBJ_ALLOCED_FAST(o,b)		(*(void**)(o) && (*(char**)(o) < (b) || *(char**)(o) >= (b) + ms_block_size))

 static void
 scan_card_table_for_block (MSBlockInfo *block, CardTableScanType scan_type, ScanCopyContext ctx)
 {
 	SgenGrayQueue *queue = ctx.queue;
 	ScanObjectFunc scan_func = ctx.ops->scan_object;
+	/*
+	 * FIXME: On systems with very large pages, we allocate fairly large
+	 * arrays on the stack here. This shouldn't be a problem once block
+	 * size is no longer required to be a multiple of the system page size.
+	 */
 #ifndef SGEN_HAVE_OVERLAPPING_CARDS
-	guint8 cards_copy [CARDS_PER_BLOCK];
+	guint8 *cards_copy = alloca (sizeof (guint8) * CARDS_PER_BLOCK);
 #endif
-	guint8 cards_preclean [CARDS_PER_BLOCK];
+	guint8 *cards_preclean = alloca (sizeof (guint8) * CARDS_PER_BLOCK);
 	gboolean small_objects;
 	int block_obj_size;
 	char *block_start;
@@ -2681,7 +2722,7 @@ update_cardtable_mod_union (void)
 		if (has_dirty_cards) {
 			size_t num_cards;
 			guint8 *mod_union = get_cardtable_mod_union_for_block (block, TRUE);
-			sgen_card_table_update_mod_union (mod_union, MS_BLOCK_FOR_BLOCK_INFO (block), MS_BLOCK_SIZE, &num_cards);
+			sgen_card_table_update_mod_union (mod_union, MS_BLOCK_FOR_BLOCK_INFO (block), ms_block_size, &num_cards);
 			SGEN_ASSERT (6, num_cards == CARDS_PER_BLOCK, "Number of cards calculation is wrong");
 		}
 	} END_FOREACH_BLOCK_NO_LOCK;
@@ -2693,7 +2734,6 @@ static void
 post_param_init (SgenMajorCollector *collector)
 {
 	collector->sweeps_lazily = lazy_sweep;
-	collector->needs_thread_pool = concurrent_mark || concurrent_sweep;
 }

 /* We are guaranteed to be called by the worker in question */
@@ -2712,12 +2752,23 @@ sgen_worker_init_callback (gpointer worker_untyped)
 	mono_native_tls_set_value (worker_block_free_list_key, worker_free_blocks);
 }

+static void
+thread_pool_init_func (void *data_untyped)
+{
+	sgen_client_thread_register_worker ();
+}
+
 static void
 sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurrent, gboolean is_parallel)
 {
 	int i;

-	sgen_register_fixed_internal_mem_type (INTERNAL_MEM_MS_BLOCK_INFO, sizeof (MSBlockInfo));
+	ms_block_size = mono_pagesize ();
+
+	if (ms_block_size < MS_BLOCK_SIZE_MIN)
+		ms_block_size = MS_BLOCK_SIZE_MIN;
+
+	sgen_register_fixed_internal_mem_type (INTERNAL_MEM_MS_BLOCK_INFO, SIZEOF_MS_BLOCK_INFO);

 	num_block_obj_sizes = ms_calculate_block_obj_sizes (MS_BLOCK_OBJ_SIZE_FACTOR, NULL);
 	block_obj_sizes = (int *)sgen_alloc_internal_dynamic (sizeof (int) * num_block_obj_sizes, INTERNAL_MEM_MS_TABLES, TRUE);
@@ -2748,6 +2799,12 @@ sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurr
 	for (i = 0; i < MS_NUM_FAST_BLOCK_OBJ_SIZE_INDEXES * 8; ++i)
 		g_assert (MS_BLOCK_OBJ_SIZE_INDEX (i) == ms_find_block_obj_size_index (i));

+	/* We can do this because we always init the minor before the major */
+	if (is_parallel || sgen_get_minor_collector ()->is_parallel) {
+		mono_native_tls_alloc (&worker_block_free_list_key, NULL);
+		collector->worker_init_cb = sgen_worker_init_callback;
+	}
+
 	mono_counters_register ("# major blocks allocated", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_major_blocks_alloced);
 	mono_counters_register ("# major blocks freed", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_major_blocks_freed);
 	mono_counters_register ("# major blocks lazy swept", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_major_blocks_lazy_swept);
@@ -2756,12 +2813,11 @@ sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurr
 	mono_counters_register ("# major blocks freed individually", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_major_blocks_freed_individual);
 	mono_counters_register ("# major blocks allocated less ideally", MONO_COUNTER_GC | MONO_COUNTER_ULONG, &stat_major_blocks_alloced_less_ideal);

-	collector->section_size = MAJOR_SECTION_SIZE;
+	collector->section_size = ms_block_size;

 	concurrent_mark = is_concurrent;
 	collector->is_concurrent = is_concurrent;
 	collector->is_parallel = is_parallel;
-	collector->needs_thread_pool = is_concurrent || concurrent_sweep;
 	collector->get_and_reset_num_major_objects_marked = major_get_and_reset_num_major_objects_marked;
 	collector->supports_cardtable = TRUE;

@@ -2807,6 +2863,7 @@ sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurr
 	collector->is_valid_object = major_is_valid_object;
 	collector->describe_pointer = major_describe_pointer;
 	collector->count_cards = major_count_cards;
+	collector->get_sweep_pool = major_get_sweep_pool;

 	collector->major_ops_serial.copy_or_mark_object = major_copy_or_mark_object_canonical;
 	collector->major_ops_serial.scan_object = major_scan_object_with_evacuation;
@@ -2837,10 +2894,6 @@ sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurr
 			collector->major_ops_conc_par_finish.scan_vtype = major_scan_vtype_par_with_evacuation;
 			collector->major_ops_conc_par_finish.scan_ptr_field = major_scan_ptr_field_par_with_evacuation;
 			collector->major_ops_conc_par_finish.drain_gray_stack = drain_gray_stack_par;
-
-			collector->worker_init_cb = sgen_worker_init_callback;
-
-			mono_native_tls_alloc (&worker_block_free_list_key, NULL);
 		}
 	}

@@ -2869,7 +2922,13 @@ sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurr
 	SGEN_ASSERT (0, SGEN_MAX_SMALL_OBJ_SIZE <= MS_BLOCK_FREE / 2, "MAX_SMALL_OBJ_SIZE must be at most MS_BLOCK_FREE / 2");

 	/*cardtable requires major pages to be 8 cards aligned*/
-	g_assert ((MS_BLOCK_SIZE % (8 * CARD_SIZE_IN_BYTES)) == 0);
+	g_assert ((ms_block_size % (8 * CARD_SIZE_IN_BYTES)) == 0);
+
+	if (concurrent_sweep) {
+		SgenThreadPool **thread_datas = &sweep_pool;
+		sweep_pool = &sweep_pool_inst;
+		sgen_thread_pool_init (sweep_pool, 1, thread_pool_init_func, NULL, NULL, NULL, (SgenThreadPoolData**)&thread_datas);
+	}
 }

 void
--- a/mono/sgen/sgen-memory-governor.c
+++ b/mono/sgen/sgen-memory-governor.c
@@ -20,10 +20,10 @@

 #include "mono/sgen/sgen-gc.h"
 #include "mono/sgen/sgen-memory-governor.h"
-#include "mono/sgen/sgen-thread-pool.h"
+#include "mono/sgen/sgen-workers.h"
 #include "mono/sgen/sgen-client.h"

-#define MIN_MINOR_COLLECTION_ALLOWANCE	((mword)(DEFAULT_NURSERY_SIZE * default_allowance_nursery_size_ratio))
+#define MIN_MINOR_COLLECTION_ALLOWANCE	((mword)(SGEN_DEFAULT_NURSERY_SIZE * default_allowance_nursery_size_ratio))

 static SgenPointerQueue log_entries = SGEN_POINTER_QUEUE_INIT (INTERNAL_MEM_TEMPORARY);
 static MonoCoopMutex log_entries_mutex;
@@ -459,7 +459,7 @@ gboolean
 sgen_memgov_try_alloc_space (mword size, int space)
 {
 	if (sgen_memgov_available_free_space () < size) {
-		SGEN_ASSERT (4, !sgen_thread_pool_is_thread_pool_thread (mono_native_thread_id_get ()), "Memory shouldn't run out in worker thread");
+		SGEN_ASSERT (4, !sgen_workers_is_worker_thread (mono_native_thread_id_get ()), "Memory shouldn't run out in worker thread");
 		return FALSE;
 	}

@@ -492,11 +492,11 @@ sgen_memgov_init (size_t max_heap, size_t soft_limit, gboolean debug_allowance,
 		max_heap = soft_limit;
 	}

-	if (max_heap < sgen_nursery_size * 4) {
+	if (max_heap < SGEN_DEFAULT_NURSERY_SIZE * 4) {
 		sgen_env_var_error (MONO_GC_PARAMS_NAME, "Setting to minimum.", "`max-heap-size` must be at least 4 times as large as `nursery size`.");
-		max_heap = sgen_nursery_size * 4;
+		max_heap = SGEN_DEFAULT_NURSERY_SIZE * 4;
 	}
-	max_heap_size = max_heap - sgen_nursery_size;
+	max_heap_size = max_heap - SGEN_DEFAULT_NURSERY_SIZE;

 	if (allowance_ratio)
 		default_allowance_nursery_size_ratio = allowance_ratio;
--- a/mono/sgen/sgen-minor-copy-object.h
+++ b/mono/sgen/sgen-minor-copy-object.h
@@ -14,6 +14,11 @@

 #if defined(SGEN_SIMPLE_NURSERY)

+#ifdef SGEN_SIMPLE_PAR_NURSERY
+/* Not supported with concurrent major yet */
+#define SERIAL_COPY_OBJECT simple_par_nursery_copy_object
+#define SERIAL_COPY_OBJECT_FROM_OBJ simple_par_nursery_copy_object_from_obj
+#else
 #ifdef SGEN_CONCURRENT_MAJOR
 #define SERIAL_COPY_OBJECT simple_nursery_serial_with_concurrent_major_copy_object
 #define SERIAL_COPY_OBJECT_FROM_OBJ simple_nursery_serial_with_concurrent_major_copy_object_from_obj
@@ -21,6 +26,7 @@
 #define SERIAL_COPY_OBJECT simple_nursery_serial_copy_object
 #define SERIAL_COPY_OBJECT_FROM_OBJ simple_nursery_serial_copy_object_from_obj
 #endif
+#endif

 #elif defined (SGEN_SPLIT_NURSERY)

@@ -108,7 +114,11 @@ SERIAL_COPY_OBJECT (GCObject **obj_slot, SgenGrayQueue *queue)

 	HEAVY_STAT (++stat_objects_copied_nursery);

+#ifdef SGEN_SIMPLE_PAR_NURSERY
+	copy = copy_object_no_checks_par (obj, queue);
+#else
 	copy = copy_object_no_checks (obj, queue);
+#endif
 	SGEN_UPDATE_REFERENCE (obj_slot, copy);
 }

@@ -214,7 +224,11 @@ SERIAL_COPY_OBJECT_FROM_OBJ (GCObject **obj_slot, SgenGrayQueue *queue)

 	HEAVY_STAT (++stat_objects_copied_nursery);

+#ifdef SGEN_SIMPLE_PAR_NURSERY
+	copy = copy_object_no_checks_par (obj, queue);
+#else
 	copy = copy_object_no_checks (obj, queue);
+#endif
 #ifdef SGEN_CONCURRENT_MAJOR
 	/*
 	 * If an object is evacuated to the major heap and a reference to it, from the major
--- a/mono/sgen/sgen-minor-scan-object.h
+++ b/mono/sgen/sgen-minor-scan-object.h
@@ -18,6 +18,12 @@ extern guint64 stat_scan_object_called_nursery;

 #if defined(SGEN_SIMPLE_NURSERY)

+#ifdef SGEN_SIMPLE_PAR_NURSERY
+#define SERIAL_SCAN_OBJECT simple_par_nursery_serial_scan_object
+#define SERIAL_SCAN_VTYPE simple_par_nursery_serial_scan_vtype
+#define SERIAL_SCAN_PTR_FIELD simple_par_nursery_serial_scan_ptr_field
+#define SERIAL_DRAIN_GRAY_STACK simple_par_nursery_serial_drain_gray_stack
+#else
 #ifdef SGEN_CONCURRENT_MAJOR
 #define SERIAL_SCAN_OBJECT simple_nursery_serial_with_concurrent_major_scan_object
 #define SERIAL_SCAN_VTYPE simple_nursery_serial_with_concurrent_major_scan_vtype
@@ -29,6 +35,7 @@ extern guint64 stat_scan_object_called_nursery;
 #define SERIAL_SCAN_PTR_FIELD simple_nursery_serial_scan_ptr_field
 #define SERIAL_DRAIN_GRAY_STACK simple_nursery_serial_drain_gray_stack
 #endif
+#endif

 #elif defined (SGEN_SPLIT_NURSERY)

@@ -104,16 +111,31 @@ SERIAL_SCAN_PTR_FIELD (GCObject *full_object, GCObject **ptr, SgenGrayQueue *que
 static gboolean
 SERIAL_DRAIN_GRAY_STACK (SgenGrayQueue *queue)
 {
-        for (;;) {
-                GCObject *obj;
-                SgenDescriptor desc;
+#ifdef SGEN_SIMPLE_PAR_NURSERY
+	int i;
+	/*
+	 * We do bounded iteration so we can switch to optimized context
+	 * when we are the last worker remaining.
+	 */
+	for (i = 0; i < 32; i++) {
+#else
+	for (;;) {
+#endif
+		GCObject *obj;
+		SgenDescriptor desc;

-                GRAY_OBJECT_DEQUEUE_SERIAL (queue, &obj, &desc);
-                if (!obj)
-                        return TRUE;
+#ifdef SGEN_SIMPLE_PAR_NURSERY
+		GRAY_OBJECT_DEQUEUE_PARALLEL (queue, &obj, &desc);
+#else
+		GRAY_OBJECT_DEQUEUE_SERIAL (queue, &obj, &desc);
+#endif
+		if (!obj)
+			return TRUE;

-                SERIAL_SCAN_OBJECT (obj, desc, queue);
-        }
+		SERIAL_SCAN_OBJECT (obj, desc, queue);
+	}
+
+	return FALSE;
 }

 #define FILL_MINOR_COLLECTOR_SCAN_OBJECT(ops)	do {			\
--- a/mono/sgen/sgen-nursery-allocator.c
+++ b/mono/sgen/sgen-nursery-allocator.c
@@ -66,16 +66,21 @@ static SgenFragmentAllocator mutator_allocator;
 /* freeelist of fragment structures */
 static SgenFragment *fragment_freelist = NULL;

-/* Allocator cursors */
-static char *nursery_last_pinned_end = NULL;
-
 char *sgen_nursery_start;
 char *sgen_nursery_end;

-#ifdef USER_CONFIG
-size_t sgen_nursery_size = (1 << 22);
-int sgen_nursery_bits = 22;
-#endif
+/* good sizes are 512KB-1MB: larger ones increase a lot memzeroing time */
+size_t sgen_nursery_size;
+/*
+ * Maximum size that we can resize the nursery to.
+ * If sgen_nursery_default_size == sgen_nursery_max_size then we are not
+ * dynamically resizing the nursery
+ */
+size_t sgen_nursery_max_size;
+size_t sgen_nursery_min_size;
+/* The number of trailing 0 bits in sgen_nursery_max_size */
+int sgen_nursery_bits;
+

 char *sgen_space_bitmap;
 size_t sgen_space_bitmap_size;
@@ -345,7 +350,7 @@ par_alloc_from_fragment (SgenFragmentAllocator *allocator, SgenFragment *frag, s
 	char *p = frag->fragment_next;
 	char *end = p + size;

-	if (end > frag->fragment_end)
+	if (end > frag->fragment_end || end > (sgen_nursery_start + sgen_nursery_size))
 		return NULL;

 	/* p = frag->fragment_next must happen before */
@@ -439,9 +444,14 @@ sgen_fragment_allocator_par_alloc (SgenFragmentAllocator *allocator, size_t size

 restart:
 	for (frag = (SgenFragment *)unmask (allocator->alloc_head); unmask (frag); frag = (SgenFragment *)unmask (frag->next)) {
+		size_t frag_size = frag->fragment_end - frag->fragment_next;
+
+		if (frag->fragment_next >= (sgen_nursery_start + sgen_nursery_size))
+			continue;
+
 		HEAVY_STAT (++stat_alloc_iterations);

-		if (size <= (size_t)(frag->fragment_end - frag->fragment_next)) {
+		if (size <= frag_size) {
 			void *p = par_alloc_from_fragment (allocator, frag, size);
 			if (!p) {
 				HEAVY_STAT (++stat_alloc_retries);
@@ -456,33 +466,6 @@ restart:
 	return NULL;
 }

-void*
-sgen_fragment_allocator_serial_alloc (SgenFragmentAllocator *allocator, size_t size)
-{
-	SgenFragment *frag;
-	SgenFragment **previous;
-#ifdef NALLOC_DEBUG
-	InterlockedIncrement (&alloc_count);
-#endif
-
-	previous = &allocator->alloc_head;
-
-	for (frag = *previous; frag; frag = *previous) {
-		char *p = (char *)serial_alloc_from_fragment (previous, frag, size);
-
-		HEAVY_STAT (++stat_alloc_iterations);
-
-		if (p) {
-#ifdef NALLOC_DEBUG
-			add_alloc_record (p, size, FIXED_ALLOC);
-#endif
-			return p;
-		}
-		previous = &frag->next;
-	}
-	return NULL;
-}
-
 void*
 sgen_fragment_allocator_serial_range_alloc (SgenFragmentAllocator *allocator, size_t desired_size, size_t minimum_size, size_t *out_alloc_size)
 {
@@ -551,6 +534,9 @@ restart:
 	for (frag = (SgenFragment *)unmask (allocator->alloc_head); frag; frag = (SgenFragment *)unmask (frag->next)) {
 		size_t frag_size = frag->fragment_end - frag->fragment_next;

+		if (frag->fragment_next >= (sgen_nursery_start + sgen_nursery_size))
+			continue;
+
 		HEAVY_STAT (++stat_alloc_range_iterations);

 		if (desired_size <= frag_size) {
@@ -578,9 +564,8 @@ restart:

 	if (min_frag) {
 		void *p;
-		size_t frag_size;
+		size_t frag_size = min_frag->fragment_end - min_frag->fragment_next;

-		frag_size = min_frag->fragment_end - min_frag->fragment_next;
 		if (frag_size < minimum_size)
 			goto restart;

@@ -705,6 +690,26 @@ fragment_list_reverse (SgenFragmentAllocator *allocator)
 	allocator->region_head = allocator->alloc_head = prev;
 }

+/*
+ * We split fragments at the border of the current nursery limit. When we
+ * allocate from the nursery we only consider fragments that start in the
+ * current nursery section. We build fragments for the entire nursery in
+ * order to facilitate scanning it for objects (adding a nursery frag also
+ * marks a region in the nursery as being free)
+ */
+static void
+add_nursery_frag_checks (SgenFragmentAllocator *allocator, char *frag_start, char *frag_end)
+{
+	char *nursery_limit = sgen_nursery_start + sgen_nursery_size;
+
+	if (frag_start < nursery_limit && frag_end > nursery_limit) {
+		add_nursery_frag (allocator, nursery_limit - frag_start, frag_start, nursery_limit);
+		add_nursery_frag (allocator, frag_end - nursery_limit, nursery_limit, frag_end);
+	} else {
+		add_nursery_frag (allocator, frag_end - frag_start, frag_start, frag_end);
+	}
+}
+
 mword
 sgen_build_nursery_fragments (GCMemSection *nursery_section, SgenGrayQueue *unpin_queue)
 {
@@ -765,7 +770,7 @@ sgen_build_nursery_fragments (GCMemSection *nursery_section, SgenGrayQueue *unpi
 		g_assert (frag_size >= 0);
 		g_assert (size > 0);
 		if (frag_size && size)
-			add_nursery_frag (&mutator_allocator, frag_size, frag_start, frag_end);	
+			add_nursery_frag_checks (&mutator_allocator, frag_start, frag_end);

 		frag_size = size;
 #ifdef NALLOC_DEBUG
@@ -774,11 +779,10 @@ sgen_build_nursery_fragments (GCMemSection *nursery_section, SgenGrayQueue *unpi
 		frag_start = frag_end + frag_size;
 	}

-	nursery_last_pinned_end = frag_start;
 	frag_end = sgen_nursery_end;
 	frag_size = frag_end - frag_start;
 	if (frag_size)
-		add_nursery_frag (&mutator_allocator, frag_size, frag_start, frag_end);
+		add_nursery_frag_checks (&mutator_allocator, frag_start, frag_end);

 	/* Now it's safe to release the fragments exclude list. */
 	sgen_minor_collector.build_fragments_release_exclude_head ();
@@ -799,13 +803,6 @@ sgen_build_nursery_fragments (GCMemSection *nursery_section, SgenGrayQueue *unpi
 	return fragment_total;
 }

-char *
-sgen_nursery_alloc_get_upper_alloc_bound (void)
-{
-	/*FIXME we need to calculate the collector upper bound as well, but this must be done in the previous GC. */
-	return sgen_nursery_end;
-}
-
 /*** Nursery memory allocation ***/
 void
 sgen_nursery_retire_region (void *address, ptrdiff_t size)
@@ -897,21 +894,59 @@ sgen_nursery_alloc_prepare_for_major (void)
 }

 void
-sgen_nursery_allocator_set_nursery_bounds (char *start, char *end)
+sgen_nursery_allocator_set_nursery_bounds (char *start, size_t min_size, size_t max_size)
 {
 	sgen_nursery_start = start;
-	sgen_nursery_end = end;
+	sgen_nursery_end = start + max_size;
+
+	sgen_nursery_size = min_size;
+	sgen_nursery_min_size = min_size;
+	sgen_nursery_max_size = max_size;
+
+	sgen_nursery_bits = 0;
+	while (ONE_P << (++ sgen_nursery_bits) != sgen_nursery_max_size)
+		;

 	/*
 	 * This will not divide evenly for tiny nurseries (<4kb), so we make sure to be on
 	 * the right side of things and round up.  We could just do a MIN(1,x) instead,
 	 * since the nursery size must be a power of 2.
 	 */
-	sgen_space_bitmap_size = (end - start + SGEN_TO_SPACE_GRANULE_IN_BYTES * 8 - 1) / (SGEN_TO_SPACE_GRANULE_IN_BYTES * 8);
+	sgen_space_bitmap_size = (sgen_nursery_end - sgen_nursery_start + SGEN_TO_SPACE_GRANULE_IN_BYTES * 8 - 1) / (SGEN_TO_SPACE_GRANULE_IN_BYTES * 8);
 	sgen_space_bitmap = (char *)g_malloc0 (sgen_space_bitmap_size);

 	/* Setup the single first large fragment */
-	sgen_minor_collector.init_nursery (&mutator_allocator, start, end);
+	sgen_minor_collector.init_nursery (&mutator_allocator, sgen_nursery_start, sgen_nursery_end);
 }

+void
+sgen_resize_nursery (gboolean need_shrink)
+{
+	size_t major_size;
+
+	if (sgen_nursery_min_size == sgen_nursery_max_size)
+		return;
+
+	major_size = major_collector.get_num_major_sections () * major_collector.section_size + los_memory_usage;
+	/*
+	 * We attempt to use a larger nursery size, as long as it doesn't
+	 * exceed a certain percentage of the major heap.
+	 *
+	 * FIXME
+	 * Commit memory when expanding and release it when shrinking (which
+	 * would only be possible if there aren't any pinned objects in the
+	 * section).
+	 */
+	if ((sgen_nursery_size * 2) < (major_size / SGEN_DEFAULT_ALLOWANCE_NURSERY_SIZE_RATIO) &&
+			(sgen_nursery_size * 2) <= sgen_nursery_max_size && !need_shrink) {
+		if ((nursery_section->end_data - nursery_section->data) == sgen_nursery_size)
+			nursery_section->end_data += sgen_nursery_size;
+		sgen_nursery_size *= 2;
+	} else if ((sgen_nursery_size > (major_size / SGEN_DEFAULT_ALLOWANCE_NURSERY_SIZE_RATIO) || need_shrink) &&
+			(sgen_nursery_size / 2) >= sgen_nursery_min_size) {
+		sgen_nursery_size /= 2;
+	}
+}
+
+
 #endif
--- a/mono/sgen/sgen-protocol.c
+++ b/mono/sgen/sgen-protocol.c
@@ -16,7 +16,7 @@
 #include "sgen-gc.h"
 #include "sgen-protocol.h"
 #include "sgen-memory-governor.h"
-#include "sgen-thread-pool.h"
+#include "sgen-workers.h"
 #include "sgen-client.h"
 #include "mono/utils/mono-membar.h"
 #include "mono/utils/mono-proclib.h"
@@ -365,11 +365,17 @@ protocol_entry (unsigned char type, gpointer data, int size)
 	buffer->buffer [index++] = type;
 	/* We should never change the header format */
 	if (include_worker_index) {
+		int worker_index;
+		MonoNativeThreadId tid = mono_native_thread_id_get ();
 		/*
 		 * If the thread is not a worker thread we insert 0, which is interpreted
 		 * as gc thread. Worker indexes are 1 based.
 		 */
-		buffer->buffer [index++] = (unsigned char) sgen_thread_pool_is_thread_pool_thread (mono_native_thread_id_get ());
+		worker_index = sgen_workers_is_worker_thread (tid);
+		if (!worker_index)
+			worker_index = sgen_thread_pool_is_thread_pool_thread (major_collector.get_sweep_pool (), tid);
+		/* FIXME Consider using different index bases for different thread pools */
+		buffer->buffer [index++] = (unsigned char) worker_index;
 	}
 	memcpy (buffer->buffer + index, data, size);
 	index += size;
--- a/mono/sgen/sgen-simple-nursery.c
+++ b/mono/sgen/sgen-simple-nursery.c
@@ -28,6 +28,19 @@ alloc_for_promotion (GCVTable vtable, GCObject *obj, size_t objsize, gboolean ha
 	return major_collector.alloc_object (vtable, objsize, has_references);
 }

+static inline GCObject*
+alloc_for_promotion_par (GCVTable vtable, GCObject *obj, size_t objsize, gboolean has_references)
+{
+	/*
+	 * FIXME
+	 * Note that the stat is not precise. total_promoted_size incrementing is not atomic and
+	 * even in that case, the same object might be promoted simultaneously by different workers
+	 * leading to one of the allocated major object to be discarded.
+	 */
+	total_promoted_size += objsize;
+	return major_collector.alloc_object_par (vtable, objsize, has_references);
+}
+
 static SgenFragment*
 build_fragments_get_exclude_head (void)
 {
@@ -57,7 +70,14 @@ clear_fragments (void)
 static void
 init_nursery (SgenFragmentAllocator *allocator, char *start, char *end)
 {
-	sgen_fragment_allocator_add (allocator, start, end);
+	char *nursery_limit = sgen_nursery_start + sgen_nursery_size;
+
+	if (start < nursery_limit && end > nursery_limit) {
+		sgen_fragment_allocator_add (allocator, start, nursery_limit);
+		sgen_fragment_allocator_add (allocator, nursery_limit, end);
+	} else {
+		sgen_fragment_allocator_add (allocator, start, end);
+	}
 }


@@ -65,7 +85,9 @@ init_nursery (SgenFragmentAllocator *allocator, char *start, char *end)

 #define collector_pin_object(obj, queue) sgen_pin_object (obj, queue);
 #define COLLECTOR_SERIAL_ALLOC_FOR_PROMOTION alloc_for_promotion
+#define COLLECTOR_PARALLEL_ALLOC_FOR_PROMOTION alloc_for_promotion_par

+#define COPY_OR_MARK_PARALLEL
 #include "sgen-copy-object.h"

 #define SGEN_SIMPLE_NURSERY
@@ -80,6 +102,19 @@ fill_serial_ops (SgenObjectOperations *ops)
 	FILL_MINOR_COLLECTOR_SCAN_OBJECT (ops);
 }

+#define SGEN_SIMPLE_PAR_NURSERY
+
+#include "sgen-minor-copy-object.h"
+#include "sgen-minor-scan-object.h"
+
+static void
+fill_parallel_ops (SgenObjectOperations *ops)
+{
+	ops->copy_or_mark_object = SERIAL_COPY_OBJECT;
+	FILL_MINOR_COLLECTOR_SCAN_OBJECT (ops);
+}
+
+#undef SGEN_SIMPLE_PAR_NURSERY
 #define SGEN_CONCURRENT_MAJOR

 #include "sgen-minor-copy-object.h"
@@ -93,11 +128,13 @@ fill_serial_with_concurrent_major_ops (SgenObjectOperations *ops)
 }

 void
-sgen_simple_nursery_init (SgenMinorCollector *collector)
+sgen_simple_nursery_init (SgenMinorCollector *collector, gboolean parallel)
 {
 	collector->is_split = FALSE;
+	collector->is_parallel = parallel;

 	collector->alloc_for_promotion = alloc_for_promotion;
+	collector->alloc_for_promotion_par = alloc_for_promotion_par;

 	collector->prepare_to_space = prepare_to_space;
 	collector->clear_fragments = clear_fragments;
@@ -108,6 +145,7 @@ sgen_simple_nursery_init (SgenMinorCollector *collector)

 	fill_serial_ops (&collector->serial_ops);
 	fill_serial_with_concurrent_major_ops (&collector->serial_ops_with_concurrent_major);
+	fill_parallel_ops (&collector->parallel_ops);
 }


--- a/mono/sgen/sgen-split-nursery.c
+++ b/mono/sgen/sgen-split-nursery.c
@@ -452,6 +452,7 @@ void
 sgen_split_nursery_init (SgenMinorCollector *collector)
 {
 	collector->is_split = TRUE;
+	collector->is_parallel = FALSE;

 	collector->alloc_for_promotion = minor_alloc_for_promotion;

--- a/mono/sgen/sgen-thread-pool.c
+++ b/mono/sgen/sgen-thread-pool.c
@@ -12,31 +12,7 @@

 #include "mono/sgen/sgen-gc.h"
 #include "mono/sgen/sgen-thread-pool.h"
-#include "mono/sgen/sgen-pointer-queue.h"
 #include "mono/utils/mono-os-mutex.h"
-#ifndef SGEN_WITHOUT_MONO
-#include "mono/utils/mono-threads.h"
-#endif
-
-#define MAX_NUM_THREADS 8
-
-static mono_mutex_t lock;
-static mono_cond_t work_cond;
-static mono_cond_t done_cond;
-
-static int threads_num = 0;
-static MonoNativeThreadId threads [MAX_NUM_THREADS];
-
-/* Only accessed with the lock held. */
-static SgenPointerQueue job_queue;
-
-static SgenThreadPoolThreadInitFunc thread_init_func;
-static SgenThreadPoolIdleJobFunc idle_job_func;
-static SgenThreadPoolContinueIdleJobFunc continue_idle_job_func;
-static SgenThreadPoolShouldWorkFunc should_work_func;
-
-static volatile gboolean threadpool_shutdown;
-static volatile int threads_finished = 0;

 enum {
 	STATE_WAITING,
@@ -46,10 +22,10 @@ enum {

 /* Assumes that the lock is held. */
 static SgenThreadPoolJob*
-get_job_and_set_in_progress (void)
+get_job_and_set_in_progress (SgenThreadPool *pool)
 {
-	for (size_t i = 0; i < job_queue.next_slot; ++i) {
-		SgenThreadPoolJob *job = (SgenThreadPoolJob *)job_queue.data [i];
+	for (size_t i = 0; i < pool->job_queue.next_slot; ++i) {
+		SgenThreadPoolJob *job = (SgenThreadPoolJob *)pool->job_queue.data [i];
 		if (job->state == STATE_WAITING) {
 			job->state = STATE_IN_PROGRESS;
 			return job;
@@ -60,10 +36,10 @@ get_job_and_set_in_progress (void)

 /* Assumes that the lock is held. */
 static ssize_t
-find_job_in_queue (SgenThreadPoolJob *job)
+find_job_in_queue (SgenThreadPool *pool, SgenThreadPoolJob *job)
 {
-	for (ssize_t i = 0; i < job_queue.next_slot; ++i) {
-		if (job_queue.data [i] == job)
+	for (ssize_t i = 0; i < pool->job_queue.next_slot; ++i) {
+		if (pool->job_queue.data [i] == job)
 			return i;
 	}
 	return -1;
@@ -71,45 +47,47 @@ find_job_in_queue (SgenThreadPoolJob *job)

 /* Assumes that the lock is held. */
 static void
-remove_job (SgenThreadPoolJob *job)
+remove_job (SgenThreadPool *pool, SgenThreadPoolJob *job)
 {
 	ssize_t index;
 	SGEN_ASSERT (0, job->state == STATE_DONE, "Why are we removing a job that's not done?");
-	index = find_job_in_queue (job);
+	index = find_job_in_queue (pool, job);
 	SGEN_ASSERT (0, index >= 0, "Why is the job we're trying to remove not in the queue?");
-	job_queue.data [index] = NULL;
-	sgen_pointer_queue_remove_nulls (&job_queue);
+	pool->job_queue.data [index] = NULL;
+	sgen_pointer_queue_remove_nulls (&pool->job_queue);
 	sgen_thread_pool_job_free (job);
 }

 static gboolean
-continue_idle_job (void *thread_data)
+continue_idle_job (SgenThreadPool *pool, void *thread_data)
 {
-	if (!continue_idle_job_func)
+	if (!pool->continue_idle_job_func)
 		return FALSE;
-	return continue_idle_job_func (thread_data);
+	return pool->continue_idle_job_func (thread_data);
 }

 static gboolean
-should_work (void *thread_data)
+should_work (SgenThreadPool *pool, void *thread_data)
 {
-	if (!should_work_func)
+	if (!pool->should_work_func)
 		return TRUE;
-	return should_work_func (thread_data);
+	return pool->should_work_func (thread_data);
 }

 static mono_native_thread_return_t
-thread_func (void *thread_data)
+thread_func (SgenThreadPoolData *thread_data)
 {
-	thread_init_func (thread_data);
+	SgenThreadPool *pool = thread_data->pool;

-	mono_os_mutex_lock (&lock);
+	pool->thread_init_func (thread_data);
+
+	mono_os_mutex_lock (&pool->lock);
 	for (;;) {
 		gboolean do_idle;
 		SgenThreadPoolJob *job;

-		if (!should_work (thread_data)) {
-			mono_os_cond_wait (&work_cond, &lock);
+		if (!should_work (pool, thread_data) && !pool->threadpool_shutdown) {
+			mono_os_cond_wait (&pool->work_cond, &pool->lock);
 			continue;
 		}
 		/*
@@ -118,51 +96,51 @@ thread_func (void *thread_data)
 		 * main thread might then set continue idle and signal us before we can take
 		 * the lock, and we'd lose the signal.
 		 */
-		do_idle = continue_idle_job (thread_data);
-		job = get_job_and_set_in_progress ();
+		do_idle = continue_idle_job (pool, thread_data);
+		job = get_job_and_set_in_progress (pool);

-		if (!job && !do_idle && !threadpool_shutdown) {
+		if (!job && !do_idle && !pool->threadpool_shutdown) {
 			/*
 			 * pthread_cond_wait() can return successfully despite the condition
 			 * not being signalled, so we have to run this in a loop until we
 			 * really have work to do.
 			 */
-			mono_os_cond_wait (&work_cond, &lock);
+			mono_os_cond_wait (&pool->work_cond, &pool->lock);
 			continue;
 		}

-		mono_os_mutex_unlock (&lock);
+		mono_os_mutex_unlock (&pool->lock);

 		if (job) {
 			job->func (thread_data, job);

-			mono_os_mutex_lock (&lock);
+			mono_os_mutex_lock (&pool->lock);

 			SGEN_ASSERT (0, job->state == STATE_IN_PROGRESS, "The job should still be in progress.");
 			job->state = STATE_DONE;
-			remove_job (job);
+			remove_job (pool, job);
 			/*
 			 * Only the main GC thread will ever wait on the done condition, so we don't
 			 * have to broadcast.
 			 */
-			mono_os_cond_signal (&done_cond);
+			mono_os_cond_signal (&pool->done_cond);
 		} else if (do_idle) {
-			SGEN_ASSERT (0, idle_job_func, "Why do we have idle work when there's no idle job function?");
+			SGEN_ASSERT (0, pool->idle_job_func, "Why do we have idle work when there's no idle job function?");
 			do {
-				idle_job_func (thread_data);
-				do_idle = continue_idle_job (thread_data);
-			} while (do_idle && !job_queue.next_slot);
+				pool->idle_job_func (thread_data);
+				do_idle = continue_idle_job (pool, thread_data);
+			} while (do_idle && !pool->job_queue.next_slot);

-			mono_os_mutex_lock (&lock);
+			mono_os_mutex_lock (&pool->lock);

 			if (!do_idle)
-				mono_os_cond_signal (&done_cond);
+				mono_os_cond_signal (&pool->done_cond);
 		} else {
-			SGEN_ASSERT (0, threadpool_shutdown, "Why did we unlock if no jobs and not shutting down?");
-			mono_os_mutex_lock (&lock);
-			threads_finished++;
-			mono_os_cond_signal (&done_cond);
-			mono_os_mutex_unlock (&lock);
+			SGEN_ASSERT (0, pool->threadpool_shutdown, "Why did we unlock if no jobs and not shutting down?");
+			mono_os_mutex_lock (&pool->lock);
+			pool->threads_finished++;
+			mono_os_cond_signal (&pool->done_cond);
+			mono_os_mutex_unlock (&pool->lock);
 			return 0;
 		}
 	}
@@ -171,41 +149,49 @@ thread_func (void *thread_data)
 }

 void
-sgen_thread_pool_init (int num_threads, SgenThreadPoolThreadInitFunc init_func, SgenThreadPoolIdleJobFunc idle_func, SgenThreadPoolContinueIdleJobFunc continue_idle_func, SgenThreadPoolShouldWorkFunc should_work_func_p, void **thread_datas)
+sgen_thread_pool_init (SgenThreadPool *pool, int num_threads, SgenThreadPoolThreadInitFunc init_func, SgenThreadPoolIdleJobFunc idle_func, SgenThreadPoolContinueIdleJobFunc continue_idle_func, SgenThreadPoolShouldWorkFunc should_work_func_p, SgenThreadPoolData **thread_datas)
 {
 	int i;

-	threads_num = (num_threads < MAX_NUM_THREADS) ? num_threads : MAX_NUM_THREADS;
+	SGEN_ASSERT (0, num_threads > 0, "Why are we creating a threadpool with no threads?");

-	mono_os_mutex_init (&lock);
-	mono_os_cond_init (&work_cond);
-	mono_os_cond_init (&done_cond);
+	pool->threads_num = (num_threads < MAX_NUM_THREADS) ? num_threads : MAX_NUM_THREADS;

-	thread_init_func = init_func;
-	idle_job_func = idle_func;
-	continue_idle_job_func = continue_idle_func;
-	should_work_func = should_work_func_p;
+	mono_os_mutex_init (&pool->lock);
+	mono_os_cond_init (&pool->work_cond);
+	mono_os_cond_init (&pool->done_cond);

-	for (i = 0; i < threads_num; i++)
-		mono_native_thread_create (&threads [i], thread_func, thread_datas ? thread_datas [i] : NULL);
+	pool->thread_init_func = init_func;
+	pool->idle_job_func = idle_func;
+	pool->continue_idle_job_func = continue_idle_func;
+	pool->should_work_func = should_work_func_p;
+
+	sgen_pointer_queue_init (&pool->job_queue, 0);
+	pool->threads_finished = 0;
+	pool->threadpool_shutdown = FALSE;
+
+	for (i = 0; i < pool->threads_num; i++) {
+		thread_datas [i]->pool = pool;
+		mono_native_thread_create (&pool->threads [i], thread_func, thread_datas [i]);
+	}
 }

 void
-sgen_thread_pool_shutdown (void)
+sgen_thread_pool_shutdown (SgenThreadPool *pool)
 {
-	if (!threads_num)
+	if (!pool)
 		return;

-	mono_os_mutex_lock (&lock);
-	threadpool_shutdown = TRUE;
-	mono_os_cond_broadcast (&work_cond);
-	while (threads_finished < threads_num)
-		mono_os_cond_wait (&done_cond, &lock);
-	mono_os_mutex_unlock (&lock);
+	mono_os_mutex_lock (&pool->lock);
+	pool->threadpool_shutdown = TRUE;
+	mono_os_cond_broadcast (&pool->work_cond);
+	while (pool->threads_finished < pool->threads_num)
+		mono_os_cond_wait (&pool->done_cond, &pool->lock);
+	mono_os_mutex_unlock (&pool->lock);

-	mono_os_mutex_destroy (&lock);
-	mono_os_cond_destroy (&work_cond);
-	mono_os_cond_destroy (&done_cond);
+	mono_os_mutex_destroy (&pool->lock);
+	mono_os_cond_destroy (&pool->work_cond);
+	mono_os_cond_destroy (&pool->done_cond);
 }

 SgenThreadPoolJob*
@@ -226,74 +212,77 @@ sgen_thread_pool_job_free (SgenThreadPoolJob *job)
 }

 void
-sgen_thread_pool_job_enqueue (SgenThreadPoolJob *job)
+sgen_thread_pool_job_enqueue (SgenThreadPool *pool, SgenThreadPoolJob *job)
 {
-	mono_os_mutex_lock (&lock);
+	mono_os_mutex_lock (&pool->lock);

-	sgen_pointer_queue_add (&job_queue, job);
-	mono_os_cond_signal (&work_cond);
+	sgen_pointer_queue_add (&pool->job_queue, job);
+	mono_os_cond_signal (&pool->work_cond);

-	mono_os_mutex_unlock (&lock);
+	mono_os_mutex_unlock (&pool->lock);
 }

 void
-sgen_thread_pool_job_wait (SgenThreadPoolJob *job)
+sgen_thread_pool_job_wait (SgenThreadPool *pool, SgenThreadPoolJob *job)
 {
 	SGEN_ASSERT (0, job, "Where's the job?");

-	mono_os_mutex_lock (&lock);
+	mono_os_mutex_lock (&pool->lock);

-	while (find_job_in_queue (job) >= 0)
-		mono_os_cond_wait (&done_cond, &lock);
+	while (find_job_in_queue (pool, job) >= 0)
+		mono_os_cond_wait (&pool->done_cond, &pool->lock);

-	mono_os_mutex_unlock (&lock);
+	mono_os_mutex_unlock (&pool->lock);
 }

 void
-sgen_thread_pool_idle_signal (void)
+sgen_thread_pool_idle_signal (SgenThreadPool *pool)
 {
-	SGEN_ASSERT (0, idle_job_func, "Why are we signaling idle without an idle function?");
+	SGEN_ASSERT (0, pool->idle_job_func, "Why are we signaling idle without an idle function?");

-	mono_os_mutex_lock (&lock);
+	mono_os_mutex_lock (&pool->lock);

-	if (continue_idle_job_func (NULL))
-		mono_os_cond_broadcast (&work_cond);
+	if (pool->continue_idle_job_func (NULL))
+		mono_os_cond_broadcast (&pool->work_cond);

-	mono_os_mutex_unlock (&lock);
+	mono_os_mutex_unlock (&pool->lock);
 }

 void
-sgen_thread_pool_idle_wait (void)
+sgen_thread_pool_idle_wait (SgenThreadPool *pool)
 {
-	SGEN_ASSERT (0, idle_job_func, "Why are we waiting for idle without an idle function?");
+	SGEN_ASSERT (0, pool->idle_job_func, "Why are we waiting for idle without an idle function?");

-	mono_os_mutex_lock (&lock);
+	mono_os_mutex_lock (&pool->lock);

-	while (continue_idle_job_func (NULL))
-		mono_os_cond_wait (&done_cond, &lock);
+	while (pool->continue_idle_job_func (NULL))
+		mono_os_cond_wait (&pool->done_cond, &pool->lock);

-	mono_os_mutex_unlock (&lock);
+	mono_os_mutex_unlock (&pool->lock);
 }

 void
-sgen_thread_pool_wait_for_all_jobs (void)
+sgen_thread_pool_wait_for_all_jobs (SgenThreadPool *pool)
 {
-	mono_os_mutex_lock (&lock);
+	mono_os_mutex_lock (&pool->lock);

-	while (!sgen_pointer_queue_is_empty (&job_queue))
-		mono_os_cond_wait (&done_cond, &lock);
+	while (!sgen_pointer_queue_is_empty (&pool->job_queue))
+		mono_os_cond_wait (&pool->done_cond, &pool->lock);

-	mono_os_mutex_unlock (&lock);
+	mono_os_mutex_unlock (&pool->lock);
 }

 /* Return 0 if is not a thread pool thread or the thread number otherwise */
 int
-sgen_thread_pool_is_thread_pool_thread (MonoNativeThreadId some_thread)
+sgen_thread_pool_is_thread_pool_thread (SgenThreadPool *pool, MonoNativeThreadId some_thread)
 {
 	int i;

-	for (i = 0; i < threads_num; i++) {
-		if (some_thread == threads [i])
+	if (!pool)
+		return 0;
+
+	for (i = 0; i < pool->threads_num; i++) {
+		if (some_thread == pool->threads [i])
 			return i + 1;
 	}

--- a/Show More
+++ b/Show More