Determine any memory/CPU limitations from sysfs cgroup (#21280)

Add capability to interrogate cgroup limitations when determining CP and memory limits

This code has been adapted from coreCLR. It has been modified from C++ but uses the same naming conventions in the event of a unified mechanism that can be shared between both runtimes being developed. The code has been tested on Ubuntu 20.04 and CentOS 7 with cgroupv1 and cgroupv2.

This code is required in the event of running runtime in a container as the current limitations being discovered by the mono runtime are purely for the machine and not in a container which may have lower quotas.

* src/mono/CMakeLists.txt
  - Set the HAVE_CGROUP_SUPPORT for Linux hosts

* src/mono/cmake/config.h.in
  - Place holder for HAVE_CGROU_SUPPORT definition

* src/mono/mono/metadata/icall.c
* src/mono/mono/sgen/sgen-marksweep.c
* src/mono/mono/sgen/sgen-simple-nursery.c
  - Use mono_cpu_limit() instead of mono_cpu_count()

* src/mono/mono/utils/CMakeLists.txt
  - Add mono-cgroup.c to the build

* src/mono/mono/utils/memfuncs.c
  - Call `getRestrictedPhysicalMemoryLimit()` or `getPhyscalMemoryAvail()`

* src/mono/mono/utils/memfuncs.h
  - Add prototypes for the new APIs

* src/mono/mono/utils/mono-cgroup.c
  - Code adapted from coreCLR to interrogate sysfs to determine any limitations on memory or CPU

* src/mono/mono/utils/mono-proclib.c
  - Add call to `getCpuLimit()`

* src/mono/mono/utils/mono-proclib.h
  - Add prototype for the new API

src/mono/CMakeLists.txt

@@ -215,6 +215,7 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
   set(HOST_LINUX 1)
   add_definitions(-D_GNU_SOURCE -D_REENTRANT)
   add_definitions(-D_THREAD_SAFE)
+  set(HAVE_CGROUP_SUPPORT 1)
   # Enable the "full RELRO" options (RELRO & BIND_NOW) at link time
   add_link_options(-Wl,-z,relro)
   add_link_options(-Wl,-z,now)

src/mono/cmake/config.h.in

@@ -888,6 +888,9 @@
 /* Define to 1 if you have /usr/include/malloc.h. */
 #cmakedefine HAVE_USR_INCLUDE_MALLOC_H 1
 
+/* Define to 1 if you have linux cgroups */
+#cmakedefine HAVE_CGROUP_SUPPORT 1
+
 /* The architecture this is running on */
 #define MONO_ARCHITECTURE @MONO_ARCHITECTURE@
 

src/mono/mono/metadata/icall.c

@@ -7189,7 +7189,7 @@ ves_icall_System_Threading_Thread_YieldInternal (void)
 gint32
 ves_icall_System_Environment_get_ProcessorCount (void)
 {
-	return mono_cpu_count ();
+	return mono_cpu_limit ();
 }
 
 // Generate wrappers.

src/mono/mono/sgen/sgen-marksweep.c

@@ -2861,7 +2861,7 @@ sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurr
 
 	sgen_register_fixed_internal_mem_type (INTERNAL_MEM_MS_BLOCK_INFO, SIZEOF_MS_BLOCK_INFO);
 
-	if (mono_cpu_count () <= 1)
+	if (mono_cpu_limit () <= 1)
 		is_parallel = FALSE;
 
 	num_block_obj_sizes = ms_calculate_block_obj_sizes (MS_BLOCK_OBJ_SIZE_FACTOR, NULL);
@@ -3027,7 +3027,7 @@ sgen_marksweep_init_internal (SgenMajorCollector *collector, gboolean is_concurr
 
 #ifndef DISABLE_SGEN_MAJOR_MARKSWEEP_CONC
 	if (is_concurrent && is_parallel)
-		sgen_workers_create_context (GENERATION_OLD, mono_cpu_count ());
+		sgen_workers_create_context (GENERATION_OLD, mono_cpu_limit ());
 	else if (is_concurrent)
 		sgen_workers_create_context (GENERATION_OLD, 1);
 

src/mono/mono/sgen/sgen-simple-nursery.c

@@ -148,7 +148,7 @@ fill_parallel_with_concurrent_major_ops (SgenObjectOperations *ops)
 void
 sgen_simple_nursery_init (SgenMinorCollector *collector, gboolean parallel)
 {
-	if (mono_cpu_count () <= 1)
+	if (mono_cpu_limit () <= 1)
 		parallel = FALSE;
 
 #ifdef DISABLE_SGEN_MAJOR_MARKSWEEP_CONC

src/mono/mono/utils/CMakeLists.txt

@@ -32,6 +32,7 @@ set(utils_common_sources
     mono-sha1.c
     mono-logger.c
     mono-logger-internals.h
+    mono-cgroup.c
     mono-codeman.c
     mono-counters.c
     mono-compiler.h

src/mono/mono/utils/memfuncs.c

@@ -27,6 +27,7 @@
 #include <config.h>
 #include <glib.h>
 #include <string.h>
+#include <errno.h>
 
 #if defined (__APPLE__)
 #include <mach/message.h>
@@ -273,24 +274,59 @@ mono_determine_physical_ram_size (void)
 
 	return (guint64)value;
 #elif defined (HAVE_SYSCONF)
-	gint64 page_size = -1, num_pages = -1;
+	guint64 page_size = 0, num_pages = 0, memsize;
 
 	/* sysconf works on most *NIX operating systems, if your system doesn't have it or if it
 	 * reports invalid values, please add your OS specific code below. */
 #ifdef _SC_PAGESIZE
-	page_size = (gint64)sysconf (_SC_PAGESIZE);
+	page_size = (guint64)sysconf (_SC_PAGESIZE);
 #endif
 
 #ifdef _SC_PHYS_PAGES
-	num_pages = (gint64)sysconf (_SC_PHYS_PAGES);
+	num_pages = (guint64)sysconf (_SC_PHYS_PAGES);
 #endif
 
-	if (page_size == -1 || num_pages == -1) {
+	if (!page_size || !num_pages) {
 		g_warning ("Your operating system's sysconf (3) function doesn't correctly report physical memory size!");
 		return _DEFAULT_MEM_SIZE;
 	}
 
-	return (guint64)page_size * (guint64)num_pages;
+#if defined(_SC_AVPHYS_PAGES)
+	memsize = sysconf(_SC_AVPHYS_PAGES) * page_size;
+#else
+	memsize = page_size * num_pages;	/* Calculate physical memory size */
+#endif
+
+#if HAVE_CGROUP_SUPPORT
+	gint64 restricted_limit = mono_get_restricted_memory_limit();	/* Check for any cgroup limit */
+	if (restricted_limit != 0) {
+		gchar *heapHardLimit = getenv("DOTNET_GCHeapHardLimit");	/* See if user has set a limit */
+		if (heapHardLimit == NULL)
+			heapHardLimit = getenv("COMPlus_GCHeapHardLimit");	/* Check old envvar name */
+		errno = 0;
+		if (heapHardLimit != NULL) {
+			guint64 gcLimit = strtoull(heapHardLimit, NULL, 16);
+			if ((errno == 0) && (gcLimit != 0))
+				restricted_limit = (restricted_limit < gcLimit ? restricted_limit : (gint64) gcLimit);
+		} else {
+			gchar *heapHardLimitPct = getenv("DOTNET_GCHeapHardLimitPercent"); /* User % limit? */
+			if (heapHardLimitPct == NULL)
+				heapHardLimitPct = getenv("COMPlus_GCHeapHardLimitPercent");	/* Check old envvar name */
+			if (heapHardLimitPct != NULL) {
+				int gcLimit = strtoll(heapHardLimitPct, NULL, 16);
+				if ((gcLimit > 0) && (gcLimit <= 100)) 
+					restricted_limit = (gcLimit * restricted_limit) / 100;
+				else
+					restricted_limit = (3 * restricted_limit) / 4;	/* Use 75% limit of container */
+			} else {
+				restricted_limit = (3 * restricted_limit) / 4;	/* Use 75% limit of container */
+			}
+		}
+		return (restricted_limit < 209715200 ? 209715200 : 	/* Use at least 20MB */
+			(restricted_limit < memsize ? restricted_limit : memsize));
+	}
+#endif
+	return memsize;
 #else
 	return _DEFAULT_MEM_SIZE;
 #endif
@@ -343,25 +379,28 @@ mono_determine_physical_ram_available_size (void)
 	host_page_size (host, &page_size);
 	return (guint64) vmstat.free_count * page_size;
 
+#elif HAVE_CGROUP_SUPPORT
+	return (mono_get_memory_avail());
+
 #elif defined (HAVE_SYSCONF)
-	gint64 page_size = -1, num_pages = -1;
+	guint64 page_size = 0, num_pages = 0;
 
 	/* sysconf works on most *NIX operating systems, if your system doesn't have it or if it
 	 * reports invalid values, please add your OS specific code below. */
 #ifdef _SC_PAGESIZE
-	page_size = (gint64)sysconf (_SC_PAGESIZE);
+	page_size = (guint64)sysconf (_SC_PAGESIZE);
 #endif
 
 #ifdef _SC_AVPHYS_PAGES
-	num_pages = (gint64)sysconf (_SC_AVPHYS_PAGES);
+	num_pages = (guint64)sysconf (_SC_AVPHYS_PAGES);
 #endif
 
-	if (page_size == -1 || num_pages == -1) {
+	if (!page_size || !num_pages) {
 		g_warning ("Your operating system's sysconf (3) function doesn't correctly report physical memory size!");
 		return _DEFAULT_MEM_SIZE;
 	}
 
-	return (guint64)page_size * (guint64)num_pages;
+	return page_size * num_pages;
 #else
 	return _DEFAULT_MEM_SIZE;
 #endif

src/mono/mono/utils/memfuncs.h

@@ -24,5 +24,10 @@ MONO_COMPONENT_API void mono_gc_memmove_atomic (void *dest, const void *src, siz
 void mono_gc_memmove_aligned (void *dest, const void *src, size_t size);
 guint64 mono_determine_physical_ram_size (void);
 guint64 mono_determine_physical_ram_available_size (void);
+#if HAVE_CGROUP_SUPPORT
+size_t mono_get_restricted_memory_limit(void);
+gboolean mono_get_memory_used(size_t *);
+size_t mono_get_memory_avail(void);
+#endif
 
 #endif