Add guarded virtual memory support for z/OS

Add guarded virtual memory support to the z/OS vmem path so that object
heap and sparse virtual memory reservations can use guarded storage for
Balanced GC. This reduces MEMLIMIT pressure by keeping reserved regions
guarded until they are committed for use.

Introduce the OMRPORT_VMEM_MEMORY_MODE_GUARDED flag (currently supported
only for Balanced GC) to indicate guarding of VLHGC heap and sparse
virtual memory reservations. Guarded 1 MB pageable and 4 KB
above-the-bar allocations are supported. Guard and unguard operations
are implemented using IARV64 guarded storage support.

Signed-off-by: Shubham Verma <shubhamv.sv@gmail.com>

Author: VermaSh

gc/base/MemoryManager.cpp

@@ -80,6 +80,15 @@ MM_MemoryManager::createVirtualMemoryForHeap(MM_EnvironmentBase *env, MM_MemoryH
 
 	uintptr_t allocateSize = size;
 
+	/*
+	 * Guard heap memory for Balanced.
+	 * This feature is supported on ZOS only.
+	 * Guarding object heap and sparse heap is necessary to not exceed MEMLIMIT.
+	 */
+	if (extensions->isVLHGC()) {
+		mode |= OMRPORT_VMEM_MEMORY_MODE_GUARDED;
+	}
+
 	uintptr_t concurrentScavengerPageSize = 0;
 	if (extensions->isConcurrentScavengerHWSupported()) {
 		OMRPORT_ACCESS_FROM_ENVIRONMENT(env);

gc/base/SparseVirtualMemory.hpp

@@ -87,7 +87,7 @@ class MM_SparseVirtualMemory : public MM_VirtualMemory {
 	MM_SparseVirtualMemory(MM_EnvironmentBase *env, uintptr_t pageSize, uintptr_t pageFlags, MM_Heap *in_heap)
 		: MM_VirtualMemory(
 				env, in_heap->getHeapRegionManager()->getRegionSize(), pageSize, pageFlags, 0,
-				OMRPORT_VMEM_MEMORY_MODE_READ | OMRPORT_VMEM_MEMORY_MODE_WRITE | OMRPORT_VMEM_MEMORY_MODE_VIRTUAL)
+				OMRPORT_VMEM_MEMORY_MODE_READ | OMRPORT_VMEM_MEMORY_MODE_WRITE | OMRPORT_VMEM_MEMORY_MODE_VIRTUAL | OMRPORT_VMEM_MEMORY_MODE_GUARDED)
 		, _heap(in_heap)
 		, _sparseDataPool(NULL)
 		, _largeObjectVirtualMemoryMutex(NULL)

include_core/omrport.h

@@ -235,6 +235,7 @@
 #define OMRPORT_VMEM_ALLOCATE_TOP_DOWN 0x00000020
 #define OMRPORT_VMEM_ALLOCATE_PERSIST 0x00000040
 #define OMRPORT_VMEM_NO_AFFINITY 0x00000080
+#define OMRPORT_VMEM_MEMORY_MODE_GUARDED 0x00000100
 /** @} */
 
 /**

port/zos390/omriarv64.c

@@ -153,6 +153,61 @@ void * omrallocate_1M_pageable_pages_above_bar(int *numMBSegments, int *userExte
 	return (void *)origin;
 }
 
+#pragma prolog(omrallocate_1M_pageable_pages_guarded_above_bar,"MYPROLOG")
+#pragma epilog(omrallocate_1M_pageable_pages_guarded_above_bar,"MYEPILOG")
+
+__asm(" IARV64 PLISTVER=MAX,MF=(L,SGETSTOR)":"DS"(sgetstor));
+
+/*
+ * Allocate 1MB pageable guarded pages above 2GB bar using IARV64 system macro.
+ * Memory allocated is freed using omrfree_memory_above_bar().
+ *
+ * @params[in] numMBSegments Number of 1MB segments to be allocated
+ * @params[in] userExtendedPrivateAreaMemoryType capability of OS: 0 - general, 1 - 2G-32G range, 2 - 2G-64G range
+ *
+ * @return pointer to memory allocated, NULL on failure.
+ */
+void *omrallocate_1M_pageable_pages_guarded_above_bar(int *numMBSegments, int *userExtendedPrivateAreaMemoryType, const char *ttkn) {
+	long segments;
+	long origin;
+	long useMemoryType = *userExtendedPrivateAreaMemoryType;
+	int  iarv64_rc = 0;
+
+	__asm(" IARV64 PLISTVER=MAX,MF=(L,SGETSTOR)":"DS"(wgetstor));
+
+	segments = *numMBSegments;
+	wgetstor = ngetstor;
+
+	switch (useMemoryType) {
+	case ZOS64_VMEM_ABOVE_BAR_GENERAL:
+		__asm(" IARV64 REQUEST=GETSTOR,COND=YES,SADMP=NO,CONTROL=UNAUTH,"\
+				"GUARDSIZE64=(%2),"\
+				"PAGEFRAMESIZE=PAGEABLE1MEG,TYPE=PAGEABLE,SEGMENTS=(%2),"\
+				"ORIGIN=(%1),TTOKEN=(%4),RETCODE=%0,MF=(E,(%3))"\
+				::"m"(iarv64_rc),"r"(&origin),"r"(&segments),"r"(&wgetstor),"r"(ttkn));
+		break;
+	case ZOS64_VMEM_2_TO_32G:
+		__asm(" IARV64 REQUEST=GETSTOR,COND=YES,SADMP=NO,CONTROL=UNAUTH,USE2GTO32G=YES,"\
+				"GUARDSIZE64=(%2),"\
+				"PAGEFRAMESIZE=PAGEABLE1MEG,TYPE=PAGEABLE,SEGMENTS=(%2),"\
+				"ORIGIN=(%1),TTOKEN=(%4),RETCODE=%0,MF=(E,(%3))"\
+				::"m"(iarv64_rc),"r"(&origin),"r"(&segments),"r"(&wgetstor),"r"(ttkn));
+		break;
+	case ZOS64_VMEM_2_TO_64G:
+		__asm(" IARV64 REQUEST=GETSTOR,COND=YES,SADMP=NO,CONTROL=UNAUTH,USE2GTO64G=YES,"\
+				"GUARDSIZE64=(%2),"\
+				"PAGEFRAMESIZE=PAGEABLE1MEG,TYPE=PAGEABLE,SEGMENTS=(%2),"\
+				"ORIGIN=(%1),TTOKEN=(%4),RETCODE=%0,MF=(E,(%3))"\
+				::"m"(iarv64_rc),"r"(&origin),"r"(&segments),"r"(&wgetstor),"r"(ttkn));
+		break;
+	}
+
+	if (0 != iarv64_rc) {
+		return (void *)0;
+	}
+	return (void *)origin;
+}
+
 #pragma prolog(omrallocate_2G_pages,"MYPROLOG")
 #pragma epilog(omrallocate_2G_pages,"MYEPILOG")
 
@@ -253,6 +308,54 @@ void * omrallocate_4K_pages_in_userExtendedPrivateArea(int *numMBSegments, int *
 	return (void *)origin;
 }
 
+#pragma prolog(omrallocate_4K_pages_guarded_in_userExtendedPrivateArea,"MYPROLOG")
+#pragma epilog(omrallocate_4K_pages_guarded_in_userExtendedPrivateArea,"MYEPILOG")
+
+__asm(" IARV64 PLISTVER=MAX,MF=(L,TGETSTOR)":"DS"(tgetstor));
+
+/*
+ * Allocate 4KB pages guarded in 2G-32G range using IARV64 system macro.
+ * Memory allocated is freed using omrfree_memory_above_bar().
+ *
+ * @params[in] numMBSegments Number of 1MB segments to be allocated
+ * @params[in] userExtendedPrivateAreaMemoryType capability of OS: 0 - general, 1 - 2G-32G range, 2 - 2G-64G range
+ *
+ * @return pointer to memory allocated, NULL on failure.
+ */
+void *omrallocate_4K_pages_guarded_in_userExtendedPrivateArea(int *numMBSegments, int *userExtendedPrivateAreaMemoryType, const char *ttkn) {
+	long segments;
+	long origin;
+	long useMemoryType = *userExtendedPrivateAreaMemoryType;
+	int  iarv64_rc = 0;
+
+	__asm(" IARV64 PLISTVER=MAX,MF=(L,TGETSTOR)":"DS"(wgetstor));
+
+	segments = *numMBSegments;
+	wgetstor = tgetstor;
+
+	switch (useMemoryType) {
+	case ZOS64_VMEM_ABOVE_BAR_GENERAL:
+		break;
+	case ZOS64_VMEM_2_TO_32G:
+		__asm(" IARV64 REQUEST=GETSTOR,COND=YES,SADMP=NO,CONTROL=UNAUTH,USE2GTO32G=YES,"\
+				"GUARDSIZE64=(%2),PAGEFRAMESIZE=4K,SEGMENTS=(%2),"\
+				"ORIGIN=(%1),TTOKEN=(%4),RETCODE=%0,MF=(E,(%3))"\
+				::"m"(iarv64_rc),"r"(&origin),"r"(&segments),"r"(&wgetstor),"r"(ttkn));
+		break;
+	case ZOS64_VMEM_2_TO_64G:
+		__asm(" IARV64 REQUEST=GETSTOR,COND=YES,SADMP=NO,CONTROL=UNAUTH,USE2GTO64G=YES,"\
+				"GUARDSIZE64=(%2),PAGEFRAMESIZE=4K,SEGMENTS=(%2),"\
+				"ORIGIN=(%1),TTOKEN=(%4),RETCODE=%0,MF=(E,(%3))"\
+				::"m"(iarv64_rc),"r"(&origin),"r"(&segments),"r"(&wgetstor),"r"(ttkn));
+		break;
+	}
+
+	if (0 != iarv64_rc) {
+		return (void *)0;
+	}
+	return (void *)origin;
+}
+
 #pragma prolog(omrallocate_4K_pages_above_bar,"MYPROLOG")
 #pragma epilog(omrallocate_4K_pages_above_bar,"MYEPILOG")
 
@@ -287,6 +390,33 @@ void * omrallocate_4K_pages_above_bar(int *numMBSegments, const char * ttkn) {
 	return (void *)origin;
 }
 
+#pragma prolog(omrallocate_4K_pages_guarded_above_bar,"MYPROLOG")
+#pragma epilog(omrallocate_4K_pages_guarded_above_bar,"MYEPILOG")
+
+__asm(" IARV64 PLISTVER=MAX,MF=(L,EGETSTOR)":"DS"(egetstor));
+
+void *omrallocate_4K_pages_guarded_above_bar(int *numMBSegments, const char *ttkn) {
+	long segments;
+	long origin;
+	int  iarv64_rc = 0;
+
+	__asm(" IARV64 PLISTVER=MAX,MF=(L,EGETSTOR)":"DS"(wgetstor));
+
+	segments = *numMBSegments;
+	wgetstor = rgetstor;
+
+	__asm(" IARV64 REQUEST=GETSTOR,COND=YES,SADMP=NO,"\
+			"GUARDSIZE=(%2),"\
+			"CONTROL=UNAUTH,PAGEFRAMESIZE=4K,"\
+			"SEGMENTS=(%2),ORIGIN=(%1),TTOKEN=(%4),RETCODE=%0,MF=(E,(%3))"\
+			::"m"(iarv64_rc),"r"(&origin),"r"(&segments),"r"(&wgetstor),"r"(ttkn));
+
+	if (0 != iarv64_rc) {
+		return (void *)0;
+	}
+	return (void *)origin;
+}
+
 #pragma prolog(omrfree_memory_above_bar,"MYPROLOG")
 #pragma epilog(omrfree_memory_above_bar,"MYEPILOG")
 
@@ -313,6 +443,67 @@ int omrfree_memory_above_bar(void *address, const char * ttkn){
 	return iarv64_rc;
 }
 
+#pragma prolog(omrremove_guard,"MYPROLOG")
+#pragma epilog(omrremove_guard,"MYEPILOG")
+
+__asm(" IARV64 PLISTVER=MAX,MF=(L,FGETSTOR)":"DS"(fgetstor));
+
+/*
+ * Remove guard for memory allocated using IARV64 system macro.
+ *
+ * @params[in] address pointer to memory region to be freed
+  * @params[in] numMBSegments Number of 1MB segments to be allocated
+ *
+ * @return non-zero if memory is not freed successfully, 0 otherwise.
+ */
+int omrremove_guard(void *address, int *numMBSegments){
+	void * memObjConvertStart;
+	int  iarv64_rc = 0;
+	long segments;
+
+	__asm(" IARV64 PLISTVER=MAX,MF=(L,FGETSTOR)":"DS"(wgetstor));
+
+	memObjConvertStart = address;
+	wgetstor = fgetstor;
+	segments = *numMBSegments;
+
+	__asm(" IARV64 REQUEST=CHANGEGUARD,CONVERT=FROMGUARD,COND=YES,"\
+			"CONVERTSTART=(%2),CONVERTSIZE64=(%3),"\
+			"RETCODE=%0,MF=(E,(%1))"\
+			::"m"(iarv64_rc),"r"(&wgetstor),"r"(&memObjConvertStart),"r"(&segments));
+	return iarv64_rc;
+}
+
+#pragma prolog(omradd_guard,"MYPROLOG")
+#pragma epilog(omradd_guard,"MYEPILOG")
+
+__asm(" IARV64 PLISTVER=MAX,MF=(L,DGETSTOR)":"DS"(dgetstor));
+
+/*
+ * Ass guard to memory allocated using IARV64 system macro.
+ *
+ * @params[in] address pointer to memory region to be freed
+ *
+ * @return non-zero if memory is not freed successfully, 0 otherwise.
+ */
+int omradd_guard(void *address, int *numMBSegments) {
+	void * memObjConvertStart;
+	int  iarv64_rc = 0;
+	long segments;
+
+	__asm(" IARV64 PLISTVER=MAX,MF=(L,DGETSTOR)":"DS"(wgetstor));
+
+	memObjConvertStart = address;
+	wgetstor = dgetstor;
+	segments = *numMBSegments;
+
+	__asm(" IARV64 REQUEST=CHANGEGUARD,CONVERT=TOGUARD,COND=YES,"\
+			"CONVERTSTART=(%2),CONVERTSIZE64=(%3),"\
+			"RETCODE=%0,MF=(E,(%1))"\
+			::"m"(iarv64_rc),"r"(&wgetstor),"r"(&memObjConvertStart),"r"(&segments));
+	return iarv64_rc;
+}
+
 #pragma prolog(omrdiscard_data,"MYPROLOG")
 #pragma epilog(omrdiscard_data,"MYEPILOG")