From db8bc872fb7a132cb3c2363dcb3a7aa8b0a5827e Mon Sep 17 00:00:00 2001
From: eapen <zhangyipeng7@huawei.com>
Date: Mon, 25 Jul 2022 16:41:24 +0800
Subject: [PATCH 26/33] I68TO2: support numactl for hadoop yarn
---
 hotspot/make/aix/makefiles/mapfile-vers-debug      |   1 +
 hotspot/make/aix/makefiles/mapfile-vers-product    |   1 +
 hotspot/make/linux/makefiles/mapfile-vers-debug    |   1 +
 hotspot/make/linux/makefiles/mapfile-vers-product  |   1 +
 hotspot/make/solaris/makefiles/mapfile-vers        |   1 +
 hotspot/make/windows/jvmexp.lcf                    |   1 +
 hotspot/make/windows/jvmexp_g.lcf                  |   1 +
 hotspot/make/windows/makefiles/vm.make             |   1 +
 hotspot/src/os/linux/vm/os_linux.cpp               |  91 +++++++++++++++++++
 hotspot/src/os/linux/vm/os_linux.hpp               |  51 +++++++++++
 hotspot/src/share/vm/prims/jni.cpp                 |   6 ++
 hotspot/src/share/vm/prims/jni.h                   |   3 +
 hotspot/src/share/vm/runtime/globals.hpp           |  13 +++
 .../runtime/containers/docker/CPUSetsReader.java   |   9 ++
 .../runtime/containers/docker/TestNUMANodes.java   | 102 +++++++++++++++++++++
 jdk/src/share/bin/java.c                           |   2 +
 jdk/src/share/bin/java.h                           |   3 +
 jdk/src/share/javavm/export/jni.h                  |   3 +
 jdk/src/solaris/bin/java_md_solinux.c              |   7 ++
 19 files changed, 298 insertions(+)
 create mode 100644 hotspot/test/runtime/containers/docker/TestNUMANodes.java

diff --git a/hotspot/make/aix/makefiles/mapfile-vers-debug b/hotspot/make/aix/makefiles/mapfile-vers-debug
index 760f955..127794c 100644
--- a/hotspot/make/aix/makefiles/mapfile-vers-debug
+++ b/hotspot/make/aix/makefiles/mapfile-vers-debug
@@ -28,6 +28,7 @@ SUNWprivate_1.1 {
         global:
                 # JNI
                 JNI_CreateJavaVM;
+                JNI_SetCParam;
                 JNI_GetCreatedJavaVMs;
                 JNI_GetDefaultJavaVMInitArgs;

diff --git a/hotspot/make/aix/makefiles/mapfile-vers-product b/hotspot/make/aix/makefiles/mapfile-vers-product
index e84b671..2bbfb32 100644
--- a/hotspot/make/aix/makefiles/mapfile-vers-product
+++ b/hotspot/make/aix/makefiles/mapfile-vers-product
@@ -28,6 +28,7 @@ SUNWprivate_1.1 {
         global:
                 # JNI
                 JNI_CreateJavaVM;
+                JNI_SetCParam;
                 JNI_GetCreatedJavaVMs;
                 JNI_GetDefaultJavaVMInitArgs;

diff --git a/hotspot/make/linux/makefiles/mapfile-vers-debug b/hotspot/make/linux/makefiles/mapfile-vers-debug
index 48b4f9d..1ebe436 100644
--- a/hotspot/make/linux/makefiles/mapfile-vers-debug
+++ b/hotspot/make/linux/makefiles/mapfile-vers-debug
@@ -28,6 +28,7 @@ SUNWprivate_1.1 {
         global:
                 # JNI
                 JNI_CreateJavaVM;
+                JNI_SetCParam;
                 JNI_GetCreatedJavaVMs;
                 JNI_GetDefaultJavaVMInitArgs;

diff --git a/hotspot/make/linux/makefiles/mapfile-vers-product b/hotspot/make/linux/makefiles/mapfile-vers-product
index d4100d7..75e5278 100644
--- a/hotspot/make/linux/makefiles/mapfile-vers-product
+++ b/hotspot/make/linux/makefiles/mapfile-vers-product
@@ -28,6 +28,7 @@ SUNWprivate_1.1 {
         global:
                 # JNI
                 JNI_CreateJavaVM;
+                JNI_SetCParam;
                 JNI_GetCreatedJavaVMs;
                 JNI_GetDefaultJavaVMInitArgs;

diff --git a/hotspot/make/solaris/makefiles/mapfile-vers b/hotspot/make/solaris/makefiles/mapfile-vers
index 26288b7..41045dd 100644
--- a/hotspot/make/solaris/makefiles/mapfile-vers
+++ b/hotspot/make/solaris/makefiles/mapfile-vers
@@ -28,6 +28,7 @@ SUNWprivate_1.1 {
         global:
                 # JNI
                 JNI_CreateJavaVM;
+                JNI_SetCParam;
                 JNI_GetCreatedJavaVMs;
                 JNI_GetDefaultJavaVMInitArgs;

diff --git a/hotspot/make/windows/jvmexp.lcf b/hotspot/make/windows/jvmexp.lcf
index 6489d02..4bba798 100644
--- a/hotspot/make/windows/jvmexp.lcf
+++ b/hotspot/make/windows/jvmexp.lcf
@@ -1,5 +1,6 @@
 -export:JNI_GetDefaultJavaVMInitArgs
 -export:JNI_CreateJavaVM
+-export:JNI_SetCParam;
 -export:JNI_GetCreatedJavaVMs

 -export:jio_snprintf
diff --git a/hotspot/make/windows/jvmexp_g.lcf b/hotspot/make/windows/jvmexp_g.lcf
index 6489d02..4bba798 100644
--- a/hotspot/make/windows/jvmexp_g.lcf
+++ b/hotspot/make/windows/jvmexp_g.lcf
@@ -1,5 +1,6 @@
 -export:JNI_GetDefaultJavaVMInitArgs
 -export:JNI_CreateJavaVM
+-export:JNI_SetCParam;
 -export:JNI_GetCreatedJavaVMs

 -export:jio_snprintf
diff --git a/hotspot/make/windows/makefiles/vm.make b/hotspot/make/windows/makefiles/vm.make
index 5322a4b..fd5e5d2 100644
--- a/hotspot/make/windows/makefiles/vm.make
+++ b/hotspot/make/windows/makefiles/vm.make
@@ -86,6 +86,7 @@ AGCT_EXPORT=/export:AsyncGetCallTrace
 LD_FLAGS=$(LD_FLAGS) $(STACK_SIZE) /subsystem:windows /dll /base:0x8000000 \
   /export:JNI_GetDefaultJavaVMInitArgs       \
   /export:JNI_CreateJavaVM                   \
+  /export:JNI_SetCParam                      \
   /export:JVM_FindClassFromBootLoader        \
   /export:JNI_GetCreatedJavaVMs              \
   /export:jio_snprintf                       \
diff --git a/hotspot/src/os/linux/vm/os_linux.cpp b/hotspot/src/os/linux/vm/os_linux.cpp
index 1ec68ab..ab28ee3 100644
--- a/hotspot/src/os/linux/vm/os_linux.cpp
+++ b/hotspot/src/os/linux/vm/os_linux.cpp
@@ -133,6 +133,8 @@ PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC
 #define LARGEPAGES_BIT (1 << 6)
 ////////////////////////////////////////////////////////////////////////////////
 // global variables
+extern char** argv_for_execvp;
+
 julong os::Linux::_physical_memory = 0;

 address   os::Linux::_initial_thread_stack_bottom = NULL;
@@ -3129,6 +3131,77 @@ void* os::Linux::libnuma_v2_dlsym(void* handle, const char* name) {
   return dlvsym(handle, name, "libnuma_1.2");
 }

+void os::Linux::parse_numa_nodes() {
+  if (NUMANodes == NULL && NUMANodesRandom == 0) {
+    return;
+  }
+  const char* numa_nodes = NUMANodes;
+  // Max length for "%d-%d" is 24
+  char buf[24] = {0};
+  if (NUMANodesRandom != 0) {
+    int nodes_to_bind = NUMANodesRandom;
+    int nodes_num = Linux::numa_max_node() + 1;
+    const int MAX_NUMA = 1000000;
+    if (nodes_num > 0 &&
+        nodes_num < MAX_NUMA &&
+        nodes_to_bind > 0 &&
+        nodes_to_bind < nodes_num) {
+      int bound = 1;
+      while (bound < nodes_to_bind) {
+        bound *= 2;
+      }
+      struct timeval tv;
+      gettimeofday(&tv,NULL);
+      srand(tv.tv_usec);
+      int first = 0;
+      if (nodes_num > bound) {
+        first = rand() % (nodes_num / bound) * bound;
+      }
+      if (bound != nodes_to_bind) {
+        first += rand() % (1 + bound - nodes_to_bind);
+      }
+      sprintf(buf, "%d-%d", first, first + nodes_to_bind - 1);
+      numa_nodes = buf;
+      if (LogNUMANodes) {
+        warning("NUMANodes is converted to %s, with total %d nodes!", buf, nodes_num);
+      }
+    } else {
+      if (LogNUMANodes) {
+        warning("The count of nodes to bind should be less that the count of all nodes, Skip!");
+      }
+      return;
+    }
+  }
+  bitmask* mask = os::Linux::numa_parse_nodestring_all(numa_nodes);
+  if (!mask) {
+    if (LogNUMANodes) {
+      warning("<%s> is invalid", numa_nodes);
+    }
+    return;
+  }
+  if (os::Linux::numa_bitmask_equal(mask, os::Linux::_numa_membind_bitmask)) {
+    os::Linux::numa_bitmask_free(mask);
+    if (LogNUMANodes) {
+      warning("Mempolicy is not changed, param: %s",  numa_nodes);
+    }
+    return;
+  }
+  errno = 0;
+  os::Linux::numa_run_on_node_mask(mask);
+  if (errno) {
+    perror("sched_setaffinity");
+  }
+  errno = 0;
+  os::Linux::numa_set_membind(mask);
+  int errtmp = errno;
+  os::Linux::numa_bitmask_free(mask);
+  if (errtmp) {
+    perror("numa_set_membind");
+  } else {
+    execvp(*argv_for_execvp, argv_for_execvp);
+  }
+}
+
 bool os::Linux::libnuma_init() {
   // sched_getcpu() should be in libc.
   set_sched_getcpu(CAST_TO_FN_PTR(sched_getcpu_func_t,
@@ -3169,6 +3242,16 @@ bool os::Linux::libnuma_init() {
                                          libnuma_dlsym(handle, "numa_move_pages")));
       set_numa_run_on_node(CAST_TO_FN_PTR(numa_run_on_node_func_t,
                                          libnuma_dlsym(handle, "numa_run_on_node")));
+      set_numa_parse_nodestring_all(CAST_TO_FN_PTR(numa_parse_nodestring_all_func_t,
+                                         libnuma_dlsym(handle, "numa_parse_nodestring_all")));
+      set_numa_run_on_node_mask(CAST_TO_FN_PTR(numa_run_on_node_mask_func_t,
+                                         libnuma_v2_dlsym(handle, "numa_run_on_node_mask")));
+      set_numa_bitmask_equal(CAST_TO_FN_PTR(numa_bitmask_equal_func_t,
+                                         libnuma_v2_dlsym(handle, "numa_bitmask_equal")));
+      set_numa_set_membind(CAST_TO_FN_PTR(numa_set_membind_func_t,
+                                         libnuma_v2_dlsym(handle, "numa_set_membind")));
+      set_numa_bitmask_free(CAST_TO_FN_PTR(numa_bitmask_free_func_t,
+                                         libnuma_dlsym(handle, "numa_bitmask_free")));

       if (numa_available() != -1) {
         set_numa_all_nodes((unsigned long*)libnuma_dlsym(handle, "numa_all_nodes"));
@@ -3176,6 +3259,9 @@ bool os::Linux::libnuma_init() {
         set_numa_nodes_ptr((struct bitmask **)libnuma_dlsym(handle, "numa_nodes_ptr"));
         set_numa_interleave_bitmask(_numa_get_interleave_mask());
         set_numa_membind_bitmask(_numa_get_membind());
+        if (isbound_to_all_node()) {
+          parse_numa_nodes();
+        }
         // Create an index -> node mapping, since nodes are not always consecutive
         _nindex_to_node = new (ResourceObj::C_HEAP, mtInternal) GrowableArray<int>(0, true);
         rebuild_nindex_to_node_map();
@@ -3295,6 +3381,11 @@ os::Linux::numa_get_membind_func_t os::Linux::_numa_get_membind;
 os::Linux::numa_get_interleave_mask_func_t os::Linux::_numa_get_interleave_mask;
 os::Linux::numa_move_pages_func_t os::Linux::_numa_move_pages;
 os::Linux::numa_run_on_node_func_t os::Linux::_numa_run_on_node;
+os::Linux::numa_parse_nodestring_all_func_t os::Linux::_numa_parse_nodestring_all;
+os::Linux::numa_run_on_node_mask_func_t os::Linux::_numa_run_on_node_mask;
+os::Linux::numa_bitmask_equal_func_t os::Linux::_numa_bitmask_equal;
+os::Linux::numa_set_membind_func_t os::Linux::_numa_set_membind;
+os::Linux::numa_bitmask_free_func_t os::Linux::_numa_bitmask_free;
 os::Linux::NumaAllocationPolicy os::Linux::_current_numa_policy;
 unsigned long* os::Linux::_numa_all_nodes;
 struct bitmask* os::Linux::_numa_all_nodes_ptr;
diff --git a/hotspot/src/os/linux/vm/os_linux.hpp b/hotspot/src/os/linux/vm/os_linux.hpp
index 4ee2c9b..18ac68f 100644
--- a/hotspot/src/os/linux/vm/os_linux.hpp
+++ b/hotspot/src/os/linux/vm/os_linux.hpp
@@ -195,6 +195,7 @@ class Linux {
   static bool is_floating_stack()             { return _is_floating_stack; }

   static void libpthread_init();
+  static void parse_numa_nodes();
   static bool libnuma_init();
   static void* libnuma_dlsym(void* handle, const char* name);
   // libnuma v2 (libnuma_1.2) symbols
@@ -283,6 +284,11 @@ private:
   typedef struct bitmask* (*numa_get_interleave_mask_func_t)(void);
   typedef long (*numa_move_pages_func_t)(int pid, unsigned long count, void **pages, const int *nodes, int *status, int flags);
   typedef int (*numa_run_on_node_func_t)(int node);
+  typedef struct bitmask* (*numa_parse_nodestring_all_func_t)(const char*);
+  typedef int (*numa_run_on_node_mask_func_t)(struct bitmask* mask);
+  typedef void (*numa_set_membind_func_t)(struct bitmask* mask);
+  typedef int (*numa_bitmask_equal_func_t)(struct bitmask* mask, struct bitmask* mask1);
+  typedef void (*numa_bitmask_free_func_t)(struct bitmask* mask);

   typedef void (*numa_set_bind_policy_func_t)(int policy);
   typedef int (*numa_bitmask_isbitset_func_t)(struct bitmask *bmp, unsigned int n);
@@ -303,6 +309,11 @@ private:
   static numa_get_interleave_mask_func_t _numa_get_interleave_mask;
   static numa_move_pages_func_t _numa_move_pages;
   static numa_run_on_node_func_t _numa_run_on_node;
+  static numa_parse_nodestring_all_func_t _numa_parse_nodestring_all;
+  static numa_run_on_node_mask_func_t _numa_run_on_node_mask;
+  static numa_bitmask_equal_func_t _numa_bitmask_equal;
+  static numa_set_membind_func_t _numa_set_membind;
+  static numa_bitmask_free_func_t _numa_bitmask_free;

   static unsigned long* _numa_all_nodes;
   static struct bitmask* _numa_all_nodes_ptr;
@@ -325,6 +336,11 @@ private:
   static void set_numa_get_interleave_mask(numa_get_interleave_mask_func_t func) { _numa_get_interleave_mask = func; }
   static void set_numa_move_pages(numa_move_pages_func_t func) { _numa_move_pages = func; }
   static void set_numa_run_on_node(numa_run_on_node_func_t func) { _numa_run_on_node = func; }
+  static void set_numa_parse_nodestring_all(numa_parse_nodestring_all_func_t func) { _numa_parse_nodestring_all = func; }
+  static void set_numa_run_on_node_mask(numa_run_on_node_mask_func_t func) { _numa_run_on_node_mask = func; }
+  static void set_numa_bitmask_equal(numa_bitmask_equal_func_t func) { _numa_bitmask_equal = func; }
+  static void set_numa_set_membind(numa_set_membind_func_t func) { _numa_set_membind = func; }
+  static void set_numa_bitmask_free(numa_bitmask_free_func_t func) { _numa_bitmask_free = func; }
   static void set_numa_all_nodes(unsigned long* ptr) { _numa_all_nodes = ptr; }
   static void set_numa_all_nodes_ptr(struct bitmask **ptr) { _numa_all_nodes_ptr = (ptr == NULL ? NULL : *ptr); }
   static void set_numa_nodes_ptr(struct bitmask **ptr) { _numa_nodes_ptr = (ptr == NULL ? NULL : *ptr); }
@@ -472,6 +488,41 @@ public:
   static mallinfo_retval_t get_mallinfo(glibc_mallinfo2* out);
 #endif

+  static bool isbound_to_all_node() {
+    if (_numa_membind_bitmask != NULL && _numa_max_node != NULL && _numa_bitmask_isbitset != NULL) {
+      unsigned int highest_node_number = _numa_max_node();
+      for (unsigned int node = 0; node <= highest_node_number; node++) {
+        if (!_numa_bitmask_isbitset(_numa_membind_bitmask, node)) {
+          return false;
+        }
+      }
+    }
+    return true;
+  }
+
+  static bitmask* numa_parse_nodestring_all(const char* s) {
+    return _numa_parse_nodestring_all != NULL ? _numa_parse_nodestring_all(s) : NULL;
+  }
+
+  static int numa_run_on_node_mask(bitmask* bitmask) {
+    return _numa_run_on_node_mask != NULL ? _numa_run_on_node_mask(bitmask) : -1;
+  }
+
+  static int numa_bitmask_equal(bitmask* bitmask, struct bitmask* bitmask1) {
+    return _numa_bitmask_equal != NULL ? _numa_bitmask_equal(bitmask, bitmask1) : 1;
+  }
+
+  static void numa_set_membind(bitmask* bitmask) {
+    if (_numa_set_membind != NULL) {
+      _numa_set_membind(bitmask);
+    }
+  }
+
+  static void numa_bitmask_free(bitmask* bitmask) {
+    if (_numa_bitmask_free != NULL) {
+      _numa_bitmask_free(bitmask);
+    }
+  }
 };


diff --git a/hotspot/src/share/vm/prims/jni.cpp b/hotspot/src/share/vm/prims/jni.cpp
index dde3975..dd40c2c 100644
--- a/hotspot/src/share/vm/prims/jni.cpp
+++ b/hotspot/src/share/vm/prims/jni.cpp
@@ -5188,6 +5188,12 @@ DT_RETURN_MARK_DECL(CreateJavaVM, jint
                     , HOTSPOT_JNI_CREATEJAVAVM_RETURN(_ret_ref));
 #endif /* USDT2 */

+const char** argv_for_execvp;
+
+_JNI_IMPORT_OR_EXPORT_ void JNICALL JNI_SetCParam(char** raw_argv) {
+  argv_for_execvp = (const char**)raw_argv;
+}
+
 _JNI_IMPORT_OR_EXPORT_ jint JNICALL JNI_CreateJavaVM(JavaVM **vm, void **penv, void *args) {
 #ifndef USDT2
   HS_DTRACE_PROBE3(hotspot_jni, CreateJavaVM__entry, vm, penv, args);
diff --git a/hotspot/src/share/vm/prims/jni.h b/hotspot/src/share/vm/prims/jni.h
index 582f2c9..1c910e8 100644
--- a/hotspot/src/share/vm/prims/jni.h
+++ b/hotspot/src/share/vm/prims/jni.h
@@ -1937,6 +1937,9 @@ JNI_GetDefaultJavaVMInitArgs(void *args);
 _JNI_IMPORT_OR_EXPORT_ jint JNICALL
 JNI_CreateJavaVM(JavaVM **pvm, void **penv, void *args);

+_JNI_IMPORT_OR_EXPORT_ void JNICALL
+JNI_SetCParam(char**raw_argv);
+
 _JNI_IMPORT_OR_EXPORT_ jint JNICALL
 JNI_GetCreatedJavaVMs(JavaVM **, jsize, jsize *);

diff --git a/hotspot/src/share/vm/runtime/globals.hpp b/hotspot/src/share/vm/runtime/globals.hpp
index 2631971..3dd4c51 100644
--- a/hotspot/src/share/vm/runtime/globals.hpp
+++ b/hotspot/src/share/vm/runtime/globals.hpp
@@ -613,6 +613,19 @@ class CommandLineFlags {
   product(uintx, NUMAPageScanRate, 256,                                     \
           "Maximum number of pages to include in the page scan procedure")  \
                                                                             \
+  product(bool, LogNUMANodes, false,                                        \
+          "Print NUMANodes")                                                \
+                                                                            \
+  product(ccstr, NUMANodes, NULL,                                           \
+          "This parameter provides the same functionality as"               \
+          "'numactl --all -N <nodes> -m <nodes>'."                          \
+          "<nodes> can be '0-2', '0,1,2', 'all' and so on.")                \
+                                                                            \
+  product(uintx, NUMANodesRandom, 0,                                        \
+          "Number of continuous nodes to bind"                              \
+          "with the first node randomly chosen."                            \
+          "NUMANodesRandom has higher priority than NUMANodes")             \
+                                                                            \
   product_pd(bool, NeedsDeoptSuspend,                                       \
           "True for register window machines (sparc/ia64)")                 \
                                                                             \
diff --git a/hotspot/test/runtime/containers/docker/CPUSetsReader.java b/hotspot/test/runtime/containers/docker/CPUSetsReader.java
index f6fa93e..cb8ced2 100644
--- a/hotspot/test/runtime/containers/docker/CPUSetsReader.java
+++ b/hotspot/test/runtime/containers/docker/CPUSetsReader.java
@@ -51,6 +51,15 @@ public class CPUSetsReader {
         Asserts.assertEquals(listToString(parseCpuSet(cpuSet)), expectedResult);
     }

+    public static int getNumCpus() {
+        String path = "/proc/cpuinfo";
+        try {
+            Stream<String> stream = Files.lines(Paths.get(path));
+            return (int) stream.filter(line -> line.startsWith("processor")).count();
+        } catch (IOException e) {
+            return 0;
+        }
+    }

     public static String readFromProcStatus(String setType) {
         String path = PROC_SELF_STATUS_PATH;
diff --git a/hotspot/test/runtime/containers/docker/TestNUMANodes.java b/hotspot/test/runtime/containers/docker/TestNUMANodes.java
new file mode 100644
index 0000000..b781484
--- /dev/null
+++ b/hotspot/test/runtime/containers/docker/TestNUMANodes.java
@@ -0,0 +1,102 @@
+/**
+ * @test TestNUMANodes.java
+ * @library /testlibrary
+ * @build CPUSetsReader TestNUMANodes
+ * @run main/othervm TestNUMANodes 1 -XX:+UseNUMA -XX:NUMANodes=0 -XX:-LogNUMANodes
+ * @run main/othervm TestNUMANodes 2 -XX:+UseNUMA -XX:NUMANodes=all -XX:+LogNUMANodes
+ * @run main/othervm TestNUMANodes 3 -XX:+UseNUMA -XX:NUMANodesRandom=1 -XX:+LogNUMANodes
+ * @run main/othervm TestNUMANodes 4 -XX:+UseNUMA -XX:NUMANodesRandom=4 -XX:+LogNUMANodes
+ * @run main/othervm TestNUMANodes 5 -XX:+UseNUMA -XX:NUMANodes=100-200 -XX:+LogNUMANodes
+ * @summary test numanodes
+ * @author zhoulei
+ */
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+import java.util.stream.Stream;
+import com.oracle.java.testlibrary.Asserts;
+import com.oracle.java.testlibrary.OutputAnalyzer;
+import com.oracle.java.testlibrary.ProcessTools;
+
+public class TestNUMANodes {
+
+    private static int getNUMAs() throws Exception {
+        final String[] arguments = {"numactl", "-H"};
+        OutputAnalyzer output = ProcessTools.executeProcess(new ProcessBuilder(arguments));
+        String[] numainfo = output.getStdout().split("\n");
+        Optional<String> o = Arrays.asList(numainfo).stream()
+                            .filter(line -> line.contains("available"))
+                            .findFirst();
+        String numas = o.get();
+        return Integer.valueOf(numas.substring(11, 12));
+    }
+
+    private static class ExeTest {
+        public static void main(String[] str) throws Exception {
+            int numCpus = CPUSetsReader.getNumCpus();
+            String cpuSetStr = CPUSetsReader.readFromProcStatus("Cpus_allowed_list");
+            String[] cpus = cpuSetStr.split(",");
+            int total = 0;
+            for (String cpu : cpus) {
+                String[] c = cpu.split("-");
+                int start = Integer.valueOf(c[0]);
+                int end = Integer.valueOf(c[1]);
+                total += end - start + 1;
+            }
+            System.err.print(total);
+        }
+    }
+
+    private static OutputAnalyzer forkProcess(String[] args) throws Exception {
+        final String[] arguments = {
+                    args[1],
+                    args[2],
+                    args[3],
+                    ExeTest.class.getName(),
+                    args[0]
+            };
+
+        ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(arguments);
+        OutputAnalyzer output = new OutputAnalyzer(pb.start());
+        output.shouldHaveExitValue(0);
+        return output;
+    }
+
+    public static void main(String[] args)throws Exception {
+        OutputAnalyzer output = forkProcess(args);
+        String err = output.getStderr();
+        String out = output.getStdout();
+        int c = Integer.parseInt(args[0]);
+        int numas = TestNUMANodes.getNUMAs();
+        int numCpus = CPUSetsReader.getNumCpus();
+        switch(c) {
+            case 1:
+                int cpuUsed = Integer.valueOf(err);
+                Asserts.assertTrue(cpuUsed * numas == numCpus);
+                break;
+            case 2:
+                Asserts.assertTrue(err.contains("Mempolicy is not changed"));
+                break;
+            case 3:
+                if (numas > 1) {
+                    Asserts.assertTrue(err.contains("NUMANodes is converted to"));
+                }
+                break;
+            case 4:
+                Asserts.assertTrue(err.contains("The count of nodes to bind should be"));
+                break;
+            case 5:
+                Asserts.assertTrue(err.contains("is invalid"));
+                break;
+            default:
+                break;
+        }
+    }
+}
diff --git a/jdk/src/share/bin/java.c b/jdk/src/share/bin/java.c
index d74b185..c3d3b1b 100644
--- a/jdk/src/share/bin/java.c
+++ b/jdk/src/share/bin/java.c
@@ -245,6 +245,7 @@ JLI_Launch(int argc, char ** argv,              /* main argc, argc */

     ifn.CreateJavaVM = 0;
     ifn.GetDefaultJavaVMInitArgs = 0;
+    ifn.raw_argv = argv;

     if (JLI_IsTraceLauncher()) {
         start = CounterGet();
@@ -1237,6 +1238,7 @@ InitializeJVM(JavaVM **pvm, JNIEnv **penv, InvocationFunctions *ifn)
                    i, args.options[i].optionString);
     }

+    ifn->SetCParam(ifn->raw_argv);
     r = ifn->CreateJavaVM(pvm, (void **)penv, &args);
     JLI_MemFree(options);
     return r == JNI_OK;
diff --git a/jdk/src/share/bin/java.h b/jdk/src/share/bin/java.h
index 9dc0e16..9a8b839 100644
--- a/jdk/src/share/bin/java.h
+++ b/jdk/src/share/bin/java.h
@@ -78,13 +78,16 @@
  * Pointers to the needed JNI invocation API, initialized by LoadJavaVM.
  */
 typedef jint (JNICALL *CreateJavaVM_t)(JavaVM **pvm, void **env, void *args);
+typedef void (JNICALL *SetCParam_t)(char** raw_argv);
 typedef jint (JNICALL *GetDefaultJavaVMInitArgs_t)(void *args);
 typedef jint (JNICALL *GetCreatedJavaVMs_t)(JavaVM **vmBuf, jsize bufLen, jsize *nVMs);

 typedef struct {
     CreateJavaVM_t CreateJavaVM;
+    SetCParam_t SetCParam;
     GetDefaultJavaVMInitArgs_t GetDefaultJavaVMInitArgs;
     GetCreatedJavaVMs_t GetCreatedJavaVMs;
+    char** raw_argv;
 } InvocationFunctions;

 int
diff --git a/jdk/src/share/javavm/export/jni.h b/jdk/src/share/javavm/export/jni.h
index 2e83cb7..8567766 100644
--- a/jdk/src/share/javavm/export/jni.h
+++ b/jdk/src/share/javavm/export/jni.h
@@ -1937,6 +1937,9 @@ JNI_GetDefaultJavaVMInitArgs(void *args);
 _JNI_IMPORT_OR_EXPORT_ jint JNICALL
 JNI_CreateJavaVM(JavaVM **pvm, void **penv, void *args);

+_JNI_IMPORT_OR_EXPORT_ void JNICALL
+JNI_SetCParam(char** raw_argv);
+
 _JNI_IMPORT_OR_EXPORT_ jint JNICALL
 JNI_GetCreatedJavaVMs(JavaVM **, jsize, jsize *);

diff --git a/jdk/src/solaris/bin/java_md_solinux.c b/jdk/src/solaris/bin/java_md_solinux.c
index a967137..9865f9d 100644
--- a/jdk/src/solaris/bin/java_md_solinux.c
+++ b/jdk/src/solaris/bin/java_md_solinux.c
@@ -903,6 +903,13 @@ LoadJavaVM(const char *jvmpath, InvocationFunctions *ifn)
         return JNI_FALSE;
     }

+    ifn->SetCParam = (SetCParam_t)
+        dlsym(libjvm, "JNI_SetCParam");
+    if (ifn->SetCParam == NULL) {
+        JLI_ReportErrorMessage(DLL_ERROR2, jvmpath, dlerror());
+        return JNI_FALSE;
+    }
+
     ifn->GetDefaultJavaVMInitArgs = (GetDefaultJavaVMInitArgs_t)
         dlsym(libjvm, "JNI_GetDefaultJavaVMInitArgs");
     if (ifn->GetDefaultJavaVMInitArgs == NULL) {
--
1.8.3.1
