It throws StackOverflowError when executing Runtime.exec() by JNI after glog initialized.
kaka11chen opened this issue · comments
Issue
It throws StackOverflowError when executing Runtime.exec() by JNI after glog initialized.
Exception in thread "process reaper" Done spawn.
java.lang.StackOverflowError
at java.base/jdk.internal.misc.Unsafe.compareAndSetLong(Native Method)
at java.base/java.util.concurrent.ConcurrentHashMap.addCount(ConcurrentHashMap.java:2327)
at java.base/java.util.concurrent.ConcurrentHashMap.putVal(ConcurrentHashMap.java:1075)
at java.base/java.util.concurrent.ConcurrentHashMap.putIfAbsent(ConcurrentHashMap.java:1541)
at java.base/java.lang.invoke.MethodType$ConcurrentWeakInternSet.add(MethodType.java:1398)
at java.base/java.lang.invoke.MethodType.makeImpl(MethodType.java:357)
at java.base/java.lang.invoke.MethodTypeForm.<init>(MethodTypeForm.java:197)
at java.base/java.lang.invoke.MethodTypeForm.findForm(MethodTypeForm.java:223)
at java.base/java.lang.invoke.MethodType.makeImpl(MethodType.java:356)
at java.base/java.lang.invoke.MethodTypeForm.canonicalize(MethodTypeForm.java:253)
at java.base/java.lang.invoke.MethodTypeForm.findForm(MethodTypeForm.java:220)
at java.base/java.lang.invoke.MethodType.makeImpl(MethodType.java:356)
at java.base/java.lang.invoke.MethodType.insertParameterTypes(MethodType.java:448)
at java.base/java.lang.invoke.VarHandle$AccessDescriptor.<init>(VarHandle.java:1982)
at java.base/java.lang.invoke.MethodHandleNatives.varHandleOperationLinkerMethod(MethodHandleNatives.java:541)
at java.base/java.lang.invoke.MethodHandleNatives.linkMethodImpl(MethodHandleNatives.java:485)
at java.base/java.lang.invoke.MethodHandleNatives.linkMethod(MethodHandleNatives.java:473)
at java.base/java.util.concurrent.CompletableFuture.completeValue(CompletableFuture.java:306)
at java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2146)
at java.base/java.lang.ProcessHandleImpl$1.run(ProcessHandleImpl.java:171)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:833)
This issue will not occur in jdk 1.8, but both java 17 and java 21 have this issue.
Workaround
The root cause I analyzed is that the glog allocated and initialized some vars described by following source code in glog-0.6.0/src/logging.cc.
The workaround is reduce memory usage by decrease LogMessage::kMaxLogMessageLen from 30000 to 1024.
const size_t LogMessage::kMaxLogMessageLen = 30000;
struct LogMessage::LogMessageData {
LogMessageData();
int preserved_errno_; // preserved errno
// Buffer space; contains complete message text.
char message_text_[LogMessage::kMaxLogMessageLen+1];
LogStream stream_;
char severity_; // What level is this LogMessage logged at?
int line_; // line number where logging call is.
void (LogMessage::*send_method_)(); // Call this in destructor to send
union { // At most one of these is used: union to keep the size low.
LogSink* sink_; // NULL or sink to send message to
std::vector<std::string>* outvec_; // NULL or vector to push message onto
std::string* message_; // NULL or string to write message into
};
size_t num_prefix_chars_; // # of chars of prefix in this message
size_t num_chars_to_log_; // # of chars of msg to send to log
size_t num_chars_to_syslog_; // # of chars of msg to send to syslog
const char* basename_; // basename of file that called LOG
const char* fullname_; // fullname of file that called LOG
bool has_been_flushed_; // false => data has not been flushed
bool first_fatal_; // true => this was first fatal msg
private:
LogMessageData(const LogMessageData&);
void operator=(const LogMessageData&);
};
static LogMessage::LogMessageData fatal_msg_data_exclusive;
static LogMessage::LogMessageData fatal_msg_data_shared;Steps to Reproduce
- Build glog and glags to static libs (or shared libs).
- Build ForkTest.java by "javac ForkTest.java".
- Build ForkTest.cpp by
clang++ ForkTest.cpp -I/mnt/datadisk0/chenqi/jdk-17.0.5/include -I/mnt/datadisk0/chenqi/jdk-17.0.5/include/linux -I/mnt/datadisk0/chenqi/include/ -lglog -lgflags -L/mnt/datadisk0/chenqi/glog-0.6.0/build/ -L/mnt/datadisk0/chenqi/gflags-2.2.2/build/lib -ldl -lpthread -o forktest - export JAVA_HOME=''
- forktest java
Run multiple times, the error will occur sometime.
ForkTest.cpp:
#include <jni.h>
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <string>
#include <memory>
#include <mutex>
#include <filesystem>
#include <dlfcn.h>
#include <glog/logging.h>
class LibJVMLoader {
public:
LibJVMLoader(const LibJVMLoader&) = delete;
LibJVMLoader& operator=(const LibJVMLoader&) = delete;
static LibJVMLoader& instance();
int load();
using JNI_GetCreatedJavaVMsPointer = std::add_pointer_t<decltype(::JNI_GetCreatedJavaVMs)>;
static JNI_GetCreatedJavaVMsPointer JNI_GetCreatedJavaVMs;
using JNI_CreateJavaVMPointer = std::add_pointer_t<decltype(::JNI_CreateJavaVM)>;
static JNI_CreateJavaVMPointer JNI_CreateJavaVM;
private:
explicit LibJVMLoader(std::string_view library)
: _library(library), _handle(nullptr, nullptr) {}
const std::string _library;
std::unique_ptr<void, void (*)(void*)> _handle;
};
_JNI_IMPORT_OR_EXPORT_ jint JNICALL JNI_GetCreatedJavaVMs(JavaVM** vm_buf, jsize bufLen,
jsize* numVMs) {
return LibJVMLoader::JNI_GetCreatedJavaVMs(vm_buf, bufLen, numVMs);
}
_JNI_IMPORT_OR_EXPORT_ jint JNICALL JNI_CreateJavaVM(JavaVM** pvm, void** penv, void* args) {
return LibJVMLoader::JNI_CreateJavaVM(pvm, penv, args);
}
namespace {
#ifndef __APPLE__
#define LIBJVM_SO "libjvm.so"
#else
#define LIBJVM_SO "libjvm.dylib"
#endif
template <typename T>
int resolve_symbol(T& pointer, void* handle, const char* symbol) {
pointer = reinterpret_cast<T>(dlsym(handle, symbol));
return (pointer != nullptr)
? 0
: -1;
}
} // namespace
LibJVMLoader::JNI_GetCreatedJavaVMsPointer LibJVMLoader::JNI_GetCreatedJavaVMs = nullptr;
LibJVMLoader::JNI_CreateJavaVMPointer LibJVMLoader::JNI_CreateJavaVM = nullptr;
LibJVMLoader& LibJVMLoader::instance() {
static std::once_flag find_library;
static std::string library;
std::call_once(find_library, []() {
const auto* java_home = getenv("JAVA_HOME");
if (!java_home) {
return;
}
std::string path(java_home);
for (const auto& entry : std::filesystem::recursive_directory_iterator(path)) {
if (entry.path().filename() == LIBJVM_SO) {
library = entry.path().string();
break;
}
}
fprintf(stderr, "path: %s\n", path.c_str());
});
static LibJVMLoader loader(library);
return loader;
}
int LibJVMLoader::load() {
if (_library.empty()) {
fprintf(stderr, "_library is empty\n");
return -1;
}
static std::once_flag resolve_symbols;
static int status;
std::call_once(resolve_symbols, [this]() {
_handle = std::unique_ptr<void, void (*)(void*)>(dlopen(_library.c_str(), RTLD_LAZY),
[](void* handle) { dlclose(handle); });
if (!_handle) {
status = -1;
return;
}
if (-1 == resolve_symbol(JNI_GetCreatedJavaVMs, _handle.get(), "JNI_GetCreatedJavaVMs")) {
status = -1;
return;
}
if (-1 == resolve_symbol(JNI_CreateJavaVM, _handle.get(), "JNI_CreateJavaVM")) {
status = -1;
return;
}
});
return status;
}
// Create the Java virtual machine, and start a test object.
static JNIEnv* createJVM(void) {
if (LibJVMLoader::instance().load() < 0) {
return nullptr;
}
JavaVM* jvm;
JNIEnv* env;
JavaVMOption options[20];
int numOptions = 0;
options[numOptions++].optionString = "-Djava.class.path=.";
options[numOptions++].optionString = "-Xss1m";
JavaVMInitArgs args;
args.version = JNI_VERSION_1_2;
args.options = options;
args.nOptions = (jint)numOptions;
args.ignoreUnrecognized = 0;
if (JNI_CreateJavaVM(&jvm, (void **)&env, &args)) {
fprintf(stderr, "Can't create JVM.\n");
}
fprintf(stderr, "JVM created.\n");
return env;
}
void testJavaFork(JNIEnv *env) {
jclass clazz = env->FindClass("ForkTest");
if (env->ExceptionOccurred() || (clazz == 0)) {
fprintf(stderr, "Can't find Java class.\n");
exit(1);
}
jmethodID init = env->GetMethodID(clazz, "<init>", "()V");
if (env->ExceptionOccurred() || (init == 0)) {
fprintf(stderr, "Can't find Java method.\n");
exit(1);
}
jobject obj = env->NewObject(clazz, init);
if (env->ExceptionOccurred() || (obj == 0)) {
fprintf(stderr, "Java object created.\n");
exit(1);
}
fprintf(stderr, "Java object created.\n");
}
void testCFork(void) {
pid_t pid;
if ((pid = fork()) == 0) {
// Child process
execl("/bin/ls", "ls", (char *)0);
fprintf(stderr, "Error in C exec.\n");
exit(1);
}
// Parent process
fprintf(stderr, "Child PID = %d\n", pid);
}
int main(int argc, char **argv) {
google::LogToStderr();
google::InitGoogleLogging("fork_test");
LOG(INFO) << "hello world";
JNIEnv* env = createJVM();
if (!env) {
exit(1);
}
if (argc > 1) {
testJavaFork(env);
} else {
testCFork();
}
LOG(INFO) << "Waiting...";
sleep(3);
LOG(INFO) << "Running Successfully.";
google::ShutdownGoogleLogging();
return 0;
}ForkTest.java:
import java.io.*;
public class ForkTest {
public static void main(String[] args) throws Exception {
new ForkTest();
}
public ForkTest() {
exec();
}
/**
* Spawn a process to run a ticker shell script.
*/
public void exec() {
try {
System.out.println("Java Fork Test....");
Process proc = Runtime.getRuntime().exec("ls");
proc.waitFor();
System.out.println("Java ForkTest exit status = " + proc.exitValue());
} catch (Exception e) {
System.err.println("Java ForkTest failed " + e.getMessage());
return;
}
}
}
Thanks for the report.
Your Java backtrace shows a lot of unrelated code that indicates the backtrace does not stem from your reproducer. Could please provide the backtrace showing where exactly the problem occurs both in Java and in the native code for the reproducer only?
The full log of error case
If we use jdk-17.0.5 as JAVA_HOME.
[chenqi@VM-0-101-centos ~]$ export JAVA_HOME=/mnt/datadisk0/chenqi/jdk-17.0.5
[chenqi@VM-0-101-centos ~]$ ./forktest java
I20231016 00:51:14.832365 3518955 ForkTest.cpp:176] hello world
path: /mnt/datadisk0/chenqi/jdk-17.0.5
JVM created.
Java Fork Test....
Exception in thread "process reaper" java.lang.StackOverflowError
at java.base/jdk.internal.misc.Unsafe.compareAndSetLong(Native Method)
at java.base/java.util.concurrent.ConcurrentHashMap.addCount(ConcurrentHashMap.java:2327)
at java.base/java.util.concurrent.ConcurrentHashMap.putVal(ConcurrentHashMap.java:1075)
at java.base/java.util.concurrent.ConcurrentHashMap.putIfAbsent(ConcurrentHashMap.java:1541)
at java.base/java.lang.invoke.MethodType$ConcurrentWeakInternSet.add(MethodType.java:1398)
at java.base/java.lang.invoke.MethodType.makeImpl(MethodType.java:357)
at java.base/java.lang.invoke.MethodTypeForm.<init>(MethodTypeForm.java:197)
at java.base/java.lang.invoke.MethodTypeForm.findForm(MethodTypeForm.java:223)
at java.base/java.lang.invoke.MethodType.makeImpl(MethodType.java:356)
at java.base/java.lang.invoke.MethodTypeForm.canonicalize(MethodTypeForm.java:253)
at java.base/java.lang.invoke.MethodTypeForm.findForm(MethodTypeForm.java:220)
at java.base/java.lang.invoke.MethodType.makeImpl(MethodType.java:356)
at java.base/java.lang.invoke.MethodType.insertParameterTypes(MethodType.java:448)
at java.base/java.lang.invoke.VarHandle$AccessDescriptor.<init>(VarHandle.java:1982)
at java.base/java.lang.invoke.MethodHandleNatives.varHandleOperationLinkerMethod(MethodHandleNatives.java:541)
at java.base/java.lang.invoke.MethodHandleNatives.linkMethodImpl(MethodHandleNatives.java:485)
at java.base/java.lang.invoke.MethodHandleNatives.linkMethod(MethodHandleNatives.java:473)
at java.base/java.util.concurrent.CompletableFuture.completeValue(CompletableFuture.java:306)
at java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2146)
at java.base/java.lang.ProcessHandleImpl$1.run(ProcessHandleImpl.java:171)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:833)
Exception in thread "process reaper" java.lang.StackOverflowError
at java.base/java.util.Arrays.equals(Arrays.java:2969)
at java.base/java.lang.invoke.MethodType.equals(MethodType.java:858)
at java.base/java.lang.invoke.MethodType.equals(MethodType.java:850)
at java.base/java.util.concurrent.ConcurrentHashMap.get(ConcurrentHashMap.java:947)
at java.base/java.lang.invoke.MethodType$ConcurrentWeakInternSet.get(MethodType.java:1368)
at java.base/java.lang.invoke.MethodType.makeImpl(MethodType.java:341)
at java.base/java.lang.invoke.MethodTypeForm.<init>(MethodTypeForm.java:197)
at java.base/java.lang.invoke.MethodTypeForm.findForm(MethodTypeForm.java:223)
at java.base/java.lang.invoke.MethodType.makeImpl(MethodType.java:356)
at java.base/java.lang.invoke.MethodTypeForm.canonicalize(MethodTypeForm.java:253)
at java.base/java.lang.invoke.MethodTypeForm.findForm(MethodTypeForm.java:220)
at java.base/java.lang.invoke.MethodType.makeImpl(MethodType.java:356)
at java.base/java.lang.invoke.MethodType.insertParameterTypes(MethodType.java:448)
at java.base/java.lang.invoke.VarHandle$AccessDescriptor.<init>(VarHandle.java:1982)
at java.base/java.lang.invoke.MethodHandleNatives.varHandleOperationLinkerMethod(MethodHandleNatives.java:541)
at java.base/java.lang.invoke.MethodHandleNatives.linkMethodImpl(MethodHandleNatives.java:485)
at java.base/java.lang.invoke.MethodHandleNatives.linkMethod(MethodHandleNatives.java:473)
at java.base/java.util.concurrent.SynchronousQueue$TransferStack.casHead(SynchronousQueue.java:317)
at java.base/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:371)
at java.base/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903)
at java.base/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:833)
The full log of correct case
If we use jdk1.8.0_131 as JAVA_HOME or modify LogMessage::kMaxLogMessageLen to 1024.
[chenqi@VM-0-101-centos ~]$ ./forktest java
I20231016 00:47:37.484891 3504232 ForkTest.cpp:176] hello world
path: /mnt/datadisk0/chenqi/jdk1.8.0_131
JVM created.
Java Fork Test....
Java ForkTest exit status = 0
Java object created.
I20231016 00:47:37.600541 3504232 ForkTest.cpp:188] Waiting...
I20231016 00:47:40.600647 3504232 ForkTest.cpp:190] Running Successfully.
Thanks. I can repro the issue with glog 0.6.0 and Java 21 as well. Disabling thread-local storage by setting WITH_TLS=OFF lets the problem go away.
It looks like Java is indeed exhausting the whole stack. I would not touch the size of the message buffer since this looks like a Java issue.
Thanks. I can repro the issue with glog 0.6.0 and Java 21 as well. Disabling thread-local storage by setting
WITH_TLS=OFFlets the problem go away.It looks like Java is indeed exhausting the whole stack. I would not touch the size of the message buffer since this looks like a Java issue.
Thanks very much. I have reported this issue to oracle, and I will update this TLS issue.
Let me close the issue since at this point there is nothing do be done.