Implement v8 OOM error handling in Go

v8.go

@@ -156,6 +156,9 @@ func NewIsolate() *Isolate {
 	v8_init_once.Do(func() { C.v8_init() })
 	iso := &Isolate{ptr: C.v8_Isolate_New(C.StartupData{ptr: nil, len: 0}, nil)}
 	runtime.SetFinalizer(iso, (*Isolate).release)
+
+	addIsolate(iso)
+	setOOMErrorHandler(iso.ptr)
 	return iso
 }
 
@@ -165,19 +168,41 @@ func NewIsolateWithSnapshot(s *Snapshot) *Isolate {
 	v8_init_once.Do(func() { C.v8_init() })
 	iso := &Isolate{ptr: C.v8_Isolate_New(s.data, nil), s: s}
 	runtime.SetFinalizer(iso, (*Isolate).release)
+
+	addIsolate(iso)
+	setOOMErrorHandler(iso.ptr)
 	return iso
 }
 
-type ResourceConstraints struct {
+type IsolateOptions struct {
+	Snapshot *Snapshot
+	// MaxOldSpaceSize sets the maximum size of the old object heap in MiB.
 	MaxOldSpaceSize int
 }
 
-func NewIsolateWithConstraints(constraints ResourceConstraints) *Isolate {
+// NewIsolateWithOptions creates a new V8 Isolate applying additional options
+// like resource constraints or using the supplied Snapshot to initialize all
+// Contexts created from this Isolate.
+func NewIsolateWithOptions(opts IsolateOptions) (*Isolate, error) {
+	var startupData = C.StartupData{ptr: nil, len: 0}
+	var resourceConstraints C.ResourceConstraints
+
 	v8_init_once.Do(func() { C.v8_init() })
-	var c = C.ResourceConstraints{max_old_space_size: C.int(constraints.MaxOldSpaceSize)}
-	iso := &Isolate{ptr: C.v8_Isolate_New(C.StartupData{ptr: nil, len: 0}, &c)}
+	if opts.Snapshot != nil {
+		startupData = opts.Snapshot.data
+	}
+	if opts.MaxOldSpaceSize < 3 {
+		return nil, errors.New("MaxOldSpaceSize is too small to initialize v8")
+	}
+	if opts.MaxOldSpaceSize > 0 {
+		resourceConstraints = C.ResourceConstraints{max_old_space_size: C.int(opts.MaxOldSpaceSize)}
+	}
+	iso := &Isolate{ptr: C.v8_Isolate_New(startupData, &resourceConstraints)}
 	runtime.SetFinalizer(iso, (*Isolate).release)
-	return iso
+
+	addIsolate(iso)
+	setOOMErrorHandler(iso.ptr)
+	return iso, nil
 }
 
 // NewContext creates a new, clean V8 Context within this Isolate.
@@ -217,6 +242,42 @@ func (i *Isolate) convertErrorMsg(error_msg C.Error) error {
 	return err
 }
 
+type OOMErrorCallback func(location string, isHeapOOM bool)
+
+var oomErrorCallbackMutex sync.RWMutex
+var oomErrorHandler OOMErrorCallback
+
+//export go_oom_error_handler
+func go_oom_error_handler(location C.String, heapOOM C.int) {
+	if oomErrorHandler != nil {
+		var isHeapOOM bool
+		b := C.int(heapOOM)
+		if b == 1 {
+			isHeapOOM = true
+		}
+		oomErrorCallbackMutex.RLock()
+		oomErrorHandler(C.GoString(location.ptr), isHeapOOM)
+		oomErrorCallbackMutex.RUnlock()
+	}
+}
+
+func SetOOMErrorHandler(fn OOMErrorCallback) {
+	oomErrorCallbackMutex.Lock()
+	oomErrorHandler = fn
+	oomErrorCallbackMutex.Unlock()
+	for ptr := range isolates {
+		C.v8_Isolate_SetOOMErrorHandler(ptr)
+	}
+}
+
+func setOOMErrorHandler(ptr C.IsolatePtr) {
+	oomErrorCallbackMutex.RLock()
+	if oomErrorHandler != nil {
+		C.v8_Isolate_SetOOMErrorHandler(ptr)
+	}
+	oomErrorCallbackMutex.RUnlock()
+}
+
 // Context is a sandboxed js environment with its own set of built-in objects
 // and functions.  Values and javascript operations within a context are visible
 // only within that context unless the Go code explicitly moves values from one
@@ -535,6 +596,11 @@ var contexts = map[int]*refCount{}
 var contextsMutex sync.RWMutex
 var nextContextId int
 
+var (
+	isolatesMutex sync.RWMutex
+	isolates      = map[C.IsolatePtr]*Isolate{}
+)
+
 type refCount struct {
 	ptr   *Context
 	count int
@@ -544,7 +610,7 @@ func addRef(ctx *Context) {
 	contextsMutex.Lock()
 	ref := contexts[ctx.id]
 	if ref == nil {
-		ref = &refCount{ctx, 0}
+		ref = &refCount{ptr: ctx, count: 0}
 		contexts[ctx.id] = ref
 	}
 	ref.count++
@@ -561,6 +627,12 @@ func decRef(ctx *Context) {
 	contextsMutex.Unlock()
 }
 
+func addIsolate(iso *Isolate) {
+	isolatesMutex.Lock()
+	isolates[iso.ptr] = iso
+	isolatesMutex.Unlock()
+}
+
 //export go_callback_handler
 func go_callback_handler(
 	cbIdStr C.String,

v8_c_bridge.cc

@@ -1,4 +1,5 @@
 #include "v8_c_bridge.h"
+#include "_cgo_export.h"
 
 #include "libplatform/libplatform.h"
 #include "v8.h"
@@ -9,6 +10,8 @@
 #include <sstream>
 #include <stdio.h>
 
+#include <functional>
+
 #define ISOLATE_SCOPE(iso) \
   v8::Isolate* isolate = (iso);                                                               \
   v8::Locker locker(isolate);                            /* Lock to current thread.        */ \
@@ -24,6 +27,8 @@
 extern "C" ValueTuple go_callback_handler(
     String id, CallerInfo info, int argc, ValueTuple* argv);
 
+extern "C" void go_oom_error_handler(const char *location, bool is_heap_oom);
+
 // We only need one, it's stateless.
 auto allocator = v8::ArrayBuffer::Allocator::NewDefaultAllocator();
 
@@ -182,6 +187,11 @@ StartupData v8_CreateSnapshotDataBlob(const char* js) {
   return StartupData{data.data, data.raw_size};
 }
 
+void v8_Isolate_SetOOMErrorHandler(IsolatePtr isolate_ptr) {
+  v8::Isolate* isolate = static_cast<v8::Isolate*>(isolate_ptr);
+  isolate->SetOOMErrorHandler(go_oom_error_handler);
+}
+
 IsolatePtr v8_Isolate_New(StartupData startup_data, ResourceConstraints* resource_constraints) {
   v8::Isolate::CreateParams create_params;
   create_params.array_buffer_allocator = allocator;

v8_c_bridge.h

@@ -122,6 +122,7 @@ extern IsolatePtr v8_Isolate_New(StartupData data, ResourceConstraints* resource
 extern ContextPtr v8_Isolate_NewContext(IsolatePtr isolate);
 extern void       v8_Isolate_Terminate(IsolatePtr isolate);
 extern void       v8_Isolate_Release(IsolatePtr isolate);
+extern void       v8_Isolate_SetOOMErrorHandler(IsolatePtr isolate);
 
 extern HeapStatistics       v8_Isolate_GetHeapStatistics(IsolatePtr isolate);
 extern void                 v8_Isolate_LowMemoryNotification(IsolatePtr isolate);

v8_test.go

@@ -1541,3 +1541,89 @@ func TestPanicHandling(t *testing.T) {
 	_ = NewIsolate()
 	_ = *f
 }
+
+func TestNewIsolateWithResourceConstraints(t *testing.T) {
+	// Creates a v8 runtime where the memory is limited to 3MB and memory is
+	// allocated with a small script until v8 runs out of memory.
+	t.Parallel()
+	isolate, err := NewIsolateWithOptions(IsolateOptions{MaxOldSpaceSize: 3})
+	if err != nil {
+		t.Fatal(err)
+	}
+	SetOOMErrorHandler(func(location string, isHeapOOM bool) {
+		// Mark test as skipped, otherwise it would crash the process
+		t.SkipNow()
+	})
+	ctx := isolate.NewContext()
+	for i := 0; i < 10; i++ {
+		_, err := ctx.Eval(fmt.Sprintf("var array%d = new Array(100000); array%d.fill(1);", i, i), "test.js")
+		if err != nil {
+			t.Fatalf("Error evaluating javascript, err: %v", err)
+		}
+	}
+}
+
+func TestNewIsolateWithOptions(t *testing.T) {
+	var newIsolateWithOptionsTests = []struct {
+		name string
+		opts IsolateOptions
+		err  string
+	}{
+		{
+			name: "negative max old space size",
+			opts: IsolateOptions{
+				MaxOldSpaceSize: -1,
+			},
+			err: "MaxOldSpaceSize is too small to initialize v8",
+		},
+		{
+			name: "too little memory to initialize v8",
+			opts: IsolateOptions{
+				MaxOldSpaceSize: 2,
+			},
+			err: "MaxOldSpaceSize is too small to initialize v8",
+		},
+	}
+	for _, tt := range newIsolateWithOptionsTests {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			_, err := NewIsolateWithOptions(tt.opts)
+			if tt.err != "" && err == nil {
+				t.Fatalf("expected error %s, actual %v", tt.err, err)
+			}
+			if err.Error() != tt.err {
+				t.Fatalf("expected error %s, actual %v", tt.err, err)
+			}
+		})
+	}
+}
+
+func TestNewIsolateWithResourceConstraintsMultipleIsolates(t *testing.T) {
+	// Creates a v8 runtime where the memory is limited to 3MB and memory is
+	// allocated with a small script until v8 runs out of memory.
+	t.Parallel()
+	_, err := NewIsolateWithOptions(IsolateOptions{MaxOldSpaceSize: 3})
+	if err != nil {
+		t.Fatal(err)
+	}
+	SetOOMErrorHandler(func(location string, isHeapOOM bool) {
+		// Mark test as skipped, otherwise it would crash the process
+		t.SkipNow()
+	})
+	_, err = NewIsolateWithOptions(IsolateOptions{MaxOldSpaceSize: 3})
+	if err != nil {
+		t.Fatal(err)
+	}
+	isolate, err := NewIsolateWithOptions(IsolateOptions{MaxOldSpaceSize: 3})
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx := isolate.NewContext()
+	for i := 0; i < 10; i++ {
+		_, err := ctx.Eval(fmt.Sprintf("var array%d = new Array(100000); array%d.fill(1);", i, i), "test.js")
+		if err != nil {
+			t.Fatalf("Error evaluating javascript, err: %v", err)
+		}
+	}
+}