Lua hot reload

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/blugelabs/bluge v0.2.2
 	github.com/blugelabs/bluge_segment_api v0.2.0
 	github.com/blugelabs/query_string v0.3.0
+	github.com/dietsche/rfsnotify v0.0.0-20200716145600-b37be6e4177f
 	github.com/dop251/goja v0.0.0-20221003171542-5ea1285e6c91
 	github.com/gofrs/uuid v4.3.0+incompatible
 	github.com/golang-jwt/jwt/v4 v4.4.2
@@ -29,6 +30,7 @@ require (
 	google.golang.org/grpc v1.50.0
 	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.2.0
 	google.golang.org/protobuf v1.28.1
+	gopkg.in/fsnotify.v1 v1.4.7
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0-20190411184413-94d9e492cc53
 	gopkg.in/yaml.v3 v3.0.1
 )

go.sum

@@ -131,6 +131,8 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/denisenkom/go-mssqldb v0.9.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc h1:8WFBn63wegobsYAX0YjD+8suexZDga5CctH4CCTx2+8=
 github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw=
+github.com/dietsche/rfsnotify v0.0.0-20200716145600-b37be6e4177f h1:b3QvpXLSx1U13VM79rSkA+6Xv4lmT/urEMzA36Yma0U=
+github.com/dietsche/rfsnotify v0.0.0-20200716145600-b37be6e4177f/go.mod h1:ztitxkMUaBsHRey1tS5xFCd4gm/zAQwA9yfCP5y4cAA=
 github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
 github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo=
 github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
@@ -151,6 +153,7 @@ github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8S
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -956,6 +959,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
 gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0-20190411184413-94d9e492cc53 h1:7D4Fu4wpNSw/l+7Y5tL2ocLV1YC6BSOLs/o5OP1MfQI=

server/runtime_lua.go

@@ -27,7 +27,9 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
+	"github.com/dietsche/rfsnotify"
 	"github.com/gofrs/uuid"
 	"github.com/heroiclabs/nakama-common/api"
 	"github.com/heroiclabs/nakama-common/rtapi"
@@ -38,6 +40,7 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/proto"
+	"gopkg.in/fsnotify.v1"
 )
 
 const LTSentinel = lua.LValueType(-1)
@@ -65,23 +68,201 @@ type RuntimeLuaCallbacks struct {
 	SubscriptionNotificationGoogle *lua.LFunction
 }
 
+const (
+	luaDisableHotfixMarker = "---@disable-hotfix"
+)
+
 type RuntimeLuaModule struct {
-	Name    string
-	Path    string
-	Content []byte
+	HotfixDisabled *atomic.Bool
+	Name           string
+	Path           string
+	Content        []byte
+}
+
+func (m *RuntimeLuaModule) Hotfix(vm *lua.LState) error {
+	vm.Push(vm.NewFunction(lua.OpenString))
+	vm.Push(lua.LString(lua.StringLibName))
+	vm.Call(1, 0)
+
+	f, err := vm.Load(bytes.NewReader(m.Content), m.Name)
+	if err != nil {
+		return err
+	}
+	vm.Push(f)
+	if err = vm.PCall(0, -1, nil); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *RuntimeLuaModule) Patch(vm *lua.LState) error {
+	// Update preload function
+	preload := vm.GetField(vm.GetField(vm.Get(lua.EnvironIndex), "package"), "preload")
+	f, err := vm.Load(bytes.NewReader(m.Content), m.Name)
+	if err != nil {
+		return err
+	}
+	vm.SetField(preload, m.Name, f)
+
+	// Update module which alreade loaded
+	loaded := vm.GetField(vm.Get(lua.RegistryIndex), "_LOADED")
+	lv := vm.GetField(loaded, m.Name)
+	if !lua.LVAsBool(lv) {
+		// Not yet evaluated
+		return nil
+	}
+
+	vm.Push(f)
+	return vm.PCall(0, -1, nil)
 }
 
 type RuntimeLuaModuleCache struct {
-	Names   []string
-	Modules map[string]*RuntimeLuaModule
+	sync.RWMutex
+	names   []string
+	modules map[string]*RuntimeLuaModule
+}
+
+func NewRuntimeLuaModuleCache() *RuntimeLuaModuleCache {
+	return &RuntimeLuaModuleCache{
+		names:   make([]string, 0),
+		modules: make(map[string]*RuntimeLuaModule, 0),
+	}
 }
 
 func (mc *RuntimeLuaModuleCache) Add(m *RuntimeLuaModule) {
-	mc.Names = append(mc.Names, m.Name)
-	mc.Modules[m.Name] = m
+	mc.Lock()
+	defer mc.Unlock()
+
+	if old, ok := mc.modules[m.Name]; !ok {
+		mc.names = append(mc.names, m.Name)
+		// Ensure modules will be listed in ascending order of names.
+		sort.Strings(mc.names)
+	} else {
+		// Preserve hotfix disabled state.
+		m.HotfixDisabled.Swap(old.HotfixDisabled.Load())
+	}
 
-	// Ensure modules will be listed in ascending order of names.
-	sort.Strings(mc.Names)
+	mc.modules[m.Name] = m
+}
+
+func (mc *RuntimeLuaModuleCache) Len() int {
+	mc.RLock()
+	defer mc.RUnlock()
+	return len(mc.names)
+}
+
+func (mc *RuntimeLuaModuleCache) List() []string {
+	mc.RLock()
+	defer mc.RUnlock()
+	clone := make([]string, len(mc.names))
+	copy(clone, mc.names)
+	return clone
+}
+
+func (mc *RuntimeLuaModuleCache) Get(name string) (*RuntimeLuaModule, bool) {
+	mc.RLock()
+	defer mc.RUnlock()
+	if m, ok := mc.modules[name]; ok {
+		return m, ok
+	}
+	return nil, false
+}
+
+func (mc *RuntimeLuaModuleCache) Watch(startupLogger, logger *zap.Logger,
+	runtimeProviderLua *RuntimeProviderLua, modulePatchRegistry *LocalRuntimeLuaModulePatchRegistry) {
+	watcher, err := rfsnotify.NewWatcher()
+	if err != nil {
+		startupLogger.Fatal("Failed to create runtime directory watcher", zap.Error(err))
+	}
+	go func() {
+		ctx := context.Background()
+		for {
+			select {
+			case <-ctx.Done():
+				// Context cancelled
+				return
+			case event, ok := <-watcher.Events:
+				if !ok {
+					return
+				}
+				if strings.ToLower(filepath.Ext(event.Name)) != ".lua" {
+					break
+				}
+				var name, relPath string
+				relPath, _ = filepath.Rel(lua.LuaLDir, event.Name)
+				name = strings.TrimSuffix(relPath, filepath.Ext(relPath))
+				// Make paths Lua friendly.
+				name = strings.Replace(name, string(os.PathSeparator), ".", -1)
+				switch event.Op {
+				case fsnotify.Write:
+					// Skip static files.
+					if m, ok := mc.Get(name); ok && m.HotfixDisabled.Load() {
+						break
+					}
+					fallthrough
+				case fsnotify.Create:
+					content, err := os.ReadFile(event.Name)
+					if err != nil {
+						logger.Warn("An error occurred while reading lua module", zap.Error(err))
+						break
+					}
+					static := strings.HasPrefix(string(content), luaDisableHotfixMarker)
+					m := &RuntimeLuaModule{
+						HotfixDisabled: atomic.NewBool(static),
+						Name:           name,
+						Path:           event.Name,
+						Content:        content,
+					}
+					completes := uint32(0)
+					// Loop through the pool and hotfix each runtime.
+				HotfixProcess:
+					for {
+						select {
+						case <-ctx.Done():
+							// Context cancelled
+							return
+						case r := <-runtimeProviderLua.poolCh:
+							runtimeProviderLua.poolCh <- r
+							// Discard hotfix if the first attempt fails.
+							if completes == 0 {
+								if err := m.Hotfix(r.vm); err != nil {
+									logger.Warn("An error occurred while patching lua module",
+										zap.String("module", name), zap.Error(err))
+									break HotfixProcess
+								}
+								// Post the hotfix to match registry.
+								modulePatchRegistry.Post(m)
+							} else if err := m.Patch(r.vm); err != nil {
+								logger.Warn("An error occurred while patching lua module",
+									zap.String("module", name), zap.Error(err))
+								break HotfixProcess
+							}
+							completes++
+							if completes < runtimeProviderLua.currentCount.Load() {
+								continue
+							}
+							mc.Add(m)
+							logger.Info("Lua runtime patched",
+								zap.String("module", name),
+								zap.Uint32("runtimes", completes))
+							break HotfixProcess
+						default:
+							time.Sleep(100 * time.Millisecond)
+						}
+					}
+				}
+			case err, ok := <-watcher.Errors:
+				if !ok {
+					return
+				}
+				logger.Error("An error occurred while watching directory", zap.Error(err))
+			}
+		}
+	}()
+	if err = watcher.AddRecursive(lua.LuaLDir); err != nil {
+		startupLogger.Fatal("An error occurred while watching directory", zap.Error(err))
+	}
+	startupLogger.Info("Watching runtime directory", zap.String("path", lua.LuaLDir))
 }
 
 type RuntimeProviderLua struct {
@@ -99,6 +280,7 @@ type RuntimeProviderLua struct {
 	metrics              Metrics
 	router               MessageRouter
 	stdLibs              map[string]lua.LGFunction
+	modulePatchRegistry  *LocalRuntimeLuaModulePatchRegistry
 
 	once         *sync.Once
 	poolCh       chan *RuntimeLua
@@ -119,6 +301,11 @@ func NewRuntimeProviderLua(logger, startupLogger *zap.Logger, db *sql.DB, protoj
 		return nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, err
 	}
 
+	modulePatchRegistry := &LocalRuntimeLuaModulePatchRegistry{
+		MapOf:  &MapOf[uuid.UUID, chan *RuntimeLuaModule]{},
+		logger: startupLogger,
+	}
+
 	once := &sync.Once{}
 	localCache := NewRuntimeLuaLocalCache()
 	rpcFunctions := make(map[string]RuntimeRpcFunction, 0)
@@ -149,6 +336,7 @@ func NewRuntimeProviderLua(logger, startupLogger *zap.Logger, db *sql.DB, protoj
 		leaderboardRankCache: leaderboardRankCache,
 		sessionRegistry:      sessionRegistry,
 		matchRegistry:        matchRegistry,
+		modulePatchRegistry:  modulePatchRegistry,
 		tracker:              tracker,
 		metrics:              metrics,
 		router:               router,
@@ -165,7 +353,7 @@ func NewRuntimeProviderLua(logger, startupLogger *zap.Logger, db *sql.DB, protoj
 
 	matchProvider.RegisterCreateFn("lua",
 		func(ctx context.Context, logger *zap.Logger, id uuid.UUID, node string, stopped *atomic.Bool, name string) (RuntimeMatchCore, error) {
-			return NewRuntimeLuaMatchCore(logger, name, db, protojsonMarshaler, protojsonUnmarshaler, config, version, socialClient, leaderboardCache, leaderboardRankCache, leaderboardScheduler, sessionRegistry, sessionCache, statusRegistry, matchRegistry, tracker, metrics, streamManager, router, stdLibs, once, localCache, eventFn, nil, nil, id, node, stopped, name, matchProvider)
+			return NewRuntimeLuaMatchCore(logger, name, db, protojsonMarshaler, protojsonUnmarshaler, config, version, socialClient, leaderboardCache, leaderboardRankCache, leaderboardScheduler, sessionRegistry, sessionCache, statusRegistry, matchRegistry, tracker, metrics, streamManager, router, stdLibs, once, localCache, eventFn, nil, nil, id, node, stopped, name, matchProvider, modulePatchRegistry)
 		},
 	)
 
@@ -1145,6 +1333,8 @@ func NewRuntimeProviderLua(logger, startupLogger *zap.Logger, db *sql.DB, protoj
 		return nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, err
 	}
 
+	moduleCache.Watch(startupLogger, logger, runtimeProviderLua, modulePatchRegistry)
+
 	if config.GetRuntime().GetLuaReadOnlyGlobals() {
 		// Capture shared globals from reference state.
 		sharedGlobals = r.vm.NewTable()
@@ -1175,6 +1365,11 @@ func NewRuntimeProviderLua(logger, startupLogger *zap.Logger, db *sql.DB, protoj
 			loadedTable.Metatable = vm.GetField(stateRegistry, "_LOADED")
 			vm.SetField(stateRegistry, "_LOADED", loadedTable)
 
+			// Metatable for literal string object.
+			vm.Push(vm.NewFunction(lua.OpenString))
+			vm.Push(lua.LString(lua.StringLibName))
+			vm.Call(1, 0)
+
 			r := &RuntimeLua{
 				logger:    logger,
 				node:      config.GetName(),
@@ -1201,7 +1396,7 @@ func NewRuntimeProviderLua(logger, startupLogger *zap.Logger, db *sql.DB, protoj
 
 	// Warm up the pool.
 	startupLogger.Info("Allocating minimum Lua runtime pool", zap.Int("count", config.GetRuntime().GetLuaMinCount()))
-	if len(moduleCache.Names) > 0 {
+	if moduleCache.Len() > 0 {
 		// Only if there are runtime modules to load.
 		for i := 0; i < config.GetRuntime().GetLuaMinCount(); i++ {
 			runtimeProviderLua.poolCh <- runtimeProviderLua.newFn()
@@ -1232,10 +1427,7 @@ func CheckRuntimeProviderLua(logger *zap.Logger, config Config, version string,
 }
 
 func openLuaModules(logger *zap.Logger, rootPath string, paths []string) (*RuntimeLuaModuleCache, []string, map[string]lua.LGFunction, error) {
-	moduleCache := &RuntimeLuaModuleCache{
-		Names:   make([]string, 0),
-		Modules: make(map[string]*RuntimeLuaModule, 0),
-	}
+	moduleCache := NewRuntimeLuaModuleCache()
 	modulePaths := make([]string, 0)
 
 	// Override before Package library is invoked.
@@ -1264,10 +1456,12 @@ func openLuaModules(logger *zap.Logger, rootPath string, paths []string) (*Runti
 		// Make paths Lua friendly.
 		name = strings.Replace(name, string(os.PathSeparator), ".", -1)
 
+		static := strings.HasPrefix(string(content), luaDisableHotfixMarker)
 		moduleCache.Add(&RuntimeLuaModule{
-			Name:    name,
-			Path:    path,
-			Content: content,
+			HotfixDisabled: atomic.NewBool(static),
+			Name:           name,
+			Path:           path,
+			Content:        content,
 		})
 		modulePaths = append(modulePaths, relPath)
 	}
@@ -2089,8 +2283,8 @@ func (r *RuntimeLua) loadModules(moduleCache *RuntimeLuaModuleCache) error {
 
 	preload := r.vm.GetField(r.vm.GetField(r.vm.Get(lua.EnvironIndex), "package"), "preload")
 	fns := make(map[string]*lua.LFunction)
-	for _, name := range moduleCache.Names {
-		module, ok := moduleCache.Modules[name]
+	for _, name := range moduleCache.List() {
+		module, ok := moduleCache.Get(name)
 		if !ok {
 			r.logger.Fatal("Failed to find named module in cache", zap.String("name", name))
 		}
@@ -2103,7 +2297,7 @@ func (r *RuntimeLua) loadModules(moduleCache *RuntimeLuaModuleCache) error {
 		fns[module.Name] = f
 	}
 
-	for _, name := range moduleCache.Names {
+	for _, name := range moduleCache.List() {
 		fn, ok := fns[name]
 		if !ok {
 			r.logger.Fatal("Failed to find named module in prepared functions", zap.String("name", name))
@@ -2288,8 +2482,8 @@ func checkRuntimeLuaVM(logger *zap.Logger, config Config, version string, stdLib
 	vm.PreloadModule("nakama", nakamaModule.Loader)
 
 	preload := vm.GetField(vm.GetField(vm.Get(lua.EnvironIndex), "package"), "preload")
-	for _, name := range moduleCache.Names {
-		module, ok := moduleCache.Modules[name]
+	for _, name := range moduleCache.List() {
+		module, ok := moduleCache.Get(name)
 		if !ok {
 			logger.Fatal("Failed to find named module in cache", zap.String("name", name))
 		}