Server keeps terminating on heavy load

[Asiapay-iraq]

Hi,
i have a unity game i created using nakama,
the game uses relayed matchmaker and for some custom server code i used lua, but recently the server keeps stoping since now i have a lot of users.

i think this issue has to do with my lua code but i can't find where.

some additional info:

1. deployment - AWS 2Xlarge instance
2. nakama version 3.15
3. 8 CPU
4. 16 GB Ram
5. Client Unity
6. game type, turn based game between 2 players (relayed multiplayer)

error after server termination

server code includes only one lua file

local nk = require("nakama")

--UPDATE WALLET AMOUNT -------

local function Update_Wallet(context, payload)
 
   local json = nk.json_decode(payload)
    local changeset = {
      coins =   json.coins
    }
     local  metadata = { }

  
    nk.wallet_update(context.user_id, changeset, metadata, true)
  end


nk.register_rpc(Update_Wallet, "Update_Wallet")

-----------------Update-XP------------------
local function Update_XP(context, payload)
 
  local json = nk.json_decode(payload)
   local changeset = {
     xp =   json.xp
   }
       metadata = { }

 
   nk.wallet_update(context.user_id, changeset, metadata, true)
 end


nk.register_rpc(Update_XP, "Update_XP")
---------------- Read And Write Waits ---------------------
local function WriteData(context, payload)

local global_id = "00000000-0000-0000-0000-000000000000" -- Some user ID.

local user_id = nk.json_decode(payload)
local userid = string.format(user_id.userid)

 nk.logger_info(string.format("Payload: %q", payload))

local new_objects = {
  { collection = "wait", key = "Data", user_id = global_id, value = { userID = userid}, permission_read = 2, permission_write = 1 }
}

nk.storage_write(new_objects)

end

nk.register_rpc(WriteData, "WriteData")

---------------- Read And Write Match List ---------------------
local function WriteToMachList(context, payload)

local global_id = "00000000-0000-0000-0000-000000000000" -- Some user ID.

local payloadData = nk.json_decode(payload)
local match_count = string.format(payloadData.count)

 nk.logger_info(string.format("Payload: %q", payload))

local new_objects = {
  { collection = "MatchList", key = "Data", user_id = global_id, value = { matchCount = match_count}, permission_read = 2, permission_write = 1 }
}

nk.storage_write(new_objects)

end

nk.register_rpc(WriteToMachList, "WriteToMachList")
------- Push Notification --------------
 
local function send_push_notification(context, payload)
  nk.logger_info(string.format("Payload: %q", payload))

  local auth = "OGE0ZTNkNTktMDA0OS00ZDRlLTg0ZGItZjM1NDA4YWYxMTMy"

  local url = "https://onesignal.com/api/v1/notifications"
  local method = "POST"
  local headers = {
    ["Authorization"] = string.format("Basic %s",auth),
    ["content-type"] = "application/json",
    ["accept"] = "application/json",
    
  }
  local user_id = nk.json_decode(payload)
  local matchid = string.format(user_id.matchid)
  local Otherid = string.format(context.user_id)
  local userid = string.format(user_id.userid)
  local Otherusername = string.format(context.username)
  local myusername = string.format(user_id.username)
  local content = nk.json_encode({
    app_id = "32981917-3b1d-44c1-bc23-d817a1114f02",
    contents = {
       en = "Hey " .. myusername .. " 🏆\n" ..Otherusername.." Sent You Challenge Request !🎲"
    },
        include_external_user_ids = {
         userid
       
    },
    url = "unitydl://?" .. matchid .. "+" .. Otherid,
  })

  local success, code, headers, resp_body = pcall(nk.http_request, url, method, headers, content)
  if (not success) then
    nk.logger_error(string.format("Failed to send push messages: %q",code))
 
  elseif (code >= 400) then
    nk.logger_error(string.format("Failed to send push message: %q %q",code , resp_body))
 
  else
    nk.logger_info(string.format("Success %q %q",code , resp_body))
 
  end
end
nk.register_rpc(send_push_notification, "one_signal")


local function push_notification_Waiting(context, payload)
  nk.logger_info(string.format("Payload: %q", payload))

  local auth = ""

  local url = "https://onesignal.com/api/v1/notifications"
  local method = "POST"
  local headers = {
    ["Authorization"] = string.format("Basic %s",auth),
    ["content-type"] = "application/json",
    ["accept"] = "application/json",
    
  }
  local user_id = nk.json_decode(payload)
  local userid = string.format(user_id.userid)
  local board = string.format(user_id.Board)
  local username = string.format(context.username)

  local content = nk.json_encode({
    app_id = "",
    contents = {
      en = "Your Next Opponent Is Ready! \n Play With Them Now! 🎲🏆"},
        include_external_user_ids = {
         userid
       
    },
    url = "unitydl://?" .. board, 
  })

  local success, code, headers, resp_body = pcall(nk.http_request, url, method, headers, content)
  if (not success) then
    nk.logger_error(string.format("Failed to send push messages: %q",code))
 
  elseif (code >= 400) then
    nk.logger_error(string.format("Failed to send push message: %q %q",code , resp_body))
 
  else
    nk.logger_info(string.format("Success %q %q",code , resp_body))
 
  end
end
nk.register_rpc(push_notification_Waiting, "one_signal_wating")
------- DELETE ACCOUNT -------------
local function delete(context , payload)
   
nk.account_delete_id(context.user_id, false)
end 
nk.register_rpc(delete, "delete")

-- STREAMS ------
local stream_model = {

streams = {
    online_stream_id = {
        mode = 1000
    }
}

}
local function join(context, payload)

        local stream_id = { mode = 1000, label = "Online users" }
	local hidden = false
	local persistence = false
	nk.stream_user_join(context.user_id, context.session_id, stream_model.streams.online_stream_id, hidden, persistence)
end
nk.register_rpc(join, "join")



local nk = require("nakama")

local function OnlineUsers(context, payload)
          local count = nk.stream_count(stream_model.streams.online_stream_id)
return tostring(count)
end

nk.register_rpc(OnlineUsers, "OnlineUsers")

local function List_stream_users(context, payload)

local presences = nk.stream_user_list(stream_model.streams.online_stream_id)

local index =""
local data =""
local user=""
for i, presence in ipairs(presences) do
      user="{\"id\":"..nk.json_encode(presence.user_id).."}"
      data =data ..","  .. user
      index ="{\"client\":[" .. data .. "]}"
end
return index
end
nk.register_rpc(List_stream_users,"List_stream_users")

local function leave(context, payload)
	nk.stream_user_leave(context.user_id, context.session_id, stream_model.streams.online_stream_id)
end
nk.register_rpc(leave, "leave")
-- INTIALIZE WALLET -------

local function initialize_Wallet(context, payload)
 
   if payload.created then
 
    local changeset = {
      coins = 1000, xp=0, -- Add 10 coins to the user's wallet.

    }
    local metadata = {}
    nk.wallet_update(context.user_id, changeset, metadata, true)
  end
end

nk.register_req_after(initialize_Wallet, "AuthenticateDevice")
nk.register_req_after(initialize_Wallet, "AuthenticateFacebook")
nk.register_req_after(initialize_Wallet, "AuthenticateEmail")
nk.register_req_after(initialize_Wallet, "AuthenticateCustom") 

local id = "level1"
local authoritative = false
local sort = "desc"
local operator = "best"
local reset = "0 0 * 12 0"
local metadata = {
     weather_conditions = "rain"
}
nk.leaderboard_create(id, authoritative, sort, operator, reset,  metadata)

 

local function users(context , payload)

local rows = nk.sql_query([[select id from users where  display_name not like 'Player%' and  display_name IS NOT NULL ORDER BY RANDOM() ]])

local index =""
local data =""

     for i, row in pairs(rows)
    do
      data =data .. ","  .. nk.json_encode(row) 
      index ="{\"client\":[" .. data .. "]}"
 
    end
return index
  
    end

nk.register_rpc(users, "users")

local function custom_rpc_func(context, payload)
  nk.logger_info(string.format("Payload: %q", payload))

  -- "payload" is bytes sent by the client we'll JSON decode it.
--  local json = nk.json_decode(payload)
 
  -- local user_id = nk.json_encode(json)
  -- local sender_id = nk.json_encode(context)-- "nil" for server sent.

  local content = {
    item_id = "Matchid",
   
  }

local user_id = nk.json_decode(payload)
local match_id = nk.json_decode(payload)

local userid = string.format(user_id.userid)
local matchid = string.format(match_id.matchid)
nk.logger_info(matchid)
local code = 102
local persistent = true

 nk.notification_send(userid, matchid , content, code, context.user_id, persistent)
 
  return userid
  
end
nk.register_rpc(custom_rpc_func, "custom_rpc_func_id")
local function custom_rpc_func_reject(context, payload)
  -- payload is bytes sent by the client we'll JSON decode it.
  --  local json = nk.json_decode(payload)
 
  -- local user_id = nk.json_encode(json)
  -- local sender_id = nk.json_encode(context)-- nil for server sent.

  local content = {
    item_id = Matchid,
   
  }

local user_id = nk.json_decode(payload)

local userid = string.format(user_id.userid)
local message = string.format("Other Player is Busy!")

local code = 103
local persistent = true

 nk.notification_send(userid, message , content, code, context.user_id, persistent)
 
  return userid
  
end

nk.register_rpc(custom_rpc_func_reject, "custom_rpc_func_reject")
 
local function custom_rpc_func_Accept(context, payload)

  local content = {
    item_id = "Matchid",
   
  }

local user_id = nk.json_decode(payload)

local userid = string.format(user_id.userid)
local message = string.format("Other Player Accepted Your Request!")

local code = 104
local persistent = true

 nk.notification_send(userid, message , content, code,context.user_id
, persistent)
 
  return userid
  
end

nk.register_rpc(custom_rpc_func_Accept, "custom_rpc_func_Accept")

local function test_rpc(context, payload)

  local User_name = nk.json_decode(payload)
  local Uname = string.format(User_name.name)
 nk.logger_info(string.format("Payload: %q", payload))
 nk.logger_info(string.format("Username: %q", Uname))
 
local rows = nk.sql_query([[SELECT id  FROM users WHERE REPLACE(username, ' ', '') ILIKE REPLACE($1, ' ', '')]], {Uname})

local index =""
local data =""

     for i, row in pairs(rows)
    do
      data =data .. ","  .. nk.json_encode(row)
      index ="{\"client\":[" .. data .. "]}"

    end
return index

    end

nk.register_rpc(test_rpc, "test_rpc")

some nakama configuration

socket:
  server_key: "defaultkey"
  port: 7350
  max_request_size_bytes: 8000
  max_message_size_bytes: 8000
  read_timeout_ms: 10000
  write_timeout_ms: 10000
  idle_timeout_ms: 60000
  write_wait_ms: 5000
  pong_wait_ms: 10000
  ping_period_ms: 8000
  outgoing_queue_size: 32
  read_buffer_size_bytes: 8192
  write_buffer_size_bytes: 8192


session:
  encryption_key: "defaultencryptionkey"
  token_expiry_sec: 10800
  refresh_encryption_key: "defaultrefreshencryptionkey"
  refresh_token_expiry_sec: 10800
  single_socket: true
  single_match: true

matchmaker:
  max_tickets: 1
  interval_sec: 15
  max_intervals: 1

logger:
  stdout: false
  level: "debug"
  file: "/home/ec2-user/logfile.log"

[linear]

NK-549 Server keeps terminating on heavy load

[zyro]

@Asiapay-iraq Start by updating to the latest Nakama release as there have been many fixes and improvements since the 3.15.0 you're using.

If that still causes you issues it would help to see more complete logs. What you've shared is too little to draw conclusions from.

[Asiapay-iraq]

@Asiapay-iraq Start by updating to the latest Nakama release as there have been many fixes and improvements since the 3.15.0 you're using.

If that still causes you issues it would help to see more complete logs. What you've shared is too little to draw conclusions from.

Dear, i did as you recommended , i updated the nakama version to 3.21 but i still face the same issue as before, the nakama service stoped this morning again

here is the full server log

one more thing i suspect causing the issue is, i have written a custom query in lua to fetch some random rows from the database so the user can send them friend requests

might that cause the problem ????

[zyro]

@Asiapay-iraq You can try using the nk.users_get_random(...) function instead of your own custom query if you want to be sure.

Your updated screenshots also do not look like complete server logs at all, just a fragment of a goroutine dump. We can't try to help unless you share complete logs, and please use a text format even attach text files if needed. Screenshots of code or logs are hard to use.

[Asiapay-iraq]

@Asiapay-iraq You can try using the nk.users_get_random(...) function instead of your own custom query if you want to be sure.

Your updated screenshots also do not look like complete server logs at all, just a fragment of a goroutine dump. We can't try to help unless you share complete logs, and please use a text format even attach text files if needed. Screenshots of code or logs are hard to use.

It happened again today around 5:34 and i restarted it in 6:01, i also stress tested the random RPC by sending 1000 calls in around 2 minutes but didn't cause the server to terminate

i attached the server logs for 15/5/2024
log.txt

[zyro]

Your log doesn't show anything useful, and also doesn't seem to contain anything that's in your screenshots.

At this point it's looking like it's an issue with your setup or server environment, which I'm not sure we can help debug since we don't control or have any insight into it.