DRAFT: Make POST/PUT payload visible in requests.json

src/backtrace-support/mtev-http-observer-module.lua

@@ -117,7 +117,9 @@ end
 --   uint64_t response_complete_ns;
 --   uint64_t inbytes;
 --   uint64_t outbytes;
---   mtev_hash_table info;                     128
+--   mtev_hash_table info;                     128 (vasu got 80d (0x50) for this)
+--   void *payload;                            144d (0x90)
+--   int64_t payload_length;
 -- } http_entry_t;
 --
 local function walk_hash(a_hash)

src/backtrace-support/mtev-http1-module.lua

@@ -2,9 +2,10 @@ function L(...)
   pmodule.log(pmodule.log_level.info, string.format(...))
 end
 
-local function bt2str(bt_var)
+local function bt2str(bt_var, len)
+  len = len or 256
   local addr = pmodule.variable_from_bt(bt_var):value()
-  return pmodule.address_read_string(addr, 256)
+  return pmodule.address_read_string(addr, len)
 end
 
 local function bt2val(bt_var)
@@ -20,12 +21,17 @@ local function variable_http1_cb(pt_var, bt_var)
   local qs = bt2str(bt_var.orig_qs)
   qs = qs and ("?" .. qs) or ""
   local length = bt2val(bt_var.content_length) or -1
+  local payload_len = bt2val(bt_var.upload.size)
+  local payload = bt2str(bt_var.upload.data, payload_len + 1)
   pt_var:thread():annotate(
     pmodule.annotation.comment,
     string.format("mtev_http1_request: %s %s%s (%d)", method, uri, qs, length))
   pt_var:backtrace():add_kv_string(
     "mtev_http_request",
     string.format("%s %s%s (%d)", method, uri, qs, length))
+  pt_var:backtrace():add_kv_string(
+    "mtev_http_payload_in",
+    string.format("%s", payload))
 end
 
 function pm_mtev_http1_req()

src/backtrace-support/mtev-http2-module.lua

@@ -2,9 +2,10 @@ local function L(...)
   pmodule.log(pmodule.log_level.info, string.format(...))
 end
 
-local function bt2str(bt_var)
+local function bt2str(bt_var, len)
+  len = len or 256
   local addr = pmodule.variable_from_bt(bt_var):value()
-  return pmodule.address_read_string(addr, 256)
+  return pmodule.address_read_string(addr, len)
 end
 
 local function bt2val(bt_var)
@@ -18,13 +19,18 @@ local function variable_http2_cb(pt_var, bt_var)
   local qs = bt2str(bt_var.orig_qs)
   qs = qs and ("?" .. qs) or ""
   local length = bt2val(bt_var.content_length) or -1
+  local payload_len = bt2val(bt_var.upload.size)
+  local payload = bt2str(bt_var.upload.data, payload_len + 1)
   pt_var:thread():annotate(
     pmodule.annotation.comment,
     string.format("mtev_http2_request: %s %s%s (%d)", method, uri, qs, length))
   pt_var:backtrace():add_kv_string(
     "mtev_http_request",
     string.format("%s %s%s (%d)", method, uri, qs, length))
-end
+  pt_var:backtrace():add_kv_string(
+    "mtev_http_payload_in",
+    string.format("%s", payload))
+  end
 
 local function pm_mtev_http2_req()
   L("module-mtev-http2: load")

src/modules/http_observer.c

@@ -42,7 +42,7 @@
 #include <pcre.h>
 
 static mtev_log_stream_t debugls, errorls;
-static uint32_t max_count = 10000, max_age = 30;
+static uint32_t max_count = 10000, max_age = 30, max_payload = 256 * 1024;
 static uint64_t global_id;
 static mtev_hash_table lookup, hdrin_extract, hdrout_extract;
 
@@ -64,11 +64,14 @@ typedef struct {
   uint64_t inbytes;
   uint64_t outbytes;
   mtev_hash_table info;
+  void *payload;
+  int64_t payload_length;
 } http_entry_t;
 
 static void http_entry_free(void *ve) {
   http_entry_t *e = ve;
   if(e == NULL) return;
+  free(e->payload);
   mtev_hash_destroy(&e->info, mtev_memory_safe_free, mtev_memory_safe_free);
 }
 
@@ -94,6 +97,8 @@ allocate_entry(mtev_http_session_ctx *ctx) {
   newe->request_complete_ns = timeofday_nanos();
   newe->id = ck_pr_faa_64(&global_id, 1);
   mtev_hash_init(&newe->info);
+  newe->payload = NULL;
+  newe->payload_length = 0;
   mtev_hash_replace(&lookup, (const char *)&newe->ctx, sizeof(newe->ctx), newe, NULL, mtev_memory_safe_free);
   return newe;
 }
@@ -178,6 +183,50 @@ http_observer_prrp(void *closure, mtev_http_session_ctx *ctx) {
   return MTEV_HOOK_CONTINUE;
 }
 
+static mtev_hook_return_t
+http_observer_prpr(void *closure, mtev_http_request *req, const void *payload, int64_t payload_length) {
+  (void)closure;
+  if (payload_length <= 0) {
+    payload = "(empty)";
+    payload_length = 7;
+  }
+  if (payload_length > (int64_t)max_payload) {
+    payload_length = (int64_t)max_payload;
+  }
+  char *payload_ptr = (char *)payload;
+  for (int64_t i = 0; i < payload_length;  i++, payload_ptr++)
+  {
+    if (!isprint(*payload_ptr)) {
+      payload = "(binary)";
+      payload_length = 8;
+      break;
+    }
+  }
+  payload_ptr = NULL;
+  if (payload) {
+    payload_ptr = malloc(payload_length);
+    memcpy(payload_ptr, payload, payload_length);
+  }
+
+  mtev_memory_begin();
+  mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
+  while(mtev_hash_adv_spmc(&lookup, &iter)) {
+    http_entry_t *entry = (http_entry_t *)iter.value.ptr;
+    mtev_http_session_ctx *ctx = entry->ctx;
+    if (req == mtev_http_session_request(ctx)) {
+      if (entry->payload) {
+        free(entry->payload);
+      }
+      entry->payload = payload_ptr;
+      entry->payload_length = payload_ptr ? payload_length : 0;
+      http_entry_update(entry, ctx);
+      break;
+    }
+  }
+  mtev_memory_end();
+  return MTEV_HOOK_CONTINUE;
+}
+
 static mtev_hook_return_t
 http_observer_rs(void *closure, mtev_http_session_ctx *ctx) {
   (void)closure;
@@ -304,6 +353,9 @@ static void http_entry_json(mtev_http_session_ctx *ctx, http_entry_t *entry) {
     MJ_KV(o, "response_complete_offset_ns", MJ_INT64(entry->response_complete_ns - entry->request_start_ns));
   MJ_KV(o, "received_bytes", MJ_INT64(entry->inbytes));
   MJ_KV(o, "sent_bytes", MJ_INT64(entry->outbytes));
+  if (entry->payload) {
+    MJ_KV(o, "payload_in", MJ_STRN(entry->payload, entry->payload_length));
+  }
   mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
   while(mtev_hash_adv_spmc(&entry->info, &iter)) {
     MJ_KV(o, iter.key.str, MJ_STR(iter.value.str));
@@ -401,6 +453,9 @@ http_observer_driver_config(mtev_dso_generic_t *img, mtev_hash_table *options) {
   if(mtev_hash_retr_str(options, "max_age", strlen("max_age"), &vstr)) {
     max_age = atoi(vstr);
   }
+  if(mtev_hash_retr_str(options, "max_payload", strlen("max_payload"), &vstr)) {
+    max_payload = atoi(vstr);
+  }
   mtev_hash_iter iter = MTEV_HASH_ITER_ZERO;
   while(mtev_hash_adv(options, &iter)) {
     if(!strncmp(iter.key.str, "header_in_", 10)) {
@@ -424,6 +479,7 @@ http_observer_driver_init(mtev_dso_generic_t *img) {
   http_response_send_hook_register("http_observer", http_observer_rs, NULL);
   http_request_log_hook_register("http_observer", http_observer_rl, NULL);
   http_post_request_read_payload_hook_register("http_observer", http_observer_prrp, NULL);
+  http_post_request_payload_retrieved_hook_register("http_observer", http_observer_prpr, NULL);
 
   mtev_rest_mountpoint_t *rule = mtev_http_rest_new_rule(
     "GET", "/module/http_observer/", "^requests.json$", requests_json_handler

src/mtev_http.c

@@ -68,6 +68,12 @@ MTEV_HOOK_IMPL(http_post_request_read_payload,
   (void *closure, mtev_http_session_ctx *ctx),
   (closure, ctx))
 
+MTEV_HOOK_IMPL(http_post_request_payload_retrieved,
+  (mtev_http_request *req, const void *payload, int64_t payload_length),
+  void *, closure,
+  (void *closure, mtev_http_request *req, const void *payload, int64_t payload_length),
+  (closure, req, payload, payload_length))
+
 MTEV_HOOK_IMPL(http_response_starting,
   (mtev_http_session_ctx *ctx),
   void *, closure,

src/mtev_http.h

@@ -279,6 +279,11 @@ MTEV_HOOK_PROTO(http_post_request_read_payload,
                 void *, closure,
                 (void *closure, mtev_http_session_ctx *ctx))
 
+MTEV_HOOK_PROTO(http_post_request_payload_retrieved,
+                (mtev_http_request *req, const void *payload, int64_t payload_length),
+                void *, closure,
+                (void *closure, mtev_http_request *req, const void *payload, int64_t payload_length))
+
 MTEV_HOOK_PROTO(http_response_starting,
                 (mtev_http_session_ctx *ctx),
                 void *, closure,

src/mtev_http1.c

@@ -371,6 +371,7 @@ mtev_http1_request_set_upload(mtev_http1_request *req,
 const void *
 mtev_http1_request_get_upload(mtev_http1_request *req, int64_t *size) {
   if(size) *size = req->upload.size;
+  (void)http_post_request_payload_retrieved_hook_invoke((mtev_http_request *)req, req->upload.data, req->upload.size);
   return req->upload.data;
 }
 const char *

src/mtev_http2.c

@@ -402,6 +402,7 @@ mtev_http2_request_set_upload(mtev_http2_request *req,
 const void *
 mtev_http2_request_get_upload(mtev_http2_request *req, int64_t *size) {
   if(size) *size = req->upload.size;
+  (void)http_post_request_payload_retrieved_hook_invoke((mtev_http_request *)req, req->upload.data, req->upload.size);
   return req->upload.data;
 }
 const char *