Read fails when using large buffers (starting from around 4GB)

[Th30n]

Description
libssh2_channel_read starts misbehaving when using large buffers, around 4GB or more in size. It seems that the API accepts size_t buflen which is 64bit (on my machine), but that gets converted to uint32_t in the implementation causing various problems. One of the errors manifests when adjusting the receive window, e.g. Unable to send transfer-window adjustment packet, deferring, while the SSH server reports something like sshd[29804]: fatal: channel 0: adjust 4294967295 overflows remote window 2097152

Code To Reproduce

#include <libssh2.h>

#include <arpa/inet.h>
#include <unistd.h>

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

// Use your own SSH auth setup
static const char *username = "<username>";

int main(int argc, char *argv[]) {
  uint32_t hostaddr = inet_addr("127.0.0.1");
  libssh2_socket_t sock;
  struct sockaddr_in sin;
  int rc;
  LIBSSH2_SESSION *session = NULL;
  LIBSSH2_CHANNEL *channel = NULL;
  LIBSSH2_AGENT *agent = NULL;
  struct libssh2_agent_publickey *identity = NULL;

  rc = libssh2_init(0);
  if(rc) {
    fprintf(stderr, "libssh2 initialization failed (%d)\n", rc);
    return 1;
  }
  sock = socket(AF_INET, SOCK_STREAM, 0);
  if(sock == LIBSSH2_INVALID_SOCKET) {
    fprintf(stderr, "failed to create socket!\n");
    goto shutdown;
  }
  sin.sin_family = AF_INET;
  sin.sin_port = htons(22);
  sin.sin_addr.s_addr = hostaddr;
  if(connect(sock, (struct sockaddr*)(&sin), sizeof(struct sockaddr_in))) {
    fprintf(stderr, "failed to connect!\n");
    goto shutdown;
  }
  session = libssh2_session_init();
  if(!session) {
    fprintf(stderr, "Could not initialize SSH session!\n");
    goto shutdown;
  }
  libssh2_session_set_blocking(session, 1);
  rc = libssh2_session_handshake(session, sock);
  if(rc) {
    fprintf(stderr, "Failure establishing SSH session: %d\n", rc);
    goto shutdown;
  }
  agent = libssh2_agent_init(session);
  if(!agent) {
    fprintf(stderr, "Failure initializing ssh-agent support\n");
    rc = 1;
    goto shutdown;
  }
  if(libssh2_agent_connect(agent)) {
    fprintf(stderr, "Failure connecting to ssh-agent\n");
    rc = 1;
    goto shutdown;
  }
  if(libssh2_agent_list_identities(agent)) {
    fprintf(stderr, "Failure requesting identities to ssh-agent\n");
    rc = 1;
    goto shutdown;
  }
  rc = libssh2_agent_get_identity(agent, &identity, NULL);
  if(rc) {
    fprintf(stderr, "Failure obtaining identity from ssh-agent support %d\n", rc);
    goto shutdown;
  }
  if(libssh2_agent_userauth(agent, username, identity)) {
    fprintf(stderr, "Authentication with username %s and "
        "public key %s failed!\n",
        username, identity->comment);
  }
  channel = libssh2_channel_open_session(session);
  if(!channel) {
    fprintf(stderr, "Error\n");
    goto shutdown;
  }
  FILE *file = fopen("/tmp/large_data", "w");
  if (!file) {
    fprintf(stderr, "file open error\n");
    goto shutdown;
  }
  char buf[1024 * 1024 /* 1 MB */];
  memset(buf, 'a', sizeof(buf));
  for (int i = 0; i < 4 * 1024 /* GB */; ++i) {
    if(fwrite(buf, sizeof(buf[0]), sizeof(buf), file) < sizeof(buf)) {
      fprintf(stderr, "file write error\n");
      fclose(file);
      goto shutdown;
    }
  }
  fclose(file);
  rc = libssh2_channel_exec(channel, "cat /tmp/large_data");
  if(rc) {
    fprintf(stderr, "exec error\n");
    goto shutdown;
  }
  // 4GB - 1byte reliably causes an error on my machine.
  // You can play with different sizes and see when it fails for you, or if incomplete data is read. 
  size_t buflen = (2l << 31) - 1;
  fprintf(stderr, "sizeof(size_t): %ld, buflen: %ld\n", sizeof(size_t), buflen);
  char *buffer = malloc(buflen);
  ssize_t nread = libssh2_channel_read(channel, buffer, buflen);
  fprintf(stderr, "libssh2_channel_read returned: %ld\n", nread);
  if (nread < 0) {
    char *errmsg = NULL;
    libssh2_session_last_error(session, &errmsg, NULL, 0);
    fprintf(stderr, "%s\n", errmsg);
  }
  free(buffer);
shutdown:
  if(agent) {
    libssh2_agent_disconnect(agent);
    libssh2_agent_free(agent);
  }
  if(session) {
    libssh2_session_disconnect(session, "Normal Shutdown");
    libssh2_session_free(session);
  }
  if(sock != LIBSSH2_INVALID_SOCKET) {
    shutdown(sock, 2);
    close(sock);
  }
  fprintf(stderr, "all done\n");
  libssh2_exit();
  return 0;
}

Expected behavior
libssh2 should handle u32 overflows gracefully

Version (please complete the following information):

• OS and version: 5.15.0-75-generic # 82-Ubuntu SMP Tue Jun 6 23:10:23 UTC 2023 x86_64 GNU/Linux
• libssh2 version: 1.11.0
• crypto backend and version: OpenSSL 3.0.9

[vszakats]

Possibly related: #1025

[rmsh1216]

Is this issue fixed?

[vszakats]

No, it isn't.