Not displaying syslogs

[dbradley01]

Hi - I'm not seeing syslogs being displayed in the libreNMS UI. I am publishing on a non-standard port (515) and I confirmed the docker host is receiving the syslogs on the expected port.

Thanks in advance.

docker-compose

version: "3.5"

services:
  db:
    image: mariadb:10.5
    container_name: librenms_db
    command:
      - "mysqld"
      - "--innodb-file-per-table=1"
      - "--lower-case-table-names=0"
      - "--character-set-server=utf8mb4"
      - "--collation-server=utf8mb4_unicode_ci"
    volumes:
      - "./db:/var/lib/mysql"
    environment:
      - "TZ=${TZ}"
      - "MYSQL_ALLOW_EMPTY_PASSWORD=yes"
      - "MYSQL_DATABASE=${MYSQL_DATABASE}"
      - "MYSQL_USER=${MYSQL_USER}"
      - "MYSQL_PASSWORD=${MYSQL_PASSWORD}"
    restart: always

  memcached:
    image: memcached:alpine
    container_name: librenms_memcached
    environment:
      - "TZ=${TZ}"
    restart: always

  redis:
    image: redis:5.0-alpine
    container_name: librenms_redis
    environment:
      - "TZ=${TZ}"
    restart: always

  msmtpd:
    image: crazymax/msmtpd:latest
    container_name: librenms_msmtpd
    env_file:
      - "./msmtpd.env"
    restart: always

  librenms:
    image: librenms/librenms:latest
    container_name: librenms
    hostname: librenms
    cap_add:
      - NET_ADMIN
      - NET_RAW
    ports:
      - target: 8000
        published: 65010
        protocol: tcp
    depends_on:
      - db
      - memcached
      - msmtpd
    volumes:
      - "./librenms:/data"
    env_file:
      - "./librenms.env"
    environment:
      - "TZ=${TZ}"
      - "PUID=${PUID}"
      - "PGID=${PGID}"
      - "DB_HOST=db"
      - "DB_NAME=${MYSQL_DATABASE}"
      - "DB_USER=${MYSQL_USER}"
      - "DB_PASSWORD=${MYSQL_PASSWORD}"
      - "DB_TIMEOUT=60"
      - "REDIS_HOST=redis"
      - "REDIS_PORT=6379"
      - "REDIS_DB=0"
    restart: always

  dispatcher:
    image: librenms/librenms:latest
    container_name: librenms_dispatcher
    hostname: librenms-dispatcher
    cap_add:
      - NET_ADMIN
      - NET_RAW
    depends_on:
      - librenms
      - redis
    volumes:
      - "./librenms:/data"
    env_file:
      - "./librenms.env"
    environment:
      - "TZ=${TZ}"
      - "PUID=${PUID}"
      - "PGID=${PGID}"
      - "DB_HOST=db"
      - "DB_NAME=${MYSQL_DATABASE}"
      - "DB_USER=${MYSQL_USER}"
      - "DB_PASSWORD=${MYSQL_PASSWORD}"
      - "DB_TIMEOUT=60"
      - "DISPATCHER_NODE_ID=dispatcher1"
      - "REDIS_HOST=redis"
      - "REDIS_PORT=6379"
      - "REDIS_DB=0"
      - "SIDECAR_DISPATCHER=1"
    restart: always

  syslogng:
    image: librenms/librenms:latest
    container_name: librenms_syslogng
    hostname: librenms-syslogng
    cap_add:
      - NET_ADMIN
      - NET_RAW
    depends_on:
      - librenms
    ports:
      - target: 514
        published: 515
        protocol: tcp
      - target: 514
        published: 515
        protocol: udp
    volumes:
      - "./librenms:/data"
    env_file:
      - "./librenms.env"
    environment:
      - "TZ=${TZ}"
      - "PUID=${PUID}"
      - "PGID=${PGID}"
      - "DB_HOST=db"
      - "DB_NAME=${MYSQL_DATABASE}"
      - "DB_USER=${MYSQL_USER}"
      - "DB_PASSWORD=${MYSQL_PASSWORD}"
      - "DB_TIMEOUT=60"
      - "REDIS_HOST=redis"
      - "REDIS_PORT=6379"
      - "REDIS_DB=0"
      - "SIDECAR_SYSLOGNG=1"
    restart: always

  snmptrapd:
    image: librenms/librenms:latest
    container_name: librenms_snmptrapd
    hostname: librenms-snmptrapd
    cap_add:
      - NET_ADMIN
      - NET_RAW
    depends_on:
      - librenms
    ports:
      - target: 162
        published: 162
        protocol: tcp
      - target: 162
        published: 162
        protocol: udp
    volumes:
      - "./librenms:/data"
    env_file:
      - "./librenms.env"
    environment:
      - "TZ=${TZ}"
      - "PUID=${PUID}"
      - "PGID=${PGID}"
      - "DB_HOST=db"
      - "DB_NAME=${MYSQL_DATABASE}"
      - "DB_USER=${MYSQL_USER}"
      - "DB_PASSWORD=${MYSQL_PASSWORD}"
      - "DB_TIMEOUT=60"
      - "SIDECAR_SNMPTRAPD=1"
    restart: always

validate.php

====================================
Component | Version
--------- | -------
LibreNMS  | 21.8.0
DB Schema | 2021_25_01_0127_create_isis_adjacencies_table (213)
PHP       | 7.4.23
Python    | 3.9.5
MySQL     | 10.5.12-MariaDB-1:10.5.12+maria~focal
RRDTool   | 1.7.2
SNMP      | NET-SNMP 5.9
====================================

[OK]    Installed from the official Docker image; no Composer required
[OK]    Database connection successful
[OK]    Database schema correct
[WARN]  IPv6 is disabled on your server, you will not be able to add IPv6 devices.
[WARN]  Updates are managed through the official Docker image`

sudo docker ps

c8a73582a002   librenms/librenms:latest                 "/init"                  4 hours ago    Up 3 hours             514/tcp, 0.0.0.0:162->162/tcp, 0.0.0.0:162->162/udp, 8000/tcp, 514/udp   librenms_snmptrapd
713391f8a6c1   librenms/librenms:latest                 "/init"                  4 hours ago    Up 3 hours             162/tcp, 8000/tcp, 162/udp, 0.0.0.0:515->514/tcp, 0.0.0.0:515->514/udp   librenms_syslogng
9676ac9d54fb   librenms/librenms:latest                 "/init"                  4 hours ago    Up 3 hours             162/tcp, 162/udp, 514/tcp, 8000/tcp, 514/udp                             librenms_dispatcher
1cf44169cb80   librenms/librenms:latest                 "/init"                  4 hours ago    Up 3 hours             162/tcp, 162/udp, 514/tcp, 514/udp, 0.0.0.0:65010->8000/tcp              librenms
7c778ad2fd24   redis:5.0-alpine                         "docker-entrypoint.s…"   5 hours ago    Up 3 hours             6379/tcp                                                                 librenms_redis
2c56feb73e98   memcached:alpine                         "docker-entrypoint.s…"   5 hours ago    Up 3 hours             11211/tcp                                                                librenms_memcached
f090e762a35f   crazymax/msmtpd:latest                   "/init"                  5 hours ago    Up 3 hours (healthy)   2500/tcp                                                                 librenms_msmtpd
5445b0134e28   mariadb:10.5                             "docker-entrypoint.s…"   5 hours ago    Up 3 hours             3306/tcp                                                                 librenms_db

packet capture on host

18:00:19.359146 IP network_device-MGMT.local.62964 > docker_host.local.515: UDP, length 96
18:00:19.359146 IP network_device-MGMT.local.62964 > docker_host.local.515: UDP, length 96
18:00:35.911609 IP network_device-MGMT.local.62964 > docker_host.local.515: UDP, length 105
18:00:35.911609 IP network_device-MGMT.local.62964 > docker_host.local.515: UDP, length 105
18:00:57.806713 IP network_device-MGMT.local.62964 > docker_host.local.515: UDP, length 104
18:00:57.806713 IP network_device-MGMT.local.62964 > docker_host.local.515: UDP, length 104
18:01:09.205734 IP network_device-MGMT.local.62964 > docker_host.local.515: UDP, length 105
18:01:09.205734 IP network_device-MGMT.local.62964 > docker_host.local.515: UDP, length 105
18:01:14.762049 IP network_device-MGMT.local.62964 > docker_host.local.515: UDP, length 97`

bash-5.1# cat /data/config/syslog.php 
<?php
$config['enable_syslog'] = 1;

bash-5.1# 

under global settings:
enable_syslog 1

[ziodona85]

Hi,
if it can help you, I have enabled it on config.php with $config['enable_syslog'] = 1; and after it appear.

docker-composer is same of examples.

syslogng:
image: librenms/librenms:latest
container_name: librenms_syslogng
hostname: librenms-syslogng
cap_add:
- NET_ADMIN
- NET_RAW
depends_on:
- librenms
ports:
- target: 514
published: 514
protocol: tcp
- target: 514
published: 514
protocol: udp
volumes:
- "./librenms:/data"
env_file:
- "./librenms.env"
environment:
- "TZ=${TZ}"
- "PUID=${PUID}"
- "PGID=${PGID}"
- "DB_HOST=10.63.."
- "DB_NAME=${MYSQL_DATABASE}"
- "DB_USER=${MYSQL_USER}"
- "DB_PASSWORD=${MYSQL_PASSWORD}"
- "DB_TIMEOUT=60"
- "REDIS_HOST=redis"
- "REDIS_PORT=6379"
- "REDIS_DB=0"
- "SIDECAR_SYSLOGNG=1"
restart: always

bash-5.1# pwd
/opt/librenms
bash-5.1# more config.php
..........
$config['allow_duplicate_sysName'] = true;
$config['enable_syslog'] = 1;
...........

`

[Illunis]

From the sight of the service container all your incoming packets are coming from the ingress network, if you are running in swarm mode. All syslog messages appear from the same ip from the ingress range (like 10.0.0.2). LibreNMS only shows messages for created devices. All other are disacrded.

moby/moby#25526 is the right disussion

https://github.com/newsnowlabs/docker-ingress-routing-daemon works as workaround for me

[cltnoc]

I am having the same issue, the log reaches the server, I can see on tcpdump

16:16:47.519668 IP 172.19.10.3.38514 > librenmsadmin-04.syslog: SYSLOG local7.critical, length: 273
16:16:48.799388 IP 172.19.10.3.38514 > librenmsadmin-04.syslog: SYSLOG local7.info, length: 296
16:16:49.517080 IP 172.19.10.3.38514 > librenmsadmin-04.syslog: SYSLOG local7.critical, length: 273
16:16:51.523285 IP 172.19.10.3.38514 > librenmsadmin-04.syslog: SYSLOG local7.critical, length: 292
16:16:56.318224 IP 172.19.10.3.38514 > librenmsadmin-04.syslog: SYSLOG local7.warning, length: 242
16:17:14.151236 IP 172.19.10.3.38514 > librenmsadmin-04.syslog: SYSLOG local7.critical, length: 292
16:17:19.158080 IP 172.19.10.3.38514 > librenmsadmin-04.syslog: SYSLOG local7.critical, length: 273
16:17:19.904175 IP 172.19.10.3.38514 > librenmsadmin-04.syslog: SYSLOG local7.warning, length: 244
16:17:23.266257 IP 172.19.10.3.38514 > librenmsadmin-04.syslog: SYSLOG local7.critical, length: 273
16:17:25.949921 IP 172.19.10.3.38514 > librenmsadmin-04.syslog: SYSLOG local7.warning, length: 245
16:17:54.978141 IP 172.19.10.3.38514 > librenmsadmin-04.syslog: SYSLOG local7.warning, length: 245

Why its not showing in librenms, how can I check whether it's reaching the container?

[hugalafutro]

I do believe container has to run as privileged to get access to ports 1-1024, can you try on port above 1024? I had similar issue in graylog docker.

[LoveSkylark]

looking the this it looks like the docker is NATing the incoming traffic (172.19.10.3 is a docker proxy IP), haven't done this in docker (using k3s) but what you need to do is make sure docker is using source based load balancing so that the container inside sees the actual IP of the device and can match it against its database.

[Olen]

If you have IPv6 on your host but not in the container, docker will set up nat64 automatically, and that might cause this.

I can not guarantee that this is the same problem, but I have a similar issue. I have verified that the syslog messages actually reach the container from the right IP.
apk add tcpdump in the container lets you packet dump the traffic.

19:11:45.242990 eth0  In  IP 167.x.x.201.42254 > 10.42.42.6.514: SYSLOG user.notice, length: 131

netstat shows that syslog-ng is listening

Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name    
udp     1536      0 0.0.0.0:514             0.0.0.0:*                           554/syslog-ng

But as you can see, the receive-q is growing for each log-message:

Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name    
udp     2304      0 0.0.0.0:514             0.0.0.0:*                           554/syslog-ng

So the process is not able to read the messages, for some reason

What I see is that there are two syslog-ng-processes in the container:

bash-5.1# ps axunw | grep syslog
       0     552  0.0  0.0   1088   648 ?        S    19:06   0:00 s6-supervise syslogng
       0     554  0.0  0.1   7968  5388 ?        Ss   19:06   0:00 /usr/sbin/syslog-ng -F
       0     586 87.0  0.0   7968   864 ?        R    19:06   8:22 /usr/sbin/syslog-ng -F

And the last one is using a lot of cpu cycles

After installing strace I can see why...

# apk add strace
(...)
# strace -p 586
close(151842925)                        = -1 EBADF (Bad file descriptor)
close(151842924)                        = -1 EBADF (Bad file descriptor)
close(151842923)                        = -1 EBADF (Bad file descriptor)
close(151842922)                        = -1 EBADF (Bad file descriptor)
close(151842921)                        = -1 EBADF (Bad file descriptor)
close(151842920)                        = -1 EBADF (Bad file descriptor)
close(151842919)                        = -1 EBADF (Bad file descriptor)
close(151842918)                        = -1 EBADF (Bad file descriptor)
close(151842917)                        = -1 EBADF (Bad file descriptor)
close(151842916)                        = -1 EBADF (Bad file descriptor)
close(151842915)                        = -1 EBADF (Bad file descriptor)
(...)

And it was counting and counting and counting... Until

close(22)                               = -1 EBADF (Bad file descriptor)                                                                                                                      
close(21)                               = -1 EBADF (Bad file descriptor)                                                                                                                      
close(20)                               = -1 EBADF (Bad file descriptor)                                                                                                                      
close(19)                               = -1 EBADF (Bad file descriptor)                                                                                                                      
close(18)                               = 0                                                                                                                                                   
close(17)                               = 0                                                                                                                                                   
close(16)                               = -1 EBADF (Bad file descriptor)                                                                                                                      
close(15)                               = -1 EBADF (Bad file descriptor)                                                                                                                      
close(14)                               = 0                                                                                                                                                   
close(13)                               = 0                                                                                                                                                   
close(12)                               = 0                                                                                                                                                   
close(11)                               = 0                                                                                                                                                   
close(10)                               = 0                                                                                                                                                   
close(9)                                = 0                                                                                                                                                   
close(8)                                = 0                                                                                                                                                   
close(7)                                = 0                                                                                                                                                   
close(6)                                = 0                                                                                                                                                   
close(5)                                = 0                                                                                                                                                   
close(4)                                = 0                                                                                                                                                   
close(3)                                = 0                                                                                                                                                   
execve("/bin/sh", ["/bin/sh", "-c", "/opt/librenms/syslog.php"], 0x7fdabd59ba40 /* 29 vars */) = 0                                                                                            
arch_prctl(ARCH_SET_FS, 0x7f15efce9b48) = 0                                                                                                                                                   
set_tid_address(0x7f15efce9fb0)         = 586
brk(NULL)                               = 0x5581e642e000
brk(0x5581e6430000)                     = 0x5581e6430000
mmap(0x5581e642e000, 4096, PROT_NONE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x5581e642e000
mprotect(0x7f15efce6000, 4096, PROT_READ) = 0
mprotect(0x5581e445c000, 16384, PROT_READ) = 0
getuid()                                = 0
(...)

And then all of a sudden, it started parsing syslog-messages...

But something is definitely not playing well here...since it first tries to close millions of un-existing file descriptors...

[vonom]

I'm also having issue with displaying syslog data in LibreNMS.
I confirmed with tcpdump that librenms_syslogng container is receiving logs from remote host.
I confirmed that librenms_db container is logging them in database.
I even confirmed that when I perform "docker container exec -it librenms /bin/bash" and manually add "$config['enable_syslog'] = 1;" to config.php then without reloading anything Syslog data is displayed in all correct places in LibreNMS.

Problem is that it should work according to instruction from https://github.com/librenms/docker :
"You have to create a configuration file to enable syslog in LibreNMS too. Create a file called for example /data/config/syslog.yaml with this content :

enable_syslog: true"
and it isn't.

I tried it in config.php.yaml
"
$config['bad_iftype'][] = "ppp"; - THIS ONE IS WORKING - I can find it in config table in librenms database
$config['enable_syslog'] = 1; - NOT WORKING
$config['syslog_purge'] = 30; - PROBABLY NOT WORKING
enable_syslog: true - NOT WORKING
"

I tried in syslog.yaml
"
enable_syslog: true
"
Of course I'm restarting containers after editing those files.

Is it broken or am I putting this "enable_syslog: true" in wrong place?

[hugalafutro]

Hi,
inside your /data/librenms/config create file my.php and add your custom config options in it to persist container rebuilds

my my.php has this in it for example:

<?php

$config['snmp']['community'] = array('cthulhu');
$config['rrd_purge'] = 30;
$config['ports_purge'] = true;
$config['bad_if_regexp'][] = '/^br-\w{12}/';
$config['bad_if_regexp'][] = '/^veth\w{7}/';
$config['bad_if_regexp'][] = '/^lo$/';
$config['bad_if_regexp'][] = '/^docker0$/';
$config['bad_if_regexp'][] = '/^kube-bridge$/';
$config['bad_if_regexp'][] = '/^kube-dummy-if$/';
$config['enable_billing'] = 1;
$config['billing_data_purge'] = 12;

I cannot actually comment on the syslog thing as I use Graylog and LibreNMS Graylog integration which when enabled replaces syslog.

[vonom]

@hugalafutro thank you. Confirmed.

If file is named anythinghere.php and have full structure like in your example it's working flawlessly.

[kyob]

I tried enable syslog according to https://github.com/librenms/docker#syslog-ng-container but without success.
Only way it worked is when I modify config.php directly in container but thats not solution.

[hugalafutro]

@kyob read the 2 posts above yours. The instructions for modifying config.php in docker install are different.