Segmentation Fault

[zem]

when starting sheepdog on debian

Jul 16 16:42:53 EMERG [main] crash_handler(288) sheep exits unexpectedly (Segmentation fault), si pid 0, uid 0, errno 0, code 1
Jul 16 16:42:53 EMERG [main] sd_backtrace(849) sheep(+0xaec2) [0x564f32330ec2]
Jul 16 16:42:53 EMERG [main] sd_backtrace(849) /lib/x86_64-linux-gnu/libpthread.so.0(+0x11f4f) [0x7f47423e9f4f]
Jul 16 16:42:53 EMERG [main] sd_backtrace(849) /lib/x86_64-linux-gnu/libc.so.6(strtok_r+0x11) [0x7f47410beea1]
Jul 16 16:42:53 EMERG [main] sd_backtrace(849) sheep(+0x2630b) [0x564f3234c30b]
Jul 16 16:42:53 EMERG [main] sd_backtrace(849) sheep(+0x9556) [0x564f3232f556]
Jul 16 16:42:53 EMERG [main] sd_backtrace(849) /lib/x86_64-linux-gnu/libc.so.6(__libc_start_main+0xe6) [0x7f4741059a86]
Jul 16 16:42:53 EMERG [main] sd_backtrace(849) sheep(+0xab99) [0x564f32330b99]
Jul 16 16:42:53 DEBUG [main] gdb_cmd(769) cannot find gdb
Jul 16 16:42:53 DEBUG [main] gdb_cmd(769) cannot find gdb

NOTE: It is a 8.3 formatted empty sheepdog volume.

[vtolstov]

Please, install dbg package and gdb. So we can see what happened.

[zem]

==
== Jul 21 00:26:05 
== program: /usr/sbin/sheep
== command: thread apply all where full
==

Thread 1 (Thread 0x7fc15f267fc0 (LWP 3600)):
#0  0x00007fc15db4c0e7 in __GI___waitpid (pid=3625, stat_loc=stat_loc@entry=0x7ffe196bed98, options=options@entry=0) at ../sysdeps/unix/sysv/linux/waitpid.c:30
        resultvar = 18446744073709551104
#1  0x00007fc15daca14f in do_system (line=line@entry=0x7ffe196bef20 "gdb -nw /usr/sbin/sheep 3600 -batch >/dev/null 2>&1 -ex 'set logging on' -ex 'echo \\n' -ex 'echo ==\\n' -ex 'echo == Jul 21 00:26:05 \\n' -ex 'echo == program: /usr/sbin/sheep\\n' -ex 'echo == command: t"...) at ../sysdeps/posix/system.c:149
        __result = <optimized out>
        _buffer = {__routine = 0x7fc15daca430 <cancel_handler>, __arg = 0x7ffe196bed9c, __canceltype = 0, __prev = 0x0}
        _avail = 1
        status = 0
        save = <optimized out>
        pid = 3625
        sa = {__sigaction_handler = {sa_handler = 0x1, sa_sigaction = 0x1}, sa_mask = {__val = {65536, 0 <repeats 15 times>}}, sa_flags = 0, sa_restorer = 0xe10}
        omask = {__val = {0, 94245247458460, 140729324924704, 140729327021952, 3600, 140468477810677, 206158430256, 140729324924696, 140729324924464, 2330847666222263808, 0, 140729324924597, 1, 0, 0, 140468501770120}}
#2  0x00007fc15daca51a in __libc_system (line=line@entry=0x7ffe196bef20 "gdb -nw /usr/sbin/sheep 3600 -batch >/dev/null 2>&1 -ex 'set logging on' -ex 'echo \\n' -ex 'echo ==\\n' -ex 'echo == Jul 21 00:26:05 \\n' -ex 'echo == program: /usr/sbin/sheep\\n' -ex 'echo == command: t"...) at ../sysdeps/posix/system.c:185
No locals.
#3  0x000055b72e7d92f6 in gdb_cmd (cmd=cmd@entry=0x55b72e7f1c9c "thread apply all where full") at logger.c:790
        time_str = "Jul 21 00:26:05 ", '\000' <repeats 200 times>...
        cmd_str = <error reading variable cmd_str (value requires 2097152 bytes, which is more than max-value-size)>
        ti = 1532132765
        tm = {tm_sec = 5, tm_min = 26, tm_hour = 0, tm_mday = 21, tm_mon = 6, tm_year = 118, tm_wday = 6, tm_yday = 201, tm_isdst = 0, tm_gmtoff = 0, tm_zone = 0x55b73163b830 "UTC"}
        __func__ = "gdb_cmd"
#4  0x000055b72e7d9c5d in dump_stack_frames () at logger.c:804
No locals.
#5  sd_backtrace () at logger.c:854
        addrs = {0x55b72e7d9b17 <sd_backtrace+55>, 0x55b72e7abec3 <crash_handler+83>, 0x7fc15ee39f50 <__restore_rt>, 0x7fc15db0eea2 <__GI___strtok_r+18>, 0x55b72e7c730c <init_global_pathnames+108>, 0x55b72e7aa557 <main+2103>, 0x7fc15daa9a87 <__libc_start_main+231>, 0x55b72e7abb9a <_start+42>, 0x0, 0x12e7f1c0f, 0x7ffe198bf180, 0x7ffe198bf190, 0x7fc15f26fd58, 0x0, 0x0, 0x0, 0xffffffff, 0x0, 0x7fc15da8ef00, 0x7fc15f26fa00, 0x0, 0x0, 0x0, 0x3000000018, 0x7ffe198bf820, 0x7ffe198bf760, 0x0, 0x7fc15de3c038, 0x7ffe198bf650, 0x7fc15de38ee0 <_IO_mem_jumps>, 0x55b73163b360, 0x7ffe198bf740, 0x14, 0x7fc15f056703 <_dl_fixup+211>, 0x1, 0x0, 0x0, 0x7fc15da8ef00, 0x7ffe198bf5d0, 0x7fc15f05d2ca <_dl_runtime_resolve_xsave+138>, 0xd68, 0x780, 0x7fc15de38760 <_IO_helper_jumps>, 0x35, 0x55b73163d360, 0x4, 0x7ffe198bf5c4, 0x8, 0x1, 0x6f00000063, 0x770000007c, 0x2ffff00001f80, 0x0, 0x2058d59a62b9ee00, 0x7ffe198bf2f0, 0x7ffe198bf2f0, 0x7ffe198bf420, 0x55b73163b80b, 0x7ffe198bf740, 0x7fc15de3d4f0 <tzname>, 0x7fc15dc0674e, 0x7fc15daf819d <_IO_vsscanf+125>, 0x7fc1fbad8001, 0x55b73163b80c, 0x55b73163b80c, 0x55b73163b80b, 0x55b73163b80b, 0x55b73163b80b, 0x0, 0xff0000, 0x2525252525252525, 0x2525252525252525, 0x0, 0x0, 0x0, 0x0, 0x0, 0xff0000, 0xffffffffffffffff, 0xffffffffffffffff, 0x7fc15f2723e0, 0x7fc15f2728f0, 0x7fc15f2719e0, 0x7fc15f271ee0, 0x0 <repeats 16 times>, 0x7fc15de39b40 <_nl_C_LC_TIME>, 0x7ffe198bf617, 0x55b73163b830, 0x1, 0x0, 0x7ffe198bf4c5, 0x1, 0x7ffe198bf524, 0x7ffe198bf520, 0x7ffe0000000c, 0x1007ffe198bf800, 0x4, 0x2, 0x0 <repeats 18 times>, 0x2058d59a62b9ee00, 0x0, 0x55b73163d350, 0x55b73163f360, 0x7fc15de3cc40 <main_arena>, 0x0, 0x40, 0x2010, 0x7fc15db09e91 <_int_realloc+609>, 0x0, 0x0, 0x55b73163d360, 0x35, 0x55b73163d350, 0x55b73163b360, 0x40, 0x7fc15db0ad7d <__GI___libc_realloc+205>, 0x0, 0x7fc15de3cc40 <main_arena>, 0x0, 0x55b73163b360, 0x7ffe198bf650, 0x2058d59a62b9ee00, 0x55b73163b360, 0x55b73163b360, 0x0, 0x2058d59a62b9ee00, 0x55b73163b360, 0x1, 0x1, 0x11, 0x55b73163b360, 0x7ffe198bf740, 0x14, 0x7fc15db79395 <__GI___vsyslog_chk+709>, 0x0, 0x55b72e7f1bf8, 0x0, 0x5b527d9d, 0x55b73163d360, 0x34, 0x1a00000005, 0x1500000000, 0x7600000006, 0xc900000006, 0x0, 0x0, 0x55b73163b830, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2058d59a62b9ee00, 0x0, 0x7fc15d420000, 0x7fc15f260000, 0x7ffe198c1a70, 0x7ffe198bf820, 0x88001, 0x7ffe198c18d0, 0x7fc15db7979f <__syslog_chk+143>, 0x3000000020, 0x7ffe198bf820, 0x7ffe198bf760, 0x2058d59a62b9ee00, 0x0, 0x0, 0x0, 0xe11, 0x7fc15f267fc0, 0x7fc15de3c240 <pa_next_type>, 0x0 <repeats 14 times>...}
        i = <optimized out>
        n = <optimized out>
        __func__ = "sd_backtrace"
#6  0x000055b72e7abec3 in crash_handler (signo=11, info=<optimized out>, context=<optimized out>) at sheep.c:290
        __func__ = "crash_handler"
#7  <signal handler called>
No locals.
#8  __GI___strtok_r (s=0x0, delim=delim@entry=0x55b72e7e8ea9 ",", save_ptr=save_ptr@entry=0x7fc15de3ea68 <olds>) at strtok_r.c:49
        end = <optimized out>
#9  0x00007fc15db0ee8c in strtok (s=<optimized out>, delim=delim@entry=0x55b72e7e8ea9 ",") at strtok.c:34
        olds = 0x0
#10 0x000055b72e7c730c in init_obj_path (argp=<optimized out>, base_path=0x55b73163c350 "/tmp") at store/common.c:318
        p = <optimized out>
        len = 9
#11 init_global_pathnames (d=0x55b73163c350 "/tmp", argp=<optimized out>) at store/common.c:447
No locals.
#12 0x000055b72e7aa557 in main (argc=<optimized out>, argv=<optimized out>) at sheep.c:1112
        ch = <optimized out>
        longindex = 0
        ret = <optimized out>
        port = 7000
        io_port = 7000
        rc = 1
        dirp = <optimized out>
        short_options = 0x55b72ed19700 <sopts> "b:c:Dfghi:j:l:np:P:R:uvV:w:Wx:y:z:"
        pid_file = 0x0
        bindaddr = 0x7ffe198c4d9e "172.23.1.1"
        log_path = "/var/log/sheep.log", '\000' <repeats 647 times>...
        argp = 0x0
        explicit_addr = true
        daemonize = <optimized out>
        nr_vnodes = 128
        zone = 1
        max_dynamic_threads = <optimized out>
        long_options = 0x55b72ed176e0 <lopts>
        sheep_info = {port = 7000}
        logdir_st = {st_dev = 65024, st_ino = 1177523, st_nlink = 9, st_mode = 16877, st_uid = 0, st_gid = 0, __pad0 = 0, st_rdev = 0, st_size = 4096, st_blksize = 4096, st_blocks = 8, st_atim = {tv_sec = 1532132611, tv_nsec = 417830334}, st_mtim = {tv_sec = 1532067901, tv_nsec = 764667396}, st_ctim = {tv_sec = 1532067901, tv_nsec = 764667396}, __glibc_reserved = {0, 0, 0}}
        log_dst_type = <optimized out>
        __func__ = "main"

==
== Jul 21 00:26:05 
== program: /usr/sbin/sheep
== command: p __sys
==

[zem]

I think the segfault might be caused by calling sheep without a storage parameter. but it also tells me the following, at least thats what I experience, I will make a few more tests tomorrow, but there is another problem rising in the Logs:

Jul 21 00:32:33 EMERG [main] migrate_from_v4_to_v5(502) PANIC: not implemented yet
Jul 21 00:32:33 EMERG [main] crash_handler(288) sheep exits unexpectedly (Aborted), si pid 3760, uid 0, errno 0, code -6