Skip to content

Commit

Permalink
tools/memleak: Fix the data error caused by the same key in map (#4970)
Browse files Browse the repository at this point in the history
Followings are the way to generate data error issue and the result after applying this patch.

File test.cpp

  #include <iostream>
  #include <thread>
  #include <unistd.h>

  void alloc() {
    for (int i = 0; i < 100000; ++i) {
      int* a = (int*)malloc(4);
    }
  }

  int main() {
    sleep(100);
    std::thread t1 {&alloc};
    std::thread t2 {&alloc};
    t1.join();
    t2.join();
    sleep(50);
    return 0;
  }

Build the test file

  $ g++ -g -o test -lpthread test.cpp

Run this with --combined-only:

sudo ./memleak.py -c ./test --combined-only
Executing './test' and tracing the resulting process.
Attaching to pid 194273, Ctrl+C to quit.
[23:36:43] Top 10 stacks with outstanding allocations:
        576 bytes in 2 allocations from stack
                __GI__dl_allocate_tls+0x2c [ld-2.28.so]
        799992 bytes in 199998 allocations from stack
                alloc()+0x22 [test]
                void std::__invoke_impl<void, void (*)()>(std::__invoke_other, void (*&&)())+0x1d [test]
                std::__invoke_result<void (*)()>::type std::__invoke<void (*)()>(void (*&&)())+0x20 [test]
                decltype (__invoke((_S_declval<0ul>)())) std::thread::_Invoker<std::tuple<void (*)()> >::_M_invoke<0ul>(std::_Index_tuple<0ul>)+0x28 [test]
                std::thread::_Invoker<std::tuple<void (*)()> >::operator()()+0x18 [test]
                std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)()> > >::_M_run()+0x1c [test]
                [unknown] [libstdc++.so.6.0.25]
        8392704 bytes in 1 allocations from stack
                pthread_create+0x893 [libpthread-2.28.so]
                std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)())+0x19 [libstdc++.so.6.0.25]
                main+0x49 [test]
                __libc_start_main+0xf3 [libc-2.28.so]
                [unknown]
        8392704 bytes in 1 allocations from stack
                pthread_create+0x893 [libpthread-2.28.so]
                std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)())+0x19 [libstdc++.so.6.0.25]
                main+0x2e [test]
                __libc_start_main+0xf3 [libc-2.28.so]
                [unknown]
        134217728 bytes in 1 allocations from stack
                new_heap+0xa7 [libc-2.28.so]

Problem:
We can see 799992 bytes alloced in 199998 allocations from stack while when we read the code, we will find that there should be 800000 bytes alloced in 200000 allocations.

Reason:
After Tracing, we can find that `malloc()` may call `mmap()`, so there will be two continuous call of `gen_alloc_enter(struct pt_regs *ctx, size_t size)` in a same process.
For example, in our `test` process, when it first call `int* a = (int*)malloc(4);`, `gen_alloc_enter(struct pt_regs *ctx, size_t size)` will be called, and there will be a
pair of data <tid, 4> in BPF_HASH sizes; then a `mmap()` will be called, also `gen_alloc_enter(struct pt_regs *ctx, size_t size)` be called, which make <tid, 4> change into
<tid, MMAP_SIZE>. This will make the call of `gen_alloc_exit()` for the first `malloc()` will return early because the `tid` key will be deleted after the call of
`gen_alloc_exit()` caused by `mmap()`, which finally cause data error.

The callchain:
malloc()->gen_alloc_enter()->mmap()->gen_alloc_enter()->mmap_return()->gen_alloc_exit()->malloc_return()->gen_alloc_exit();

Solution:
We can add type_index to help distinguish calling sources and reduce key conflicts.

After Applying this patch, run memleak.py with --combined-only:

sudo ./memleak.py -c ./test --combined-only
Executing './test' and tracing the resulting process.
Attaching to pid 194659, Ctrl+C to quit.
[23:37:16] Top 10 stacks with outstanding allocations:
        576 bytes in 2 allocations from stack
                __GI__dl_allocate_tls+0x2c [ld-2.28.so]
        800000 bytes in 200000 allocations from stack
                alloc()+0x22 [test]
                void std::__invoke_impl<void, void (*)()>(std::__invoke_other, void (*&&)())+0x1d [test]
                std::__invoke_result<void (*)()>::type std::__invoke<void (*)()>(void (*&&)())+0x20 [test]
                decltype (__invoke((_S_declval<0ul>)())) std::thread::_Invoker<std::tuple<void (*)()> >::_M_invoke<0ul>(std::_Index_tuple<0ul>)+0x28 [test]
                std::thread::_Invoker<std::tuple<void (*)()> >::operator()()+0x18 [test]
                std::thread::_State_impl<std::thread::_Invoker<std::tuple<void (*)()> > >::_M_run()+0x1c [test]
                [unknown] [libstdc++.so.6.0.25]
        8392704 bytes in 1 allocations from stack
                pthread_create+0x893 [libpthread-2.28.so]
                std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)())+0x19 [libstdc++.so.6.0.25]
                main+0x49 [test]
                __libc_start_main+0xf3 [libc-2.28.so]
                [unknown]
        8392704 bytes in 1 allocations from stack
                pthread_create+0x893 [libpthread-2.28.so]
                std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)())+0x19 [libstdc++.so.6.0.25]
                main+0x2e [test]
                __libc_start_main+0xf3 [libc-2.28.so]
                [unknown]
        134217728 bytes in 1 allocations from stack
                new_heap+0xa7 [libc-2.28.so]
  • Loading branch information
AshinZ committed May 9, 2024
1 parent 07e3102 commit 1150523
Showing 1 changed file with 53 additions and 38 deletions.
91 changes: 53 additions & 38 deletions tools/memleak.py
Expand Up @@ -161,7 +161,20 @@ def run_command_get_pid(command):
u64 number_of_allocs;
};
BPF_HASH(sizes, u32, u64);
#define KERNEL 0
#define MALLOC 1
#define CALLOC 2
#define REALLOC 3
#define MMAP 4
#define POSIX_MEMALIGN 5
#define VALLOC 6
#define MEMALIGN 7
#define PVALLOC 8
#define ALIGNED_ALLOC 9
#define FREE 10
#define MUNMAP 11
BPF_HASH(sizes, u64, u64);
BPF_HASH(allocs, u64, struct alloc_info_t, 1000000);
BPF_HASH(memptrs, u32, u64);
BPF_STACK_TRACE(stack_traces, 10240);
Expand Down Expand Up @@ -197,7 +210,7 @@ def run_command_get_pid(command):
}
}
static inline int gen_alloc_enter(struct pt_regs *ctx, size_t size) {
static inline int gen_alloc_enter(struct pt_regs *ctx, size_t size, u32 type_index) {
SIZE_FILTER
if (SAMPLE_EVERY_N > 1) {
u64 ts = bpf_ktime_get_ns();
Expand All @@ -207,23 +220,25 @@ def run_command_get_pid(command):
u32 tid = bpf_get_current_pid_tgid();
u64 size64 = size;
sizes.update(&tid, &size64);
u64 key = (uint64_t)type_index << 32 | tid;
sizes.update(&key, &size64);
if (SHOULD_PRINT)
bpf_trace_printk("alloc entered, size = %u\\n", size);
return 0;
}
static inline int gen_alloc_exit2(struct pt_regs *ctx, u64 address) {
static inline int gen_alloc_exit2(struct pt_regs *ctx, u64 address, u32 type_index) {
u32 tid = bpf_get_current_pid_tgid();
u64* size64 = sizes.lookup(&tid);
u64 key = (uint64_t)type_index << 32 | tid;
u64* size64 = sizes.lookup(&key);
struct alloc_info_t info = {0};
if (size64 == 0)
return 0; // missed alloc entry
info.size = *size64;
sizes.delete(&tid);
sizes.delete(&key);
if (address != 0) {
info.timestamp_ns = bpf_ktime_get_ns();
Expand All @@ -239,8 +254,8 @@ def run_command_get_pid(command):
return 0;
}
static inline int gen_alloc_exit(struct pt_regs *ctx) {
return gen_alloc_exit2(ctx, PT_REGS_RC(ctx));
static inline int gen_alloc_exit(struct pt_regs *ctx, u32 type_index) {
return gen_alloc_exit2(ctx, PT_REGS_RC(ctx), type_index);
}
static inline int gen_free_enter(struct pt_regs *ctx, void *address) {
Expand All @@ -260,41 +275,41 @@ def run_command_get_pid(command):
}
int malloc_enter(struct pt_regs *ctx, size_t size) {
return gen_alloc_enter(ctx, size);
return gen_alloc_enter(ctx, size, MALLOC);
}
int malloc_exit(struct pt_regs *ctx) {
return gen_alloc_exit(ctx);
return gen_alloc_exit2(ctx, PT_REGS_RC(ctx), MALLOC);
}
int free_enter(struct pt_regs *ctx, void *address) {
return gen_free_enter(ctx, address);
}
int calloc_enter(struct pt_regs *ctx, size_t nmemb, size_t size) {
return gen_alloc_enter(ctx, nmemb * size);
return gen_alloc_enter(ctx, nmemb * size, CALLOC);
}
int calloc_exit(struct pt_regs *ctx) {
return gen_alloc_exit(ctx);
return gen_alloc_exit(ctx, CALLOC);
}
int realloc_enter(struct pt_regs *ctx, void *ptr, size_t size) {
gen_free_enter(ctx, ptr);
return gen_alloc_enter(ctx, size);
return gen_alloc_enter(ctx, size, REALLOC);
}
int realloc_exit(struct pt_regs *ctx) {
return gen_alloc_exit(ctx);
return gen_alloc_exit(ctx, REALLOC);
}
int mmap_enter(struct pt_regs *ctx) {
size_t size = (size_t)PT_REGS_PARM2(ctx);
return gen_alloc_enter(ctx, size);
return gen_alloc_enter(ctx, size, MMAP);
}
int mmap_exit(struct pt_regs *ctx) {
return gen_alloc_exit(ctx);
return gen_alloc_exit(ctx, MMAP);
}
int munmap_enter(struct pt_regs *ctx, void *address) {
Expand All @@ -307,7 +322,7 @@ def run_command_get_pid(command):
u32 tid = bpf_get_current_pid_tgid();
memptrs.update(&tid, &memptr64);
return gen_alloc_enter(ctx, size);
return gen_alloc_enter(ctx, size, POSIX_MEMALIGN);
}
int posix_memalign_exit(struct pt_regs *ctx) {
Expand All @@ -324,39 +339,39 @@ def run_command_get_pid(command):
return 0;
u64 addr64 = (u64)(size_t)addr;
return gen_alloc_exit2(ctx, addr64);
return gen_alloc_exit2(ctx, addr64, POSIX_MEMALIGN);
}
int aligned_alloc_enter(struct pt_regs *ctx, size_t alignment, size_t size) {
return gen_alloc_enter(ctx, size);
return gen_alloc_enter(ctx, size, ALIGNED_ALLOC);
}
int aligned_alloc_exit(struct pt_regs *ctx) {
return gen_alloc_exit(ctx);
return gen_alloc_exit(ctx, ALIGNED_ALLOC);
}
int valloc_enter(struct pt_regs *ctx, size_t size) {
return gen_alloc_enter(ctx, size);
return gen_alloc_enter(ctx, size, VALLOC);
}
int valloc_exit(struct pt_regs *ctx) {
return gen_alloc_exit(ctx);
return gen_alloc_exit(ctx, VALLOC);
}
int memalign_enter(struct pt_regs *ctx, size_t alignment, size_t size) {
return gen_alloc_enter(ctx, size);
return gen_alloc_enter(ctx, size, MEMALIGN);
}
int memalign_exit(struct pt_regs *ctx) {
return gen_alloc_exit(ctx);
return gen_alloc_exit(ctx, MEMALIGN);
}
int pvalloc_enter(struct pt_regs *ctx, size_t size) {
return gen_alloc_enter(ctx, size);
return gen_alloc_enter(ctx, size, PVALLOC);
}
int pvalloc_exit(struct pt_regs *ctx) {
return gen_alloc_exit(ctx);
return gen_alloc_exit(ctx, PVALLOC);
}
"""

Expand All @@ -365,15 +380,15 @@ def run_command_get_pid(command):
TRACEPOINT_PROBE(kmem, kmalloc_node) {
if (WORKAROUND_MISSING_FREE)
gen_free_enter((struct pt_regs *)args, (void *)args->ptr);
gen_alloc_enter((struct pt_regs *)args, args->bytes_alloc);
return gen_alloc_exit2((struct pt_regs *)args, (size_t)args->ptr);
gen_alloc_enter((struct pt_regs *)args, args->bytes_alloc, KERNEL);
return gen_alloc_exit2((struct pt_regs *)args, (size_t)args->ptr, KERNEL);
}
TRACEPOINT_PROBE(kmem, kmem_cache_alloc_node) {
if (WORKAROUND_MISSING_FREE)
gen_free_enter((struct pt_regs *)args, (void *)args->ptr);
gen_alloc_enter((struct pt_regs *)args, args->bytes_alloc);
return gen_alloc_exit2((struct pt_regs *)args, (size_t)args->ptr);
gen_alloc_enter((struct pt_regs *)args, args->bytes_alloc, KERNEL);
return gen_alloc_exit2((struct pt_regs *)args, (size_t)args->ptr, KERNEL);
}
"""

Expand All @@ -382,8 +397,8 @@ def run_command_get_pid(command):
TRACEPOINT_PROBE(kmem, kmalloc) {
if (WORKAROUND_MISSING_FREE)
gen_free_enter((struct pt_regs *)args, (void *)args->ptr);
gen_alloc_enter((struct pt_regs *)args, args->bytes_alloc);
return gen_alloc_exit2((struct pt_regs *)args, (size_t)args->ptr);
gen_alloc_enter((struct pt_regs *)args, args->bytes_alloc, KERNEL);
return gen_alloc_exit2((struct pt_regs *)args, (size_t)args->ptr, KERNEL);
}
TRACEPOINT_PROBE(kmem, kfree) {
Expand All @@ -393,17 +408,17 @@ def run_command_get_pid(command):
TRACEPOINT_PROBE(kmem, kmem_cache_alloc) {
if (WORKAROUND_MISSING_FREE)
gen_free_enter((struct pt_regs *)args, (void *)args->ptr);
gen_alloc_enter((struct pt_regs *)args, args->bytes_alloc);
return gen_alloc_exit2((struct pt_regs *)args, (size_t)args->ptr);
gen_alloc_enter((struct pt_regs *)args, args->bytes_alloc, KERNEL);
return gen_alloc_exit2((struct pt_regs *)args, (size_t)args->ptr, KERNEL);
}
TRACEPOINT_PROBE(kmem, kmem_cache_free) {
return gen_free_enter((struct pt_regs *)args, (void *)args->ptr);
}
TRACEPOINT_PROBE(kmem, mm_page_alloc) {
gen_alloc_enter((struct pt_regs *)args, PAGE_SIZE << args->order);
return gen_alloc_exit2((struct pt_regs *)args, args->pfn);
gen_alloc_enter((struct pt_regs *)args, PAGE_SIZE << args->order, KERNEL);
return gen_alloc_exit2((struct pt_regs *)args, args->pfn, KERNEL);
}
TRACEPOINT_PROBE(kmem, mm_page_free) {
Expand All @@ -414,8 +429,8 @@ def run_command_get_pid(command):
bpf_source_percpu = """
TRACEPOINT_PROBE(percpu, percpu_alloc_percpu) {
gen_alloc_enter((struct pt_regs *)args, args->size);
return gen_alloc_exit2((struct pt_regs *)args, (size_t)args->ptr);
gen_alloc_enter((struct pt_regs *)args, args->size, KERNEL);
return gen_alloc_exit2((struct pt_regs *)args, (size_t)args->ptr, KERNEL);
}
TRACEPOINT_PROBE(percpu, percpu_free_percpu) {
Expand Down

0 comments on commit 1150523

Please sign in to comment.