Out of memory while deleting files

[filmil]

I'm experimenting with upspinserver-drive and have been writing files over and over again.

I'm encountering out-of-memory error like so, on a standard GCP instance.

At the moment I'm not quite sure where the issue is. Filing this bug so you're aware that it exists, and if I find more information, I'll backfill.

   1 2020/01/06 06:55:58.666934 dir/server.Put: Error removing group file f@<REDACTED>/Group/sharing: access.RemoveGroup: item does not exist: group does not exist
   2 fatal error: runtime: out of memory                                                                                                                                                                                                                                                                                     
   3                                                                                                     
   4 runtime stack:                                                                                      
   5 runtime.throw(0xb7b41e, 0x16)                                                                       
   6     /home/fmil/local/opt/go/src/runtime/panic.go:774 +0x72                                          
   7 runtime.sysMap(0xc014000000, 0x4000000, 0x10e7db8)                                                  
   8     /home/fmil/local/opt/go/src/runtime/mem_linux.go:169 +0xc5                                      
   9 runtime.(*mheap).sysAlloc(0x10cf620, 0x800000, 0x0, 0x0)                                            
  10     /home/fmil/local/opt/go/src/runtime/malloc.go:701 +0x1cd                                        
  11 runtime.(*mheap).grow(0x10cf620, 0x400, 0xffffffff)                                                 
  12     /home/fmil/local/opt/go/src/runtime/mheap.go:1255 +0xa3                                         
  13 runtime.(*mheap).allocSpanLocked(0x10cf620, 0x400, 0x10e7dc8, 0x0)                                  
  14     /home/fmil/local/opt/go/src/runtime/mheap.go:1170 +0x266                                        
  15 runtime.(*mheap).alloc_m(0x10cf620, 0x400, 0x7efd50cd0101, 0x7efd3e26fdc8)                          
  16     /home/fmil/local/opt/go/src/runtime/mheap.go:1022 +0xc2                                         
  17 runtime.(*mheap).alloc.func1()                                                                      
  18     /home/fmil/local/opt/go/src/runtime/mheap.go:1093 +0x4c                                         
  19 runtime.(*mheap).alloc(0x10cf620, 0x400, 0xc000010101, 0x7efd50cd83a8)                              
  20     /home/fmil/local/opt/go/src/runtime/mheap.go:1092 +0x8a                                         
  21 runtime.largeAlloc(0x800000, 0x7efd50cb0101, 0x7efd50cd83a8)                                        
  22     /home/fmil/local/opt/go/src/runtime/malloc.go:1138 +0x97                                        
  23 runtime.mallocgc.func1()                                                                            
  24     /home/fmil/local/opt/go/src/runtime/malloc.go:1033 +0x46                                        
  25 runtime.systemstack(0x0)                                                                            
  26     /home/fmil/local/opt/go/src/runtime/asm_amd64.s:370 +0x66                                       
  27 runtime.mstart()                                                                                    
  28     /home/fmil/local/opt/go/src/runtime/proc.go:1146                                                
  29                                                                                                     
  30 goroutine 2462 [running]:                                                                           
  31 runtime.systemstack_switch()                                                                        
  32     /home/fmil/local/opt/go/src/runtime/asm_amd64.s:330 fp=0xc003fb9560 sp=0xc003fb9558 pc=0x45a830 
  33 runtime.mallocgc(0x800000, 0xa53260, 0xc003fb9601, 0x40e2c8)                                        
  34     /home/fmil/local/opt/go/src/runtime/malloc.go:1032 +0x895 fp=0xc003fb9600 sp=0xc003fb9560 pc=0x40dfd5
  35 runtime.makeslice(0xa53260, 0x0, 0x800000, 0x910b07)                                                
  36     /home/fmil/local/opt/go/src/runtime/slice.go:49 +0x6c fp=0xc003fb9630 sp=0xc003fb9600 pc=0x4451fc
  37 google.golang.org/api/internal/gensupport.NewMediaBuffer(...)                                       
  38     /home/fmil/code/go/pkg/mod/google.golang.org/api@v0.15.0/internal/gensupport/buffer.go:28       
  39 google.golang.org/api/internal/gensupport.PrepareUpload(0xc53f00, 0xc0016aaa80, 0x800000, 0x18, 0xc53f00, 0xc0016aaa80, 0xb7c5e3)
  40     /home/fmil/code/go/pkg/mod/google.golang.org/api@v0.15.0/internal/gensupport/media.go:207 +0x4e fp=0xc003fb9688 sp=0xc003fb9630 pc=0x98c92e
  41 google.golang.org/api/internal/gensupport.NewInfoFromMedia(0xc53f00, 0xc0016aaa80, 0xc003fb9768, 0x1, 0x1, 0xc540e0)
  42     /home/fmil/code/go/pkg/mod/google.golang.org/api@v0.15.0/internal/gensupport/media.go:237 +0x8c fp=0xc003fb96e8 sp=0xc003fb9688 pc=0x98cafc
  43 google.golang.org/api/drive/v3.(*FilesCreateCall).Media(...)                                        
  44     /home/fmil/code/go/pkg/mod/google.golang.org/api@v0.15.0/drive/v3/drive-gen.go:5953             
  45 drive.upspin.io/cloud/storage/drive.(*driveImpl).Put(0xc0001a71a0, 0xc004048c40, 0x40, 0xc001708000, 0xc8e, 0xd80, 0x40, 0xba0020)
  46     /home/fmil/code/go/pkg/mod/drive.upspin.io@v0.0.0-20180516213836-bf1608f3f80a/cloud/storage/drive/drive.go:134 +0x385 fp=0xc003fb97c0 sp=0xc003fb96e8 pc=0x9de645
  47 upspin.io/store/server.(*server).Put(0xc00008ce60, 0xc001708000, 0xc8e, 0xd80, 0x0, 0x0, 0x0)       
  48     /home/fmil/code/go/src/upspin.io/store/server/server.go:71 +0x33a fp=0xc003fb9910 sp=0xc003fb97c0 pc=0x8f56ba
  49 upspin.io/serverutil/perm.(*storeWrapper).Put(0xc0016aa210, 0xc001708000, 0xc8e, 0xd80, 0x18, 0xc003fb9a30, 0x2)
  50     /home/fmil/code/go/src/upspin.io/serverutil/perm/store.go:72 +0x1a5 fp=0xc003fb9960 sp=0xc003fb9910 pc=0x8daef5
  51 upspin.io/rpc/storeserver.(*server).Put(0xc00026c0c0, 0xc62be0, 0xc00057d020, 0xc0016f0000, 0xc91, 0xe00, 0xc003fb9d70, 0xc00179b1ec, 0x0, 0x0)
  52     /home/fmil/code/go/src/upspin.io/rpc/storeserver/server.go:93 +0x222 fp=0xc003fb9a60 sp=0xc003fb9960 pc=0x8d7642
  53 upspin.io/rpc/storeserver.(*server).Put-fm(0xc62be0, 0xc00057d020, 0xc0016f0000, 0xc91, 0xe00, 0xe00, 0x0, 0x0, 0xb)
  54     /home/fmil/code/go/src/upspin.io/rpc/storeserver/server.go:85 +0x63 fp=0xc003fb9ac0 sp=0xc003fb9a60 pc=0x8d7fe3
  55 upspin.io/rpc.(*serverImpl).ServeHTTP(0xc000061b00, 0xc60920, 0xc0017c9340, 0xc0017b3b00)           
  56     /home/fmil/code/go/src/upspin.io/rpc/server.go:177 +0x3ec fp=0xc003fb9bb8 sp=0xc003fb9ac0 pc=0x8ca75c
  57 net/http.(*ServeMux).ServeHTTP(0x10c8ec0, 0xc60920, 0xc0017c9340, 0xc0017b3b00)                     
  58     /home/fmil/local/opt/go/src/net/http/server.go:2387 +0x1bd fp=0xc003fb9c18 sp=0xc003fb9bb8 pc=0x76133d
  59 net/http.serverHandler.ServeHTTP(0xc00028c0e0, 0xc60920, 0xc0017c9340, 0xc0017b3b00)                
  60     /home/fmil/local/opt/go/src/net/http/server.go:2802 +0xa4 fp=0xc003fb9c48 sp=0xc003fb9c18 pc=0x7628b4
  61 net/http.(*conn).serve(0xc000544f00, 0xc627a0, 0xc003fc0840)                                        
  62     /home/fmil/local/opt/go/src/net/http/server.go:1890 +0x875 fp=0xc003fb9fc8 sp=0xc003fb9c48 pc=0x75e255
  63 runtime.goexit()                                                                                    
  64     /home/fmil/local/opt/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0xc003fb9fd0 sp=0xc003fb9fc8 pc=0x45c901
  65 created by net/http.(*Server).Serve                                                                 
  66     /home/fmil/local/opt/go/src/net/http/server.go:2927 +0x38e                                      
  67