{"payload":{"feedbackUrl":"https://github.com/orgs/community/discussions/53140","repo":{"id":511691380,"defaultBranch":"main","name":"lance","ownerLogin":"lancedb","currentUserCanPush":false,"isFork":false,"isEmpty":false,"createdAt":"2022-07-07T22:29:29.000Z","ownerAvatar":"https://avatars.githubusercontent.com/u/108903835?v=4","public":true,"private":false,"isOrgOwned":true},"refInfo":{"name":"","listCacheKey":"v0:1716577454.0","currentOid":""},"activityList":{"items":[{"before":"006fe0b344dc635726d5464ae9d6fd33c7460e5c","after":"02cf8a441c3ee61bcce83bf0fe25aadecb752ec8","ref":"refs/heads/jack/public-inserts","pushedAt":"2024-05-25T07:50:20.000Z","pushType":"push","commitsCount":3,"pusher":{"login":"jacketsj","name":"jacketsj","path":"/jacketsj","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3720257?s=80&v=4"},"commit":{"message":"Add TODO for new_size mult","shortMessageHtmlLink":"Add TODO for new_size mult"}},{"before":"8fb73549319882c4d0b45385981ea3f1099b2b90","after":"006fe0b344dc635726d5464ae9d6fd33c7460e5c","ref":"refs/heads/jack/public-inserts","pushedAt":"2024-05-25T02:08:58.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jacketsj","name":"jacketsj","path":"/jacketsj","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3720257?s=80&v=4"},"commit":{"message":"Make insert code more concise","shortMessageHtmlLink":"Make insert code more concise"}},{"before":"29f59afbab5190f5e5732d602ca6a3f800006f5e","after":"8fb73549319882c4d0b45385981ea3f1099b2b90","ref":"refs/heads/jack/public-inserts","pushedAt":"2024-05-25T00:59:14.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jacketsj","name":"jacketsj","path":"/jacketsj","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3720257?s=80&v=4"},"commit":{"message":"retrigger checks","shortMessageHtmlLink":"retrigger checks"}},{"before":"4e8e5d7c8754552fdef528f7c64a1894ace65021","after":"29f59afbab5190f5e5732d602ca6a3f800006f5e","ref":"refs/heads/jack/public-inserts","pushedAt":"2024-05-25T00:43:59.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jacketsj","name":"jacketsj","path":"/jacketsj","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3720257?s=80&v=4"},"commit":{"message":"Make clippy happier","shortMessageHtmlLink":"Make clippy happier"}},{"before":"7d490f803d973491fb25bb87313eef83e42beaea","after":"4e8e5d7c8754552fdef528f7c64a1894ace65021","ref":"refs/heads/jack/public-inserts","pushedAt":"2024-05-25T00:38:12.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jacketsj","name":"jacketsj","path":"/jacketsj","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3720257?s=80&v=4"},"commit":{"message":"Fix compilation after upstream changes","shortMessageHtmlLink":"Fix compilation after upstream changes"}},{"before":"94dce777b28209c5f08046e6886b9e48728df600","after":"7d490f803d973491fb25bb87313eef83e42beaea","ref":"refs/heads/jack/public-inserts","pushedAt":"2024-05-25T00:29:11.000Z","pushType":"push","commitsCount":22,"pusher":{"login":"jacketsj","name":"jacketsj","path":"/jacketsj","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3720257?s=80&v=4"},"commit":{"message":"Merge branch 'main' into jack/public-inserts","shortMessageHtmlLink":"Merge branch 'main' into jack/public-inserts"}},{"before":"12c42a3c0f0f6653c91db5d0df6ca072045f58ad","after":"94dce777b28209c5f08046e6886b9e48728df600","ref":"refs/heads/jack/public-inserts","pushedAt":"2024-05-25T00:16:20.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jacketsj","name":"jacketsj","path":"/jacketsj","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3720257?s=80&v=4"},"commit":{"message":"Swap from mutex to rwlock","shortMessageHtmlLink":"Swap from mutex to rwlock"}},{"before":"1e1aa18ddecd11c74171c5b8f706299b2f895ea3","after":"12c42a3c0f0f6653c91db5d0df6ca072045f58ad","ref":"refs/heads/jack/public-inserts","pushedAt":"2024-05-25T00:04:22.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jacketsj","name":"jacketsj","path":"/jacketsj","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3720257?s=80&v=4"},"commit":{"message":"Add global rwlock to nodes and change the size when needed","shortMessageHtmlLink":"Add global rwlock to nodes and change the size when needed"}},{"before":"2b5cb5989028059d5811d59d6539a1e3c9a5e777","after":"1e1aa18ddecd11c74171c5b8f706299b2f895ea3","ref":"refs/heads/jack/public-inserts","pushedAt":"2024-05-24T19:16:13.000Z","pushType":"push","commitsCount":2,"pusher":{"login":"jacketsj","name":"jacketsj","path":"/jacketsj","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3720257?s=80&v=4"},"commit":{"message":"build takes capacity","shortMessageHtmlLink":"build takes capacity"}},{"before":"8f13ab54cc6a187ff40844aa3522b545965983e7","after":null,"ref":"refs/heads/rmeng/exec-api","pushedAt":"2024-05-24T19:04:14.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"chebbyChefNEQ","name":"Rob Meng","path":"/chebbyChefNEQ","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/12615154?s=80&v=4"}},{"before":"fc9cfd0cc62213511f0dc0ca02349fa25235c15d","after":"72b6715449d247383d06c1e2cdc30d3bbce8726b","ref":"refs/heads/main","pushedAt":"2024-05-24T19:04:13.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"chebbyChefNEQ","name":"Rob Meng","path":"/chebbyChefNEQ","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/12615154?s=80&v=4"},"commit":{"message":"chore: expose exec node internal APIs (#2390)\n\nfor https://github.com/lancedb/sophon/issues/1947","shortMessageHtmlLink":"chore: expose exec node internal APIs (#2390)"}},{"before":"68b45c32992b7d63ce0310e9edee6fdb170263aa","after":"fc9cfd0cc62213511f0dc0ca02349fa25235c15d","ref":"refs/heads/main","pushedAt":"2024-05-24T16:25:45.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"westonpace","name":"Weston Pace","path":"/westonpace","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/1696093?s=80&v=4"},"commit":{"message":"feat: make the index cache size (in bytes) available (#2381)\n\nThis PR makes extensive use of the deepsize crate for tracking the deep\r\nsize of standard structures (Vec, BTreeMap, etc.). This crate does\r\nappear to be abandoned (no updates in 3 years). However, I'm unable to\r\nfind anything similar, the code is fairly small should we need to vendor\r\nit in the future, and it is unlikely to be doing anything dangerous.\r\n\r\nIt turns out we store many many things in the cache (transitively)\r\nthough most of them are probably small.","shortMessageHtmlLink":"feat: make the index cache size (in bytes) available (#2381)"}},{"before":null,"after":"8f13ab54cc6a187ff40844aa3522b545965983e7","ref":"refs/heads/rmeng/exec-api","pushedAt":"2024-05-24T15:50:00.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"chebbyChefNEQ","name":"Rob Meng","path":"/chebbyChefNEQ","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/12615154?s=80&v=4"},"commit":{"message":"chore: expose exec node internal APIs","shortMessageHtmlLink":"chore: expose exec node internal APIs"}},{"before":"371e32631776205b4520dedccb37f671db39a441","after":"68b45c32992b7d63ce0310e9edee6fdb170263aa","ref":"refs/heads/main","pushedAt":"2024-05-24T15:41:31.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"westonpace","name":"Weston Pace","path":"/westonpace","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/1696093?s=80&v=4"},"commit":{"message":"feat: general compression for value page buffer (#2368)\n\nThis PR introduces general compression for value page buffers, starting\r\nwith zstd, to reduce the on-disk size of all types of value arrays.\r\n\r\nHere are the key details:\r\n1. After some code exploration, I implemented this as a buffer encoder\r\nfor `ValueEncoder` instead of as an independent physical encoder. Please\r\nlet me know if this approach is suitable.\r\n2. Enhancements to `ValuePageScheduler.schedule_ranges` allow it to read\r\nthe entire buffer range for compressed buffers. To support this, I added\r\na new `buffer_size` metadata to several buffer structs, populating these\r\nvariables using metadata from Lance.\r\n3. Currently, only zstd compression with the default level (level 3) is\r\nimplemented. If this approach is deemed suitable, we can consider adding\r\nmore general compression methods in the future.\r\n4. In a specific test case, the original Lance file was 13MB. After\r\napplying zstd compression, its size was reduced to 7.4MB.\r\n\r\n---------\r\n\r\nCo-authored-by: Weston Pace ","shortMessageHtmlLink":"feat: general compression for value page buffer (#2368)"}},{"before":"8c1ee00fda619762d5030d745ff9c54d87590bf6","after":"371e32631776205b4520dedccb37f671db39a441","ref":"refs/heads/main","pushedAt":"2024-05-24T13:21:44.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"westonpace","name":"Weston Pace","path":"/westonpace","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/1696093?s=80&v=4"},"commit":{"message":"feat: add some schema utility methods to the v2 reader/writer (#2389)","shortMessageHtmlLink":"feat: add some schema utility methods to the v2 reader/writer (#2389)"}},{"before":"7dbf32d0efa51e225b7d24b05f521c85fd31cd22","after":"2b5cb5989028059d5811d59d6539a1e3c9a5e777","ref":"refs/heads/jack/public-inserts","pushedAt":"2024-05-24T08:34:49.000Z","pushType":"push","commitsCount":1,"pusher":{"login":"jacketsj","name":"jacketsj","path":"/jacketsj","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3720257?s=80&v=4"},"commit":{"message":"Public insert calls in HNSW, and also allow building an empty HNSW index","shortMessageHtmlLink":"Public insert calls in HNSW, and also allow building an empty HNSW index"}},{"before":null,"after":"7dbf32d0efa51e225b7d24b05f521c85fd31cd22","ref":"refs/heads/jack/public-inserts","pushedAt":"2024-05-24T08:32:38.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"jacketsj","name":"jacketsj","path":"/jacketsj","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/3720257?s=80&v=4"},"commit":{"message":"fix: minor bug fixes for v2 (#2361)\n\nFix an issue that could occur when a batch of lists has zero size (even\r\nif the page does not)\r\n\r\nAdd test cases for above.\r\n\r\nFix list priority which got messed up when we changed to using a struct\r\ndecoder","shortMessageHtmlLink":"fix: minor bug fixes for v2 (#2361)"}},{"before":"f5b96181523165002906516b5e72702a8ac7d3a8","after":null,"ref":"refs/heads/rmeng/revert-2384","pushedAt":"2024-05-23T20:52:40.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"chebbyChefNEQ","name":"Rob Meng","path":"/chebbyChefNEQ","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/12615154?s=80&v=4"}},{"before":"8a1388cd27018556a7fb8d2e71a3f5965f68a621","after":"8c1ee00fda619762d5030d745ff9c54d87590bf6","ref":"refs/heads/main","pushedAt":"2024-05-23T20:52:39.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"chebbyChefNEQ","name":"Rob Meng","path":"/chebbyChefNEQ","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/12615154?s=80&v=4"},"commit":{"message":"Revert \"fix: concat batches before writing to avoid small IO slow down (#2384)\" (#2387)\n\nThis reverts commit 571a70a77951aa1d1e437179c7908d697f4db917.","shortMessageHtmlLink":"Revert \"fix: concat batches before writing to avoid small IO slow down ("}},{"before":null,"after":"f5b96181523165002906516b5e72702a8ac7d3a8","ref":"refs/heads/rmeng/revert-2384","pushedAt":"2024-05-23T20:15:44.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"chebbyChefNEQ","name":"Rob Meng","path":"/chebbyChefNEQ","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/12615154?s=80&v=4"},"commit":{"message":"Revert \"fix: concat batches before writing to avoid small IO slow down (#2384)\"\n\nThis reverts commit 571a70a77951aa1d1e437179c7908d697f4db917.","shortMessageHtmlLink":"Revert \"fix: concat batches before writing to avoid small IO slow down ("}},{"before":"571a70a77951aa1d1e437179c7908d697f4db917","after":"8a1388cd27018556a7fb8d2e71a3f5965f68a621","ref":"refs/heads/main","pushedAt":"2024-05-23T18:34:35.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"BubbleCal","name":null,"path":"/BubbleCal","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/159225399?s=80&v=4"},"commit":{"message":"fix: low recall if the num partitions is more than num rows (#2386)\n\nSigned-off-by: BubbleCal ","shortMessageHtmlLink":"fix: low recall if the num partitions is more than num rows (#2386)"}},{"before":"efb1e16956fa5ce77e8b02b4241aae983aea2bcb","after":null,"ref":"refs/heads/rmeng/fix-shuffler-perf","pushedAt":"2024-05-23T15:59:51.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"chebbyChefNEQ","name":"Rob Meng","path":"/chebbyChefNEQ","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/12615154?s=80&v=4"}},{"before":"3dd63b8e7190256a99da7d431f504a2cf5f90794","after":"571a70a77951aa1d1e437179c7908d697f4db917","ref":"refs/heads/main","pushedAt":"2024-05-23T15:59:50.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"chebbyChefNEQ","name":"Rob Meng","path":"/chebbyChefNEQ","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/12615154?s=80&v=4"},"commit":{"message":"fix: concat batches before writing to avoid small IO slow down (#2384)\n\nreduce number of IOs to improve shuffle performance.","shortMessageHtmlLink":"fix: concat batches before writing to avoid small IO slow down (#2384)"}},{"before":null,"after":"efb1e16956fa5ce77e8b02b4241aae983aea2bcb","ref":"refs/heads/rmeng/fix-shuffler-perf","pushedAt":"2024-05-23T04:32:03.000Z","pushType":"branch_creation","commitsCount":0,"pusher":{"login":"chebbyChefNEQ","name":"Rob Meng","path":"/chebbyChefNEQ","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/12615154?s=80&v=4"},"commit":{"message":"fix: concat batches before writing to avoid small IO slow down","shortMessageHtmlLink":"fix: concat batches before writing to avoid small IO slow down"}},{"before":"131a62b669e0276233ec2d639cd81e98d084bb61","after":null,"ref":"refs/heads/gh/chebbyChefNEQ/5/head","pushedAt":"2024-05-23T04:12:11.000Z","pushType":"branch_deletion","commitsCount":0,"pusher":{"login":"chebbyChefNEQ","name":"Rob Meng","path":"/chebbyChefNEQ","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/12615154?s=80&v=4"}},{"before":"83ecc0156a82a28b2524163da3465dc56b2bcec5","after":"3dd63b8e7190256a99da7d431f504a2cf5f90794","ref":"refs/heads/main","pushedAt":"2024-05-23T04:12:11.000Z","pushType":"pr_merge","commitsCount":1,"pusher":{"login":"chebbyChefNEQ","name":"Rob Meng","path":"/chebbyChefNEQ","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/12615154?s=80&v=4"},"commit":{"message":"feat: index v3 starting point (#2380) (#2378) (#2377) (#2376)\n\nThis is a commit of a stack of PR\r\n* closes #2380\r\n* closes #2378\r\n* closes #2377\r\n* closes #2376","shortMessageHtmlLink":"feat: index v3 starting point (#2380) (#2378) (#2377) (#2376)"}},{"before":"1d7b4107fbf5035988f77c5229318d7e88a7ec05","after":"2b93027a415f4c993c0afc2cc7d04e739ed3089c","ref":"refs/heads/gh/chebbyChefNEQ/1/orig","pushedAt":"2024-05-23T03:39:26.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"chebbyChefNEQ","name":"Rob Meng","path":"/chebbyChefNEQ","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/12615154?s=80&v=4"},"commit":{"message":"feat: define IVFShuffler trait\n\nghstack-source-id: 8c40ce30c935005b6490aefd48911bdaa3e999a4\nPull Request resolved: https://github.com/lancedb/lance/pull/2376","shortMessageHtmlLink":"feat: define IVFShuffler trait"}},{"before":"88c771a8977734fd67388ef040c6e92aadc205b2","after":"9724dfaf409ee0bacf091a8e02e089c80884f51c","ref":"refs/heads/gh/chebbyChefNEQ/3/orig","pushedAt":"2024-05-23T03:39:26.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"chebbyChefNEQ","name":"Rob Meng","path":"/chebbyChefNEQ","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/12615154?s=80&v=4"},"commit":{"message":"feat: add sub index trait\n\nghstack-source-id: 4cbcc031f97adf08f1f8570691388007671a888a\nPull Request resolved: https://github.com/lancedb/lance/pull/2378","shortMessageHtmlLink":"feat: add sub index trait"}},{"before":"9c6519453cae476c51e8b5c7c7735f1280f49bf7","after":"bcce0ae4f5eeacc34b27965b5934165301bfad49","ref":"refs/heads/gh/chebbyChefNEQ/5/orig","pushedAt":"2024-05-23T03:39:26.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"chebbyChefNEQ","name":"Rob Meng","path":"/chebbyChefNEQ","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/12615154?s=80&v=4"},"commit":{"message":"feat: define Flat index as a scan over VectorStorage\n\nghstack-source-id: 8c523fd8abb4f69352fa6998a25090e8c2ea1e6c\nPull Request resolved: https://github.com/lancedb/lance/pull/2380","shortMessageHtmlLink":"feat: define Flat index as a scan over VectorStorage"}},{"before":"a83fd3cc26c89fb71ee48ad05f4dfd2ca8d14c1c","after":"ca024d521fefc8c1e96234acfda716e7b47de456","ref":"refs/heads/gh/chebbyChefNEQ/2/orig","pushedAt":"2024-05-23T03:39:26.000Z","pushType":"force_push","commitsCount":0,"pusher":{"login":"chebbyChefNEQ","name":"Rob Meng","path":"/chebbyChefNEQ","primaryAvatarUrl":"https://avatars.githubusercontent.com/u/12615154?s=80&v=4"},"commit":{"message":"chore: move VectorStorage and DistCalculator to v3 module\n\nghstack-source-id: e007346d4c37136b959239dc3bd6ddd094256d6e\nPull Request resolved: https://github.com/lancedb/lance/pull/2377","shortMessageHtmlLink":"chore: move VectorStorage and DistCalculator to v3 module"}}],"hasNextPage":true,"hasPreviousPage":false,"activityType":"all","actor":null,"timePeriod":"all","sort":"DESC","perPage":30,"cursor":"djE6ks8AAAAEU50FlQA","startCursor":null,"endCursor":null}},"title":"Activity ยท lancedb/lance"}