Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Change LRU cache and other hash tables to use kcas #2281

Open
wants to merge 4 commits into
base: eio
Choose a base branch
from
Open

Change LRU cache and other hash tables to use kcas #2281

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3e945d5
irmin: Move LRU cache to kcas
clecat Feb 23, 2024
4e42315
irmin-pack: Change hash tables to kcas
clecat Feb 23, 2024
9c7be8d
irmin: Bring iter back
clecat Feb 23, 2024
8c3de90
Fmt
clecat Feb 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
8 changes: 8 additions & 0 deletions irmin.opam
View file
Open in desktop
Expand Up @@ -22,6 +22,8 @@ depends: [
"uutf"
"jsonm" {>= "1.0.0"}
"eio" {>= "1.0"}
"kcas" {= "dev"}
"kcas_data" {= "dev"}
"lwt" {>= "5.6.1"}
"digestif" {>= "0.9.0"}
"ocamlgraph"
Expand All @@ -41,6 +43,12 @@ depends: [
]

pin-depends: [
# Fix segv in kcas
[ "kcas.dev" "git+https://git@github.com/ocaml-multicore/kcas#5f3a39dfc72189e2b83f96c3754d402d5e7d6bc5"]
[ "kcas_data.dev" "git+https://git@github.com/ocaml-multicore/kcas#5f3a39dfc72189e2b83f96c3754d402d5e7d6bc5"]
# Metrics may have been unnecessarily constrained in opam-repository
[ "metrics.dev" "git+https://github.com/mirage/metrics#995eb18d2837df02c8ead719c00fb156cf475ab5"]
[ "metrics-unix.dev" "git+https://github.com/mirage/metrics#995eb18d2837df02c8ead719c00fb156cf475ab5"]
# Fix race in formatters
[ "alcotest.dev" "git+https://github.com/haesbaert/alcotest#99030be1df12a17fc2a0f3d00b181cd31cf6b0d4" ]
]
Expand Down
46 changes: 10 additions & 36 deletions src/irmin-pack/io/atomic_write.ml
View file
Open in desktop
@@ -1,45 +1,19 @@
open Import
include Irmin_pack.Atomic_write

module UnsafeTbl (K : Irmin.Type.S) = Hashtbl.Make (struct
type t = K.t [@@deriving irmin ~short_hash ~equal]

let hash = short_hash ?seed:None
end)

(** Safe but might be incredibly slow. *)
module Table (K : Irmin.Type.S) = struct
module Unsafe = UnsafeTbl (K)

type 'a t = { lock : Eio.Mutex.t; data : 'a Unsafe.t }

let create n =
let lock = Eio.Mutex.create () in
let data = Unsafe.create n in
{ lock; data }

let add { lock; data } k v =
Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.add data k v

let mem { lock; data } k =
Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.mem data k

let find_opt { lock; data } k =
Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.find_opt data k

let find t k = match find_opt t k with Some v -> v | None -> raise Not_found
module K = struct
include K

let replace { lock; data } k v =
Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.replace data k v
type t = K.t [@@deriving irmin ~short_hash ~equal]

let remove { lock; data } k =
Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.remove data k
let hash = short_hash ?seed:None
let equal = Irmin.Type.(unstage (equal K.t))
end

let reset { lock; data } =
Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.reset data
include Kcas_data.Hashtbl

let fold f { lock; data } init =
Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.fold f data init
let create min_buckets = create ~hashed_type:(module K) ~min_buckets ()
end

module Make_persistent (Io : Io_intf.S) (K : Irmin.Type.S) (V : Value.S) =
Expand All @@ -53,8 +27,8 @@ struct
type watch = W.watch

type t = {
index : int63 Tbl.t;
cache : V.t Tbl.t;
index : (K.t, int63) Tbl.t;
cache : (K.t, V.t) Tbl.t;
block : Io.t;
mutable block_size : int63;
w : W.t;
Expand Down
36 changes: 8 additions & 28 deletions src/irmin-pack/io/pack_store.ml
View file
Open in desktop
Expand Up @@ -24,37 +24,17 @@ exception Dangling_hash
let invalid_read fmt = Fmt.kstr (fun s -> raise (Invalid_read s)) fmt
let corrupted_store fmt = Fmt.kstr (fun s -> raise (Corrupted_store s)) fmt

module UnsafeTbl (K : Irmin.Hash.S) = Hashtbl.Make (struct
type t = K.t

let hash = K.short_hash
let equal = Irmin.Type.(unstage (equal K.t))
end)

(** Safe but might be incredibly slow. *)
module Table (K : Irmin.Hash.S) = struct
module Unsafe = UnsafeTbl (K)

type 'a t = { lock : Eio.Mutex.t; data : 'a Unsafe.t }

let create n =
let lock = Eio.Mutex.create () in
let data = Unsafe.create n in
{ lock; data }
module K = struct
include K

let add { lock; data } k v =
Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.add data k v

let mem { lock; data } k =
Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.mem data k

let find_opt { lock; data } k =
Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.find_opt data k
let hash = short_hash
let equal = Irmin.Type.(unstage (equal K.t))
end

let find t k = match find_opt t k with Some v -> v | None -> raise Not_found
include Kcas_data.Hashtbl

let clear { lock; data } =
Eio.Mutex.use_rw ~protect:true lock @@ fun () -> Unsafe.clear data
let create min_buckets = create ~hashed_type:(module K) ~min_buckets ()
end

module Make_without_close_checks
Expand Down Expand Up @@ -86,7 +66,7 @@ struct

type 'a t = {
lru : Lru.t;
staging : Val.t Tbl.t;
staging : (Hash.t, Val.t) Tbl.t;
indexing_strategy : Irmin_pack.Indexing_strategy.t;
fm : Fm.t;
dict : Dict.t;
Expand Down
2 changes: 2 additions & 0 deletions src/irmin/dune
View file
Open in desktop
Expand Up @@ -11,6 +11,8 @@
logs
logs.fmt
eio
kcas
kcas_data
mtime
ocamlgraph
uri
Expand Down