Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
102 commits
Select commit Hold shift + click to select a range
8a42dce
Convert Irmin from Lwt to direct-style with Eio
patricoferris Jun 5, 2022
e687490
Restore Test_tree.clear tree size
patricoferris Jun 5, 2022
5f560ed
Fix map_p, filter_map_p and convert irmin-fs
patricoferris Jun 5, 2022
b90b3a5
Add eio to opam file
patricoferris Jun 5, 2022
7c26ef3
Convert irmin-pack to direct-style
patricoferris Jun 5, 2022
45f1069
Update to latest eio
patricoferris Sep 2, 2022
28f2504
Working FS unix backend
patricoferris Oct 8, 2022
9f95409
Convert irmin-containers to direct-style
patricoferris Oct 8, 2022
52d1258
Remove pin-depends for eio
patricoferris Nov 29, 2022
2dad4b1
Fix irmin-fs file info
patricoferris Nov 29, 2022
e2b202c
Fix ignoring commit in store
patricoferris Nov 29, 2022
73fb8b1
Fix generic-key tests
patricoferris Nov 29, 2022
4ce5ae8
Remove traceln
patricoferris Dec 9, 2022
7b57ac1
Update to eio.0.7 and catch IO errors
patricoferris Jan 10, 2023
2887b69
Fix after rebase
art-w May 31, 2023
b3bc47e
Fix irmin-pack bench
art-w May 31, 2023
55c7582
Fix irmin-pack examples
art-w May 31, 2023
2610f9f
Fix irmin-http using lwt_eio
art-w Jun 1, 2023
0785a31
Fix irmin-graphql using lwt_eio
art-w Jun 2, 2023
7d9925a
Fix irmin-git using lwt_eio
art-w Jun 2, 2023
dbf96b2
Fix irmin-cli using lwt_eio
art-w Jun 2, 2023
d1b8b75
Fix irmin-mirage using lwt_eio
art-w Jun 2, 2023
46ed87a
Fix examples using lwt_eio
art-w Jun 2, 2023
5eb5668
Fix for libirmin
art-w Jun 5, 2023
4d880e7
Fix opam dependencies to lwt_eio
art-w Jun 29, 2023
b4278db
Add multicore tests
ElectreAAS Jun 5, 2023
c4305c8
Add lock on branch
ElectreAAS May 30, 2023
d25729d
Fix LRU hashtable with a mutex
ElectreAAS Jun 5, 2023
9721f49
Fixed data race in stat counting
ElectreAAS Jun 5, 2023
6488dca
Fixed data race in tree counters
ElectreAAS Jun 5, 2023
f7b13f9
Fixed again stats, this time not in unix
ElectreAAS Jun 5, 2023
3e359de
Update multicore testing
ElectreAAS Jun 12, 2023
38a5f74
Added locks on unix/pack-store, in Tbl
ElectreAAS Jun 19, 2023
9e16d78
Added locks in irmin-pack atomic_write
ElectreAAS Jun 19, 2023
dfabba2
Added locks in control-file
ElectreAAS Jun 19, 2023
18165b7
Remove unsafe index stats in irmin-pack io
ElectreAAS Jun 19, 2023
c4521e6
Added locks in store
ElectreAAS Jun 19, 2023
251b7a1
Safe pack_key promotion
art-w Jun 21, 2023
3a21400
Optimize control file payload
art-w Jun 22, 2023
2c9d59a
Remove unsafe usage of Index.Stats
art-w Jun 22, 2023
73d80e8
Fix Tree findv_cache
art-w Jun 23, 2023
c992ee4
Fix Tree value cache
art-w Jun 23, 2023
8477be4
Remove spinlock on Tree stats
art-w Jun 23, 2023
c4642f3
Fix irmin-pack inode partial_ptr
art-w Jun 23, 2023
c788c64
irmin-pack: add multicore unit tests
art-w Jul 3, 2023
96c8e2e
Fix alcotest dependency
art-w Jul 3, 2023
ff364f3
Fix irmin Tree lazy length
art-w Jul 3, 2023
752c202
Fix irmin Tree Contents mutables
art-w Jul 3, 2023
0e3679d
Fix irmin proof
art-w Jul 3, 2023
d544f9e
Fix irmin Tree add and remove
art-w Jul 3, 2023
d64e91c
Fix irmin-pack Append_only_file buffer length
art-w Jul 3, 2023
e73dc68
Fix irmin-pack Gc cancellation on close
art-w Jul 3, 2023
4d3fa83
Fix irmin-pack Async Gc status handling
art-w Jul 3, 2023
f7eca87
Fix for append_only_file buffer length
art-w Jul 4, 2023
7f5cf6c
Fix index-unix flushing from another thread
art-w Jul 4, 2023
5e72e98
Fix irmin-pack Gc locks
art-w Jul 4, 2023
15283dc
Add commit unit test
art-w Jul 4, 2023
60c3192
Fix irmin-pack batch in batch test
art-w Jul 4, 2023
0a39c9e
Fix irmin store mutable tree
art-w Jul 4, 2023
85e45a7
Fix commit write lock
art-w Jul 5, 2023
2e926d0
Fix parallel tree hash computation
art-w Jul 5, 2023
76c7dc4
irmin-pack: Add Store.Tree.list test
clecat Jul 5, 2023
87736c7
irmin-pack: Add tests on commits for of_hash & parents
clecat Jul 5, 2023
3c138d7
irmin-pack: Add Commit.v test & fix it with a lock
clecat Jul 5, 2023
7b87f49
Fix libirmin: at_exit executes outside of eio
art-w Jul 5, 2023
0c46c1e
Add half and full diamond multicore benchmarks
art-w Jul 10, 2023
cda46d6
Optimize Tree findv_cache for add/remove
art-w Jul 10, 2023
966591e
Fix formatting and remove old tests
art-w Jul 10, 2023
58fd010
Fix irmin_fsck
art-w Jul 12, 2023
f442acb
Fix opam dependencies
art-w Jul 12, 2023
02c9d38
eio: irmin-server using lwt_eio
art-w Aug 24, 2023
b949ff5
irmin-pack: remove Io.Unix constraint
art-w Aug 21, 2023
1bbc03c
upgrade to eio.0.12
art-w Sep 4, 2023
241965a
irmin-pack: generic io lib without unix dependency
art-w Sep 6, 2023
a83b4b9
irmin-pack: extract unix Async
art-w Sep 6, 2023
436a2e8
irmin-pack: extract remaining Unix functions
art-w Sep 6, 2023
e6fb82e
irmin-pack: fix Gc.on_finalise
art-w Sep 6, 2023
bfa6230
mirage: irmin-pack.unix is optional
art-w Sep 8, 2023
9f97659
irmin-pack.io: remove rusage dependency
art-w Sep 8, 2023
b28bd83
mirage: remove dependency to fmt_tty
art-w Sep 8, 2023
1214345
mirage: remove dependency to mtime.clock.os
art-w Sep 11, 2023
32ffd16
mirage: remove dependency to progress
art-w Sep 11, 2023
807087e
fix progress dependency injection
art-w Sep 20, 2023
c279a8a
irmin-pack: use checkseum.c
art-w Oct 7, 2023
ce7c584
Update opam dependencies
art-w Sep 20, 2023
35599e2
Fix irmin-client tests
art-w Feb 21, 2024
08a82fe
Simplify concurrent list operations
art-w Feb 28, 2024
a22a24d
Fix CI opam dependencies
art-w Mar 5, 2024
6381c4e
Upgrade dune and fix new warnings
art-w Mar 5, 2024
675277a
macos: reduce number of open files in unix-client test to avoid EMFIL…
art-w Mar 6, 2024
e5d7d5c
Upgrade to Eio 1.0
art-w Mar 26, 2024
b20b407
Update CHANGES
art-w Apr 9, 2024
07330f8
Temporarily disable CI for specific platforms issues
art-w Apr 12, 2024
962d878
Update opam for OCaml 5.1.0
art-w May 6, 2024
b5b41dd
Update CI for OCaml 5.2
art-w Jun 25, 2024
fbaf446
Fix multicore race in unit test
art-w Jun 25, 2024
ff46c79
Remove pin-depend to alcotest
clecat Jun 24, 2025
e788310
Apply fmt
clecat Jun 24, 2025
bffabb8
Fix irmin-client test
clecat Jun 27, 2025
7280946
Remove pin-depends to progress and terminal and remove irmin-http.opa…
lyrm Jul 7, 2025
a5478b0
Remove useless lwt dependencies.
lyrm Jul 7, 2025
d9eabf4
Remove _lwt from function names
clecat Jul 10, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
2 changes: 1 addition & 1 deletion .ocamlformat
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
version = 0.26.2
version = 0.27.0
profile = conventional

ocaml-version = 4.08.0
Expand Down
6 changes: 6 additions & 0 deletions CHANGES.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,9 @@
## 4.0.0

### Changed

- Convert to direct-style with Eio (#2149, @patricoferris, @ElectreAAS, @clecat, @art-w)

## 3.11.0 (2025-06-19)

### Added
Expand Down
19 changes: 8 additions & 11 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -178,8 +178,6 @@ Git-based, filesystem-backed store.

<!-- $MDX file=examples/readme.ml -->
```ocaml
open Lwt.Syntax

(* Irmin store with string contents *)
module Store = Irmin_git_unix.FS.KV (Irmin.Contents.String)

Expand All @@ -192,25 +190,24 @@ let author = "Example <example@example.com>"
(* Commit information *)
let info fmt = Irmin_git_unix.info ~author fmt

let main =
let main () =
(* Open the repo *)
let* repo = Store.Repo.v config in
let repo = Store.Repo.v config in

(* Load the main branch *)
let* t = Store.main repo in
let t = Store.main repo in

(* Set key "foo/bar" to "testing 123" *)
let* () =
Store.set_exn t ~info:(info "Updating foo/bar") [ "foo"; "bar" ]
"testing 123"
in
Store.set_exn t ~info:(info "Updating foo/bar") [ "foo"; "bar" ] "testing 123";

(* Get key "foo/bar" and print it to stdout *)
let+ x = Store.get t [ "foo"; "bar" ] in
let x = Store.get t [ "foo"; "bar" ] in
Printf.printf "foo/bar => '%s'\n" x

(* Run the program *)
let () = Lwt_main.run main
let () =
Eio_main.run @@ fun env ->
Lwt_eio.with_event_loop ~clock:env#clock @@ fun _ -> main ()
```

The example is contained in [examples/readme.ml](./examples/readme.ml) It can
Expand Down
11 changes: 11 additions & 0 deletions bench.Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
FROM ocaml/opam:debian-ocaml-5.1
RUN sudo ln -sf /usr/bin/opam-2.1 /usr/bin/opam
RUN mkdir bench-dir && chown opam:opam bench-dir
WORKDIR bench-dir
RUN sudo chown opam .
COPY *.opam ./
RUN opam remote add origin https://github.com/ocaml/opam-repository.git && opam update
RUN opam pin -yn --with-version=dev .
RUN opam install -y --deps-only --with-test .
COPY . ./
ENTRYPOINT opam exec -- make bench
16 changes: 8 additions & 8 deletions bench/irmin-pack/bench_common.ml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*)

open Irmin.Export_for_backends
module Mtime = Import.Mtime

let c0 = Mtime_clock.counter ()
let now_us () = Mtime.span_to_us (Mtime_clock.count c0)
Expand Down Expand Up @@ -78,7 +78,7 @@ let prepare_artefacts_dir path =

let with_timer f =
let t0 = Sys.time () in
let+ a = f () in
let a = f () in
let t1 = Sys.time () -. t0 in
(t1, a)

Expand Down Expand Up @@ -164,30 +164,30 @@ struct
let add_chain_trees depth nb tree =
let path = key 2 in
let rec aux i tree =
if i >= nb then Lwt.return tree
if i >= nb then tree
else
let* tree = chain_tree tree depth path in
let tree = chain_tree tree depth path in
aux (i + 1) tree
in
aux 0 tree

let large_tree path tree width =
let rec aux i tree =
if i >= width then Lwt.return tree
if i >= width then tree
else
let k = path @ [ random_key () ] in
let* tree = Store.Tree.add tree k (random_blob ()) in
let tree = Store.Tree.add tree k (random_blob ()) in
aux (i + 1) tree
in
aux 0 tree

let add_large_trees width nb tree =
let path = key 1 in
let rec aux i tree =
if i >= nb then Lwt.return tree
if i >= nb then tree
else
let path = path @ [ random_key () ] in
let* tree = large_tree path tree width in
let tree = large_tree path tree width in
aux (i + 1) tree
in
aux 0 tree
Expand Down
8 changes: 5 additions & 3 deletions bench/irmin-pack/bench_common.mli
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,14 @@
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*)

module Mtime : module type of Import.Mtime

val default_artefacts_dir : string
val prepare_artefacts_dir : string -> unit
val reporter : ?prefix:string -> unit -> Logs.reporter
val setup_log : Fmt.style_renderer option -> Logs.level option -> unit
val reset_stats : unit -> unit
val with_timer : (unit -> 'a Lwt.t) -> (float * 'a) Lwt.t
val with_timer : (unit -> 'a) -> float * 'a

val with_progress_bar :
message:string -> n:int -> unit:string -> ((int -> unit) -> 'a) -> 'a
Expand All @@ -40,11 +42,11 @@ end

module Generate_trees
(Store : Irmin.Generic_key.KV with type Schema.Contents.t = bytes) : sig
val add_chain_trees : int -> int -> Store.tree -> Store.tree Lwt.t
val add_chain_trees : int -> int -> Store.tree -> Store.tree
(** [add_chain_trees depth nb tree] adds [nb] random contents to [tree],
depthwise. *)

val add_large_trees : int -> int -> Store.tree -> Store.tree Lwt.t
val add_large_trees : int -> int -> Store.tree -> Store.tree
(** [add_large_trees width nb tree] adds [nb] random contents to [tree],
breadthwise. *)
end
8 changes: 2 additions & 6 deletions bench/irmin-pack/dune
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
(libraries
irmin-pack
irmin-test.bench
lwt
unix
cmdliner
logs
Expand All @@ -21,7 +20,7 @@
(library
(name bench_common)
(public_name irmin-bench.common)
(modules bench_common)
(modules bench_common import)
(libraries irmin-pack irmin-pack.unix irmin-tezos unix progress uuidm)
(preprocess
(pps ppx_irmin.internal))
Expand All @@ -42,15 +41,13 @@
trace_replay
trace_replay_intf
tezos_history_metrics
trace_stat_summary_cb
import)
trace_stat_summary_cb)
(preprocess
(pps ppx_irmin.internal ppx_repr ppx_deriving.enum))
(libraries
irmin
irmin-pack
unix
lwt
repr
ppx_repr
bentov
Expand All @@ -71,7 +68,6 @@
irmin-pack
irmin-pack.mem
irmin-test.bench
lwt
unix
cmdliner
logs
Expand Down
11 changes: 11 additions & 0 deletions bench/irmin-pack/import.ml
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,14 @@
*)

include Irmin.Export_for_backends

module Mtime = struct
include Mtime

module Span = struct
include Mtime.Span

let to_s span = Mtime.Span.to_float_ns span *. 1e-9
let to_us span = Mtime.Span.to_float_ns span *. 1e-3
end
end
3 changes: 2 additions & 1 deletion bench/irmin-pack/tezos_history_metrics.ml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@

Computed using:

https://api.tzstats.com/series/block.json?collapse=1w&columns=n_tx,n_ops,count *)
https://api.tzstats.com/series/block.json?collapse=1w&columns=n_tx,n_ops,count
*)
let weekly_stats =
[
(1529884800000, 497, 16396, 1814);
Expand Down
18 changes: 8 additions & 10 deletions bench/irmin-pack/trace_collection.ml
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,7 @@

A module [Make_replayable] has yet to be implemented. *)

open Lwt.Syntax
module Mtime = Import.Mtime
module Mtime = Bench_common.Mtime

(** Make state trace collector. *)
module Make_stat (Store : Irmin.Generic_key.KV) = struct
Expand Down Expand Up @@ -201,8 +200,7 @@ module Make_stat (Store : Irmin.Generic_key.KV) = struct

let short_op_end { t0; writer; _ } short_op =
let duration =
Mtime_clock.count t0 |> fun span ->
Mtime.span_to_s span |> Int32.bits_of_float
Mtime_clock.count t0 |> Mtime.span_to_s |> Int32.bits_of_float
in
let op =
match short_op with
Expand All @@ -217,34 +215,34 @@ module Make_stat (Store : Irmin.Generic_key.KV) = struct
Def.append_row writer op

let create_store_before tree =
let+ Store.Tree.{ nodes; leafs; skips; depth; width } =
let Store.Tree.{ nodes; leafs; skips; depth; width } =
Store.Tree.stats ~force:false tree
in
Def.{ nodes; leafs; skips; depth; width }

let create_store_after tree =
let* watched_nodes_length =
Lwt_list.map_s
let watched_nodes_length =
List.map
(fun (_, steps) -> Store.Tree.length tree steps)
Def.step_list_per_watched_node
in
Lwt.return Def.{ watched_nodes_length }
Def.{ watched_nodes_length }

let commit_begin t tree =
short_op_begin t;
let stats_before =
Bag_of_stats.create t.store_path t.prev_merge_durations
in
t.prev_merge_durations <- Index.Stats.((get ()).merge_durations);
let+ store_before = create_store_before tree in
let store_before = create_store_before tree in
t.commit_before <- (stats_before, store_before)

let commit_end t tree =
let duration = Mtime_clock.count t.t0 |> Mtime.span_to_s in
let duration = duration |> Int32.bits_of_float in
let stats_after = Bag_of_stats.create t.store_path t.prev_merge_durations in
t.prev_merge_durations <- Index.Stats.((get ()).merge_durations);
let+ store_after = create_store_after tree in
let store_after = create_store_after tree in
let op =
`Commit
Def.
Expand Down
3 changes: 2 additions & 1 deletion bench/irmin-pack/trace_definitions.ml
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,8 @@ module Stat_trace = struct
store_pack : int64;
}
[@@deriving repr]
(** Stats extracted from filesystem. Requires the path to the irmin store. *)
(** Stats extracted from filesystem. Requires the path to the irmin store.
*)

type 'pack_stats bag_of_stats_base = {
pack : 'pack_stats;
Expand Down
Loading