From 6cdf28cd9d3ef20162bff250322f1c2aec25a5f9 Mon Sep 17 00:00:00 2001 From: icristescu Date: Thu, 5 Aug 2021 19:05:43 +0200 Subject: [PATCH 1/2] Split raw and index stats --- bench/bench.ml | 26 ++++++++++++++------------ src/stats.ml | 24 ------------------------ src/stats.mli | 6 ------ src/unix/index_unix.ml | 3 +-- src/unix/index_unix.mli | 1 + src/unix/raw.ml | 2 +- src/unix/raw_stats.ml | 31 +++++++++++++++++++++++++++++++ 7 files changed, 48 insertions(+), 45 deletions(-) create mode 100644 src/unix/raw_stats.ml diff --git a/bench/bench.ml b/bench/bench.ml index 8634de31..39b9d6fb 100644 --- a/bench/bench.ml +++ b/bench/bench.ml @@ -1,3 +1,4 @@ +module Raw_stats = Index_unix.Private.Raw_stats module Stats = Index.Stats let src = @@ -12,11 +13,11 @@ let src = in let open Stats in let tags = Tags.[] in - let data t = + let data (t, (raw : Raw_stats.t)) = Data.v [ - int "bytes_read" t.bytes_read; - int "bytes_written" t.bytes_written; + int "bytes_read" raw.bytes_read; + int "bytes_written" raw.bytes_written; int "merge" t.nb_merge; int "replace" t.nb_replace; head "replace_durations" t.replace_durations; @@ -60,9 +61,11 @@ end let with_stats f = Stats.reset_stats (); + Raw_stats.reset_stats (); let _, duration = Common.with_timer f in let stats = Stats.get () in - (duration, stats) + let raw = Raw_stats.get () in + (duration, stats, raw) module Mtime = struct include Mtime @@ -91,20 +94,18 @@ module Benchmark = struct [@@deriving to_yojson] let run ~nb_entries f = - let time, stats = with_stats (fun () -> f ()) in + let time, stats, raw = with_stats (fun () -> f ()) in let time_sec = Mtime.Span.to_s time in let nb_entriesf = float_of_int nb_entries in let entry_sizef = float_of_int entry_size in let read_amplification_size = - float_of_int stats.bytes_read /. (entry_sizef *. nb_entriesf) + float_of_int raw.bytes_read /. (entry_sizef *. nb_entriesf) in - let read_amplification_calls = float_of_int stats.nb_reads /. nb_entriesf in + let read_amplification_calls = float_of_int raw.nb_reads /. nb_entriesf in let write_amplification_size = - float_of_int stats.bytes_written /. (entry_sizef *. nb_entriesf) - in - let write_amplification_calls = - float_of_int stats.nb_writes /. nb_entriesf + float_of_int raw.bytes_written /. (entry_sizef *. nb_entriesf) in + let write_amplification_calls = float_of_int raw.nb_writes /. nb_entriesf in let ops_per_sec = nb_entriesf /. time_sec in let mbs_per_sec = entry_sizef *. nb_entriesf /. 1_048_576. /. time_sec in let replace_durations = stats.replace_durations in @@ -157,7 +158,8 @@ module Index = struct let add_metrics = let no_tags x = x in - fun () -> Metrics.add src no_tags (fun m -> m (Stats.get ())) + fun () -> + Metrics.add src no_tags (fun m -> m (Stats.get (), Raw_stats.get ())) let write ~with_metrics ?(with_flush = false) ?sampling_interval bindings rw = Array.iter diff --git a/src/stats.ml b/src/stats.ml index 14239ff6..2aa5b956 100644 --- a/src/stats.ml +++ b/src/stats.ml @@ -1,10 +1,6 @@ open! Import type t = { - mutable bytes_read : int; - mutable nb_reads : int; - mutable bytes_written : int; - mutable nb_writes : int; mutable nb_merge : int; mutable merge_durations : float list; mutable nb_replace : int; @@ -15,10 +11,6 @@ type t = { let fresh_stats () = { - bytes_read = 0; - nb_reads = 0; - bytes_written = 0; - nb_writes = 0; nb_merge = 0; merge_durations = []; nb_replace = 0; @@ -31,10 +23,6 @@ let stats = fresh_stats () let get () = stats let reset_stats () = - stats.bytes_read <- 0; - stats.nb_reads <- 0; - stats.bytes_written <- 0; - stats.nb_writes <- 0; stats.nb_merge <- 0; stats.merge_durations <- []; stats.nb_replace <- 0; @@ -42,22 +30,10 @@ let reset_stats () = stats.nb_sync <- 0; stats.time_sync <- 0.0 -let incr_bytes_read n = stats.bytes_read <- stats.bytes_read + n -let incr_bytes_written n = stats.bytes_written <- stats.bytes_written + n -let incr_nb_reads () = stats.nb_reads <- succ stats.nb_reads -let incr_nb_writes () = stats.nb_writes <- succ stats.nb_writes let incr_nb_merge () = stats.nb_merge <- succ stats.nb_merge let incr_nb_replace () = stats.nb_replace <- succ stats.nb_replace let incr_nb_sync () = stats.nb_sync <- succ stats.nb_sync -let add_read n = - incr_bytes_read n; - incr_nb_reads () - -let add_write n = - incr_bytes_written n; - incr_nb_writes () - module Make (Clock : Platform.CLOCK) = struct let replace_timer = ref (Clock.counter ()) let nb_replace = ref 0 diff --git a/src/stats.mli b/src/stats.mli index 4d8455d8..d1358458 100644 --- a/src/stats.mli +++ b/src/stats.mli @@ -1,10 +1,6 @@ open! Import type t = { - mutable bytes_read : int; - mutable nb_reads : int; - mutable bytes_written : int; - mutable nb_writes : int; mutable nb_merge : int; mutable merge_durations : float list; mutable nb_replace : int; @@ -29,8 +25,6 @@ type t = { val get : unit -> t val reset_stats : unit -> unit -val add_read : int -> unit -val add_write : int -> unit val incr_nb_merge : unit -> unit val incr_nb_replace : unit -> unit val incr_nb_sync : unit -> unit diff --git a/src/unix/index_unix.ml b/src/unix/index_unix.ml index 72de479d..bf7b46a3 100644 --- a/src/unix/index_unix.ml +++ b/src/unix/index_unix.ml @@ -25,8 +25,6 @@ exception RO_not_allowed let current_version = "00000001" -module Stats = Index.Stats - module IO : Index.Platform.IO = struct let ( ++ ) = Int63.add let ( -- ) = Int63.sub @@ -426,6 +424,7 @@ module Syscalls = Syscalls module Private = struct module IO = IO module Raw = Raw + module Raw_stats = Raw_stats module Make (K : Index.Key.S) (V : Index.Value.S) = Index.Private.Make (K) (V) (Platform) diff --git a/src/unix/index_unix.mli b/src/unix/index_unix.mli index 0e549273..e8683d09 100644 --- a/src/unix/index_unix.mli +++ b/src/unix/index_unix.mli @@ -28,6 +28,7 @@ module Syscalls = Syscalls module Private : sig module IO : Index.Platform.IO module Raw = Raw + module Raw_stats = Raw_stats module Make (K : Index.Key.S) (V : Index.Value.S) (C : Index.Cache.S) : Index.Private.S with type key = K.t and type value = V.t diff --git a/src/unix/raw.ml b/src/unix/raw.ml index fc6d893f..ad462d17 100644 --- a/src/unix/raw.ml +++ b/src/unix/raw.ml @@ -1,5 +1,5 @@ open! Import -module Stats = Index.Stats +module Stats = Raw_stats let ( ++ ) = Int63.add diff --git a/src/unix/raw_stats.ml b/src/unix/raw_stats.ml new file mode 100644 index 00000000..1583ecca --- /dev/null +++ b/src/unix/raw_stats.ml @@ -0,0 +1,31 @@ +type t = { + mutable bytes_read : int; + mutable nb_reads : int; + mutable bytes_written : int; + mutable nb_writes : int; +} + +let fresh_stats () = + { bytes_read = 0; nb_reads = 0; bytes_written = 0; nb_writes = 0 } + +let stats = fresh_stats () +let get () = stats + +let reset_stats () = + stats.bytes_read <- 0; + stats.nb_reads <- 0; + stats.bytes_written <- 0; + stats.nb_writes <- 0 + +let incr_bytes_read n = stats.bytes_read <- stats.bytes_read + n +let incr_bytes_written n = stats.bytes_written <- stats.bytes_written + n +let incr_nb_reads () = stats.nb_reads <- succ stats.nb_reads +let incr_nb_writes () = stats.nb_writes <- succ stats.nb_writes + +let add_read n = + incr_bytes_read n; + incr_nb_reads () + +let add_write n = + incr_bytes_written n; + incr_nb_writes () From a8683753370bdf8ce29162ac805c77b5ae0bd1c2 Mon Sep 17 00:00:00 2001 From: icristescu Date: Thu, 5 Aug 2021 19:17:51 +0200 Subject: [PATCH 2/2] Separate stats on file names --- CHANGES.md | 2 ++ bench/bench.ml | 75 +++++++++++++++++++++++------------------- src/index.ml | 2 ++ src/index_intf.ml | 5 +++ src/platform.ml | 30 +++++++++++++++++ src/stats.ml | 43 +++++++++++++++++++++++- src/stats.mli | 4 ++- src/unix/index_unix.ml | 18 +++++++--- src/unix/raw.ml | 14 +++++--- src/unix/raw.mli | 5 ++- src/unix/raw_stats.ml | 25 +++++++------- src/unix/raw_stats.mli | 4 +++ test/unix/main.ml | 1 + test/unix/stats.ml | 67 +++++++++++++++++++++++++++++++++++++ test/unix/stats.mli | 1 + 15 files changed, 238 insertions(+), 58 deletions(-) create mode 100644 src/unix/raw_stats.mli create mode 100644 test/unix/stats.ml create mode 100644 test/unix/stats.mli diff --git a/CHANGES.md b/CHANGES.md index 91327d44..41f0381b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,6 +9,8 @@ - Changed the implementation of the write-ahead log to significantly reduce its memory usage (at the cost of some additional disk IO). (#355) +- Separate the IO stats recorded in the Raw module, from the index ones. (#353) + # 1.4.1 (2021-07-16) ## Fixed diff --git a/bench/bench.ml b/bench/bench.ml index 39b9d6fb..6072f954 100644 --- a/bench/bench.ml +++ b/bench/bench.ml @@ -1,4 +1,3 @@ -module Raw_stats = Index_unix.Private.Raw_stats module Stats = Index.Stats let src = @@ -13,11 +12,11 @@ let src = in let open Stats in let tags = Tags.[] in - let data (t, (raw : Raw_stats.t)) = + let data (t, bytes_read, bytes_written) = Data.v [ - int "bytes_read" raw.bytes_read; - int "bytes_written" raw.bytes_written; + int "bytes_read" bytes_read; + int "bytes_written" bytes_written; int "merge" t.nb_merge; int "replace" t.nb_replace; head "replace_durations" t.replace_durations; @@ -59,14 +58,6 @@ module Context = struct end end -let with_stats f = - Stats.reset_stats (); - Raw_stats.reset_stats (); - let _, duration = Common.with_timer f in - let stats = Stats.get () in - let raw = Raw_stats.get () in - (duration, stats, raw) - module Mtime = struct include Mtime @@ -78,7 +69,7 @@ module Mtime = struct end end -module Benchmark = struct +module Make_benchmark (Io_stats : Index.Platform.IO_STATS) = struct type result = { time : Mtime.Span.t; ops_per_sec : float; @@ -93,8 +84,16 @@ module Benchmark = struct } [@@deriving to_yojson] + let with_stats f = + Stats.reset (); + Io_stats.reset_all (); + let _, duration = Common.with_timer f in + let stats = Stats.get () in + let raw = Io_stats.get () in + (duration, stats, raw) + let run ~nb_entries f = - let time, stats, raw = with_stats (fun () -> f ()) in + let time, stats, (raw : Io_stats.t) = with_stats (fun () -> f ()) in let time_sec = Mtime.Span.to_s time in let nb_entriesf = float_of_int nb_entries in let entry_sizef = float_of_int entry_size in @@ -152,14 +151,18 @@ let sorted_bindings_pool = ref [||] module Index_lib = Index -module Index = struct - module Index = - Index_unix.Private.Make (Context.Key) (Context.Value) (Index.Cache.Noop) +module Index = + Index_unix.Private.Make (Context.Key) (Context.Value) (Index.Cache.Noop) + +module Io_stats = Index.Io_stats +module Benchmark = Make_benchmark (Io_stats) +module Index_bench = struct let add_metrics = let no_tags x = x in fun () -> - Metrics.add src no_tags (fun m -> m (Stats.get (), Raw_stats.get ())) + Metrics.add src no_tags (fun m -> + m (Stats.get (), Io_stats.bytes_read (), Io_stats.bytes_written ())) let write ~with_metrics ?(with_flush = false) ?sampling_interval bindings rw = Array.iter @@ -338,23 +341,27 @@ module Index = struct end let list_benches () = - let pp_bench ppf b = Fmt.pf ppf "%s\t-- %s" b.Index.name b.synopsis in - Index.suite |> Fmt.(pr "%a" (list ~sep:Fmt.(const string "\n") pp_bench)) + let pp_bench ppf b = Fmt.pf ppf "%s\t-- %s" b.Index_bench.name b.synopsis in + Index_bench.suite + |> Fmt.(pr "%a" (list ~sep:Fmt.(const string "\n") pp_bench)) let schedule p s = let todos = List.map fst in - let init = ref (s |> List.map (fun b -> (p b.Index.name, b))) in + let init = ref (s |> List.map (fun b -> (p b.Index_bench.name, b))) in let apply_dep s = let deps = s |> List.fold_left (fun acc (todo, b) -> if todo then - match b.Index.dependency with Some s -> s :: acc | None -> acc + match b.Index_bench.dependency with + | Some s -> s :: acc + | None -> acc else acc) [] in - s |> List.map (fun (todo, b) -> (todo || List.mem b.Index.name deps, b)) + s + |> List.map (fun (todo, b) -> (todo || List.mem b.Index_bench.name deps, b)) in let next = ref (apply_dep !init) in while todos !init <> todos !next do @@ -391,14 +398,14 @@ let pp_config fmt config = let cleanup root = let files = [ "data"; "log"; "lock"; "log_async"; "merge" ] in List.iter - (fun (b : Index.suite_elt) -> + (fun (b : Index_bench.suite_elt) -> let dir = root // b.name // "index" in List.iter (fun file -> let file = dir // file in if Sys.file_exists file then Unix.unlink file) files) - Index.suite + Index_bench.suite let init config = Printexc.record_backtrace true; @@ -415,8 +422,8 @@ let init config = let print fmt (config, results) = let pp_bench fmt (b, result) = - Format.fprintf fmt "@[%s@,%a@]" b.Index.synopsis Benchmark.pp_result - result + Format.fprintf fmt "@[%s@,%a@]" b.Index_bench.synopsis + Benchmark.pp_result result in Format.fprintf fmt "@[Configuration:@,%a@,@]@,@[Results:@,%a@]@." pp_config config @@ -435,7 +442,7 @@ let print_json fmt (config, results) = (fun (b, result) -> `Assoc [ - ("name", `String b.Index.name); + ("name", `String b.Index_bench.name); ("metrics", Benchmark.result_to_yojson result); ]) results) ); @@ -445,8 +452,10 @@ let print_json fmt (config, results) = let get_suite_list minimal_flag = if minimal_flag then - List.filter (fun bench -> bench.Index.speed = `Quick) Index.suite - else Index.suite + List.filter + (fun bench -> bench.Index_bench.speed = `Quick) + Index_bench.suite + else Index_bench.suite let repeat n f l = let rec aux i acc = if i = n then acc else aux (i + 1) (f l :: acc) in @@ -462,7 +471,7 @@ let mean l = (fun acc bresult -> List.fold_left2 (fun acc (tsm, resultm) (ts, result) -> - assert (Index.(tsm.name = ts.name)); + assert (Index_bench.(tsm.name = ts.name)); ( tsm, Benchmark. { @@ -536,12 +545,12 @@ let run filter root output seed with_metrics log_size nb_entries nb_exec json current_suite |> schedule name_filter |> repeat nb_exec - (List.map (fun (b : Index.suite_elt) -> + (List.map (fun (b : Index_bench.suite_elt) -> let name = match b.dependency with None -> b.name | Some name -> name in let result = - Index.run ~with_metrics ~nb_entries ~log_size ~root ~name + Index_bench.run ~with_metrics ~nb_entries ~log_size ~root ~name ~fresh:b.fresh ~readonly:b.readonly b.benchmark in (b, result))) diff --git a/src/index.ml b/src/index.ml index 4cf115d9..978325c2 100644 --- a/src/index.ml +++ b/src/index.ml @@ -73,6 +73,8 @@ struct include Stats end + module Io_stats = Platform.Io_stats + module IO = struct include Io.Extend (IO) diff --git a/src/index_intf.ml b/src/index_intf.ml index a9fe5597..bc3ebe04 100644 --- a/src/index_intf.ml +++ b/src/index_intf.ml @@ -146,6 +146,11 @@ module type S = sig include Checks.S (** @inline *) end + + module Io_stats : sig + include Platform.IO_STATS + (** @inline *) + end end module Private_types = struct diff --git a/src/platform.ml b/src/platform.ml index e592045e..c820b214 100644 --- a/src/platform.ml +++ b/src/platform.ml @@ -58,9 +58,39 @@ module type THREAD = sig (** Re-schedule the calling thread without suspending it. *) end +module type RAW_STATS = sig + (** Stats for IO per file descriptor. *) + + type t = { + mutable bytes_read : int; + mutable nb_reads : int; + mutable bytes_written : int; + mutable nb_writes : int; + } + + val fresh_stats : unit -> t + val reset : t -> unit +end + +module type IO_STATS = sig + (** Accumulated stats for all IO handlers in Index. *) + + include RAW_STATS + + val get : unit -> t + val get_by_file : string -> t + val get_all : unit -> (string * t) list + val reset_all : unit -> unit + val bytes_read : unit -> int + val bytes_written : unit -> int + val nb_reads : unit -> int + val nb_writes : unit -> int +end + module type S = sig module IO : IO module Semaphore : SEMAPHORE module Thread : THREAD module Clock : CLOCK + module Io_stats : IO_STATS end diff --git a/src/stats.ml b/src/stats.ml index 2aa5b956..166e591b 100644 --- a/src/stats.ml +++ b/src/stats.ml @@ -22,7 +22,7 @@ let fresh_stats () = let stats = fresh_stats () let get () = stats -let reset_stats () = +let reset () = stats.nb_merge <- 0; stats.merge_durations <- []; stats.nb_replace <- 0; @@ -62,3 +62,44 @@ module Make (Clock : Platform.CLOCK) = struct let span = Mtime.Span.to_us span in stats.merge_durations <- drop_head stats.merge_durations @ [ span ] end + +module Io_stats (R : Platform.RAW_STATS) = struct + include R + + let tbl : (string, t) Hashtbl.t = Hashtbl.create 13 + + let get_by_file file = + try Hashtbl.find tbl file + with Not_found -> + let stats = R.fresh_stats () in + Hashtbl.add tbl file stats; + stats + + let get_all () = + Hashtbl.fold (fun file stats acc -> (file, stats) :: acc) tbl [] + + let bytes_read () = + Hashtbl.fold (fun _file stats acc -> stats.R.bytes_read + acc) tbl 0 + + let bytes_written () = + Hashtbl.fold (fun _file stats acc -> stats.R.bytes_written + acc) tbl 0 + + let nb_reads () = + Hashtbl.fold (fun _file stats acc -> stats.R.nb_reads + acc) tbl 0 + + let nb_writes () = + Hashtbl.fold (fun _file stats acc -> stats.R.nb_writes + acc) tbl 0 + + let get () = + let acc = R.fresh_stats () in + Hashtbl.iter + (fun _file stats -> + acc.bytes_read <- stats.bytes_read + acc.bytes_read; + acc.R.nb_reads <- stats.R.nb_reads + acc.R.nb_reads; + acc.R.bytes_written <- stats.R.bytes_written + acc.R.bytes_written; + acc.R.nb_writes <- stats.R.nb_writes + acc.R.nb_writes) + tbl; + acc + + let reset_all () = Hashtbl.iter (fun _file stats -> R.reset stats) tbl +end diff --git a/src/stats.mli b/src/stats.mli index d1358458..46c0697f 100644 --- a/src/stats.mli +++ b/src/stats.mli @@ -24,7 +24,7 @@ type t = { - [time_sync] is the duration of the latest call to sync. *) val get : unit -> t -val reset_stats : unit -> unit +val reset : unit -> unit val incr_nb_merge : unit -> unit val incr_nb_replace : unit -> unit val incr_nb_sync : unit -> unit @@ -35,3 +35,5 @@ module Make (_ : Platform.CLOCK) : sig val sync_with_timer : (unit -> unit) -> unit val add_merge_duration : Mtime.Span.t -> unit end + +module Io_stats (R : Platform.RAW_STATS) : Platform.IO_STATS with type t = R.t diff --git a/src/unix/index_unix.ml b/src/unix/index_unix.ml index bf7b46a3..23422f67 100644 --- a/src/unix/index_unix.ml +++ b/src/unix/index_unix.ml @@ -25,6 +25,9 @@ exception RO_not_allowed let current_version = "00000001" +module Stats : Index.Platform.IO_STATS with type t = Raw_stats.t = + Index.Stats.Io_stats (Raw_stats) + module IO : Index.Platform.IO = struct let ( ++ ) = Int63.add let ( -- ) = Int63.sub @@ -179,10 +182,16 @@ module IO : Index.Platform.IO = struct in (aux [@tailcall]) dirname (fun () -> ()) + let raw_with_stats file fd = + (* if the file is reopened then reuse the previous stats. *) + let stats = Stats.get_by_file file in + let raw = Raw.v ~stats fd in + raw + let raw_file ~flags ~version ~offset ~generation file = let x = Unix.openfile file flags 0o644 in - let raw = Raw.v x in let header = { Raw.Header.offset; version; generation } in + let raw = raw_with_stats file x in Log.debug (fun m -> m "[%s] raw set_header %a" file Header.pp { offset; generation }); Raw.Header.set raw header; @@ -244,13 +253,13 @@ module IO : Index.Platform.IO = struct match Sys.file_exists file with | false -> let x = Unix.openfile file Unix.[ O_CREAT; O_CLOEXEC; O_RDWR ] 0o644 in - let raw = Raw.v x in + let raw = raw_with_stats file x in Raw.Header.set raw header; Raw.Fan.set_size raw fan_size; v ~fan_size ~offset:Int63.zero raw | true -> let x = Unix.openfile file Unix.[ O_EXCL; O_CLOEXEC; O_RDWR ] 0o644 in - let raw = Raw.v x in + let raw = raw_with_stats file x in if fresh then ( Raw.Header.set raw header; Raw.Fan.set_size raw fan_size; @@ -271,7 +280,7 @@ module IO : Index.Platform.IO = struct mkdir (Filename.dirname file); try let x = Unix.openfile file Unix.[ O_EXCL; O_CLOEXEC; O_RDONLY ] 0o644 in - let raw = Raw.v x in + let raw = raw_with_stats file x in try let version = Raw.Version.get raw in if version <> current_version then @@ -414,6 +423,7 @@ module Platform = struct module Semaphore = Semaphore module Thread = Thread module Clock = Mtime_clock + module Io_stats : Index.Platform.IO_STATS with type t = Raw_stats.t = Stats end module Make (K : Index.Key.S) (V : Index.Value.S) = diff --git a/src/unix/raw.ml b/src/unix/raw.ml index ad462d17..490c65d4 100644 --- a/src/unix/raw.ml +++ b/src/unix/raw.ml @@ -3,9 +3,15 @@ module Stats = Raw_stats let ( ++ ) = Int63.add -type t = { fd : Unix.file_descr } [@@unboxed] +type t = { fd : Unix.file_descr; stats : Raw_stats.t } -let v fd = { fd } +let v ?stats fd = + let stats = + match stats with None -> Raw_stats.fresh_stats () | Some stats -> stats + in + { fd; stats } + +let get_stats { stats; _ } = stats let really_write fd fd_offset buffer buffer_offset length = let rec aux fd_offset buffer_offset length = @@ -37,11 +43,11 @@ let fstat t = Unix.fstat t.fd let unsafe_write t ~off buffer buffer_offset length = let buffer = Bytes.unsafe_of_string buffer in really_write t.fd off buffer buffer_offset length; - Stats.add_write length + Stats.add_write t.stats length let unsafe_read t ~off ~len buf = let n = really_read t.fd off len buf in - Stats.add_read n; + Stats.add_read t.stats n; n let encode_int63 n = diff --git a/src/unix/raw.mli b/src/unix/raw.mli index 71e2d78e..ccdcf6fe 100644 --- a/src/unix/raw.mli +++ b/src/unix/raw.mli @@ -13,9 +13,12 @@ open! Import type t (** The type of [raw] file handles. *) -val v : Unix.file_descr -> t +val v : ?stats:Raw_stats.t -> Unix.file_descr -> t (** Construct a [raw] value from a file descriptor. *) +val get_stats : t -> Raw_stats.t +(** Get a [raw] stats. *) + val unsafe_write : t -> off:int63 -> string -> int -> int -> unit val unsafe_read : t -> off:int63 -> len:int -> bytes -> int val fsync : t -> unit diff --git a/src/unix/raw_stats.ml b/src/unix/raw_stats.ml index 1583ecca..0a9a4322 100644 --- a/src/unix/raw_stats.ml +++ b/src/unix/raw_stats.ml @@ -8,24 +8,21 @@ type t = { let fresh_stats () = { bytes_read = 0; nb_reads = 0; bytes_written = 0; nb_writes = 0 } -let stats = fresh_stats () -let get () = stats - -let reset_stats () = +let reset stats = stats.bytes_read <- 0; stats.nb_reads <- 0; stats.bytes_written <- 0; stats.nb_writes <- 0 -let incr_bytes_read n = stats.bytes_read <- stats.bytes_read + n -let incr_bytes_written n = stats.bytes_written <- stats.bytes_written + n -let incr_nb_reads () = stats.nb_reads <- succ stats.nb_reads -let incr_nb_writes () = stats.nb_writes <- succ stats.nb_writes +let incr_bytes_read stats n = stats.bytes_read <- stats.bytes_read + n +let incr_bytes_written stats n = stats.bytes_written <- stats.bytes_written + n +let incr_nb_reads stats = stats.nb_reads <- succ stats.nb_reads +let incr_nb_writes stats = stats.nb_writes <- succ stats.nb_writes -let add_read n = - incr_bytes_read n; - incr_nb_reads () +let add_read stats n = + incr_bytes_read stats n; + incr_nb_reads stats -let add_write n = - incr_bytes_written n; - incr_nb_writes () +let add_write stats n = + incr_bytes_written stats n; + incr_nb_writes stats diff --git a/src/unix/raw_stats.mli b/src/unix/raw_stats.mli new file mode 100644 index 00000000..5a7550ec --- /dev/null +++ b/src/unix/raw_stats.mli @@ -0,0 +1,4 @@ +include Index.Platform.RAW_STATS + +val add_read : t -> int -> unit +val add_write : t -> int -> unit diff --git a/test/unix/main.ml b/test/unix/main.ml index 92288330..9fa7a7b4 100644 --- a/test/unix/main.ml +++ b/test/unix/main.ml @@ -1053,4 +1053,5 @@ let () = ("filter", Filter.tests); ("flush_callback", Flush_callback.tests); ("throttle", Throttle.tests); + ("stats", Stats.tests); ] diff --git a/test/unix/stats.ml b/test/unix/stats.ml new file mode 100644 index 00000000..c0f348cc --- /dev/null +++ b/test/unix/stats.ml @@ -0,0 +1,67 @@ +module I = Index +open Common + +let root = Filename.concat "_tests" "unix.stats" + +module Context = Common.Make_context (struct + let root = root +end) + +module Stats = I.Stats +module Io_stats = Index.Io_stats +module Raw = Index_unix.Private.Raw +module Raw_stats = Index_unix.Private.Raw_stats + +let test_replace () = + let* Context.{ rw; _ } = Context.with_empty_index () in + Stats.reset (); + Io_stats.reset_all (); + (* if reset_raw then Stats_raw.reset_stats (); *) + let k1, v1, k2, v2, k3, v3 = + (Key.v (), Value.v (), Key.v (), Value.v (), Key.v (), Value.v ()) + in + Index.replace rw k1 v1; + Index.replace rw k2 v2; + Index.replace rw k3 v3; + Index.flush rw; + Alcotest.(check int) "no bytes read" 0 (Io_stats.bytes_read ()); + Alcotest.(check int) "no reads" 0 (Io_stats.nb_reads ()); + Alcotest.(check int) + "one write for entries and a separate write for offset" 2 + (Io_stats.nb_writes ()); + Alcotest.(check int) + "bytes written for 3 entries and an offset" 128 + (Io_stats.bytes_written ()) + +let test_find () = + let* Context.{ rw; _ } = Context.with_empty_index () in + let k1, v1 = (Key.v (), Value.v ()) in + Index.replace rw k1 v1; + Index.try_merge_aux ~force:true rw |> Index.await |> check_completed; + Io_stats.reset_all (); + ignore (Index.mem rw k1); + Alcotest.(check int) "bytes read" 40 (Io_stats.bytes_read ()); + Alcotest.(check int) "no reads" 1 (Io_stats.nb_reads ()); + Alcotest.(check int) "no writes" 0 (Io_stats.nb_writes ()); + Alcotest.(check int) "no bytes written" 0 (Io_stats.bytes_written ()) + +let test_multiple_raw () = + let test_single_raw = test_replace in + let tmp = Filename.concat root "_tmp" in + let x = Unix.openfile tmp Unix.[ O_CREAT; O_RDWR; O_CLOEXEC ] 0o644 in + let raw = Raw.v x in + Raw.Offset.set raw Optint.Int63.zero; + test_single_raw (); + let stats = Raw.get_stats raw in + Alcotest.(check int) "no bytes read" stats.bytes_read 0; + Alcotest.(check int) "no reads" stats.nb_reads 0; + Alcotest.(check int) "one write for offset" stats.nb_writes 1; + Alcotest.(check int) "bytes written for offset" stats.bytes_written 8; + Unix.close x + +let tests = + [ + ("stats for replace", `Quick, test_replace); + ("stats for find", `Quick, test_find); + ("stats for different raw files", `Quick, test_multiple_raw); + ] diff --git a/test/unix/stats.mli b/test/unix/stats.mli new file mode 100644 index 00000000..d38ba9a9 --- /dev/null +++ b/test/unix/stats.mli @@ -0,0 +1 @@ +val tests : unit Alcotest.test_case list