benchmark instance

[dyu]

For benchmarking mutable objects/fields.

[Hejsil]

Maybe it is worth exploring just making instancing the default and only way to do this. So we look for fields instead of declarations for configuration. We could even have a BenchmarkConfig type for most configuration. So one of our tests would look something like this:

test "benchmark generics" {
    try benchmark(struct {
        const Vec = @import("std").meta.Vector;

        config: BenchmarkConfig = BenchmarkConfig{
            .min_iterations = 1000;
            .max_iterations = 100000;
            .arg_names = &[_][]const u8{
                "vec4f16",  "vec4f32",  "vec4f64",
                "vec8f16",  "vec8f32",  "vec8f64",
                "vec16f16", "vec16f32", "vec16f64",
            };
        },

        comptime args: []const type = &[_]type{
            Vec(4, f16),  Vec(4, f32),  Vec(4, f64),
            Vec(8, f16),  Vec(8, f32),  Vec(8, f64),
            Vec(16, f16), Vec(16, f32), Vec(16, f64),
        },


        pub fn sum_vectors(comptime T: type) T {
            const info = @typeInfo(T).Vector;
            const one = @splat(info.len, @as(info.child, 1));
            const vecs = [1]T{one} ** 512;

            var res = one;
            for (vecs) |vec| {
                res += vec;
            }
            return res;
        }
    }{});
}

We have to keep args separate still to allow comptime fields, so we can have comptime arguments.

[hazeycode]

^ Note that resource acquisition and cleanup happens inside the measured function here. Not a problem in this example but not generally desired. Resource acquisition and cleanup can be separated from the measured code by passing in a context (see #15)

[Hejsil]

^ Note that resource acquisition and cleanup happens inside the measured function here. This is probably not desired. Resource acquisition and cleanup can be separated by passing in a context (see #15)

A good point here is that in the thing i proposed, you cannot access the state in the benchmark instance from the benchmarked function. We can easily allow this and the test code from #15 would look something like this:

test "benchmark allocating" {
    const Benchmark = struct {
        allocator: mem.Alllcator,

        pub fn alloc_slice(benchmark: @This()) ![]u8 {
            return try benchmark.allocator.alloc(u8, 4096);
        }
    };

    var temp_arena = std.heap.ArenaAllocator.init(testing.allocator);
    defer temp_arena.deinit();

    try benchmark(Benchmark{ .allocator = temp_arena.allocator() });
}

[hazeycode]

^ Looks good. It's passing a context either way.