libtest.c

/*
 * Copyright (c) 2016--2021  Wu, Xingbo <wuxb45@gmail.com>
 *
 * All rights reserved. No warranty, explicit or implicit, provided.
 */
#define _GNU_SOURCE

#include "lib.h"
#include "kv.h"
#include "wh.h"
#include "ht.h"
#include "ord.h"
#include "ctypes.h"
#include <sched.h>
#include <execinfo.h>
#include <signal.h>
#if defined(__FreeBSD__)
#include <pthread_np.h>
#endif

// misc {{{
  static void *
printer_th(void * ptr)
{
  (void)ptr;
  printf("thread %lx\n", (u64)pthread_self());
  return NULL;
}

  static void *
burning_th(void * ptr)
{
  (void)ptr;
here:
  goto here;
  return NULL;
}

enum dummy {DUMMY};

  static void
test_misc(void)
{
  printf("sizeof long: %zu\n", sizeof(unsigned long));
  printf("sizeof long long: %zu\n", sizeof(unsigned long long));
  printf("sizeof size_t: %zu\n", sizeof(size_t));
  printf("sizeof pthread_t: %zu\n", sizeof(pthread_t));
  printf("sizeof pt_mutex: %zu\n", sizeof(pthread_mutex_t));
  printf("sizeof pt_cond: %zu\n", sizeof(pthread_cond_t));
  printf("sizeof pt_rwlock: %zu\n", sizeof(pthread_rwlock_t));
  printf("sizeof enum: %zu\n", sizeof(enum dummy));
  logger_printf(1, "%s %s %d %u\n", __func__, __FILE__, 100, 200u);
  logger_printf(2, "%s %s %d %u\n", __func__, __FILE__, 100, 200u);
  //printf("sizeof pt_spin: %zu\n", sizeof(pthread_spinlock_t));
  u64 stackvar;
  printf("symbol %s %p\n", __func__, test_misc);
  printf("stackvar %p\n", &stackvar);
  u64 t0 = time_nsec();
  const double d0 = time_sec();
  watch_u64_usr1(&t0);
  u32 r32 = bits_reverse_u32(0x12345678u);
  printf("reverse u32 0x12345678 -> %x\n", r32);
  u64 r64 = bits_reverse_u64(0x123456789abcdef0lu);
  printf("reverse u64 0x123456789abcdef0 -> %lx\n", r64);

  printf("rss %ldkB\n", process_get_rss());
  const u32 cc = process_affinity_count();
  printf("affinity_core_count %u\n", cc);
  u32 cores[64] = {};
  const u64 nc = process_getaffinity_list(64, cores);
  printf("affinity cores: ");
  for (u64 i = 0; i < nc; i++) {
    printf(" %u", cores[i]);
  }
  printf("\n");
  printf("cpu time %lu\n", process_cpu_time_usec());
  thread_fork_join(cc, printer_th, false, NULL);

  char buf[32] = {};
  strhex_32(buf, 0x1234abcdu);
  printf("strhex_32 0x1234abcd : %8s\n", buf);
  strdec_32(buf, 1234567890u);
  printf("strdec_32 1234567890 : %10s\n", buf);
  strhex_64(buf, 0x123456789abcdef0lu);
  printf("strhex_64 123456789abcdef0 : %16s\n", buf);
  strdec_64(buf, 12345678901234567890lu);
  printf("strdec_64 12345678901234567890 : %20s\n", buf);

  char ** const toks = strtoks("    1.23 456 799\tok\nnext line     looks   ok\n", " \t\n\r");
  for (u64 i = 0; toks[i]; i++) {
    printf("token [%lu]:   %s\n", i, toks[i]);
  }
  free(toks);

  void * pg = pages_alloc_1gb(1);
  if (pg) {
    pages_unmap(pg, 1<<30);
    printf("got 1gb page\n");
  } else {
    printf("no 1gb page\n");
  }
  pg = pages_alloc_2mb(1);
  if (pg) {
    pages_unmap(pg, 1<<21);
    printf("got 2mb page\n");
  } else {
    printf("no 2mb page\n");
  }
  pg = pages_alloc_4kb(1);
  if (pg) {
    pages_unmap(pg, 1<<12);
    printf("got 4kb page\n");
  } else {
    printf("no 4kb page\n");
  }
  pg = malloc(1024);
  if (pg) {
    memset(pg, 0, 1024);
    memset(pg, 1, 1024);
    qsort_u16(pg, 512);
    qsort_u32(pg, 256);
    qsort_u64(pg, 128);
    free(pg);
  } else {
    printf("malloc failed\n");
    debug_die();
  }

  u16 v16[4] = {0, 0xfffu, 0x8fffu, 0xffffu};
  printf("cmp(%hu,%hu)=%d\n", v16[0], v16[1], compare_u16(v16, v16+1));
  printf("cmp(%hu,%hu)=%d\n", v16[0], v16[2], compare_u16(v16, v16+2));
  printf("cmp(%hu,%hu)=%d\n", v16[0], v16[3], compare_u16(v16, v16+3));

  // done
  u64 dt = time_diff_nsec(t0);
  double dd = time_diff_sec(d0);
  printf("nsec %lu sec %.9lf\n", dt, dd);
  dt = time_nsec();
  dt = time_diff_nsec(dt);
  dd = time_sec();
  dd = time_diff_sec(dd);
  printf("cost nsec %lu sec %.9lf\n", dt, dd);
  printf("Done. Good. wait gdb:\n");
  pthread_t pt;
  pthread_create(&pt, NULL, burning_th, NULL);
  debug_die();
}

  static void
test_segv(void)
{
  u64 z = 0;
  for (u64 i = 0; i < 10000; i++) {
    const u64 x = random_u64();
    const u64 * const p = (typeof(p))x;
    const u64 y = *p;
    z ^= y;
  }
  printf("crazy z = %lu\n", z);
}

  static void
test_gcd(void)
{
  printf("gcd 0 1 = %lu\n", gcd64(0, 1));
  printf("gcd 1 1 = %lu\n", gcd64(1, 1));
  printf("gcd 1 10 = %lu\n", gcd64(1, 10));
  printf("gcd 10 10 = %lu\n", gcd64(10, 10));
  printf("gcd 3 7 = %lu\n", gcd64(3, 7));
  printf("gcd 100 99 = %lu\n", gcd64(100, 99));
  printf("gcd 100 27 = %lu\n", gcd64(100, 27));
  printf("gcd 100 16 = %lu\n", gcd64(100, 16));
  printf("gcd 100 1024 = %lu\n", gcd64(100, 1024));
  printf("gcd 100 359 = %lu\n", gcd64(100, 359));
  printf("gcd 100 360 = %lu\n", gcd64(100, 360));
}

  static void
lcp_verify_perf(const u32 len, const u32 lcp)
{
  if (lcp > len)
    return;

  struct kv * const k1 = yalloc(len + sizeof(struct kv));
  struct kv * const k2 = yalloc(len + sizeof(struct kv));
  k1->klen = len;
  k2->klen = len;
  u8 * const v1 = k1->kv;
  u8 * const v2 = k2->kv;
  for (u64 i = 0; i < len; i++) {
    v1[i] = (u8)random_u64();
    v2[i] = v1[i];
  }

  if (lcp < len)
    v2[lcp] = ~v1[lcp]; // only one diff

  const u32 r1 = kv_key_lcp(k1, k2);
  if (r1 != lcp) {
    printf("WRONG LCP len %u lcp %u r1 %u\n", len, lcp, r1);
    exit(0);
  }

  u64 r = 0;
  const double t0 = time_sec();
  for (u64 i = 0; i < (1lu << 26); i++) {
    if (kv_key_lcp(k1, k2))
      r++;
  }
  const double d0 = time_diff_sec(t0);
  printf("%10lu %10u %10u time %7.4lf\n", r, len, lcp, d0);

  free(k1);
  free(k2);
}

  static void
test_lcp(void)
{
  for (u32 i = 11; i < 256; i += 11)
    for (u32 j = 7; j <= i; j += 7)
      lcp_verify_perf(i, j);
}

  static void
test_shm(void)
{
  u8 * p = NULL;
  const int fd = shm_open("/wuxb", O_RDWR, 0600);
  if (fd < 0) {
    printf("object not found\n");
    const int fd1 = shm_open("/wuxb", O_RDWR|O_CREAT, 0600);
    if (fd1 < 0) {
      printf("create failed\n");
      exit(0);
    } else {
      printf("created /dev/shm/wuxb\n");
    }
    ftruncate(fd1, 4096);
    p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, fd1, 0);
    close(fd1);
    if (p == MAP_FAILED) {
      printf("map failed\n");
      exit(0);
    } else {
      printf("Run me again to see the saved values (a,b)\n");
    }
    p[0] = 'a';
    p[4000] = 'b';
  } else {
    p = mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
    if (p == MAP_FAILED) {
      printf("map failed\n");
      exit(0);
    }
    close(fd);
    printf("read %c %c\n", p[0], p[4000]);
  }
  if (p) {
    munmap(p, 4096);
  }
}
// }}} misc

// gen {{{
  static void
genperf(struct rgen * const gi, const char * const tag)
{
  const u64 min = rgen_min(gi);
  const u64 max = rgen_max(gi);
  const u64 ts0 = time_nsec();
  for (u64 i = 0; i < (1lu << 20); i++)
    (void)rgen_next(gi);
  const u64 dts = time_diff_nsec(ts0);
  const u64 nr_ops = (100000000lu << 20) / dts;
  for (u64 x = 0; x < 3; x++) {
    printf("[%lu] %7lu %7lu %7lu %7lu %7lu %7lu ",
        x, rgen_next(gi), rgen_next(gi), rgen_next(gi), rgen_next(gi), rgen_next(gi), rgen_next(gi));
    const double t0 = time_sec();
    uint64_t r = 0;
    for (uint64_t i = 0; i < nr_ops; i++)
      r += rgen_next(gi);
    const double dt = time_diff_sec(t0);
    (void)r;
    printf("min %7lu max %7lu mops %7.3lf %s\n", min, max, ((double)nr_ops)/dt * 1e-6, tag);
  }
  u64 gmin = UINT64_MAX;
  u64 gmax = 0;
  for (u64 i = 0; i < 10000000; i++) {
    const u64 r = rgen_next(gi);
    if (r < min || r > max) {
      fprintf(stderr, "%s %s r %lu min %lu max %lu\n",
          __func__, tag, r, min, max);
      debug_die();
    }
    if (r < gmin)
      gmin = r;
    if (r > gmax)
      gmax = r;
  }
  printf("min/max after 10 million next(): %lu (%lx) %lu (%lx)\n", gmin, gmin, gmax, gmax);
  rgen_destroy(gi);
}

  static void
test_gen(void)
{
  genperf(rgen_new_rnd64(), "rnd64");
  genperf(rgen_new_rnd64s(1), "rnd64s 1");
  genperf(rgen_new_rnd64s(37), "rnd64s 37");
  genperf(rgen_new_rnd64s(0xffffffffu), "rnd64s UINT32_MAX");
  genperf(rgen_new_uniform(0, (1<< 22)), "uniform");
  genperf(rgen_new_uniform(0, (1<< 22) - (1 << 20)), "uniform");
  genperf(rgen_new_uniform(0, (1<< 22) - 1), "uniform");

  genperf(rgen_new_zipfian(0, 1000000), "zipfian");
  genperf(rgen_new_xzipfian(0, 1000000), "xzipfian");
  genperf(rgen_new_unizipf(0, 1000000, 100), "unizipf uf=100");
  genperf(rgen_new_unizipf(0, 1000000, 100), "zipfuni uf=100");
  genperf(rgen_new_latest(100), "latest (just zipfian)");

  genperf(rgen_new_incs(0, 1000000), "incs");
  genperf(rgen_new_decs(0, 1000000), "decs");
  genperf(rgen_new_skips(0, 1000000, 100), "skips +100");

  genperf(rgen_new_incu(0, 1000000), "incu");
  genperf(rgen_new_decu(0, 1000000), "decu");
  genperf(rgen_new_skipu(0, 1000000, 100), "skipu +100");

  genperf(rgen_new_skipu(0, 1000000, 1), "skipu +1");
  genperf(rgen_new_skipu(0, 1000000, -1), "skipu -1");
  genperf(rgen_new_skipu(0, 999999, 200000), "skipu +200000");
  genperf(rgen_new_skipu(0, 999999, -200000), "skipu -200000");

  genperf(rgen_new_shuffle(0, 0), "shuffle [0,0]");
  genperf(rgen_new_shuffle(0, 1), "shuffle [0,1]");
  genperf(rgen_new_shuffle(0, 2), "shuffle [0,2]");
  genperf(rgen_new_shuffle(0, 3), "shuffle [0,3]");
  genperf(rgen_new_shuffle(0, 10), "shuffle [0,10]");
  genperf(rgen_new_shuffle(0, 1000000), "shuffle [0,1000000]");
}

// gi will be freed by sample_dist
  static void
sample_dist(struct rgen * const gi, const u64 range)
{
  u64 * const ctr = calloc(range+1, sizeof(ctr[0]));
  const u64 rep = range > 10000000 ? (range * 4) : 40000000;
  for (u64 i = 0; i < rep; i++) {
    const u64 r = rgen_next(gi);
    if (r > range)
      debug_die();
    ctr[r]++;
  }
  rgen_destroy(gi);
  printf("------\n");
  qsort_u64_sample(ctr, range+1, 32, stdout);
  free(ctr);
}

  static void
test_gendist(void)
{
  //printf("====SMALL UNIFORM====\n");
  //for (u64 i = 1; i < 20; i++)
  //  sample_dist(rgen_new_uniform(0,i), i);

  // x15
  u64 ranges[20] = {100,500,1000,1000000,58644196,
      34703230,100000000,172942040,142411471,4502812,
      4077290,4179061,192678503,466551,16700946};
  for (u64 i = 0; i < 2; i++) {
    printf("====UNIFORM====\n");
    sample_dist(rgen_new_uniform(0, ranges[i]), ranges[i]);
    printf("====ZIPFIAN====\n");
    sample_dist(rgen_new_zipfian(0, ranges[i]), ranges[i]);
    printf("====XZIPFIAN====\n");
    sample_dist(rgen_new_xzipfian(0, ranges[i]), ranges[i]);
  }
}

  static void
test_gentrace(void)
{
  const char * const fn = "/tmp/lib1-test-genhead.bin";
  FILE * const out = fopen(fn, "wb");
  printf("samples to %s\n", fn);
  for (u64 i = 0; i < 40; i++) {
    const u32 r = (u32)(random_u64() % 1000lu);
    printf(" %u", r);
    fwrite(&r, sizeof(r), 1, out);
  }
  printf("\n\nrandom numbers:\n");
  fclose(out);
  struct rgen * const gen = rgen_new_trace32(fn, 1024);
  for (u64 i = 0; i < 200; i++) {
    const u32 r = (u32)rgen_next(gen);
    printf(" %u", r);
  }
  printf("\n");
  rgen_destroy(gen);
}

typedef u64(*genfunc)(struct rgen *);
  static void
run_gen(genfunc f, struct rgen * ga)
{
  rgen_async_wait(ga);
  const double t0 = time_sec();
  uint64_t r = 0;
  const u64 nr_gen = 2000000000;
  for (uint64_t i = 0; i < nr_gen; i++)
    r += f(ga);

  const double dt = time_diff_sec(t0);
  (void)r;
  const double mops = ((double)nr_gen) / (dt * 1e6);
  const double MBps = ((double)nr_gen * sizeof(u32)) / (1024.0 * 1024.0 * dt);
  printf("%s dt %.3lf    mops %.3lf    TP %.3lf MB/s\n", __func__, dt, mops, MBps);
}

  static void
test_asyncgen(void)
{
  thread_pin(0);
  //struct rgen * const g1 = rgen_new_unizipf(0, 1000000, 1024);
  struct rgen * const g1 = rgen_new_uniform(0, 1000000);
  debug_assert(g1);
  struct rgen * const g2 = rgen_fork(g1);
  debug_assert(g2);
  struct rgen * const ga = rgen_async_create(g2, 1);
  debug_assert(ga);

  printf("test sync origin\n");
  run_gen(rgen_next, g1);

  printf("test sync forked\n");
  run_gen(rgen_next, g2);

  printf("test nowait\n");
  run_gen(rgen_next_nowait, ga);

  printf("test wait\n");
  run_gen(rgen_next, ga);

  rgen_destroy(ga);
  rgen_join(g2);
  rgen_destroy(g1);
}
// }}} gen

// conc {{{
struct lock_worker_info {
  void ** plocks;
  u64 which_lock;
  u64 id_mask;
  au64 nr_ops;
  u64 end_time;
};

  static void *
lock_worker(void * const ptr)
{
  struct lock_worker_info * const info = (typeof(info))ptr;
  const u64 mask = info->id_mask;
  const u64 end_time = info->end_time;
  srandom_u64(time_nsec());
  u64 count = 0;

#define LOCK_WORKER_BATCH ((4000))
  do {
    switch (info->which_lock) {
    case 1:
      for (u64 i = 0; i < LOCK_WORKER_BATCH; i++) {
        const u64 x = random_u64() & mask;
        spinlock_lock((spinlock *)info->plocks[x]);
        spinlock_unlock((spinlock *)info->plocks[x]);
      }
      break;
    case 2:
      for (u64 i = 0; i < LOCK_WORKER_BATCH; i++) {
        const u64 x = random_u64() & mask;
        mutex_lock((mutex *)info->plocks[x]);
        mutex_unlock((mutex *)info->plocks[x]);
      }
      break;
    default:
      sleep(1);
      break;
    }
    count += LOCK_WORKER_BATCH;
  } while (time_nsec() < end_time);
  atomic_fetch_add(&(info->nr_ops), count);
  return NULL;
}

  static void
test_locks(void)
{
  printf("locks: 1 spin %zu, 2 mutex %zu\n",
      sizeof(spinlock), sizeof(pthread_mutex_t));
  const u64 ncpus = process_affinity_count();
  for (u64 p = 0; p <= 12; p+=3) { // power of 2 locks
    const u64 nlocks = 1lu << p;
    struct lock_worker_info info;
    info.plocks = malloc(sizeof(info.plocks[0]) * nlocks);
    info.id_mask = nlocks - 1;;
    for (u64 i = 0; i < nlocks; i++) {
      info.plocks[i] = yalloc(64);
    }
    for (u64 t = 1; t <= 2; t++) { // lock type
      switch (t) {
      case 1:
        for (u64 x = 0; x < nlocks; x++)
          spinlock_init(info.plocks[x]);
        break;
      case 2:
        for (u64 x = 0; x < nlocks; x++)
          pthread_mutex_init(info.plocks[x], NULL);
        break;
      }
      for (u32 th = 4; th <= ncpus; th += 2) {
        atomic_store(&(info.nr_ops), 0);
        info.which_lock = t;
        info.end_time = time_nsec() + (UINT64_C(2) << 30);
        const u64 dt = thread_fork_join(th, lock_worker, false, &info);
        const u64 nr = info.nr_ops;
        const double mops = ((double)nr) * 1e3 / ((double)dt);
        printf("TYPE %lu NTH %3u NLOCKS %3lu TP %6.2lf mops\n", t, th, nlocks, mops);
      }
    }
    for (u64 i = 0; i < nlocks; i++) {
      free(info.plocks[i]);
    }
    free(info.plocks);
  }
}

struct rwlock_worker_info {
  union {
    rwlock * rw;
    pthread_rwlock_t * pt;
  } * locks;
  u64 id_mask;
  au64 seq;
  au64 nr_w;
  au64 nr_r;
  u64 nr_writer;
  u64 end_time;
};

  static void *
rwlock_worker(void * const ptr)
{
  struct rwlock_worker_info * const info = (typeof(info))ptr;
  const u64 seq = atomic_fetch_add(&(info->seq), 1);
  const bool is_writer = seq < info->nr_writer ? true : false;
  const u64 mask = info->id_mask;
  const u64 end_time = info->end_time;
  srandom_u64(time_nsec());
  u64 count = 0;
  do {
    for (u64 i = 0; i < 1024; i++) {
      const u64 x = random_u64() & mask;
      if (is_writer) {
        rwlock_lock_write(info->locks[x].rw);
        rwlock_unlock_write(info->locks[x].rw);
      } else {
        rwlock_lock_read(info->locks[x].rw);
        rwlock_unlock_read(info->locks[x].rw);
      }
    }
    count += 1024;
  } while (time_nsec() < end_time);
  if (is_writer) {
    atomic_fetch_add(&(info->nr_w), count);
  } else {
    atomic_fetch_add(&(info->nr_r), count);
  }
  return NULL;
}

  static void *
hplock_worker(void * const ptr)
{
  struct rwlock_worker_info * const info = (typeof(info))ptr;
  const u64 seq = atomic_fetch_add(&(info->seq), 1);
  const bool is_writer = seq < info->nr_writer ? true : false;
  const u64 mask = info->id_mask;
  const u64 end_time = info->end_time;
  srandom_u64(time_nsec());
  u64 count = 0;
  do {
    for (u64 i = 0; i < 1024; i++) {
      const u64 x = random_u64() & mask;
      if (is_writer) {
        rwlock_lock_write_hp(info->locks[x].rw);
        rwlock_unlock_write(info->locks[x].rw);
      } else {
        rwlock_lock_read(info->locks[x].rw);
        rwlock_unlock_read(info->locks[x].rw);
      }
    }
    count += 1024;
  } while (time_nsec() < end_time);
  if (is_writer) {
    atomic_fetch_add(&(info->nr_w), count);
  } else {
    atomic_fetch_add(&(info->nr_r), count);
  }
  return NULL;
}

  static void *
rwlock_worker_pt(void * const ptr)
{
  struct rwlock_worker_info * const info = (typeof(info))ptr;
  const u64 seq = atomic_fetch_add(&(info->seq), 1);
  const bool is_writer = seq < info->nr_writer ? true : false;
  const u64 mask = info->id_mask;
  const u64 end_time = info->end_time;
  srandom_u64(time_nsec());
  u64 count = 0;
  do {
    for (u64 i = 0; i < 1024; i++) {
      const u64 x = random_u64() & mask;
      if (is_writer) {
        pthread_rwlock_wrlock(info->locks[x].pt);
        pthread_rwlock_unlock(info->locks[x].pt);
      } else {
        pthread_rwlock_rdlock(info->locks[x].pt);
        pthread_rwlock_unlock(info->locks[x].pt);
      }
    }
    count += 1024;
  } while (time_nsec() < end_time);
  if (is_writer) {
    atomic_fetch_add(&(info->nr_w), count);
  } else {
    atomic_fetch_add(&(info->nr_r), count);
  }
  return NULL;
}

  static void
test_rwlock(void)
{
  const u64 ncpus = process_affinity_count();
  void * (*wkr[3])(void *) = {rwlock_worker, hplock_worker, rwlock_worker_pt};
  for (u64 x = 0; x < 3; x++) {
    for (u64 w = 0; w < 3; w++) {
      for (u64 p = 0; p <= 9; p+=3) {
        const u64 nlocks = 1lu << p;
        struct rwlock_worker_info info;
        info.locks = (typeof(info.locks))malloc(sizeof(info.locks[0]) * nlocks);
        for (u64 i = 0; i < nlocks; i++) {
          info.locks[i].rw = yalloc(64);
          if (x < 2)
            rwlock_init(info.locks[i].rw);
          else
            pthread_rwlock_init(info.locks[i].pt, NULL);
        }

        info.id_mask = nlocks - 1;;
        info.seq = 0;
        info.nr_writer = w;
        for (u32 i = 4; i <= ncpus; i += 2) {
          info.seq = 0;
          atomic_store(&(info.nr_w), 0);
          atomic_store(&(info.nr_r), 0);
          info.end_time = time_nsec() + (UINT64_C(2) << 30);
          const u64 dt = thread_fork_join(i, wkr[x], false, &info);
          const u64 nr_w = atomic_load(&info.nr_w);
          const double mw = ((double)nr_w) * 1e3 / ((double)dt);
          const u64 nr_r = atomic_load(&info.nr_r);
          const double mr = ((double)nr_r) * 1e3 / ((double)dt);
          printf("%lu NTH %3u NW %3lu NL %3lu R %6.2lf W %6.2lf\n", x, i, w, nlocks, mr, mw);
        }
        for (u64 i = 0; i < nlocks; i++) {
          free(info.locks[i].rw);
        }
        free(info.locks);
      }
    }
  }
}

  static void *
rcu_worker(void * ptr_rcu)
{
  u64 sum = 0;
  u64 cnt = 0;
  struct rcu * const rcu = (typeof(rcu))ptr_rcu;
  const double t0 = time_sec();
  do {
    for (u64 i = 0; i < 1024; i++) {
      const u64 r = random_u64() % 0xff;
      u64 * ptr = (typeof(ptr))rcu_ref(rcu, r);
      sum += (*ptr);
      rcu_unref(rcu, ptr, r);
      cnt++;
    }
  } while (time_diff_sec(t0) < 5.0);
  printf("cnt %20lu sum %20lu\n", cnt, sum);
  return NULL;
}

  static void
rcu_main(const u64 power)
{
  struct rcu * const rcu = rcu_create(1lu << power);
  u64 vec[1024] = {};
  for (u64 i = 0; i < 1024; i++) {
    vec[i] = i;
  }
  rcu_update(rcu, &(vec[1023]));
  const u32 nth = process_affinity_count();
  u32 cores[1024];
  process_getaffinity_list(nth, cores);
  pthread_t pt[1024];
  printf("power %lu creating %u threads\n", power, nth);
  for (u64 i = 0; i < nth; i++) {
    thread_create_at(cores[i], &pt[i], rcu_worker, rcu);
  }
  const double t0 = time_sec();
  u64 cnt = 0;
  do {
    for (u64 i = 0; i < 64; i++) {
      const u64 r1 = random_u64() % 512;
      rcu_update(rcu, &(vec[r1]));
      const u64 r2 = (random_u64() % 512) + 512;
      rcu_update(rcu, &(vec[r2]));
      cnt += 2;
    }
  } while (time_diff_sec(t0) < 5.0);
  for (u64 i = 0; i < nth; i++) {
    pthread_join(pt[i], NULL);
  }
  printf("update %lu\n", cnt);
  free(rcu);
}

  static void
test_rcu(void)
{
  for (u64 i = 4; i <= 10; i++) {
    rcu_main(i);
  }
}

static au64 qsbr_nop = 0;
  static void *
qsbr_ref_worker(void * const ptr)
{
  struct qsbr * const q = (typeof(q))ptr;
  u64 count1 = 0;
  const u64 endtime = time_nsec() + (u64)5e9;
  do {
    struct qsbr_ref qref = {};
    for (u64 i = 0; i < 10000; i++) {
      qsbr_register(q, &qref);
      cpu_cfence();
      qsbr_unregister(q, &qref);
    }
    count1 += 10000;
  } while (time_nsec() < endtime);
  qsbr_nop += count1;
  return NULL;
}

  static void *
qsbr_park_worker(void * const ptr)
{
  struct qsbr * const q = (typeof(q))ptr;
  u64 count1 = 0;
  const u64 endtime = time_nsec() + (u64)5e9;
  do {
    struct qsbr_ref * const qref = malloc(sizeof(*qref));
    qsbr_register(q, qref);
    for (u64 i = 0; i < 10000; i++) {
      qsbr_park(qref);
      cpu_cfence();
      qsbr_resume(qref);
    }
    qsbr_unregister(q, qref);
    free(qref);
    count1 += 10000;
  } while (time_nsec() < endtime);
  qsbr_nop += count1;
  return NULL;
}

static volatile u64 * volatile qsbr_root;
  static void *
qsbr_checker(void * const ptr)
{
  struct qsbr * const q = (typeof(q))ptr;
  const double t0 = time_sec();
  u64 count = 0;
  do {
    struct qsbr_ref qref = {};
    qsbr_register(q, &qref);
    for (u64 i = 0; i < 4096; i++) {
      volatile u64 * const root = qsbr_root;
      if (*root)
        debug_die();
      qsbr_update(&qref, (u64)root);
      if (*root)
        debug_die();
      cpu_pause();
      if ((i & 0xff) == 0) {
        qsbr_park(&qref);
        cpu_pause();
        qsbr_resume(&qref);
      }
    }
    qsbr_unregister(q, &qref);
    count++;
  } while (time_diff_sec(t0) < 5.0);
  qsbr_nop += (count * 4096);
  return NULL;
}

  static void
test_qsbr(void)
{
  const u32 nth0 = process_affinity_count();
  const u32 nth = nth0 < 64 ? nth0 : 64;
  struct qsbr * const q = qsbr_create();
  qsbr_nop = 0;
  const u64 dt1 = thread_fork_join(1, qsbr_ref_worker, false, q);
  printf("  1 thread {ref/unref} mops %.2lf\n", ((double)qsbr_nop) * 1e3 / ((double)dt1));
  qsbr_nop = 0;
  const u64 dt4 = thread_fork_join(0, qsbr_ref_worker, false, q);
  printf("%3u thread {ref/unref} mops %.2lf\n", nth, ((double)qsbr_nop) * 1e3 / ((double)dt4));

  qsbr_nop = 0;
  const u64 dt1p = thread_fork_join(1, qsbr_park_worker, false, q);
  printf("  1 thread {park/resume} mops %.2lf\n", ((double)qsbr_nop) * 1e3 / ((double)dt1p));
  qsbr_nop = 0;
  const u64 dt4p = thread_fork_join(0, qsbr_park_worker, false, q);
  printf("%3u thread {park/resume} mops %.2lf\n", nth, ((double)qsbr_nop) * 1e3 / ((double)dt4p));

  qsbr_nop = 0;
  volatile u64 vars[2] = {0, 1};
  qsbr_root = vars;
  u32 cores[64];
  process_getaffinity_list(nth, cores);
  thread_pin(cores[0]);
  printf("creating %u threads\n", nth-1);
  pthread_t pt[64];
  for (u64 i = 1; i < nth; i++)
    thread_create_at(cores[i], &pt[i], qsbr_checker, q);
  const double t0 = time_sec();
  u64 cnt = 0;
  do {
    for (u64 i = 0; i < 1000; i++) {
      vars[1] = 0; // valid data == 0
      cpu_cfence();
      qsbr_root = vars+1;
      qsbr_wait(q, (u64)(vars+1));
      vars[0] = 1; // invalidate
      cpu_pause();
      cpu_cfence();
      cpu_pause();
      (void)(vars[0]);
      (void)(vars[1]);

      vars[0] = 0;
      cpu_cfence();
      qsbr_root = vars;
      qsbr_wait(q, (u64)(vars));
      vars[1] = 1;
      cpu_pause();
      cpu_cfence();
      cpu_pause();
      (void)(vars[0]);
      (void)(vars[1]);
    }
    cnt += 2000;
  } while (time_diff_sec(t0) < 5.0);
  for (u64 i = 1; i < nth; i++)
    pthread_join(pt[i], NULL);
  printf("wait per sec %lu ref/unref %lu\n", cnt/5, qsbr_nop);
  qsbr_destroy(q);
}
// }}} conc

// statistics {{{
  static void
one_damp(struct damp * const d, const double v)
{
  char c = damp_add_test(d, v) ? 't':'f';
  printf("add %lf result %c avg %lf ravg %lf\n", v, c, damp_avg(d), damp_ravg(d));
}

  static void
test_damp(void)
{
  struct damp * const d = damp_create(8, 0.004, 0.05);
  debug_assert(d);
  one_damp(d, 40.0);
  one_damp(d, 40.09);
  one_damp(d, 40.11);
  damp_clean(d);
  printf("cleaned\n");
  for (u64 i = 0; i < 10; i++) {
    one_damp(d, 55.0);
    one_damp(d, 57.0);
  }
  for (u64 i = 0; i < 10; i++) {
    one_damp(d, 56.0 + random_double());
    one_damp(d, 56.0 - random_double());
  }
  damp_destroy(d);
}

static u64 fjcount;

  static void *
fjworker(void * const ptr)
{
  atomic_fetch_add((au64 *)ptr, 1);
  return NULL;
}

  static void *
fjname_l2(void * const ptr)
{
  (void)ptr;
  char name[16] = {};
  thread_get_name(pthread_self(), name, 16);
  printf("thread name: %s\n", name);
  return NULL;
}

  static void *
fjname_l1(void * const ptr)
{
  (void)ptr;
  char name[16] = {};
  thread_get_name(pthread_self(), name, 16);
  printf("thread name: %s\n", name);
  thread_fork_join(3, fjname_l2, false, NULL);
  return NULL;
}

  static void
test_forkjoin(void)
{
  thread_fork_join(3, fjname_l1, false, NULL);

  for (u32 i = 1; i <= (1lu << 10); i <<= 1) {
    printf("fj %u\n", i);
    fjcount = 0;
    cpu_cfence();
    u64 t0 = time_nsec();
    thread_fork_join(i, fjworker, false, &fjcount);
    u64 dt = time_diff_nsec(t0);
    cpu_cfence();
    printf("dt %lu fjcount %lu\n", dt, fjcount);
  }

  for (u64 i = 0; i < 100; i++) {
    for (u64 j = 0; j < 100; j++) {
      thread_fork_join(64, fjworker, false, &fjcount);
    }
    printf("RSS %ldkB\n", process_get_rss());
  }
}
// }}} statistics

// crc {{{
  static void
test_crc32cpfx(void)
{
  u32 crc = 0xdeadbeefu;
  u8 * string = pages_alloc_4kb(1);
  double t0, dt;
  for (u64 i = 0; i < 4096; i++)
    string[i] = (u8)random_u64();

  u32 *crcx = pages_alloc_4kb(1);
  t0 = time_sec();
  for (u64 x = 0; x < 1000000; x++)
    for (u64 i = 0; i < 1024; i++)
      crcx[i] ^= (crc = crc32c_u8(crc, string[i]));

  dt = time_diff_sec(t0);
  for (u64 i = 0; i < 1024; i++)
    crc ^= crcx[i];
  printf("X%uX seq dt %.3lf\n", crc, dt);

  memset(crcx, 0, 4096);
  crc = 0xdeadbeefu;
  t0 = time_sec();
  for (u32 x = 0; x < 1000000; x++) {
    for (u32 i = 0; i < 1024; i += 4) {
      const u32 tmp = crc32c_u16(crc, *(u16 *)(&string[i]));
      crcx[i+1] ^= tmp;
      const u32 next = crc32c_u32(crc, *(u32 *)(&string[i]));
      crcx[i+3] ^= next;
      crcx[i] ^= crc32c_u8(crc, string[i]);
      crcx[i+2] ^= crc32c_u8(tmp, string[i+2]);
      crc = next;
    }
  }

  dt = time_diff_sec(t0);
  for (u64 i = 0; i < 1024; i++)
    crc ^= crcx[i];
  printf("X%uX +4 dt %.3lf\n", crc, dt);

  memset(crcx, 0, 4096);
  crc = 0xdeadbeefu;
  t0 = time_sec();
  u32 t1,t3,t5,t7,t9,t11,t13;
  for (u64 x = 0; x < 1000000; x++) {
    for (u64 i = 0; i < 1024; i += 16) {
      u8 * const curr = &string[i];

      crcx[i] ^= crc32c_u8(crc, *curr); // +1
      crcx[i+1] ^= (t1 = crc32c_u16(crc, *((u16 *)curr))); // +2
      crcx[i+3] ^= (t3 = crc32c_u32(crc, *((u32 *)curr))); // +4
      crcx[i+7] ^= (t7 = crc32c_u64(crc, *((u64 *)curr))); // +8

      crcx[i+2] ^= crc32c_u8(t1, *(curr+2)); // +1

      crcx[i+4] ^= crc32c_u8(t3, *(curr+4)); // +1
      crcx[i+5] ^= (t5 = crc32c_u16(t3, *((u16 *)(curr+4)))); // +2

      crcx[i+9] ^= (t9 = crc32c_u16(t7, *((u16 *)(curr+8)))); // +2
      crcx[i+8] ^= crc32c_u8(t7, *(curr+8)); // +1
      crcx[i+11] ^= (t11 = crc32c_u32(t7, *((u32 *)(curr+8)))); // +4

      crcx[i+6] ^= crc32c_u8(t5, *(curr+6)); // +1
      crcx[i+10] ^= crc32c_u8(t9, *(curr+10)); // +1

      crcx[i+13] ^= (t13 = crc32c_u16(t11, *((u16 *)(curr+12)))); // +2
      crcx[i+12] ^= crc32c_u8(t11, *(curr+12)); // +1
      crcx[i+15] ^= (crc = crc32c_u32(t11, *((u32 *)(curr+12)))); // +4

      crcx[i+14] ^= crc32c_u8(t13, *(curr+14)); // +1
    }
  }
  dt = time_diff_sec(t0);
  for (u64 i = 0; i < 1024; i++)
    crc ^= crcx[i];
  printf("X%uX +16 dt %.3lf\n", crc, dt);
}

  static void
test_crc32c(void)
{
  u32 crc = 0xdeedbeefu;
  u64 v64 = 0x12345678u;
  u32 v32 = 0x12345678u;
  u16 v16 = 0x1234u;
  u8 v8 = 0x12u;
  double t0, dt;

  // 8x4
  t0 = time_sec();
  for (u64 i = 0; i < 40000000; i++) {
    crc = crc32c_u64(crc, v64);
    crc = crc32c_u64(crc, v64);
    crc = crc32c_u64(crc, v64);
    crc = crc32c_u64(crc, v64);
  }
  dt = time_diff_sec(t0);
  printf("4x unrolled 64 dt %.3lf\n", dt);

  // 8x2
  t0 = time_sec();
  for (u64 i = 0; i < 80000000; i++) {
    crc = crc32c_u64(crc, v64);
    crc = crc32c_u64(crc, v64);
  }
  dt = time_diff_sec(t0);
  printf("2x unrolled 64 dt %.3lf\n", dt);

  // 8x1
  t0 = time_sec();
  for (u64 i = 0; i < 160000000; i++) {
    crc = crc32c_u64(crc, v64);
  }
  dt = time_diff_sec(t0);
  printf("1x iterate  64 dt %.3lf\n", dt);

  // 4x1
  t0 = time_sec();
  for (u64 i = 0; i < 320000000; i++) {
    crc = crc32c_u32(crc, v32);
  }
  dt = time_diff_sec(t0);
  printf("1x iterate  32 dt %.3lf\n", dt);

  // 2x1
  t0 = time_sec();
  for (u64 i = 0; i < 640000000; i++) {
    crc = crc32c_u16(crc, v16);
  }
  dt = time_diff_sec(t0);
  printf("1x iterate  16 dt %.3lf\n", dt);

  // 1x1
  t0 = time_sec();
  for (u64 i = 0; i < 1280000000; i++) {
    crc = crc32c_u8(crc, v8);
  }
  dt = time_diff_sec(t0);
  printf("1x iterate   8 dt %.3lf\n", dt);

  // 1x2
  t0 = time_sec();
  for (u64 i = 0; i < 640000000; i++) {
    crc = crc32c_u8(crc, v8);
    crc = crc32c_u8(crc, v8);
  }
  dt = time_diff_sec(t0);
  printf("2x unrolled  8 dt %.3lf\n", dt);

  // 1x4
  t0 = time_sec();
  for (u64 i = 0; i < 320000000; i++) {
    crc = crc32c_u8(crc, v8);
    crc = crc32c_u8(crc, v8);
    crc = crc32c_u8(crc, v8);
    crc = crc32c_u8(crc, v8);
  }
  dt = time_diff_sec(t0);
  printf("4x unrolled  8 dt %.3lf\n", dt);

  // 1x8
  t0 = time_sec();
  for (u64 i = 0; i < 160000000; i++) {
    crc = crc32c_u8(crc, v8);
    crc = crc32c_u8(crc, v8);
    crc = crc32c_u8(crc, v8);
    crc = crc32c_u8(crc, v8);
    crc = crc32c_u8(crc, v8);
    crc = crc32c_u8(crc, v8);
    crc = crc32c_u8(crc, v8);
    crc = crc32c_u8(crc, v8);
  }
  dt = time_diff_sec(t0);
  printf("8x unrolled  8 dt %.3lf\n", dt);

  printf("crc %u\n", crc);
}
// }}} crc

// memcmp {{{
  static u64
memcmp_64(const u8 * m1, const u8 * m2, u64 rem)
{
  u64 off = 0;
  while (rem >= sizeof(u64)) {
    if ((*((const u64 *)(m1 + off))) != (*((const u64 *)(m2 + off))))
      return false;
    off += sizeof(u64);
    rem -= sizeof(u64);
  }
  u64 x = 0;
  if (rem & 0x4lu) {
    x = ((*((const u32 *)(m1 + off))) ^ (*((const u32 *)(m2 + off))));
    off += sizeof(u32);
  }
  if (rem & 0x2lu) {
    x |= ((*((const u16 *)(m1 + off))) ^ (*((const u16 *)(m2 + off))));
    off += sizeof(u16);
  }
  if (rem & 0x1lu) {
    x |= ((*(m1 + off) ^ (*(m2 + off))));
  }
  return x;
}

  static inline u64
memcmp_128(const u8 * m1, const u8 * m2, u64 nr)
{
#ifdef __AVX2__
  u64 i = 0;
  u64 x = 0;
  const u64 nr16 = nr >> 4 << 4;

  while (i < nr16) {
    const m128 cmp = _mm_cmpeq_epi64(_mm_load_si128((m128 *)(m1+i)),
        _mm_load_si128((m128 *)(m2+i)));
    x |= (u64)(0xffff ^ _mm_movemask_epi8(cmp));
    i += 16;
  }

  if (nr & 8) {
    x |= ((*(u64*)(m1+i)) ^ (*(u64*)(m2+i)));
    i += 8;
  }

  if (nr & 7) { // little endian
    const u64 shift = (8 - (nr & 7)) << 3;
    x |= (((*(u64*)(m1+i)) ^ (*(u64*)(m2+i))) << shift);
  }

  return x;
#else
  return memcmp_64(m1, m2, nr);
#endif
}

  static inline u64
memcmp_256(const u8 * m1, const u8 * m2, u64 nr)
{
#ifdef __AVX2__
  u64 i = 0;
  u64 x = 0;
  const u64 nr32 = nr >> 5 << 5;

  while (i < nr32) {
    const __m256i cmp = _mm256_cmpeq_epi64(_mm256_load_si256((__m256i*)(m1+i)),
        _mm256_load_si256((__m256i*)(m2+i))); // 1 means eq
    x |= (u64)~_mm256_movemask_epi8(cmp);
    i += 32;
  }

  if (nr & 16) {
    const m128 cmp = _mm_cmpeq_epi64(_mm_load_si128((m128 *)(m1+i)),
        _mm_load_si128((m128 *)(m2+i)));
    x |= (u64)(0xffff ^ _mm_movemask_epi8(cmp));
    i += 16;
  }

  if (nr & 8) {
    x |= ((*(u64*)(m1+i)) ^ (*(u64*)(m2+i)));
    i += 8;
  }

  if (nr & 7) { // little endian
    const u64 shift = (8 - (nr & 7)) << 3;
    x |= (((*(u64*)(m1+i)) ^ (*(u64*)(m2+i))) << shift);
  }

  return x;
#else
  return memcmp_128(m1, m2, nr);
#endif
}

  static void
test_memcmp(void)
{
  u64 (*cmp[4])(const u8 *, const u8 *, u64);
  cmp[0] = (void *)memcmp;
  cmp[1] = memcmp_64;
  cmp[2] = memcmp_128;
  cmp[3] = memcmp_256;
  u64 (*cmpx)(const u8 *, const u8 *, u64);
  printf("exec time (seconds) of [glibc] [u64] [128] [256]\n");
  u64 x = 0;

  // 256
  const char * const key1x = "aaaaaaaabbbbbbbbccccccccddddddddeeeeeeeeffffffffgggggggghhhhhhhhaaaaaaaabbbbbbbbccccccccddddddddeeeeeeeeffffffffgggggggghhhhhhhhaaaaaaaabbbbbbbbccccccccddddddddeeeeeeeeffffffffgggggggghhhhhhhhaaaaaaaabbbbbbbbccccccccddddddddeeeeeeeeffffffffgggggggghhhhhhhh443321";
  const char * const key2x = "aaaaaaaabbbbbbbbccccccccddddddddeeeeeeeeffffffffgggggggghhhhhhhhaaaaaaaabbbbbbbbccccccccddddddddeeeeeeeeffffffffgggggggghhhhhhhhaaaaaaaabbbbbbbbccccccccddddddddeeeeeeeeffffffffgggggggghhhhhhhhaaaaaaaabbbbbbbbccccccccddddddddeeeeeeeeffffffffgggggggghhhhhhhh443322";
  u8 * key1 = yalloc(256);
  u8 * key2 = yalloc(256);
  memcpy(key1, key1x, 256);
  memcpy(key2, key2x, 256);

#pragma nounroll
  for (u64 i = 0; i <= 256; i += 1) {
    printf("[string length %4lu]", i);
#pragma nounroll
    for (u64 j = 0; j < 4; j++) {
      cmpx = cmp[j];
      //printf(" %s ", names[j]);
      double t0 = time_sec();
#pragma nounroll
      for (u64 k = 0; k < 200000000; k++)
        if (cmpx(key1, key2, i)) x++;
      double dt = time_diff_sec(t0);
      printf("%6.3lf %c ", dt, x ? 'x' : '=');
    }
    printf("\n");
  }
  free(key1);
  free(key2);
}
// }}} memcmp

// alloc {{{
struct alloc_worker_info {
  void * priv;
  void * (*alloc_func)(void * priv, size_t sz);
  void (*free_func)(void * priv, void * ptr);

  // test 1
  au64 nalloc;
  au64 nfree;
  u64 endtime;

  // test 2
  u32 conc_alloc;
  u32 conc_free;
  u64 nptrs;
  volatile au64 seq;
  void ** ptrs;
};

  static void *
alloc_worker1(void * const ptr)
{
  struct alloc_worker_info * const wi = (typeof(wi))ptr;
  const u64 nptrs = 1lu << 20; // 1 million
  const u64 mask = nptrs - 1;
  void ** ptrs = malloc(sizeof(*ptrs) * nptrs);
  u64 nptr = 0;
  u64 nalloc = 0;
  u64 nfree = 0;
  do {
    for (u64 i = 0; i < 4096; i++) {
      const u64 r = random_u64() & mask;
      if (r >= nptr) {
        u64 * const new = wi->alloc_func(wi->priv, 160);
        new[1] = i; // just force incore
        ptrs[nptr] = new;
        debug_assert(ptrs[nptr]);
        nptr++;
        nalloc++;
      } else {
        nptr--;
        wi->free_func(wi->priv, ptrs[r]);
        ptrs[r] = ptrs[nptr];
        nfree++;
      }
    }
  } while (time_nsec() < wi->endtime);

  for (u64 i = 0; i < nptr; i++)
    wi->free_func(wi->priv, ptrs[i]);
  free(ptrs);

  atomic_fetch_add(&wi->nalloc, nalloc);
  atomic_fetch_add(&wi->nfree, nfree + nptr);
  return NULL;
}

  static void *
alloc_worker2(void * const ptr)
{
  struct alloc_worker_info * const wi = (typeof(wi))ptr;
  do {
    const u64 s = atomic_fetch_add(&wi->seq, 32);
    if (s >= wi->nptrs)
      break;

    for (u64 i = 0; i < 32; i++) {
      u64 * const new = wi->alloc_func(wi->priv, 160);
      new[1] = i; // just force incore
      wi->ptrs[s+i] = new;
    }
  } while (true);
  return NULL;
}

  static void *
alloc_worker3(void * const ptr)
{
  struct alloc_worker_info * const wi = (typeof(wi))ptr;
  const u64 s = atomic_fetch_add(&wi->seq, 1);
  for (u64 i = s; i < wi->nptrs; i += wi->conc_free)
    wi->free_func(wi->priv, wi->ptrs[i]);
  return NULL;
}

  static void *
malloc_wrapper(void * x, size_t sz)
{
  (void)x;
  return malloc(sz);
}

  static void
free_wrapper(void * x, void * ptr)
{
  (void)x;
  free(ptr);
}

  static void *
slab_alloc_wrapper(void * x, size_t sz)
{
  (void)sz;
  return slab_alloc_safe(x);
}

  static void
slab_free_wrapper(void * x, void * ptr)
{
  slab_free_safe(x, ptr);
}

  static void *
arena_alloc_wrapper(void * x, size_t sz)
{
  void * const ptr = arena_alloc(x, sz);
  *(size_t *)ptr = sz; // save the size
  return ptr;
}

  static void
arena_free_wrapper(void * x, void * ptr)
{
  arena_free(x, ptr, *(size_t *)ptr);
}

  static void
test_alloc_run(struct alloc_worker_info * const wi)
{
  for (u64 e = 0; e < 2; e++) {
    const u32 ncores = process_affinity_count();
    for (u32 i = 1; i <= ncores; i++) {
      wi->conc_alloc = i;
      wi->nalloc = 0;
      wi->nfree = 0;
      wi->endtime = time_nsec() + (u64)3e9;
      const u64 dt1 = thread_fork_join(i, alloc_worker1, false, wi);
      const u64 nraf = wi->nalloc + wi->nfree;
      const double mops_af = ((double)nraf) / (double)dt1 * 1e3;
      printf("nrth=%u LOCAL alloc+free mops %.3lf\n", i, mops_af);

      wi->nptrs = 1lu << 25;
      wi->ptrs = malloc(sizeof(void *) * wi->nptrs);

      for (u32 j = 1; j <= ncores; j++) {
        wi->conc_free = j;
        wi->seq = 0;
        const u64 dt2 = thread_fork_join(i, alloc_worker2, false, wi);
        const double mops_ga = ((double)wi->nptrs) / ((double)dt2) * 1e3;
        const long rss = process_get_rss();
        wi->seq = 0;
        const u64 dt3 = thread_fork_join(j, alloc_worker3, false, wi);
        const double mops_gf = ((double)wi->nptrs) / ((double)dt3) * 1e3;
        printf("GLOBAL nptrs %lu alloc mops %.3f rss-mb %ld free nrth=%u mops %.3f\n", wi->nptrs, mops_ga, rss>>10, j, mops_gf);
      }

      free(wi->ptrs);
    }
  }
}

  static void
test_arena(void)
{
  struct alloc_worker_info wi = {};
  wi.priv = arena_create(16, 1 << 21);
  debug_assert(wi.priv);
  wi.alloc_func = arena_alloc_wrapper;
  wi.free_func = arena_free_wrapper;
  test_alloc_run(&wi);
  arena_destroy(wi.priv);
}

  static void
test_malloc(void)
{
  struct alloc_worker_info wi = {};
  wi.alloc_func = malloc_wrapper;
  wi.free_func = free_wrapper;
  test_alloc_run(&wi);
}

  static void
test_slab(void)
{
  struct alloc_worker_info wi = {};
  wi.priv = slab_create(160, 1 << 21);
  debug_assert(wi.priv);
  wi.alloc_func = slab_alloc_wrapper;
  wi.free_func = slab_free_wrapper;
  test_alloc_run(&wi);
  slab_destroy(wi.priv);
}
// }}} alloc

// co {{{
static u64 comastersp;
static u64 coworkersp;
static void coswitch_worker(void)
{
  printf("in worker 1\n");
  co_switch_stack(&coworkersp, comastersp, 1);
  printf("in worker 2\n");
  co_switch_stack(&coworkersp, comastersp, 2);
  printf("in worker 3\n");
  co_switch_stack(&coworkersp, comastersp, 3);
}

static void test_coswitch(void)
{
  u64 * const costk = yalloc(1<<16);
  u64 * rsp = costk + 8000;
  rsp[0] = (u64)coswitch_worker;
#if defined(__x86_64__)
  coworkersp = (u64)(rsp - 6);
#elif defined(__aarch64__)
  coworkersp = (u64)rsp;
#endif
  printf("master get %lu\n", co_switch_stack(&comastersp, coworkersp, 100));
  printf("master get %lu\n", co_switch_stack(&comastersp, coworkersp, 101));
  printf("master get %lu\n", co_switch_stack(&comastersp, coworkersp, 102));
  free(costk);
}

struct chaseptr {
  u64 next;
  u64 rank;
};

  static int
chaseptrcmp(const void * p1, const void * p2)
{
  const u64 r1 = ((struct chaseptr *)(p1))->rank;
  const u64 r2 = ((struct chaseptr *)(p2))->rank;
  if (r1 < r2)
    return -1;
  else if (r1 > r2)
    return 1;
  else
    return 0;
}

  static void
init_ptrmap(struct chaseptr * const ptrmap, const u64 memsize)
{
  const u64 nr = memsize / sizeof(struct chaseptr);
  for (u64 i = 0; i < nr; i++) {
    ptrmap[i].next = (i+1) % nr;
    ptrmap[i].rank = random_u64();
  }
  qsort(ptrmap, nr, sizeof(struct chaseptr), chaseptrcmp);
}

struct cochase_priv {
  struct chaseptr * ptrmap;
  u64 nr;
  double stop;
  u64 count;
};

  static void
cochase(void)
{
  struct cochase_priv * const priv = co_priv();
  const double stop = priv->stop;
  struct chaseptr * const ptrmap = priv->ptrmap;
  const u64 n = 10000;
  srandom_u64(time_nsec());

  u64 idx = random_u64() % priv->nr;
  u64 count = 0;
  do {
    for (u64 i = 0; i < n; i++) {
      __builtin_prefetch(&(ptrmap[idx]), 0, 0);
      corr_yield();
      idx = ptrmap[idx].next;
    }
    count += n;
  } while (time_sec() < stop);

  priv->count += count;
  (void)idx;
}

  static void
test_cochase(void)
{
  u64 memsize;
  struct chaseptr * const ptrmap = pages_alloc_best(1lu << 25, true, &memsize);
  init_ptrmap(ptrmap, memsize);
#define NRCO1 ((24))
  struct corr * crs[NRCO1];
  struct cochase_priv priv = {ptrmap, memsize / sizeof(struct chaseptr), 0, 0};
  u64 hostrsp;
#define COMUL ((2))
  for (u64 s = 256*COMUL; s < 512*COMUL; s += 64*COMUL) {
    for (u64 i = 1; i <= NRCO1; i++) {
      crs[0] = corr_create(s, cochase, &priv, &hostrsp);
      for (u64 j = 1; j < i; j++)
        crs[j] = corr_link(s, cochase, &priv, crs[j-1]);

      priv.count = 0;
      const double t0 = time_sec();
      priv.stop = t0 + 1.0;
      corr_enter(crs[0]);
      const double dt = time_diff_sec(t0);

      printf("stack %lu ptrchase %lu mops = %.3lf\n", s, i, ((double)priv.count) * 0.000001 / dt);
      for (u64 j = 0; j < i; j++)
        corr_destroy(crs[j]);
    }
  }
  pages_unmap(ptrmap, memsize);
}

  static void
test_co_worker(void)
{
  u64 x = 0;
  printf("%p\n", &x);
  struct co ** y = co_priv();
  struct co * to = *y;

  x = co_switch_to(to, x+1);
  printf("sw1 %p %lu\n", &x, x);
  x = co_switch_to(to, x+1);
  printf("sw2 %p %lu\n", &x, x);

  x = co_back(x+1);
  printf("bk1 %p %lu\n", &x, x);
  x = co_back(x+1);
  printf("bk2 %p %lu\n", &x, x);
  co_exit(x+1);
  return;
}

  static void
test_corr_worker(void)
{
  u64 x = 0;
  printf("1 %p\n", &x);
  corr_yield();
  printf("2 %p\n", &x);
  corr_yield();
  printf("3 %p\n", &x);
}

  static void
test_co(void)
{
  printf("test co\n");
  u64 hostrsp = 0;
  struct co * co2[2];
  co2[0] = co_create(4096, test_co_worker, &co2[1], &hostrsp);
  co2[1] = co_create(4096, test_co_worker, &co2[0], &hostrsp);
  u64 ret = 0;
  while (co_valid(co2[0]) || co_valid(co2[1])) {
    if (co_valid(co2[0])) {
      ret = co_enter(co2[0], ret + 1);
      if (!co_valid(co2[0]))
        printf("enter %p returned %lu\n", co2[0], ret);
    }
    if (co_valid(co2[1])) {
      ret = co_enter(co2[1], ret + 1);
      if (!co_valid(co2[1]))
        printf("enter %p returned %lu\n", co2[1], ret);
    }
  }
  printf("now reuse\n");
  co_reuse(co2[0], test_co_worker, &co2[1], &hostrsp);
  co_reuse(co2[1], test_co_worker, &co2[0], &hostrsp);
  while (co_valid(co2[0]) || co_valid(co2[1])) {
    if (co_valid(co2[1]))
      ret = co_enter(co2[1], ret + 1);
    if (co_valid(co2[0]))
      ret = co_enter(co2[0], ret + 1);
  }
  co_destroy(co2[0]);
  co_destroy(co2[1]);
  printf("host co ret %lu\n", ret);

  printf("test corr\n");
  struct corr * co3[3];
  co3[0] = corr_create(4096, test_corr_worker, NULL, &hostrsp);
  co3[1] = corr_link(4096, test_corr_worker, NULL, co3[0]);
  co3[2] = corr_link(4096, test_corr_worker, NULL, co3[1]);
  corr_enter(co3[0]);
  printf("corr reuse\n");
  corr_reuse(co3[0], test_corr_worker, NULL, &hostrsp);
  corr_relink(co3[1], test_corr_worker, NULL, co3[0]);
  corr_relink(co3[2], test_corr_worker, NULL, co3[1]);
  corr_enter(co3[0]);
  printf("corr returned\n");
  corr_destroy(co3[0]);
  corr_destroy(co3[1]);
  corr_destroy(co3[2]);
}
// }}} co

// misc {{{
  static void
test_gettime(void)
{
#if defined(__linux__)
  clockid_t clks[5] = {CLOCK_REALTIME, CLOCK_REALTIME_COARSE, CLOCK_MONOTONIC,
    CLOCK_MONOTONIC_COARSE, CLOCK_MONOTONIC_RAW};
  char * names[5] = {"RT", "RT_COARSE", "MONO", "MONO_COARSE", "MONO_RAW"};
  const u64 nr = 5;
#elif defined(__FreeBSD__)
  clockid_t clks[5] = {CLOCK_REALTIME, CLOCK_REALTIME_FAST, CLOCK_MONOTONIC,
    CLOCK_MONOTONIC_FAST, CLOCK_MONOTONIC_PRECISE};
  char * names[5] = {"RT", "RT_FAST", "MONO", "MONO_FAST", "MONO_PRECISE"};
  const u64 nr = 5;
#elif defined(__APPLE__) && defined(__MACH__)
  clockid_t clks[4] = {CLOCK_REALTIME, CLOCK_MONOTONIC, CLOCK_MONOTONIC_RAW,
    CLOCK_MONOTONIC_RAW_APPROX};
  char * names[4] = {"RT", "MONO", "MONO_RAW", "MONO_RAW_APP"};
  const u64 nr = 4;
#else
#error Unsupported OS
#endif
  struct timespec specs[16];

  for (u64 i = 0; i < nr; i++) {
    clockid_t clk_id = clks[i];
    clock_gettime(clk_id, &specs[0]);
    clock_gettime(clk_id, &specs[1]);
    clock_gettime(clk_id, &specs[2]);
    clock_gettime(clk_id, &specs[3]);
    clock_gettime(clk_id, &specs[4]);
    clock_gettime(clk_id, &specs[5]);
    clock_gettime(clk_id, &specs[6]);
    clock_gettime(clk_id, &specs[7]);

    clock_gettime(clk_id, &specs[8]);
    clock_gettime(clk_id, &specs[9]);
    clock_gettime(clk_id, &specs[10]);
    clock_gettime(clk_id, &specs[11]);
    clock_gettime(clk_id, &specs[12]);
    clock_gettime(clk_id, &specs[13]);
    clock_gettime(clk_id, &specs[14]);
    clock_gettime(clk_id, &specs[15]);

    printf("%12s", names[i]);
    for (u64 j = 1; j < 16; j++) {
#define E9 (1000000000lu)
      const u64 t1 = ((u64)specs[j].tv_sec * E9 + (u64)specs[j].tv_nsec);
      const u64 t0 = ((u64)specs[j-1].tv_sec * E9 + (u64)specs[j-1].tv_nsec);
      printf(" %lu", t1 - t0);
    }
    printf("\n");
  }
}

  static void
test_hashfill(void)
{
  u8 cnt [32];
  for (u64 i = 4; i <= 32; i += 4) {
    for (u64 j = 8; j <= 16; j += 8) {
      memset(cnt, 0, 32);
      for (;;) {
        const u64 idx = random_u64() % i;
        if (cnt[idx] == j) {
          printf("bucket %lu x assoc %lu\n", i, j);
          u64 count = 0;
          for (u64 x = 0; x < i; x++) {
            printf("%u ", (u32)cnt[x]);
            count += cnt[x];
          }
          printf("tot %lu %lu\n", count, i * j);
          break;
        }
        cnt[idx]++;
      }
    }
  }
}

  static void
test_rnddist(void)
{
  for (u64 x = 0; x < 5; x++) {
    srandom_u64(time_nsec() * time_nsec());
    u64 * const cs = malloc(sizeof(u64) * (1lu << 16));
    for (u64 i = 0; i < 64; i+= 16) {
      memset(cs, 0, sizeof(u64) * (1lu << 16));
      const double t0 = time_sec();
      for (u64 j = 0; j < (1lu << 28); j++) {
        const u16 r = (u16)(random_u64() >> i);
        cs[r]++;
      }
      const double dt = time_diff_sec(t0);
      qsort_u64(cs, 1 << 16);
      printf("r64 shift %2lu DT %.3lf 0 %4lu 20 %4lu 40 %4lu 60 %4lu 80 %4lu 100 %4lu\n",
          i, dt, cs[0], cs[6553*2], cs[6553*4], cs[6553*6], cs[6553*8], cs[65535]);
    }
    free(cs);
  }
}

#if defined(__x86_64__)
  static void
test_stacksize(void)
{
  u64 rsp;
  asm volatile("mov %%rsp, %0\n":"=r"(rsp)::);
  printf("rsp %lu\n", rsp);

  asm volatile (
      "loop1:\n"
      "pushq %%rax\n"
      "pushq %%rbx\n"
      "pushq %%rcx\n"
      "pushq %%rdx\n"
      "pushq %%rdi\n"
      "pushq %%rsi\n"
      "pushq %%rbp\n"
      "pushq %%rsp\n"
      "pushq %%r8\n"
      "pushq %%r9\n"
      "pushq %%r10\n"
      "pushq %%r11\n"
      "pushq %%r12\n"
      "pushq %%r13\n"
      "pushq %%r14\n"
      "pushq %%r15\n"
      "jmp loop1\n":::);
}
#endif

void (*tsigtable[4])(u64);

static void tsig1 (u64 i) { if (i) tsigtable[random_u64() % 4](i-1); }
static void tsig2 (u64 i) { if (i) tsigtable[random_u64() % 4](i-1); }
static void tsig3 (u64 i) { if (i) tsigtable[random_u64() % 4](i-1); }
static void tsig4 (u64 i) { if (i) tsigtable[random_u64() % 4](i-1); }

static volatile u64 usr1_var;

  static void
signal_handler_usr1(const int sig, siginfo_t * info, void * context)
{
  (void)sig;
  (void)info;
  (void)context;
  usr1_var++;
  //void *array[10];
  //const int size = backtrace(array, 10);
  //for (int i = 0; i < size; i++)
  //  printf("%d %p\n", i, array[i]);
}

  static void *
tsig_worker(void * ptr)
{
  struct sigaction sa = {};
  sa.sa_sigaction = signal_handler_usr1;
  sigemptyset(&(sa.sa_mask));
  sa.sa_flags = SA_RESTART | SA_SIGINFO;
  if (sigaction(SIGUSR1, &sa, NULL) == -1) {
    fprintf(stderr, "Failed to set signal handler for SIGUSR1\n");
  } else {
    fprintf(stderr, "SIGUSR1 ok %d\n", getpid());
  }
  (void)ptr;
  tsigtable[0] = tsig1;
  tsigtable[1] = tsig2;
  tsigtable[2] = tsig3;
  tsigtable[3] = tsig4;

  usr1_var = 1;
  while (true)
    tsigtable[random_u64() % 4](10);

  return NULL;
}

  static void
test_tsig(void)
{
  printf("%p\n", tsig1);
  printf("%p\n", tsig2);
  printf("%p\n", tsig3);
  printf("%p\n", tsig4);
  pthread_t pt;
  usr1_var = 0;
  thread_create_at(1, &pt, tsig_worker, NULL);
  pthread_detach(pt);
  while (usr1_var == 0);
  const double t0 = time_sec();
#define SIGNR ((100000))
  for (u64 i = 1; i < SIGNR; i++) {
    cpu_cfence();
    pthread_kill(pt, SIGUSR1);
    while (usr1_var < i);
  }
  const double dt = time_diff_sec(t0);
  printf("dt %.3lf sps %.3lf\n", dt, ((double)SIGNR) / dt);
}
// }}} misc

// clmove {{{
#if defined(__x86_64__)
  static void
clmove_memmove(u8 * const dst, u8 * const src)
{
  memmove(dst, src, 64);
}

  static void
clmove_memcpy(u8 * const dst, u8 * const src)
{
  memcpy(dst, src, 64);
}

// m128
  static void
clmove_ldst128(u8 * const dst, u8 * const src)
{
  const m128 v0 = _mm_load_si128((m128 *)src);
  const m128 v1 = _mm_load_si128(((m128 *)src) + 1);
  const m128 v2 = _mm_load_si128(((m128 *)src) + 2);
  const m128 v3 = _mm_load_si128(((m128 *)src) + 3);
  _mm_store_si128((m128 *)dst, v0);
  _mm_store_si128(((m128 *)dst) + 1, v1);
  _mm_store_si128(((m128 *)dst) + 2, v2);
  _mm_store_si128(((m128 *)dst) + 3, v3);
}

  static void
clmove_ntld128(u8 * const dst, u8 * const src)
{
  const m128 v0 = _mm_stream_load_si128((m128 *)src);
  const m128 v1 = _mm_stream_load_si128(((m128 *)src) + 1);
  const m128 v2 = _mm_stream_load_si128(((m128 *)src) + 2);
  const m128 v3 = _mm_stream_load_si128(((m128 *)src) + 3);
  _mm_store_si128((m128 *)dst, v0);
  _mm_store_si128(((m128 *)dst) + 1, v1);
  _mm_store_si128(((m128 *)dst) + 2, v2);
  _mm_store_si128(((m128 *)dst) + 3, v3);
}

  static void
clmove_ntst128(u8 * const dst, u8 * const src)
{
  const m128 v0 = _mm_load_si128((m128 *)src);
  const m128 v1 = _mm_load_si128(((m128 *)src) + 1);
  const m128 v2 = _mm_load_si128(((m128 *)src) + 2);
  const m128 v3 = _mm_load_si128(((m128 *)src) + 3);
  _mm_stream_si128((m128 *)dst, v0);
  _mm_stream_si128(((m128 *)dst) + 1, v1);
  _mm_stream_si128(((m128 *)dst) + 2, v2);
  _mm_stream_si128(((m128 *)dst) + 3, v3);
}

  static void
clmove_ldst128sf(u8 * const dst, u8 * const src)
{
  clmove_ldst128(dst, src);
  _mm_sfence();
}

  static void
clmove_ldst128fl(u8 * const dst, u8 * const src)
{
  clmove_ldst128(dst, src);
  _mm_clflush(dst);
}

#if defined(__CLFLUSHOPT__)
  static void
clmove_ldst128opt(u8 * const dst, u8 * const src)
{
  clmove_ldst128(dst, src);
  _mm_clflushopt(dst);
}

  static void
clmove_ldst128optsf(u8 * const dst, u8 * const src)
{
  clmove_ldst128(dst, src);
  _mm_clflushopt(dst);
  _mm_sfence();
}
#endif // __CLFLUSHOPT__

#if defined(__CLWB__)
  static void
clmove_ldst128wb(u8 * const dst, u8 * const src)
{
  clmove_ldst128(dst, src);
  _mm_clwb(dst);
}

  static void
clmove_ldst128wbsf(u8 * const dst, u8 * const src)
{
  clmove_ldst128(dst, src);
  _mm_clwb(dst);
  _mm_sfence();
}
#endif // __CLWB__

  static void
clmove_ldst128ntld(u8 * const dst, u8 * const src)
{
  clmove_ldst128(dst, src);
  cpu_cfence();
  const m128 v0 = _mm_stream_load_si128((m128 *)dst);
  const m128 v1 = _mm_stream_load_si128(((m128 *)dst) + 1);
  const m128 v2 = _mm_stream_load_si128(((m128 *)dst) + 2);
  const m128 v3 = _mm_stream_load_si128(((m128 *)dst) + 3);
  (void)v0;
  (void)v1;
  (void)v2;
  (void)v3;
}

  static void
clmove_ntst128sf(u8 * const dst, u8 * const src)
{
  clmove_ntst128(dst, src);
  _mm_sfence();
}
// m256
#ifdef __AVX2__
  static void
clmove_ldst256(u8 * const dst, u8 * const src)
{
  const m256 v0 = _mm256_load_si256((m256 *)src);
  const m256 v1 = _mm256_load_si256(((m256 *)src) + 1);
  _mm256_store_si256((m256 *)dst, v0);
  _mm256_store_si256(((m256 *)dst) + 1, v1);
}

  static void
clmove_ntld256(u8 * const dst, u8 * const src)
{
  const m256 v0 = _mm256_stream_load_si256((m256 *)src);
  const m256 v1 = _mm256_stream_load_si256(((m256 *)src) + 1);
  _mm256_store_si256((m256 *)dst, v0);
  _mm256_store_si256(((m256 *)dst) + 1, v1);
}

  static void
clmove_ntst256(u8 * const dst, u8 * const src)
{
  const m256 v0 = _mm256_load_si256((m256 *)src);
  const m256 v1 = _mm256_load_si256(((m256 *)src) + 1);
  _mm256_stream_si256((m256 *)dst, v0);
  _mm256_stream_si256(((m256 *)dst) + 1, v1);
}

  static void
clmove_ldst256sf(u8 * const dst, u8 * const src)
{
  clmove_ldst256(dst, src);
  _mm_sfence();
}

  static void
clmove_ldst256fl(u8 * const dst, u8 * const src)
{
  clmove_ldst256(dst, src);
  _mm_clflush(dst);
}

#if defined(__CLFLUSHOPT__)
  static void
clmove_ldst256opt(u8 * const dst, u8 * const src)
{
  clmove_ldst256(dst, src);
  _mm_clflushopt(dst);
}

  static void
clmove_ldst256optsf(u8 * const dst, u8 * const src)
{
  clmove_ldst256(dst, src);
  _mm_clflushopt(dst);
  _mm_sfence();
}
#endif // __CLFLUSHOPT__

#if defined(__CLWB__)
  static void
clmove_ldst256wb(u8 * const dst, u8 * const src)
{
  clmove_ldst256(dst, src);
  _mm_clwb(dst);
}

  static void
clmove_ldst256wbsf(u8 * const dst, u8 * const src)
{
  clmove_ldst256(dst, src);
  _mm_clwb(dst);
  _mm_sfence();
}
#endif // __CLWB__

  static void
clmove_ldst256ntld(u8 * const dst, u8 * const src)
{
  clmove_ldst256(dst, src);
  cpu_cfence();
  const m256 v0 = _mm256_stream_load_si256((m256 *)dst);
  const m256 v1 = _mm256_stream_load_si256(((m256 *)dst) + 1);
  (void)v0;
  (void)v1;
}

  static void
clmove_ntst256sf(u8 * const dst, u8 * const src)
{
  clmove_ntst256(dst, src);
  _mm_sfence();
}
#endif // __AVX2__

// m512
#ifdef __AVX512F__
  static void
clmove_ldst512(u8 * const dst, u8 * const src)
{
  _mm512_store_si512((m512 *)dst, _mm512_load_si512((m512 *)src));
}

  static void
clmove_ntld512(u8 * const dst, u8 * const src)
{
  _mm512_store_si512((m512 *)dst, _mm512_stream_load_si512((m512 *)src));
}

  static void
clmove_ntst512(u8 * const dst, u8 * const src)
{
  _mm512_stream_si512((m512 *)dst, _mm512_load_si512((m512 *)src));
}

  static void
clmove_ldst512sf(u8 * const dst, u8 * const src)
{
  clmove_ldst512(dst, src);
  _mm_sfence();
}

  static void
clmove_ldst512fl(u8 * const dst, u8 * const src)
{
  clmove_ldst512(dst, src);
  _mm_clflush(dst);
}

#if defined(__CLFLUSHOPT__)
  static void
clmove_ldst512opt(u8 * const dst, u8 * const src)
{
  clmove_ldst512(dst, src);
  _mm_clflushopt(dst);
}

  static void
clmove_ldst512optsf(u8 * const dst, u8 * const src)
{
  clmove_ldst512(dst, src);
  _mm_clflushopt(dst);
  _mm_sfence();
}
#endif // __CLFLUSHOPT__

#if defined(__CLWB__)
  static void
clmove_ldst512wb(u8 * const dst, u8 * const src)
{
  clmove_ldst512(dst, src);
  _mm_clwb(dst);
}

  static void
clmove_ldst512wbsf(u8 * const dst, u8 * const src)
{
  clmove_ldst512(dst, src);
  _mm_clwb(dst);
  _mm_sfence();
}
#endif // __CLWB__

  static void
clmove_ldst512ntld(u8 * const dst, u8 * const src)
{
  clmove_ldst512(dst, src);
  cpu_cfence();
  const m512 v0 = _mm512_stream_load_si512((m512 *)dst);
  (void)v0;
}

  static void
clmove_ntst512sf(u8 * const dst, u8 * const src)
{
  clmove_ntst512(dst, src);
  _mm_sfence();
}
#endif // __AVX512F__

  static u64
clmove_worker(u8 * const mem, const u64 msize, const u8 epoch, void (*clmove)(u8 *, u8 *))
{
  u8 * const src = yalloc(64);
  memset(src, epoch, 64);
  const u64 t0 = time_nsec();
  for (u64 i = 0; i < msize; i += 64)
    clmove(mem + i, src);
  const u64 dt = time_diff_nsec(t0);
  if (memcmp(src, mem, 64) || memcmp(mem, mem+64, msize-64))
    debug_die();
  free(src);
  return dt;
}

  static void
test_clmove(void)
{
  u64 msize = 0;
  u8 * const mem = pages_alloc_best(1lu << 30, true, &msize);
  if (!mem) {
    printf("pages allocation failed\n");
    exit(0);
  }
  memset(mem, 0x3c, msize);
  printf("memmove         dt   %10lu\n", clmove_worker(mem, msize, 1, clmove_memmove));
  printf("memmove         dt   %10lu\n", clmove_worker(mem, msize, 2, clmove_memmove));
  printf("memcpy          dt   %10lu\n", clmove_worker(mem, msize, 3, clmove_memcpy));
  printf("Running SIMD tests. Please wait patiently.\n");

  // 128
  u64 dts[3][16] = {};

  dts[0][0]  = clmove_worker(mem, msize, 30, clmove_ldst128);
  dts[0][1]  = clmove_worker(mem, msize, 31, clmove_ntld128);
  dts[0][2]  = clmove_worker(mem, msize, 32, clmove_ntst128);
  dts[0][3]  = clmove_worker(mem, msize, 33, clmove_ldst128sf);
  dts[0][4]  = clmove_worker(mem, msize, 34, clmove_ldst128fl);
#if defined(__CLFLUSHOPT__)
  dts[0][5]  = clmove_worker(mem, msize, 35, clmove_ldst128opt);
  dts[0][6]  = clmove_worker(mem, msize, 36, clmove_ldst128optsf);
#endif // __CLFLUSHOPT__

#if defined(__CLWB__)
  dts[0][7]  = clmove_worker(mem, msize, 37, clmove_ldst128wb);
  dts[0][8]  = clmove_worker(mem, msize, 38, clmove_ldst128wbsf);
#endif // __CLWB__
  dts[0][9]  = clmove_worker(mem, msize, 39, clmove_ldst128ntld);
  dts[0][10] = clmove_worker(mem, msize, 30, clmove_ntst128sf);

#ifdef __AVX2__
  dts[1][0]  = clmove_worker(mem, msize, 40, clmove_ldst256);
  dts[1][1]  = clmove_worker(mem, msize, 41, clmove_ntld256);
  dts[1][2]  = clmove_worker(mem, msize, 42, clmove_ntst256);
  dts[1][3]  = clmove_worker(mem, msize, 43, clmove_ldst256sf);
  dts[1][4]  = clmove_worker(mem, msize, 44, clmove_ldst256fl);
#if defined(__CLFLUSHOPT__)
  dts[1][5]  = clmove_worker(mem, msize, 45, clmove_ldst256opt);
  dts[1][6]  = clmove_worker(mem, msize, 46, clmove_ldst256optsf);
#endif // __CLFLUSHOPT__

#if defined(__CLWB__)
  dts[1][7]  = clmove_worker(mem, msize, 47, clmove_ldst256wb);
  dts[1][8]  = clmove_worker(mem, msize, 48, clmove_ldst256wbsf);
#endif // __CLWB__
  dts[1][9]  = clmove_worker(mem, msize, 49, clmove_ldst256ntld);
  dts[1][10] = clmove_worker(mem, msize, 40, clmove_ntst256sf);
#endif // __AVX2__

#ifdef __AVX512F__
  dts[2][0]  = clmove_worker(mem, msize, 50, clmove_ldst512);
  dts[2][1]  = clmove_worker(mem, msize, 51, clmove_ntld512);
  dts[2][2]  = clmove_worker(mem, msize, 52, clmove_ntst512);
  dts[2][3]  = clmove_worker(mem, msize, 53, clmove_ldst512sf);
  dts[2][4]  = clmove_worker(mem, msize, 54, clmove_ldst512fl);
#if defined(__CLFLUSHOPT__)
  dts[2][5]  = clmove_worker(mem, msize, 55, clmove_ldst512opt);
  dts[2][6]  = clmove_worker(mem, msize, 56, clmove_ldst512optsf);
#endif // __CLFLUSHOPT__

#if defined(__CLWB__)
  dts[2][7]  = clmove_worker(mem, msize, 57, clmove_ldst512wb);
  dts[2][8]  = clmove_worker(mem, msize, 58, clmove_ldst512wbsf);
#endif // __CLWB__
  dts[2][9]  = clmove_worker(mem, msize, 59, clmove_ldst512ntld);
  dts[2][10] = clmove_worker(mem, msize, 50, clmove_ntst512sf);
#endif // __AVX512F__

  printf("opname           dt-simd128  dt-simd256  dt-simd512\n");
  printf("ldst128          %10lu  %10lu  %10lu\n", dts[0][0], dts[1][0], dts[2][0]);
  printf("ntld128          %10lu  %10lu  %10lu\n", dts[0][1], dts[1][1], dts[2][1]);
  printf("ntst128          %10lu  %10lu  %10lu\n", dts[0][2], dts[1][2], dts[2][2]);
  printf("ldst128sf        %10lu  %10lu  %10lu\n", dts[0][3], dts[1][3], dts[2][3]);
  printf("ldst128fl        %10lu  %10lu  %10lu\n", dts[0][4], dts[1][4], dts[2][4]);
  printf("ldst128opt       %10lu  %10lu  %10lu\n", dts[0][5], dts[1][5], dts[2][5]);
  printf("ldst128optsf     %10lu  %10lu  %10lu\n", dts[0][6], dts[1][6], dts[2][6]);
  printf("ldst128wb        %10lu  %10lu  %10lu\n", dts[0][7], dts[1][7], dts[2][7]);
  printf("ldst128wbsf      %10lu  %10lu  %10lu\n", dts[0][8], dts[1][8], dts[2][8]);
  printf("ldst128ntld      %10lu  %10lu  %10lu\n", dts[0][9], dts[1][9], dts[2][9]);
  printf("ntst128sf        %10lu  %10lu  %10lu\n", dts[0][10], dts[1][10], dts[2][10]);

  pages_unmap(mem, msize);
}

#endif // __x86_64__
// }}} clmove

// bf {{{
  static void
test_bf(void)
{
  srandom_u64(time_nsec());
  struct bf * const bf = bf_create(10, 100);
  u64 ps[100];
  u64 totfp = 0;
  for (u64 i = 0; i < 10000; i++) {
    for (u64 j = 0; j < 100; j++) {
      const u64 x = random_u64();
      u64 hash = crc32c_u64(0xDEADBEEF, x);
      hash |= __builtin_bswap64(hash);
      ps[j] = hash;
      bf_add(bf, hash);
    }
    for (u64 j = 0; j < 100; j++) {
      if (!bf_test(bf, ps[j])) {
        fprintf(stderr, "bf failed\n");
      }
    }
    u64 fp = 0;
    for (u64 j = 0; j < 100; j++) {
      if (bf_test(bf, random_u64())) {
        fp++;
      }
    }
    totfp += fp;
    if (fp)
      printf("[%4lu] fp %lu\n", i, fp);
    bf_clean(bf);
  }
  printf("totfp %lu/%lu\n", totfp, (u64)(10000*100));
  bf_destroy(bf);
}
// }}} bf

// atomic {{{
#if defined(__x86_64__)
// TODO: xadd
  static void
atomic_incl(u32 * ptr)
{
  asm volatile ("lock incl %0":"=m" (*ptr): "m" (*ptr): "cc");
}

  static void
atomic_addl(u32 * ptr)
{
  asm volatile ("lock addl $1, %0":"=m" (*ptr): "m" (*ptr): "cc");
}

  static void
atomic_decl(u32 * ptr)
{
  asm volatile ("lock decl %0":"=m" (*ptr): "m" (*ptr): "cc");
}

  static void
atomic_subl(u32 * ptr)
{
  asm volatile ("lock subl $1, %0":"=m" (*ptr): "m" (*ptr): "cc");
}

#define ATOMICN ((10000000))
#define AWORKER(____afunc, ____bfunc) \
  static void * \
  ____afunc ## _w(u8 * mem) \
{ \
  const u64 mask = 0x7ffff80; \
  for (u64 i = 0; i < ATOMICN; i++) { \
    const u64 r = random_u64() & mask; \
    u32 * ptr = (u32 *)(mem + r); \
    ____afunc(ptr); \
    ____bfunc(ptr); \
  } \
  return NULL; \
}

AWORKER(atomic_incl, atomic_decl);
AWORKER(atomic_addl, atomic_subl);

  static void
test_atomicasm(void)
{
  u8 * mem = calloc(128, 1024*1024);
  for (u32 i = 0; i < 3; i++) {
    const u64 dt0 = thread_fork_join(0, (void *)atomic_incl_w, false, mem);
    const u64 dt1 = thread_fork_join(0, (void *)atomic_addl_w, false, mem);
    printf("incl time %lu   addl time %lu\n", dt0, dt1);
  }
  free(mem);
}
#endif
// }}} atomic

// mhash {{{
struct mhash32_info {
  au64 seq;
  u64 *bitmap [32]; // 32 threads
};

  static void *
mhash32_worker(void * ptr)
{
  struct mhash32_info * mi = (typeof(mi))ptr;
  const u64 seq = mi->seq++;
  u64 * bm = mi->bitmap[seq];
  const u64 start = seq * (1 << 27);
  const u64 end = start + (1 << 27);
  for (u64 i = start; i < end; i++) {
    const u32 h = mhash32((u32)i);
    if (bm[h >> 6] & (1lu << (h & 0x3f))) {
      printf("worker1 clash at %x : %x\n", (u32)i, h);
    }
    bm[h >> 6] |= (1lu << (h & 0x3f));
  }
  while (mi->seq < 32)
    cpu_pause();

  cpu_cfence();
  mi->seq++;
  cpu_cfence();
  while (mi->seq < 64)
    cpu_pause();

  cpu_cfence();
  for (u64 i = start; i < end; i += 64) {
    u64 sum = 0;
    for (u64 j = 0; j < 32; j++) {
      if (mi->bitmap[j][i>>6] & sum) {
        printf("worker2 clash at %x %x\n", (u32)i, (u32)j);
      }
      sum |= mi->bitmap[j][i>>6];
    }
    if (~sum)
      printf("worker2 found 0 at %x %lx\n", (u32)i, sum);
  }
  printf("worker2 done %lu\n", seq);
  return NULL;
}

  static void
test_mhash32(void)
{
  struct mhash32_info mi = {.seq = 0};
  size_t msize;
  for (u64 i = 0; i < 32; i++) {
#define MH32BMSZ ((1 << 29))
    mi.bitmap[i] = pages_alloc_best(MH32BMSZ, false, &msize);
    if (mi.bitmap[i] == NULL)
      debug_die();
  }
  thread_fork_join(32, mhash32_worker, false, &mi);
  for (u64 i = 0; i < 32; i++) {
    pages_unmap(mi.bitmap[i], msize);
  }
}
// }}} mhash

// kv {{{
  static struct kv *
kvapi_merge_func(struct kv * const kv0, void * const priv)
{
  if (kv0 == NULL) { // insert
    return priv;
  } else if (kv0->kv[0] == 'h') { // hello world -> priv (replace)
    return (struct kv *)priv;
  } else if (kv0->kv[0] == 'M') { // Monica Bill -> "Jobs" (inplace)
    memcpy(kv_vptr(kv0), "Jobs", 4);
    return kv0;
  } else if (kv0->kv[0] == 'e') { // can't change the law
    return kv0;
  }
  debug_die();
}

  static void
test_kvapi(void)
{
  char * names[] = {"ht2unsafe", "cc2unsafe", "skiplist", "skipsafe",
    "ht1unsafe", "bptree", "wormhole", "whunsafe", "whsafe", NULL};
  char * argx[2] = {"api", NULL};
  struct kv * const key = malloc(1024);
  struct kv * const out = malloc(1024);
  for (u64 sid = 0; names[sid]; sid++) {
    argx[1] = names[sid];
    const struct kvmap_api * api = NULL;
    void * map = NULL;
    if (kvmap_api_helper(2, argx, NULL, &api, &map) < 0)
      continue;
    backtrace_symbols_fd((void *)(&(api->get)), 1, 1);
    backtrace_symbols_fd((void *)(&(api->destroy)), 1, 1);

    // set key and value as strings
    void * const ref = kvmap_ref(api, map);
    kv_refill_str_str(key, "hello", "world");
    api->put(ref, key);
    kv_refill_str_str(key, "Monica", "Bill");
    api->put(ref, key);
    kv_refill_str_str(key, "e=", "mc2");
    api->put(ref, key);

    // check keys and values
    if (api->ordered) {
      void * const iter = api->iter_create(ref);
      api->iter_seek(iter, kref_null());
      for (struct kv * kv = api->iter_next(iter, out); kv; kv = api->iter_next(iter, out)) {
        printf("[%s] %.*s -> %.*s\n", argx[1], kv->klen, (char *)kv_kptr(kv), kv->vlen, (char *)kv_vptr(kv));
      }
      api->iter_destroy(iter);
    } else {
      kv_refill_str_str(key, "hello", "");
      kvmap_kv_get(api, ref, key, out);
      printf("<%s> %.*s -> %.*s\n", argx[1], key->klen, (char *)key->kv, out->vlen, (char *)kv_vptr(out));
      kv_refill_str_str(key, "Monica", "");
      kvmap_kv_get(api, ref, key, out);
      printf("<%s> %.*s -> %.*s\n", argx[1], key->klen, (char *)key->kv, out->vlen, (char *)kv_vptr(out));
      kv_refill_str_str(key, "e=", "");
      kvmap_kv_get(api, ref, key, out);
      printf("<%s> %.*s -> %.*s\n", argx[1], key->klen, (char *)key->kv, out->vlen, (char *)kv_vptr(out));
    }

    if (api->merge) {
      // merge
      kv_refill_str_str(key, "newkey", "haha");
      kvmap_kv_merge(api, ref, key, kvapi_merge_func, key);
      kv_refill_str_str(key, "hello", "WORLD");
      kvmap_kv_merge(api, ref, key, kvapi_merge_func, key);
      kv_refill_str_str(key, "Monica", "????");
      kvmap_kv_merge(api, ref, key, kvapi_merge_func, key);
      kv_refill_str_str(key, "e=", "????");
      kvmap_kv_merge(api, ref, key, kvapi_merge_func, key);

      // check again
      kv_refill_str_str(key, "newkey", "");
      kvmap_kv_get(api, ref, key, out);
      printf("<%s> %.*s -> %.*s\n", argx[1], key->klen, (char *)key->kv, out->vlen, (char *)kv_vptr(out));
      kv_refill_str_str(key, "hello", "");
      kvmap_kv_get(api, ref, key, out);
      printf("<%s> %.*s -> %.*s\n", argx[1], key->klen, (char *)key->kv, out->vlen, (char *)kv_vptr(out));
      kv_refill_str_str(key, "Monica", "");
      kvmap_kv_get(api, ref, key, out);
      printf("<%s> %.*s -> %.*s\n", argx[1], key->klen, (char *)key->kv, out->vlen, (char *)kv_vptr(out));
      kv_refill_str_str(key, "e=", "");
      kvmap_kv_get(api, ref, key, out);
      printf("<%s> %.*s -> %.*s\n", argx[1], key->klen, (char *)key->kv, out->vlen, (char *)kv_vptr(out));
    }

    kvmap_unref(api, ref);
    api->clean(map);
    api->destroy(map);
  }
  free(key);
  free(out);
}

  static void
run_miter_seek(struct miter * const miter)
{
  struct kv * const tmp = malloc(100);
  u64 found = 0;
  miter_seek(miter, kref_null());
  for (u64 i = 0; i < 200; i++) {
    const u32 rank = miter_rank(miter);
    struct kv * const ret = miter_next(miter, tmp);
    if (ret) {
      fprintf(stdout, "[%u] ", rank);
      kv_print(ret, "xxn", stdout);
      found++;
    }
  }
  printf("miter_next found %lu\n", found);

  // next_unique
  kv_refill_kref(tmp, kref_null());
  miter_park(miter);
  miter_kv_seek(miter, tmp);
  found = 0;
  for (u64 i = 0; i < 200; i++) {
    const u32 rank = miter_rank(miter);
    struct kv * const ret = miter_next_unique(miter, tmp);
    if (ret) {
      fprintf(stdout, "[%u] ", rank);
      kv_print(ret, "xxn", stdout);
      found++;
    }
  }
  printf("next_unique all found %lu\n", found);

  // seek 0 11 22 33
  for (u64 s = 0; s < 50; s += 11) {
    printf("seek %lu\n", s);
    const u64 x = __builtin_bswap64(s);
    kv_refill(tmp, &x, 8, &x, 8);
    miter_park(miter);
    miter_kv_seek(miter, tmp);
    found = 0;
    for (u64 i = 0; i < 200; i++) {
      struct kv * const ret = miter_next(miter, tmp);
      if (ret) {
        kv_print(ret, "xxn", stdout);
        found++;
      }
    }
    printf("seek-next at %lu found %lu\n", s, found);
  }
  free(tmp);
}

  static void
test_miter(void)
{
  // prepare api with 8 levels
#define TEST_MITER_NWAY ((8))
  const struct kvmap_api * apis[TEST_MITER_NWAY] = {};
  void * maps[TEST_MITER_NWAY] = {};
  char * argv[3] = {"api", NULL, NULL};
  argv[1] = "wormhole";
  kvmap_api_helper(2, argv, NULL, &apis[0], &maps[0]);
  kvmap_api_helper(2, argv, NULL, &apis[4], &maps[4]);
  argv[1] = "whunsafe";
  kvmap_api_helper(2, argv, NULL, &apis[1], &maps[1]);
  kvmap_api_helper(2, argv, NULL, &apis[5], &maps[5]);
  argv[1] = "bptree";
  kvmap_api_helper(2, argv, NULL, &apis[2], &maps[2]);
  kvmap_api_helper(2, argv, NULL, &apis[6], &maps[6]);
  argv[1] = "skiplist";
  kvmap_api_helper(2, argv, NULL, &apis[3], &maps[3]);
  kvmap_api_helper(2, argv, NULL, &apis[7], &maps[7]);
  void * refs[TEST_MITER_NWAY];
  for (u64 i = 0; i < TEST_MITER_NWAY; i++)
    refs[i] = kvmap_ref(apis[i], maps[i]);

  struct kv * const tmp = malloc(100);
  u64 x;
  // insert some duplicated keys: [0 0 0 0 1 1 1 1 ... 24 24 24 24]
  for (u64 i = 0; i < 100; i++) {
    x = __builtin_bswap64(i>>2);
    kv_refill(tmp, &x, 8, &i, 8);
    apis[i%TEST_MITER_NWAY]->put(refs[i%TEST_MITER_NWAY], tmp);
  }

  for (u64 i = 0; i < TEST_MITER_NWAY; i++)
    kvmap_unref(apis[i], refs[i]);

  struct miter * const miter = miter_create();

  // miter_add
  for (u64 i = 0; i < TEST_MITER_NWAY; i++)
    miter_add(miter, apis[i], maps[i]);
  run_miter_seek(miter);
  miter_clean(miter);

  for (u64 i = 0; i < TEST_MITER_NWAY; i++) {
    refs[i] = kvmap_ref(apis[i], maps[i]);
    miter_add_ref(miter, apis[i], refs[i]);
  }
  run_miter_seek(miter);
  miter_clean(miter);
  // refs are still alive

  void * iters[TEST_MITER_NWAY];
  for (u64 i = 0; i < TEST_MITER_NWAY; i++) {
    iters[i] = apis[i]->iter_create(refs[i]);
    miter_add_iter(miter, apis[i], refs[i], iters[i]);
  }
  run_miter_seek(miter);
  miter_clean(miter);
  miter_destroy(miter);

  for (u64 i = 0; i < TEST_MITER_NWAY; i++) {
    apis[i]->iter_destroy(iters[i]);
    kvmap_unref(apis[i], refs[i]);
  }

  for (u64 i = 0; i < TEST_MITER_NWAY; i++)
    apis[i]->destroy(maps[i]);

  free(tmp);
  printf("done\n");
}

  static void
test_utoa(void)
{
  char buf[32];
  u64 t0, dt;

  t0 = time_nsec();
  char x = 0;
  for (u64 i = 0; i < 10000000; i++) {
    strhex_64(buf, i);
    x+=buf[0];
  }
  dt = time_diff_nsec(t0);
  printf("strhex_64 %lu %hhd\n", dt, x);

  t0 = time_nsec();
  for (u32 i = 0; i < 10000000; i++) {
    strhex_32(buf, i);
    x+=buf[0];
  }
  dt = time_diff_nsec(t0);
  printf("strhex_32 %lu %hhd\n", dt, x);

  t0 = time_nsec();
  for (u64 i = 0; i < 10000000; i++) {
    strdec_64(buf, i);
    x+=buf[0];
  }
  dt = time_diff_nsec(t0);
  printf("strdec_64 %lu %hhd\n", dt, x);

  t0 = time_nsec();
  for (u32 i = 0; i < 10000000; i++) {
    strdec_32(buf, i);
    x+=buf[0];
  }
  dt = time_diff_nsec(t0);
  printf("strdec_32 %lu %hhd\n", dt, x);
}

  static void
test_vi128(void)
{
  // u64
  u8 * const buf = malloc(10000000 * 10);
  u8 * iter = buf;
  iter = vi128_encode_u64(iter, 0lu);
  iter = vi128_encode_u64(iter, ~0lu);
  for (u64 i = 0; i < 10000000; i++) {
    const u64 v = mhash64(i) >> 20;
    iter = vi128_encode_u64(iter, v);
  }
  u8 * end = iter;
  const u8 * iter1 = buf;
  u64 err = 0;
  u64 vd64 = 0;
  u64 t0 = time_nsec();
  iter1 = vi128_decode_u64(iter1, &vd64);
  if (0lu != vd64)
    err++;
  iter1 = vi128_decode_u64(iter1, &vd64);
  if (~0lu != vd64)
    err++;
  for (u64 i = 0; i < 10000000; i++) {
    const u64 v = mhash64(i) >> 20;
    iter1 = vi128_decode_u64(iter1, &vd64);
    if (v != vd64)
      err++;
  }
  u64 dt = time_diff_nsec(t0);
  printf("dt %lu err %lu buf %p iter %p end %p\n", dt, err, buf, iter1, end);

  // u32
  iter = buf;
  iter = vi128_encode_u32(iter, 0u);
  iter = vi128_encode_u32(iter, ~0u);
  for (u32 i = 0; i < 10000000; i++) {
    const u32 v = mhash32(i) >> 20;
    iter = vi128_encode_u32(iter, v);
  }
  end = iter;
  iter1 = buf;
  err = 0;
  u32 vd32 = 0;
  t0 = time_nsec();
  iter1 = vi128_decode_u32(iter1, &vd32);
  if (0u != vd32)
    err++;
  iter1 = vi128_decode_u32(iter1, &vd32);
  if (~0u != vd32)
    err++;
  for (u32 i = 0; i < 10000000; i++) {
    const u32 v = mhash32(i) >> 20;
    iter1 = vi128_decode_u32(iter1, &vd32);
    if (v != vd32)
      err++;
  }
  dt = time_diff_nsec(t0);
  printf("dt %lu err %lu buf %p iter %p end %p\n", dt, err, buf, iter1, end);
}

  static void
test_seekle(void)
{
  struct bptree * const tree = bptree_create(NULL);
  char * const tmp0 = malloc(100);
  struct kv * const kv = malloc(100);
  struct kv * const kv1 = malloc(100);
  for (u64 i = 0; i < 10000; i++) {
    sprintf(tmp0, "0x%08lx33", i);
    kv_refill_str_str(kv, tmp0, "");
    bptree_put(tree, kv);
  }
  struct bptree_iter * const iter = bptree_iter_create(tree);
  sprintf(tmp0, "0x%08lx22", 0lu);
  kv_refill_str_str(kv, tmp0, "");
  struct kref s = kv_kref(kv);
  bptree_iter_seek_le(iter, &s);
  if (bptree_iter_valid(iter)) {
    printf("err: valid\n");
    exit(0);
  }

  for (u64 i = 1; i < 10001; i++) {
    sprintf(tmp0, "0x%08lx22", i);
    kv_refill_str_str(kv, tmp0, "");
    struct kref s1 = kv_kref(kv);
    bptree_iter_seek_le(iter, &s1);
    if (!bptree_iter_valid(iter)) {
      printf("err: invalid\n");
      exit(0);
    }
    bptree_iter_peek(iter, kv);
    sprintf(tmp0, "0x%08lx33", i-1);
    kv_refill_str_str(kv1, tmp0, "");
    if (!kv_match(kv, kv1)) {
      printf("err: match key\n");
      exit(0);
    }
  }
  free(tmp0);
  free(kv);
  free(kv1);
  bptree_iter_destroy(iter);
  bptree_destroy(tree);
  printf("%s all correct\n", __func__);
}

  static void
test_kref(void)
{
  struct kv * kv = kv_create_str_str("hello", "world");
  struct kref s = kv_kref(kv);
  printf("%p %u %u\n", s.ptr, s.len, s.priv);
}

  static void
hashiter_helper(const struct kvmap_api * const api, void * const map)
{
  void * const ref = kvmap_ref(api, map);
  debug_assert(ref);
  struct kv * const tmp = malloc(1024);
  debug_assert(tmp);
  u64 n = 0;

  void * const i0 = api->iter_create(ref);
  api->iter_seek(i0, kref_null());
  debug_assert(i0);
  do {
    struct kv * const ret = api->iter_next(i0, tmp);
    if (!ret)
      break;
    n++;
  } while (true);
  api->iter_destroy(i0);
  printf("empty table: %lu should be 0\n", n);

  kv_refill_str_str(tmp, "breaking", "123");
  api->put(ref, tmp);
  api->put(ref, tmp);
  kv_refill_str_str(tmp, "news", "123");
  api->put(ref, tmp);
  kv_refill_str_str(tmp, "", "123");
  api->put(ref, tmp);
  kv_refill_str_str(tmp, "bye", "123");
  api->put(ref, tmp);

  void * const i1 = api->iter_create(ref);
  api->iter_seek(i1, kref_null());
  debug_assert(i1);
  do {
    struct kv * const ret = api->iter_next(i1, tmp);
    if (!ret)
      break;
    kv_print(ret, "ssn", stdout);
    n++;
  } while (true);
  api->iter_destroy(i1);
  printf("table of four: %lu should be 4\n", n);
  kvmap_unref(api, ref);
  api->destroy(map);
}

  static void
test_hashiter(void)
{
  hashiter_helper(&kvmap_api_kvmap2, kvmap2_create(NULL));
  hashiter_helper(&kvmap_api_cuckoo, cuckoo2_create(NULL));
  hashiter_helper(&kvmap_api_chainmap, chainmap_create(NULL));
}

  static void
test_psort_helper(struct kv ** const kvs, const u32 l, const u32 r)
{
  srandom_u64(time_nsec());
  shuffle_u64((u64 *)kvs, 100);
  kv_psort(kvs, 100, l, r);
  printf("psort %u %u\n", l, r);
  for (u32 i = l; i <= r; i++)
    printf("%02u == %c%c\n", i, kvs[i]->kv[0], kvs[i]->kv[1]);
}

  static void
test_psort(void)
{
  struct kv *kvs[100];
  char str[8];

  for (int i = 0; i < 100; i++) {
    sprintf(str, "  %02d", i);
    kvs[i] = kv_create(str+2, 2, str, 4);
  }
  test_psort_helper(kvs, 0, 0);
  test_psort_helper(kvs, 1, 1);
  test_psort_helper(kvs, 49, 50);
  test_psort_helper(kvs, 45, 54);
  test_psort_helper(kvs, 40, 59);
  test_psort_helper(kvs, 30, 69);
  test_psort_helper(kvs, 20, 79);
  test_psort_helper(kvs, 0, 99);
}
// }}} kv

// stream {{{
  static void *
html_worker(void * ptr)
{
  struct server_wi * const wi = (typeof(wi))ptr;
  int fd = server_worker_fd(wi);
  char html[1024];
  strcpy(html, "<html><body><h1>Hello HTML!</h1>");
  strcat(html, "<a href=https://www.google.com/>Google</a>");
  strcat(html, "</body></html>");
  dprintf(fd, "HTTP/1.0 200 OK\r\n");
  dprintf(fd, "MIME-Version: 1.0\r\n");
  dprintf(fd, "Date: Tue, 31 Mar 2020 23:59:59 GMT\r\n");
  dprintf(fd, "Content-Type: text/html\r\n");
  const size_t csize = strlen(html);
  dprintf(fd, "Content-Length: %zu\r\n\r\n", csize);
  write(fd, html, csize);
  // done; clean up
  server_worker_exit(wi);
  return NULL;
}

  static void
test_html(void)
{
  struct server * const s = server_create("localhost", "8080", html_worker, NULL);
  if (!s) {
    printf("server create failed\n");
    return;
  }
  printf("listening on localhost:8080\n");
  server_wait_destroy(s);
  printf("server done\n");
}
// }}} stream

// simd {{{
  static void
test_movemask(void)
{
  size_t sz;
  m128 * const mem = pages_alloc_best(1lu << 21, false, &sz);
  const u64 nsegs = (1lu << 21) / sizeof(m128);
  for (u64 i = 0; i < nsegs; i++)
    memset((void *)(mem+i), random_u64(), sizeof(m128));

  u64 sum = 0;
  const u64 t0 = time_nsec();

  for (u64 i = 0; i < 10000000lu; i++) {
    const u64 rnd = random_u64() % nsegs;
    sum += m128_movemask_u8(mem[rnd]);
  }

  const u64 dt = time_diff_nsec(t0);
  printf("dt %lu nsegs %lu sum %lu\n", dt, nsegs, sum);
}
// }}} simd

// aq {{{
struct aq_entry {
  struct aq_entry * next;
  struct aq_txn * txn;
};

struct aq_txn { // a transaction structure
  u64 tid; // not used
  u32 padding1;
  u16 padding2;
  bool committed;
  bool retired;
  struct aq_entry es[0]; // vla
};

struct aq_head {
  au64 aptr;
  u64 padding[7];
};

struct aq_info {
  u64 nvars; // total number of variables
  struct aq_head * qs; // queue heads
  u64 * vars; // variables
  au64 * avars; // for checking
};

  static void
aq_enqueue(struct aq_head * const head, struct aq_entry * const entry)
{
  do {
    u64 ptr0 = atomic_load_explicit(&head->aptr, MO_CONSUME);
    entry->next = (void *)ptr0;
    if (atomic_compare_exchange_weak_explicit(&head->aptr, &ptr0, (u64)entry, MO_ACQUIRE, MO_RELAXED))
      return;
  } while (true);
};

#define AQ_NTXN ((100000lu))
  static void *
aq_worker(void * ptr)
{
  struct aq_info * const ai = ptr;
  struct oalloc * const oa = oalloc_create(1lu << 16);

  for (u32 i = 0; i < AQ_NTXN; i++) { // run 10k txns
    const u64 nr = 10 + (random_u64() % 10);
    struct aq_txn * const t = oalloc_alloc(oa, sizeof(*t) + (sizeof(t->es[0]) * nr));
    t->tid = i; // not used
    for (u32 j = 0; j < nr; j++) {
      const u64 vid = random_u64() % ai->nvars;
      t->es[j].txn = t;
      aq_enqueue(&ai->qs[vid], &t->es[j]);
    }
  }

  oalloc_destroy(oa);
  return NULL;
}

  static void
test_aq(void)
{
  struct aq_info ai = {.nvars = 1000, .qs = calloc(1000, sizeof(struct aq_head)),
      .vars = calloc(1000, sizeof(u64)), .avars = calloc(1000, sizeof(au64))};

  const u32 ncores = process_affinity_count();
  for (u32 i = 1; i <= ncores; i++) {
    const u64 dt = thread_fork_join(i, aq_worker, false, &ai);
    printf("dt %lu mops %.3lf\n", dt, (double)(AQ_NTXN * 1000lu * i) / (double)dt);
  }
  free(ai.qs);
  free(ai.vars);
  free(ai.avars);
}
// }}} aq

// main {{{
  int
main(int argc, char ** argv)
{
  struct tests {
    char * name;
    void (*func)(void);
  };
#define TC(name) { #name, test_ ## name, }
  static struct tests tests[] = {
    TC(misc), TC(segv), TC(gen), TC(gcd), TC(gentrace), TC(asyncgen), TC(gendist),
    TC(locks), TC(rwlock), TC(rcu), TC(damp), TC(forkjoin),
    TC(crc32c), TC(memcmp), TC(arena), TC(malloc), TC(slab), TC(crc32cpfx),
    TC(lcp), TC(shm), TC(qsbr), TC(gettime), TC(kvapi), TC(hashfill),
    TC(rnddist), TC(tsig), TC(bf), TC(miter), TC(mhash32), TC(vi128),
    TC(seekle), TC(html), TC(kref), TC(hashiter), TC(utoa),
    TC(co), TC(cochase), TC(coswitch), TC(psort), TC(movemask), TC(aq),
#if defined(__x86_64__)
    TC(clmove), TC(stacksize), TC(atomicasm),
#endif
    {NULL, NULL, },
  };
#undef TC
  if (argc == 1) {
    printf("TEST: ");
    for (int i = 0; tests[i].name; i++) {
      printf(" %s", tests[i].name);
    }
    printf("\n");
  }
  for (int i = 1; i < argc; i++) {
    for (int j = 0; tests[j].name; j++) {
      if (0 == strcmp(tests[j].name, argv[i])) {
        tests[j].func();
        break;
      }
    }
  }
  exit(0);
}
// }}} main

// vim:fdm=marker