Page fault during incremental allocation of memory

[C4-S3]

The program below allocates memory via a loop using the syscall mmap.
In the loop, i*page_size memory is requested and released at the end of the
loop by munmap. In iteration 26 where 104kb should be allocated there is
always a page fault. My guess is that the memory pages are not properly freed
by munmap. The host app uses as default value 1mb for free-mem-size, if the
value is increased to 10mb, there is a page fault in the 67 iteration.
The summed memory allocation is 1508kb after 26 iterations and 10352kb after 67 iterations.

# ./keystone-generic-host ./test_mem ./eyrie-rt
[test_mem_mmap] i = 1, memsize = 4kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = B, s[4095] = B, &s[4095] = 0x2000000fff
[test_mem_mmap] i = 2, memsize = 8kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = C, s[8191] = C, &s[8191] = 0x2000001fff
[test_mem_mmap] i = 3, memsize = 12kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = D, s[12287] = D, &s[12287] = 0x2000002fff
[test_mem_mmap] i = 4, memsize = 16kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = E, s[16383] = E, &s[16383] = 0x2000003fff
[test_mem_mmap] i = 5, memsize = 20kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = F, s[20479] = F, &s[20479] = 0x2000004fff
[test_mem_mmap] i = 6, memsize = 24kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = G, s[24575] = G, &s[24575] = 0x2000005fff
[test_mem_mmap] i = 7, memsize = 28kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = H, s[28671] = H, &s[28671] = 0x2000006fff
[test_mem_mmap] i = 8, memsize = 32kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = I, s[32767] = I, &s[32767] = 0x2000007fff
[test_mem_mmap] i = 9, memsize = 36kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = J, s[36863] = J, &s[36863] = 0x2000008fff
[test_mem_mmap] i = 10, memsize = 40kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = K, s[40959] = K, &s[40959] = 0x2000009fff
[test_mem_mmap] i = 11, memsize = 44kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = L, s[45055] = L, &s[45055] = 0x200000afff
[test_mem_mmap] i = 12, memsize = 48kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = M, s[49151] = M, &s[49151] = 0x200000bfff
[test_mem_mmap] i = 13, memsize = 52kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = N, s[53247] = N, &s[53247] = 0x200000cfff
[test_mem_mmap] i = 14, memsize = 56kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = O, s[57343] = O, &s[57343] = 0x200000dfff
[test_mem_mmap] i = 15, memsize = 60kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = P, s[61439] = P, &s[61439] = 0x200000efff
[test_mem_mmap] i = 16, memsize = 64kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = Q, s[65535] = Q, &s[65535] = 0x200000ffff
[test_mem_mmap] i = 17, memsize = 68kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = R, s[69631] = R, &s[69631] = 0x2000010fff
[test_mem_mmap] i = 18, memsize = 72kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = S, s[73727] = S, &s[73727] = 0x2000011fff
[test_mem_mmap] i = 19, memsize = 76kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = T, s[77823] = T, &s[77823] = 0x2000012fff
[test_mem_mmap] i = 20, memsize = 80kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = U, s[81919] = U, &s[81919] = 0x2000013fff
[test_mem_mmap] i = 21, memsize = 84kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = V, s[86015] = V, &s[86015] = 0x2000014fff
[test_mem_mmap] i = 22, memsize = 88kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = W, s[90111] = W, &s[90111] = 0x2000015fff
[test_mem_mmap] i = 23, memsize = 92kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = , s[94207] = X, &s[94207] = 0x2000016fff
[test_mem_mmap] i = 24, memsize = 96kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = Y, s[98303] = Y, &s[98303] = 0x2000017fff
[test_mem_mmap] i = 25, memsize = 100kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = Z, s[102399] = Z, &s[102399] = 0x2000018fff
[test_mem_mmap] i = 26, memsize = 104kb, &s = 0x3ffffe90, s = 0x2000000000, s[0] = A, s[106495] = A, &s[106495] = [runtime] page fault at 0xffffffffc000466c on 0x4b4b4b4b4b4b4b4b (scause: 0xd)
Enclave ran successfully

test_mem.c

#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>

const size_t PAGE_SIZE = 4096;

void fill_mem(char *s, const size_t len, const char c) {
  for (size_t i = 0; i < len; ++i) {
    s[i] = c;
  }
}

char getcbyi(const char i) { return 'A' + i % 26; }

char *get_mem_mmap(const size_t size, const char c) {
  char *s = mmap(0, size, 7, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
  fill_mem(s, size, c);
  const size_t last_idx = size - 1;
  printf("&s = %p, s = %p, s[0] = %c, s[%lu] = %c, &s[%lu] = %p\n", &s, s, s[0],
         last_idx, s[last_idx], last_idx, &s[last_idx]);
  return s;
}

void test_mem_mmap() {
  for (size_t i = 1; i <= 128; ++i) {
    const size_t size = PAGE_SIZE * i;
    printf("[test_mem_mmap] i = %lu, memsize = %lukb, ", i, size / 1024);
    char *s = get_mem_mmap(size, getcbyi(i));
    munmap(s, size);
  }
}

int main() {
  test_mem_mmap();
  return EXIT_SUCCESS;
}

Build test_mem

riscv64-unknown-linux-musl-gcc test_mem.c -o test_mem -Og -g -static

Build eyrie runtime

./build.sh freemem untrusted_io_syscall linux_syscall env_setup debug strace_debug