Support include jemalloc lib automatic

checkpoint_scripts/config.yaml

@@ -1,14 +1,14 @@
 base_config:
   message: "NULL"
-  spec_app_list: null
-  spec_apps: "cactusADM,tonto"
-  elf_folder: "./jemalloc_elf"
+  spec_app_list: "/nfs/home/share/jiaxiaoyu/dual_core_checkpoint/checkpoint_scripts/checkpoint_scripts/spec2k17.lst"
+  spec_apps: null
+  elf_folder: "/nfs/home/share/jiaxiaoyu/checkpoint/spec17_checkpoint/spec17_gcc12.2.0_rv64gcb"
   times: "1,1,1"
   start_id: "0,0,0"
   emulator: "QEMU"
   build_bbl_only: true
-  max_threads: 70
-  CPU2017: false
+  max_threads: 60
+  CPU2017: true
   generate_rootfs_script_only: false
   copies: 2
   archive_id: null
@@ -17,11 +17,12 @@ base_config:
   mem_bind: 0
   bootloader: "opensbi"
   all_in_one_workload: true
-  boot_for_test: false
+  boot_for_test: true
   enable_h_ext: false
+  using_jemalloc: true
 archive_id_config:
   gcc_version: "gcc12.2.0"
   riscv_ext: "rv64gcb"
   base_or_fixed: "base"
-  special_flag: "intFppOff_for_qemu_dual_core"
+  special_flag: "intFppOff_for_spec17_dual_core"
   group: "archgroup"

checkpoint_scripts/find_dynamic_lib.py

@@ -0,0 +1,54 @@
+import os
+import stat
+import magic
+
+def is_dynamic_library(file_path):
+    try:
+        with open(file_path, 'rb') as file:
+            header = file.read(4)
+            return header.startswith(b'\x7fELF')
+    except Exception as e:
+        print(f"Could not check file type for {file_path}: {e}")
+        return False
+
+def find_dynamic_libraries(search_dir='.'):
+    dynamic_libraries = []
+    for root, _, files in os.walk(search_dir):
+        for filename in files:
+            file_path = os.path.join(root, filename)
+            try:
+                if is_dynamic_library(file_path) or os.path.islink(file_path):
+                    dynamic_libraries.append(file_path)
+            except OSError as e:
+                print(f"Error accessing file {file_path}: {e}")
+    return dynamic_libraries
+
+def get_dynamic_libraries_list(search_dir):
+    libraries = find_dynamic_libraries(search_dir)
+    initramfs_string_list = []
+    for lib in libraries:
+        initramfs_string_list.append([f"file /lib/{lib.split('/')[-1]}", lib, "755 0 0"])
+
+    return initramfs_string_list
+
+def get_jemalloc_libraries_list(riscv_toolchains_top):
+    initramfs_string_list = [
+        ["file /lib/libjemalloc.so", f"{riscv_toolchains_top}/lib/libjemalloc.so", "755 0 0"],
+        ["file /lib/libjemalloc.so.2", f"{riscv_toolchains_top}/lib/libjemalloc.so.2", "755 0 0"],
+    ]
+    return initramfs_string_list
+
+def check_file_exists(file_path):
+    if os.path.exists(file_path):
+        return True
+    return False
+
+
+def get_libraries_str(riscv_toolchains_top, using_jemalloc):
+    libraries = get_dynamic_libraries_list(os.path.join(riscv_toolchains_top, "sysroot", "lib"))
+    if using_jemalloc:
+        libraries += get_jemalloc_libraries_list(riscv_toolchains_top)
+#    for i in libraries:
+#        assert(check_file_exists(i[1]))
+    libraries_str = list(map(lambda x: f"{x[0]} {x[1]} {x[2]}", libraries))
+    return libraries_str

checkpoint_scripts/generate_bbl.py

@@ -6,6 +6,7 @@
 import shutil
 from config import BaseConfig
 from typing import Tuple, List
+from find_dynamic_lib import get_libraries_str
 import subprocess
 
 class RootfsBuilder(BaseConfig):
@@ -43,13 +44,6 @@ def __init__(self, archive_buffer_layout, spec_info, path_env_vars_to_check=None
                     "dir /root 755 0 0", "dir /var/log 755 0 0", "",
                     "nod /dev/console 644 0 0 c 5 1", "nod /dev/null 644 0 0 c 1 3", "",
                     "nod /dev/urandom 644 0 0 c 1 9", "nod /dev/random 644 0 0 c 1 8",
-                    "# libraries",
-                    "file /lib/ld-linux-riscv64-lp64d.so.1 ${RISCV}/sysroot/lib/ld-linux-riscv64-lp64d.so.1 755 0 0",
-                    "file /lib/libc.so.6 ${RISCV}/sysroot/lib/libc.so.6 755 0 0",
-                    "file /lib/libresolv.so.2 ${RISCV}/sysroot/lib/libresolv.so.2 755 0 0",
-                    "file /lib/libm.so.6 ${RISCV}/sysroot/lib/libm.so.6 755 0 0",
-                    "file /lib/libdl.so.2 ${RISCV}/sysroot/lib/libdl.so.2 755 0 0",
-                    "file /lib/libpthread.so.0 ${RISCV}/sysroot/lib/libpthread.so.0 755 0 0",
                     "", "# busybox",
                     "file /bin/busybox ${RISCV_ROOTFS_HOME}/rootfsimg/build/busybox 755 0 0",
                     "file /etc/inittab ${RISCV_ROOTFS_HOME}/rootfsimg/inittab-spec 755 0 0",
@@ -418,6 +412,7 @@ def __init__(self, archive_buffer_layout, spec_info, path_env_vars_to_check=None
             }
         )
 
+
     def prepare_rcS(self, guest_memory, guest_cpus, spec, scripts_archive_folder):
         lines = []
         lines.append("#! /bin/sh")
@@ -431,10 +426,10 @@ def prepare_rcS(self, guest_memory, guest_cpus, spec, scripts_archive_folder):
             f.writelines(map(lambda x: x + "\n", lines))
 
         self.rcs_list.append((spec, os.path.join(scripts_archive_folder, f"{spec}_rcS-spec.txt")))
-
-    def prepare_rootfs(self, spec, using_cpu2017, copies, with_nemu_trap, redirect_output, emu):
+        
+    def prepare_rootfs(self, spec, using_cpu2017, using_jemalloc, copies, with_nemu_trap, redirect_output, emu):
         archive_buffer_layout = self.config["archive_buffer_layout"]
-        self.__generate_initramfs(archive_buffer_layout["scripts"], archive_buffer_layout["elf"], spec, os.path.join(self.config["path_env_vars"]["RISCV_ROOTFS_HOME"], "rootfsimg"), using_cpu2017, copies)
+        self.__generate_initramfs(archive_buffer_layout["scripts"], archive_buffer_layout["elf"], spec, os.path.join(self.config["path_env_vars"]["RISCV_ROOTFS_HOME"], "rootfsimg"), using_cpu2017, using_jemalloc, copies)
         self.__generate_run_scripts(spec, copies, redirect_output, using_cpu2017, with_nemu_trap, os.path.join(self.config["path_env_vars"]["RISCV_ROOTFS_HOME"], "rootfsimg"), archive_buffer_layout["scripts"], emu)
 
 
@@ -471,10 +466,12 @@ def __generate_host_initramfs(self, scripts_archive_folder, spec):
         self.host_initramfs_list.append((spec, os.path.join(scripts_archive_folder, f"{spec}_host_initramfs-spec.txt")))
 
 # original func is: https://github.com/OpenXiangShan/riscv-rootfs/blob/c61a659b454e5b038b5374a9091b29ad4995f13f/rootfsimg/spec_gen.py#L558
-    def __generate_initramfs(self, scripts_archive_folder, elf_folder, spec, dest_path, using_cpu2017=False, copies=1):
+    def __generate_initramfs(self, scripts_archive_folder, elf_folder, spec, dest_path, using_cpu2017=False, using_jemalloc=True, copies=1):
         spec_config = self.config["spec_info"]
 
         lines = self.config["default_initramfs_file"].copy()
+        lines += ["# libraries"]
+        lines += get_libraries_str(self.config["path_env_vars"]["RISCV"], using_jemalloc)
         if not isinstance(lines, list):
             raise ValueError("lines not a list type value")
 
@@ -490,13 +487,13 @@ def __generate_initramfs(self, scripts_archive_folder, elf_folder, spec, dest_pa
 
         lines.append(f"file /spec0/run.sh {scripts_archive_folder}/{spec}_run.sh 755 0 0")
 
-        for i in range(0, copies):
+        for i in range(0, 1):
             elf_file_abspath = os.path.realpath(f"{elf_folder}/{spec_config[spec]['base_name']}")
             lines.append(f"file /spec{i}/{spec_config[spec]['base_name']} {elf_file_abspath} 755 0 0")
 
         for i, filename in enumerate(spec_files):
             if len(filename.split()) == 1:
-                for j in range(0, copies):
+                for j in range(0, 1):
                     target_filename = f"file /spec{j}/{filename.split('/')[-1]} {cpu20xx_run_dir}/{filename} 755 0 0"
                     lines.append(target_filename)
 
@@ -509,13 +506,13 @@ def __generate_initramfs(self, scripts_archive_folder, elf_folder, spec, dest_pa
 
                 all_dirs, all_files = self.traverse_path(f"{cpu20xx_run_dir}{path}")
 
-                for i in range(0, copies):
+                for i in range(0, 1):
                     lines.append(f"dir /spec{i}/{name} 755 0 0")
                 for sub_dir in all_dirs:
-                    for i in range(0, copies):
+                    for i in range(0, 1):
                         lines.append(f"dir /spec{i}/{name}/{sub_dir} 755 0 0")
                 for file in all_files:
-                    for i in range(0, copies):
+                    for i in range(0, 1):
                         lines.append(f"file /spec{i}/{name}/{file} {cpu20xx_run_dir}{path}/{file} 755 0 0")
             else:
                 print(f"unknown filename: {filename}")
@@ -551,14 +548,19 @@ def __generate_run_scripts(self, spec, copies, redirect_output, using_cpu2017, w
 
         lines.append(f"echo '===== Start running {SPEC_20XX} ====='")
         lines.append(f"echo '======== BEGIN {spec} ========'")
+        lines.append("export LD_LIBRARY_PATH=/lib:$LD_LIBRARY_PATH")
+        lines.append("head -c 10 /dev/random | hexdump")
         lines.append("set -x")
 
-        for i in range(0, copies):
+        for i in range(0, 1):
             lines.append(f"md5sum /spec{i}/{spec_bin}")
 
         output_redirect = (" ").join([">", "out.log", "2>", "err.log"]) if redirect_output else ""
         force_output_redirect = (" ").join([">", "out.log", "2>", "err.log"])
 
+        output_redirect_workload_list_cpu2006 = ['xalancbmk']
+        output_redirect_workload_list_cpu2017 = ['perlbench', "povray", "xalancbmk", "leela"]
+
         taskN = []
         for i in range(0, int(copies)):
             taskN.append("#!/bin/sh")
@@ -569,14 +571,20 @@ def __generate_run_scripts(self, spec, copies, redirect_output, using_cpu2017, w
             if with_nemu_trap:
                 taskN.append("/spec_common/before_workload")
 
-            if spec_bin in ['xalancbmk']:
-                taskN.append(f'cd /spec{i} && ./{spec_bin} {spec_cmd} {force_output_redirect} ')
+            if using_cpu2017:
+                if spec_bin in output_redirect_workload_list_cpu2017:
+                    taskN.append(f'cd /spec{0} && ./{spec_bin} {spec_cmd} {force_output_redirect} ')
+                else:
+                    taskN.append(f'cd /spec{0} && ./{spec_bin} {spec_cmd} {output_redirect} ')
             else:
-                taskN.append(f'cd /spec{i} && ./{spec_bin} {spec_cmd} {output_redirect} ')
+                if spec_bin in output_redirect_workload_list_cpu2006:
+                    taskN.append(f'cd /spec{0} && ./{spec_bin} {spec_cmd} {force_output_redirect} ')
+                else:
+                    taskN.append(f'cd /spec{0} && ./{spec_bin} {spec_cmd} {output_redirect} ')
 
             taskN.append("date -R")
             if with_nemu_trap:
-                if emu=="NEMU":
+                if emu == "NEMU":
                     taskN.append("/spec_common/trap")
                 else:
                     taskN.append("/spec_common/qemu_trap")
@@ -777,7 +785,7 @@ def build_opensbi(self, copies, spec, bin_suffix):
 
         _, fw_payload_bin = self.kernel_list.pop()
         fw_payload_bin_size = os.path.getsize(fw_payload_bin)
-        fw_payload_fdt_addr = (((fw_payload_bin_size + 0x800000) + 0xfffff) // 0x100000) * 0x100000
+        fw_payload_fdt_addr = (((fw_payload_bin_size + 0x1800000) + 0xfffff) // 0x100000) * 0x100000
         fw_payload_fdt_addr = fw_payload_fdt_addr + 0x80000000
         print(f"SPEC: {spec}, file size: {fw_payload_bin_size:X}, fw_payload_fdt_addr: {fw_payload_fdt_addr:X}")
 
@@ -788,7 +796,7 @@ def build_opensbi(self, copies, spec, bin_suffix):
         if copies == 1:
             fw_payload_offset = 0x100000
         else:
-            fw_payload_offset = 0x700000
+            fw_payload_offset = 0x1700000
 
         shared_opensbi_command.append(
             ["make", "-C", OPENSBI_HOME, f"O={archive_buffer_layout['opensbi']}/build", "PLATFORM=generic", f"FW_PAYLOAD_PATH={fw_payload_bin}", f"FW_FDT_PATH={XIANGSHAN_FDT}", f"FW_PAYLOAD_OFFSET={fw_payload_offset}", f"FW_PAYLOAD_FDT_ADDR={fw_payload_fdt_addr}", "-j10"])

checkpoint_scripts/generate_checkpoint.py

@@ -192,7 +192,8 @@ def main(config_ctx: GlobalConfigCtx):
                                     "LINUX_HOME", "OPENSBI_HOME",
                                     "XIANGSHAN_FDT", "GCPT_HOME",
                                     "RISCV_ROOTFS_HOME", "CPU2006_RUN_DIR",
-                                    "CPU2017_RUN_DIR", "QEMU_HOME", "NEMU_HOME"
+                                    "CPU2017_RUN_DIR", "QEMU_HOME", "NEMU_HOME",
+                                    "RISCV"
                                 ],
                                 env_vars_to_check=["ARCH"])
     else:
@@ -201,7 +202,7 @@ def main(config_ctx: GlobalConfigCtx):
                                 path_env_vars_to_check=[
                                     "RISCV_PK_HOME", "GCPT_HOME",
                                     "RISCV_ROOTFS_HOME", "CPU2006_RUN_DIR",
-                                    "CPU2017_RUN_DIR"
+                                    "CPU2017_RUN_DIR", "RISCV"
                                 ])
 
     # if not set already exists archive id, script will generate benchmark assembly, generate rootfs, build bbl, and start checkpoint
@@ -218,6 +219,7 @@ def main(config_ctx: GlobalConfigCtx):
             builder.prepare_rootfs(
                 spec_app,
                 using_cpu2017=base_config["CPU2017"],
+                using_jemalloc=base_config["using_jemalloc"],
                 copies=base_config["copies"],
                 with_nemu_trap=True,
                 redirect_output=base_config["redirect_output"],

checkpoint_scripts/take_checkpoint.py

@@ -45,6 +45,7 @@ def __init__(self, start_id, times, path_env_vars_to_check=["NEMU_HOME", "QEMU_H
                     "workload_folder": "",
                     "compile_format": "zstd",
                     "interval": "20000000",
+                    "sync-interval": "20000",
                     "workload": "",
                     "buffer": "",
                     "bin_suffix": "",
@@ -299,7 +300,7 @@ def qemu_checkpoint_command(config, is_resume_from):
         # "numactl","--cpunodebind={}".format(config["cpu_bind"]),"--membind={}".format(config["mem_bind"]),
         config["QEMU"]["QEMU"],
         "-bios", "{}/{}{}".format(config["utils"]["workload_folder"], config["utils"]["workload"], config["utils"]["bin_suffix"]),
-        "-M", f'nemu,simpoint-path={simpoint_path},workload={config["utils"]["workload"]},cpt-interval={config["utils"]["interval"]},output-base-dir={config["utils"]["buffer"]},config-name={config["checkpoint"]["config"]},checkpoint-mode={"SimpointCheckpoint"}',
+        "-M", f'nemu,simpoint-path={simpoint_path},workload={config["utils"]["workload"]},cpt-interval={config["utils"]["interval"]},sync-interval={config["utils"]["interval"]},output-base-dir={config["utils"]["buffer"]},config-name={config["checkpoint"]["config"]},checkpoint-mode={"SimpointCheckpoint"}',
         "-nographic",
         "-m", config["QEMU"]["memory"],
         "-smp", config["QEMU"]["smp"],
@@ -345,8 +346,8 @@ def cluster_func(profiling_id, cluster_id, config, is_resume_from=False, dry_run
 
     cluster_config["execute_mode"] = "cluster"
 
-    cluster_config["out-log"] = os.path.join(config["utils"]["log_folder"], "cluster-{}-{}".format(profiling_id, cluster_id), config["utils"]["workload"],"cluster.out.log")
-    cluster_config["err-log"] = os.path.join(config["utils"]["log_folder"], "cluster-{}-{}".format(profiling_id, cluster_id), config["utils"]["workload"],"cluster.err.log")
+    cluster_config["out-log"] = os.path.join(config["utils"]["log_folder"], "cluster-{}-{}".format(profiling_id, cluster_id), config["utils"]["workload"], "cluster.out.log")
+    cluster_config["err-log"] = os.path.join(config["utils"]["log_folder"], "cluster-{}-{}".format(profiling_id, cluster_id), config["utils"]["workload"], "cluster.err.log")
 
     if not dry_run:
         cluster_config["command"] = cluster_command(cluster_config, is_resume_from=is_resume_from)