diff --git a/checkpoint_scripts/config.yaml b/checkpoint_scripts/config.yaml index c3fb145..bcaa903 100644 --- a/checkpoint_scripts/config.yaml +++ b/checkpoint_scripts/config.yaml @@ -1,14 +1,14 @@ base_config: message: "NULL" - spec_app_list: null - spec_apps: "cactusADM,tonto" - elf_folder: "./jemalloc_elf" + spec_app_list: "/nfs/home/share/jiaxiaoyu/dual_core_checkpoint/checkpoint_scripts/checkpoint_scripts/spec2k17.lst" + spec_apps: null + elf_folder: "/nfs/home/share/jiaxiaoyu/checkpoint/spec17_checkpoint/spec17_gcc12.2.0_rv64gcb" times: "1,1,1" start_id: "0,0,0" emulator: "QEMU" build_bbl_only: true - max_threads: 70 - CPU2017: false + max_threads: 60 + CPU2017: true generate_rootfs_script_only: false copies: 2 archive_id: null @@ -17,11 +17,12 @@ base_config: mem_bind: 0 bootloader: "opensbi" all_in_one_workload: true - boot_for_test: false + boot_for_test: true enable_h_ext: false + using_jemalloc: true archive_id_config: gcc_version: "gcc12.2.0" riscv_ext: "rv64gcb" base_or_fixed: "base" - special_flag: "intFppOff_for_qemu_dual_core" + special_flag: "intFppOff_for_spec17_dual_core" group: "archgroup" \ No newline at end of file diff --git a/checkpoint_scripts/find_dynamic_lib.py b/checkpoint_scripts/find_dynamic_lib.py new file mode 100644 index 0000000..2279cf4 --- /dev/null +++ b/checkpoint_scripts/find_dynamic_lib.py @@ -0,0 +1,54 @@ +import os +import stat +import magic + +def is_dynamic_library(file_path): + try: + with open(file_path, 'rb') as file: + header = file.read(4) + return header.startswith(b'\x7fELF') + except Exception as e: + print(f"Could not check file type for {file_path}: {e}") + return False + +def find_dynamic_libraries(search_dir='.'): + dynamic_libraries = [] + for root, _, files in os.walk(search_dir): + for filename in files: + file_path = os.path.join(root, filename) + try: + if is_dynamic_library(file_path) or os.path.islink(file_path): + dynamic_libraries.append(file_path) + except OSError as e: + print(f"Error accessing file {file_path}: {e}") + return dynamic_libraries + +def get_dynamic_libraries_list(search_dir): + libraries = find_dynamic_libraries(search_dir) + initramfs_string_list = [] + for lib in libraries: + initramfs_string_list.append([f"file /lib/{lib.split('/')[-1]}", lib, "755 0 0"]) + + return initramfs_string_list + +def get_jemalloc_libraries_list(riscv_toolchains_top): + initramfs_string_list = [ + ["file /lib/libjemalloc.so", f"{riscv_toolchains_top}/lib/libjemalloc.so", "755 0 0"], + ["file /lib/libjemalloc.so.2", f"{riscv_toolchains_top}/lib/libjemalloc.so.2", "755 0 0"], + ] + return initramfs_string_list + +def check_file_exists(file_path): + if os.path.exists(file_path): + return True + return False + + +def get_libraries_str(riscv_toolchains_top, using_jemalloc): + libraries = get_dynamic_libraries_list(os.path.join(riscv_toolchains_top, "sysroot", "lib")) + if using_jemalloc: + libraries += get_jemalloc_libraries_list(riscv_toolchains_top) +# for i in libraries: +# assert(check_file_exists(i[1])) + libraries_str = list(map(lambda x: f"{x[0]} {x[1]} {x[2]}", libraries)) + return libraries_str diff --git a/checkpoint_scripts/generate_bbl.py b/checkpoint_scripts/generate_bbl.py index bc41532..fa5cdfc 100644 --- a/checkpoint_scripts/generate_bbl.py +++ b/checkpoint_scripts/generate_bbl.py @@ -6,6 +6,7 @@ import shutil from config import BaseConfig from typing import Tuple, List +from find_dynamic_lib import get_libraries_str import subprocess class RootfsBuilder(BaseConfig): @@ -43,13 +44,6 @@ def __init__(self, archive_buffer_layout, spec_info, path_env_vars_to_check=None "dir /root 755 0 0", "dir /var/log 755 0 0", "", "nod /dev/console 644 0 0 c 5 1", "nod /dev/null 644 0 0 c 1 3", "", "nod /dev/urandom 644 0 0 c 1 9", "nod /dev/random 644 0 0 c 1 8", - "# libraries", - "file /lib/ld-linux-riscv64-lp64d.so.1 ${RISCV}/sysroot/lib/ld-linux-riscv64-lp64d.so.1 755 0 0", - "file /lib/libc.so.6 ${RISCV}/sysroot/lib/libc.so.6 755 0 0", - "file /lib/libresolv.so.2 ${RISCV}/sysroot/lib/libresolv.so.2 755 0 0", - "file /lib/libm.so.6 ${RISCV}/sysroot/lib/libm.so.6 755 0 0", - "file /lib/libdl.so.2 ${RISCV}/sysroot/lib/libdl.so.2 755 0 0", - "file /lib/libpthread.so.0 ${RISCV}/sysroot/lib/libpthread.so.0 755 0 0", "", "# busybox", "file /bin/busybox ${RISCV_ROOTFS_HOME}/rootfsimg/build/busybox 755 0 0", "file /etc/inittab ${RISCV_ROOTFS_HOME}/rootfsimg/inittab-spec 755 0 0", @@ -418,6 +412,7 @@ def __init__(self, archive_buffer_layout, spec_info, path_env_vars_to_check=None } ) + def prepare_rcS(self, guest_memory, guest_cpus, spec, scripts_archive_folder): lines = [] lines.append("#! /bin/sh") @@ -431,10 +426,10 @@ def prepare_rcS(self, guest_memory, guest_cpus, spec, scripts_archive_folder): f.writelines(map(lambda x: x + "\n", lines)) self.rcs_list.append((spec, os.path.join(scripts_archive_folder, f"{spec}_rcS-spec.txt"))) - - def prepare_rootfs(self, spec, using_cpu2017, copies, with_nemu_trap, redirect_output, emu): + + def prepare_rootfs(self, spec, using_cpu2017, using_jemalloc, copies, with_nemu_trap, redirect_output, emu): archive_buffer_layout = self.config["archive_buffer_layout"] - self.__generate_initramfs(archive_buffer_layout["scripts"], archive_buffer_layout["elf"], spec, os.path.join(self.config["path_env_vars"]["RISCV_ROOTFS_HOME"], "rootfsimg"), using_cpu2017, copies) + self.__generate_initramfs(archive_buffer_layout["scripts"], archive_buffer_layout["elf"], spec, os.path.join(self.config["path_env_vars"]["RISCV_ROOTFS_HOME"], "rootfsimg"), using_cpu2017, using_jemalloc, copies) self.__generate_run_scripts(spec, copies, redirect_output, using_cpu2017, with_nemu_trap, os.path.join(self.config["path_env_vars"]["RISCV_ROOTFS_HOME"], "rootfsimg"), archive_buffer_layout["scripts"], emu) @@ -471,10 +466,12 @@ def __generate_host_initramfs(self, scripts_archive_folder, spec): self.host_initramfs_list.append((spec, os.path.join(scripts_archive_folder, f"{spec}_host_initramfs-spec.txt"))) # original func is: https://github.com/OpenXiangShan/riscv-rootfs/blob/c61a659b454e5b038b5374a9091b29ad4995f13f/rootfsimg/spec_gen.py#L558 - def __generate_initramfs(self, scripts_archive_folder, elf_folder, spec, dest_path, using_cpu2017=False, copies=1): + def __generate_initramfs(self, scripts_archive_folder, elf_folder, spec, dest_path, using_cpu2017=False, using_jemalloc=True, copies=1): spec_config = self.config["spec_info"] lines = self.config["default_initramfs_file"].copy() + lines += ["# libraries"] + lines += get_libraries_str(self.config["path_env_vars"]["RISCV"], using_jemalloc) if not isinstance(lines, list): raise ValueError("lines not a list type value") @@ -490,13 +487,13 @@ def __generate_initramfs(self, scripts_archive_folder, elf_folder, spec, dest_pa lines.append(f"file /spec0/run.sh {scripts_archive_folder}/{spec}_run.sh 755 0 0") - for i in range(0, copies): + for i in range(0, 1): elf_file_abspath = os.path.realpath(f"{elf_folder}/{spec_config[spec]['base_name']}") lines.append(f"file /spec{i}/{spec_config[spec]['base_name']} {elf_file_abspath} 755 0 0") for i, filename in enumerate(spec_files): if len(filename.split()) == 1: - for j in range(0, copies): + for j in range(0, 1): target_filename = f"file /spec{j}/{filename.split('/')[-1]} {cpu20xx_run_dir}/{filename} 755 0 0" lines.append(target_filename) @@ -509,13 +506,13 @@ def __generate_initramfs(self, scripts_archive_folder, elf_folder, spec, dest_pa all_dirs, all_files = self.traverse_path(f"{cpu20xx_run_dir}{path}") - for i in range(0, copies): + for i in range(0, 1): lines.append(f"dir /spec{i}/{name} 755 0 0") for sub_dir in all_dirs: - for i in range(0, copies): + for i in range(0, 1): lines.append(f"dir /spec{i}/{name}/{sub_dir} 755 0 0") for file in all_files: - for i in range(0, copies): + for i in range(0, 1): lines.append(f"file /spec{i}/{name}/{file} {cpu20xx_run_dir}{path}/{file} 755 0 0") else: print(f"unknown filename: {filename}") @@ -551,14 +548,19 @@ def __generate_run_scripts(self, spec, copies, redirect_output, using_cpu2017, w lines.append(f"echo '===== Start running {SPEC_20XX} ====='") lines.append(f"echo '======== BEGIN {spec} ========'") + lines.append("export LD_LIBRARY_PATH=/lib:$LD_LIBRARY_PATH") + lines.append("head -c 10 /dev/random | hexdump") lines.append("set -x") - for i in range(0, copies): + for i in range(0, 1): lines.append(f"md5sum /spec{i}/{spec_bin}") output_redirect = (" ").join([">", "out.log", "2>", "err.log"]) if redirect_output else "" force_output_redirect = (" ").join([">", "out.log", "2>", "err.log"]) + output_redirect_workload_list_cpu2006 = ['xalancbmk'] + output_redirect_workload_list_cpu2017 = ['perlbench', "povray", "xalancbmk", "leela"] + taskN = [] for i in range(0, int(copies)): taskN.append("#!/bin/sh") @@ -569,14 +571,20 @@ def __generate_run_scripts(self, spec, copies, redirect_output, using_cpu2017, w if with_nemu_trap: taskN.append("/spec_common/before_workload") - if spec_bin in ['xalancbmk']: - taskN.append(f'cd /spec{i} && ./{spec_bin} {spec_cmd} {force_output_redirect} ') + if using_cpu2017: + if spec_bin in output_redirect_workload_list_cpu2017: + taskN.append(f'cd /spec{0} && ./{spec_bin} {spec_cmd} {force_output_redirect} ') + else: + taskN.append(f'cd /spec{0} && ./{spec_bin} {spec_cmd} {output_redirect} ') else: - taskN.append(f'cd /spec{i} && ./{spec_bin} {spec_cmd} {output_redirect} ') + if spec_bin in output_redirect_workload_list_cpu2006: + taskN.append(f'cd /spec{0} && ./{spec_bin} {spec_cmd} {force_output_redirect} ') + else: + taskN.append(f'cd /spec{0} && ./{spec_bin} {spec_cmd} {output_redirect} ') taskN.append("date -R") if with_nemu_trap: - if emu=="NEMU": + if emu == "NEMU": taskN.append("/spec_common/trap") else: taskN.append("/spec_common/qemu_trap") @@ -777,7 +785,7 @@ def build_opensbi(self, copies, spec, bin_suffix): _, fw_payload_bin = self.kernel_list.pop() fw_payload_bin_size = os.path.getsize(fw_payload_bin) - fw_payload_fdt_addr = (((fw_payload_bin_size + 0x800000) + 0xfffff) // 0x100000) * 0x100000 + fw_payload_fdt_addr = (((fw_payload_bin_size + 0x1800000) + 0xfffff) // 0x100000) * 0x100000 fw_payload_fdt_addr = fw_payload_fdt_addr + 0x80000000 print(f"SPEC: {spec}, file size: {fw_payload_bin_size:X}, fw_payload_fdt_addr: {fw_payload_fdt_addr:X}") @@ -788,7 +796,7 @@ def build_opensbi(self, copies, spec, bin_suffix): if copies == 1: fw_payload_offset = 0x100000 else: - fw_payload_offset = 0x700000 + fw_payload_offset = 0x1700000 shared_opensbi_command.append( ["make", "-C", OPENSBI_HOME, f"O={archive_buffer_layout['opensbi']}/build", "PLATFORM=generic", f"FW_PAYLOAD_PATH={fw_payload_bin}", f"FW_FDT_PATH={XIANGSHAN_FDT}", f"FW_PAYLOAD_OFFSET={fw_payload_offset}", f"FW_PAYLOAD_FDT_ADDR={fw_payload_fdt_addr}", "-j10"]) diff --git a/checkpoint_scripts/generate_checkpoint.py b/checkpoint_scripts/generate_checkpoint.py index a6966e7..b0c6f51 100644 --- a/checkpoint_scripts/generate_checkpoint.py +++ b/checkpoint_scripts/generate_checkpoint.py @@ -192,7 +192,8 @@ def main(config_ctx: GlobalConfigCtx): "LINUX_HOME", "OPENSBI_HOME", "XIANGSHAN_FDT", "GCPT_HOME", "RISCV_ROOTFS_HOME", "CPU2006_RUN_DIR", - "CPU2017_RUN_DIR", "QEMU_HOME", "NEMU_HOME" + "CPU2017_RUN_DIR", "QEMU_HOME", "NEMU_HOME", + "RISCV" ], env_vars_to_check=["ARCH"]) else: @@ -201,7 +202,7 @@ def main(config_ctx: GlobalConfigCtx): path_env_vars_to_check=[ "RISCV_PK_HOME", "GCPT_HOME", "RISCV_ROOTFS_HOME", "CPU2006_RUN_DIR", - "CPU2017_RUN_DIR" + "CPU2017_RUN_DIR", "RISCV" ]) # if not set already exists archive id, script will generate benchmark assembly, generate rootfs, build bbl, and start checkpoint @@ -218,6 +219,7 @@ def main(config_ctx: GlobalConfigCtx): builder.prepare_rootfs( spec_app, using_cpu2017=base_config["CPU2017"], + using_jemalloc=base_config["using_jemalloc"], copies=base_config["copies"], with_nemu_trap=True, redirect_output=base_config["redirect_output"], diff --git a/checkpoint_scripts/take_checkpoint.py b/checkpoint_scripts/take_checkpoint.py index f91bc98..bf3a5a0 100644 --- a/checkpoint_scripts/take_checkpoint.py +++ b/checkpoint_scripts/take_checkpoint.py @@ -45,6 +45,7 @@ def __init__(self, start_id, times, path_env_vars_to_check=["NEMU_HOME", "QEMU_H "workload_folder": "", "compile_format": "zstd", "interval": "20000000", + "sync-interval": "20000", "workload": "", "buffer": "", "bin_suffix": "", @@ -299,7 +300,7 @@ def qemu_checkpoint_command(config, is_resume_from): # "numactl","--cpunodebind={}".format(config["cpu_bind"]),"--membind={}".format(config["mem_bind"]), config["QEMU"]["QEMU"], "-bios", "{}/{}{}".format(config["utils"]["workload_folder"], config["utils"]["workload"], config["utils"]["bin_suffix"]), - "-M", f'nemu,simpoint-path={simpoint_path},workload={config["utils"]["workload"]},cpt-interval={config["utils"]["interval"]},output-base-dir={config["utils"]["buffer"]},config-name={config["checkpoint"]["config"]},checkpoint-mode={"SimpointCheckpoint"}', + "-M", f'nemu,simpoint-path={simpoint_path},workload={config["utils"]["workload"]},cpt-interval={config["utils"]["interval"]},sync-interval={config["utils"]["interval"]},output-base-dir={config["utils"]["buffer"]},config-name={config["checkpoint"]["config"]},checkpoint-mode={"SimpointCheckpoint"}', "-nographic", "-m", config["QEMU"]["memory"], "-smp", config["QEMU"]["smp"], @@ -345,8 +346,8 @@ def cluster_func(profiling_id, cluster_id, config, is_resume_from=False, dry_run cluster_config["execute_mode"] = "cluster" - cluster_config["out-log"] = os.path.join(config["utils"]["log_folder"], "cluster-{}-{}".format(profiling_id, cluster_id), config["utils"]["workload"],"cluster.out.log") - cluster_config["err-log"] = os.path.join(config["utils"]["log_folder"], "cluster-{}-{}".format(profiling_id, cluster_id), config["utils"]["workload"],"cluster.err.log") + cluster_config["out-log"] = os.path.join(config["utils"]["log_folder"], "cluster-{}-{}".format(profiling_id, cluster_id), config["utils"]["workload"], "cluster.out.log") + cluster_config["err-log"] = os.path.join(config["utils"]["log_folder"], "cluster-{}-{}".format(profiling_id, cluster_id), config["utils"]["workload"], "cluster.err.log") if not dry_run: cluster_config["command"] = cluster_command(cluster_config, is_resume_from=is_resume_from)