diff --git a/.gitignore b/.gitignore index 71b817354..275db7547 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,35 @@ -.DS_Store +# IDE folders +.idea/ +.vs/ + +# Caches __pycache__ -.idea -.ninja_* .mypy_cache -*.exe -build -build.ninja -objdiff.json +.cache/ + +# Original files orig/*/* !orig/*/.gitkeep +*.dol +*.rel +*.elf +*.o +*.map +*.MAP + +# Build files +build/ +.ninja_* +build.ninja + +# decompctx output +ctx.* +*.ctx + +# Generated configs +objdiff.json +compile_commands.json + +# Miscellaneous /*.txt -ctx.c +*.exe diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 000000000..c20797ff5 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,12 @@ +{ + "recommendations": [ + "llvm-vs-code-extensions.vscode-clangd", + "ms-python.black-formatter", + "ms-python.flake8", + ], + "unwantedRecommendations": [ + "ms-vscode.cmake-tools", + "ms-vscode.cpptools-extension-pack", + "ms-vscode.cpptools", + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json index fd9c4e58f..09e36b599 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,30 +1,23 @@ { "[c]": { "files.encoding": "utf8", - "editor.defaultFormatter": "xaver.clang-format" + "editor.defaultFormatter": "llvm-vs-code-extensions.vscode-clangd" }, "[cpp]": { "files.encoding": "utf8", - "editor.defaultFormatter": "xaver.clang-format" + "editor.defaultFormatter": "llvm-vs-code-extensions.vscode-clangd" }, "[python]": { "editor.defaultFormatter": "ms-python.black-formatter" }, + // "editor.tabSize": 2, + "files.autoSave": "onFocusChange", "files.insertFinalNewline": true, "files.trimFinalNewlines": true, "files.associations": { - ".fantomasignore": "ignore", - "*.inc": "cpp", - "new": "cpp" + "*.inc": "c", + ".clangd": "yaml" }, - "search.useIgnoreFiles": false, - "search.exclude": { - "build/*/config.json": true, - "build/**/*.MAP": true, - "build.ninja": true, - ".ninja_*": true, - "objdiff.json": true, - ".mypy_cache": true - }, - "C_Cpp.errorSquiggles": "enabled" + // Disable C/C++ IntelliSense, use clangd instead + "C_Cpp.intelliSenseEngine": "disabled", } diff --git a/.vscode/tasks.json b/.vscode/tasks.json new file mode 100644 index 000000000..6dc12130b --- /dev/null +++ b/.vscode/tasks.json @@ -0,0 +1,16 @@ +{ + // Use Ctrl+Shift+B to run build tasks. + // Or "Run Build Task" in the Command Palette. + "version": "2.0.0", + "tasks": [ + { + "label": "ninja", + "type": "shell", + "command": "ninja", + "group": { + "kind": "build", + "isDefault": true + } + }, + ] +} diff --git a/configure.py b/configure.py index 3c01dc086..8d160a4c8 100644 --- a/configure.py +++ b/configure.py @@ -19,6 +19,7 @@ from tools.project import ( Object, + ProgressCategory, ProjectConfig, calculate_progress, generate_build, @@ -71,11 +72,6 @@ action="store_true", help="generate map file(s)", ) -parser.add_argument( - "--no-asm", - action="store_true", - help="don't incorporate .s files from asm directory", -) parser.add_argument( "--debug", action="store_true", @@ -94,6 +90,12 @@ type=Path, help="path to decomp-toolkit binary or source (optional)", ) +parser.add_argument( + "--objdiff", + metavar="BINARY | DIR", + type=Path, + help="path to objdiff-cli binary or source (optional)", +) parser.add_argument( "--sjiswrap", metavar="EXE", @@ -111,6 +113,12 @@ action="store_true", help="builds equivalent (but non-matching) or modded objects", ) +parser.add_argument( + "--no-progress", + dest="progress", + action="store_false", + help="disable progress calculation", +) args = parser.parse_args() config = ProjectConfig() @@ -120,22 +128,25 @@ # Apply arguments config.build_dir = args.build_dir config.dtk_path = args.dtk +config.objdiff_path = args.objdiff config.binutils_path = args.binutils config.compilers_path = args.compilers -config.debug = args.debug config.generate_map = args.map config.non_matching = args.non_matching config.sjiswrap_path = args.sjiswrap +config.progress = args.progress if not is_windows(): config.wrapper = args.wrapper -if args.no_asm: +# Don't build asm unless we're --non-matching +if not config.non_matching: config.asm_dir = None # Tool versions config.binutils_tag = "2.42-1" -config.compilers_tag = "20240702" -config.dtk_tag = "v1.4.0" -config.sjiswrap_tag = "v1.1.1" +config.compilers_tag = "20240706" +config.dtk_tag = "v1.4.1" +config.objdiff_tag = "v2.7.1" +config.sjiswrap_tag = "v1.2.0" config.wibo_tag = "0.6.11" # Project @@ -146,17 +157,26 @@ "--strip-local-absolute", "-I include", f"-I build/{config.version}/include", - f"--defsym version={version_num}", + f"--defsym BUILD_VERSION={version_num}", + f"--defsym VERSION_{config.version}", ] config.ldflags = [ "-fp hardware", "-nodefaults", - "-warn off", - # "-listclosure", # Uncomment for Wii linkers ] +if args.debug: + config.ldflags.append("-g") # Or -gdwarf-2 for Wii linkers +if args.map: + config.ldflags.append("-mapunused") + # config.ldflags.append("-listclosure") # For Wii linkers + # Use for any additional files that should cause a re-configure when modified config.reconfig_deps = [] +# Optional numeric ID for decomp.me preset +# Can be overridden in libraries or objects +config.scratch_preset_id = 65 # Battle for Bikini Bottom + # Base flags, common to most GC/Wii games. # Generally leave untouched, with overrides added below. cflags_base = [ @@ -179,11 +199,13 @@ "-multibyte", # For Wii compilers, replace with `-enc SJIS` "-i include", f"-i build/{config.version}/include", - f"-DVERSION={version_num}", + f"-DBUILD_VERSION={version_num}", + f"-DVERSION_{config.version}", ] # Debug flags -if config.debug: +if args.debug: + # Or -sym dwarf-2 for Wii compilers cflags_base.extend(["-sym on", "-DDEBUG=1"]) else: cflags_base.append("-DNDEBUG=1") @@ -236,7 +258,7 @@ def DolphinLib(lib_name: str, objects: List[Object]) -> Dict[str, Any]: "lib": lib_name, "mw_version": "GC/1.2.5n", "cflags": cflags_base, - "host": False, + "progress_category": "sdk", "objects": objects, } @@ -247,7 +269,7 @@ def RenderWareLib(lib_name: str, objects: List[Object]) -> Dict[str, Any]: "lib": lib_name, "mw_version": "GC/1.3.2", "cflags": cflags_base, - "host": False, + "progress_category": "sdk", "objects": objects, } @@ -258,7 +280,7 @@ def Rel(lib_name: str, objects: List[Object]) -> Dict[str, Any]: "lib": lib_name, "mw_version": "GC/1.3.2", "cflags": cflags_rel, - "host": True, + "progress_category": "game", "objects": objects, } @@ -267,6 +289,12 @@ def Rel(lib_name: str, objects: List[Object]) -> Dict[str, Any]: NonMatching = False # Object does not match and should not be linked Equivalent = config.non_matching # Object should be linked when configured with --non-matching + +# Object is only matching for specific versions +def MatchingFor(*versions): + return config.version in versions + + config.warn_missing_config = True config.warn_missing_source = False config.libs = [ @@ -274,7 +302,7 @@ def Rel(lib_name: str, objects: List[Object]) -> Dict[str, Any]: "lib": "SB", "mw_version": config.linker_version, "cflags": cflags_bfbb, - "host": True, + "progress_category": "game", "objects": [ Object(NonMatching, "SB/Core/x/xAnim.cpp", extra_cflags=["-sym on"]), Object(Matching, "SB/Core/x/xBase.cpp"), @@ -506,7 +534,7 @@ def Rel(lib_name: str, objects: List[Object]) -> Dict[str, Any]: "lib": "binkngc", "mw_version": "GC/1.3.2", "cflags": cflags_runtime, - "host": False, + "progress_category": "sdk", "objects": [ Object(NonMatching, "bink/src/sdk/decode/ngc/binkngc.c"), Object(NonMatching, "bink/src/sdk/decode/ngc/ngcsnd.c"), @@ -703,7 +731,7 @@ def Rel(lib_name: str, objects: List[Object]) -> Dict[str, Any]: "lib": "Runtime.PPCEABI.H", "mw_version": config.linker_version, "cflags": cflags_runtime, - "host": False, + "progress_category": "sdk", "objects": [ Object(NonMatching, "PowerPC_EABI_Support/Runtime/Src/__mem.c"), Object(NonMatching, "PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Src/abort_exit.c"), @@ -781,7 +809,7 @@ def Rel(lib_name: str, objects: List[Object]) -> Dict[str, Any]: "lib": "TRK_Minnow_Dolphin", "mw_version": "GC/1.3.2", "cflags": cflags_runtime, - "host": False, + "progress_category": "sdk", "objects": [ Object(NonMatching, "TRK_MINNOW_DOLPHIN/Portable/mainloop.c"), Object(NonMatching, "TRK_MINNOW_DOLPHIN/Portable/nubevent.c"), @@ -813,7 +841,7 @@ def Rel(lib_name: str, objects: List[Object]) -> Dict[str, Any]: "lib": "OdemuExi2", "mw_version": "GC/1.3.2", "cflags": cflags_runtime, - "host": False, + "progress_category": "sdk", "objects": [ Object(NonMatching, "OdemuExi2/DebuggerDriver.c"), ], @@ -834,13 +862,6 @@ def Rel(lib_name: str, objects: List[Object]) -> Dict[str, Any]: Object(NonMatching, "rwsdk/plugin/hanim/rphanim.c"), ], ), - RenderWareLib( - "rphanim", - [ - Object(NonMatching, "rwsdk/plugin/hanim/stdkey.c"), - Object(NonMatching, "rwsdk/plugin/hanim/rphanim.c"), - ], - ), RenderWareLib( "rpmatfx", [ @@ -1002,12 +1023,37 @@ def Rel(lib_name: str, objects: List[Object]) -> Dict[str, Any]: ), ] + +# Optional callback to adjust link order. This can be used to add, remove, or reorder objects. +# This is called once per module, with the module ID and the current link order. +# +# For example, this adds "dummy.c" to the end of the DOL link order if configured with --non-matching. +# "dummy.c" *must* be configured as a Matching (or Equivalent) object in order to be linked. +def link_order_callback(module_id: int, objects: List[str]) -> List[str]: + # Don't modify the link order for matching builds + if not config.non_matching: + return objects + if module_id == 0: # DOL + return objects + ["dummy.c"] + return objects + +# Uncomment to enable the link order callback. +# config.link_order_callback = link_order_callback + + +# Optional extra categories for progress tracking +# Adjust as desired for your project +config.progress_categories = [ + ProgressCategory("game", "Game Code"), + ProgressCategory("sdk", "SDK Code"), +] +config.progress_each_module = args.verbose + if args.mode == "configure": # Write build.ninja and objdiff.json generate_build(config) elif args.mode == "progress": # Print progress and write progress.json - config.progress_each_module = args.verbose calculate_progress(config) else: sys.exit("Unknown mode: " + args.mode) diff --git a/docs/splits.md b/docs/splits.md index 484e89266..a05eaa7dc 100644 --- a/docs/splits.md +++ b/docs/splits.md @@ -26,11 +26,24 @@ path/to/file.cpp: [file attributes] ### File attributes -- `comment:` Overrides the `mw_comment_version` setting in [`config.yml`](/config/GAMEID/config.example.yml) for this file. See [Comment section](comment_section.md). +- `comment:` Overrides the `mw_comment_version` setting in [`config.yml`](/config/GAMEID/config.example.yml) for this file. See [Comment section](comment_section.md). + - `comment:0` is used to disable `.comment` section generation for a file that wasn't compiled with `mwcc`. + Example: `TRK_MINNOW_DOLPHIN/ppc/Export/targsupp.s: comment:0` + This file was assembled and only contains label symbols. Generating a `.comment` section for it will crash `mwld`. + +- `order:` Allows influencing the resolved link order of objects. This is **not required**, as decomp-toolkit will generate the link order automatically. This can be used to fine-tune the link order for ambiguous cases. + Example: + ``` + file1.cpp: order:0 + ... + + file2.cpp: order:1 + ... -`comment:0` is used to disable `.comment` section generation for a file that wasn't compiled with `mwcc`. -Example: `TRK_MINNOW_DOLPHIN/ppc/Export/targsupp.s: comment:0` -This file was assembled and only contains label symbols. Generating a `.comment` section for it will crash `mwld`. + file3.cpp: order:2 + ... + ``` + This ensures that `file2.cpp` is always anchored in between 1 and 3 when resolving the final link order. ### Section attributes diff --git a/docs/symbols.md b/docs/symbols.md index 5ce4b2522..ad56a3179 100644 --- a/docs/symbols.md +++ b/docs/symbols.md @@ -30,7 +30,7 @@ All attributes are optional, and are separated by spaces. - `size:` The size of the symbol. - `scope:` The symbol's visibility. `global` (default), `local` or `weak`. - `align:` The symbol's alignment. -- `data:` The data type used when writing disassembly. `byte`, `2byte`, `4byte`, `8byte`, `float`, `double`, `string`, `wstring`, `string_table`, or `wstring_table`. +- `data:` The data type used when writing disassembly. `byte`, `2byte`, `4byte`, `8byte`, `float`, `double`, `int`, `short`, `string`, `wstring`, `string_table`, or `wstring_table`. - `hidden` Marked as "hidden" in the generated object. (Only used for extab) - `force_active` Marked as ["exported"](comment_section.md) in the generated object, and added to `FORCEACTIVE` in the generated `ldscript.lcf`. Prevents the symbol from being deadstripped. - `noreloc` Prevents the _contents_ of the symbol from being interpreted as addresses. Used for objects containing data that look like pointers, but aren't. diff --git a/tools/decompctx.py b/tools/decompctx.py index 6251f3841..f2f31dfac 100644 --- a/tools/decompctx.py +++ b/tools/decompctx.py @@ -18,67 +18,63 @@ script_dir = os.path.dirname(os.path.realpath(__file__)) root_dir = os.path.abspath(os.path.join(script_dir, "..")) src_dir = os.path.join(root_dir, "src") -include_dirs = [ - os.path.join(root_dir, "include"), - os.path.join(root_dir, "include/bink"), - os.path.join(root_dir, "include/PowerPC_EABI_Support/MSL/MSL_C/MSL_Common/Include"), - os.path.join(root_dir, "include/PowerPC_EABI_Support/MSL/MSL_C++/MSL_Common/Include"), - os.path.join(root_dir, "include/dolphin"), - os.path.join(root_dir, "include/inline"), - os.path.join(root_dir, "include/rwsdk"), - os.path.join(root_dir, "src/SB/Core/gc"), - os.path.join(root_dir, "src/SB/Core/x"), - os.path.join(root_dir, "src/SB/Game"), - # Add additional include directories here -] - -include_pattern = re.compile(r'^#include\s*[<"](.+?)[>"]$') -guard_pattern = re.compile(r"^#ifndef\s+(.*)$") +include_dirs: List[str] = [] # Set with -I flag + +include_pattern = re.compile(r'^#\s*include\s*[<"](.+?)[>"]') +guard_pattern = re.compile(r"^#\s*ifndef\s+(.*)$") +once_pattern = re.compile(r"^#\s*pragma\s+once$") defines = set() +deps = [] -def import_h_file(in_file: str, r_path: str, deps: List[str]) -> str: +def import_h_file(in_file: str, r_path: str) -> str: rel_path = os.path.join(root_dir, r_path, in_file) if os.path.exists(rel_path): - return import_c_file(rel_path, deps) + return import_c_file(rel_path) for include_dir in include_dirs: inc_path = os.path.join(include_dir, in_file) if os.path.exists(inc_path): - return import_c_file(inc_path, deps) + return import_c_file(inc_path) else: print("Failed to locate", in_file) return "" -def import_c_file(in_file: str, deps: List[str]) -> str: +def import_c_file(in_file: str) -> str: in_file = os.path.relpath(in_file, root_dir) deps.append(in_file) out_text = "" try: with open(in_file, encoding="utf-8") as file: - out_text += process_file(in_file, list(file), deps) + out_text += process_file(in_file, list(file)) except Exception: with open(in_file) as file: - out_text += process_file(in_file, list(file), deps) + out_text += process_file(in_file, list(file)) return out_text -def process_file(in_file: str, lines: List[str], deps: List[str]) -> str: +def process_file(in_file: str, lines: List[str]) -> str: out_text = "" for idx, line in enumerate(lines): - guard_match = guard_pattern.match(line.strip()) if idx == 0: + guard_match = guard_pattern.match(line.strip()) if guard_match: if guard_match[1] in defines: break defines.add(guard_match[1]) + else: + once_match = once_pattern.match(line.strip()) + if once_match: + if in_file in defines: + break + defines.add(in_file) print("Processing file", in_file) include_match = include_pattern.match(line.strip()) if include_match and not include_match[1].endswith(".s"): out_text += f'/* "{in_file}" line {idx} "{include_match[1]}" */\n' - out_text += import_h_file(include_match[1], os.path.dirname(in_file), deps) + out_text += import_h_file(include_match[1], os.path.dirname(in_file)) out_text += f'/* end "{include_match[1]}" */\n' else: out_text += line @@ -109,10 +105,19 @@ def main(): "--depfile", help="""Dependency file""", ) + parser.add_argument( + "-I", + "--include", + help="""Include directory""", + action="append", + ) args = parser.parse_args() - deps = [] - output = import_c_file(args.c_file, deps) + if args.include is None: + exit("No include directories specified") + global include_dirs + include_dirs = args.include + output = import_c_file(args.c_file) with open(os.path.join(root_dir, args.output), "w", encoding="utf-8") as f: f.write(output) diff --git a/tools/download_tool.py b/tools/download_tool.py index 4ff874be5..7adbf0d70 100644 --- a/tools/download_tool.py +++ b/tools/download_tool.py @@ -56,16 +56,7 @@ def dtk_url(tag: str) -> str: return f"{repo}/releases/download/{tag}/dtk-{system}-{arch}{suffix}" -def sjiswrap_url(tag: str) -> str: - repo = "https://github.com/encounter/sjiswrap" - return f"{repo}/releases/download/{tag}/sjiswrap-windows-x86.exe" - - -def wibo_url(tag: str) -> str: - repo = "https://github.com/decompals/wibo" - return f"{repo}/releases/download/{tag}/wibo" - -def objdiffcli_url(tag: str) -> str: +def objdiff_cli_url(tag: str) -> str: uname = platform.uname() suffix = "" system = uname.system.lower() @@ -80,15 +71,41 @@ def objdiffcli_url(tag: str) -> str: repo = "https://github.com/encounter/objdiff" return f"{repo}/releases/download/{tag}/objdiff-cli-{system}-{arch}{suffix}" + +def sjiswrap_url(tag: str) -> str: + repo = "https://github.com/encounter/sjiswrap" + return f"{repo}/releases/download/{tag}/sjiswrap-windows-x86.exe" + + +def wibo_url(tag: str) -> str: + repo = "https://github.com/decompals/wibo" + return f"{repo}/releases/download/{tag}/wibo" + + TOOLS: Dict[str, Callable[[str], str]] = { "binutils": binutils_url, "compilers": compilers_url, "dtk": dtk_url, + "objdiff-cli": objdiff_cli_url, "sjiswrap": sjiswrap_url, "wibo": wibo_url, - "objdiff-cli": objdiffcli_url } +def download(url, response, output) -> None: + if url.endswith(".zip"): + data = io.BytesIO(response.read()) + with zipfile.ZipFile(data) as f: + f.extractall(output) + # Make all files executable + for root, _, files in os.walk(output): + for name in files: + os.chmod(os.path.join(root, name), 0o755) + output.touch(mode=0o755) # Update dir modtime + else: + with open(output, "wb") as f: + shutil.copyfileobj(response, f) + st = os.stat(output) + os.chmod(output, st.st_mode | stat.S_IEXEC) def main() -> None: parser = argparse.ArgumentParser() @@ -102,22 +119,21 @@ def main() -> None: print(f"Downloading {url} to {output}") req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"}) - with urllib.request.urlopen(req) as response: - if url.endswith(".zip"): - data = io.BytesIO(response.read()) - with zipfile.ZipFile(data) as f: - f.extractall(output) - # Make all files executable - for root, _, files in os.walk(output): - for name in files: - os.chmod(os.path.join(root, name), 0o755) - output.touch(mode=0o755) # Update dir modtime - else: - with open(output, "wb") as f: - shutil.copyfileobj(response, f) - st = os.stat(output) - os.chmod(output, st.st_mode | stat.S_IEXEC) - + try: + with urllib.request.urlopen(req) as response: + download(url, response, output) + except urllib.error.URLError as e: + if str(e).find("CERTIFICATE_VERIFY_FAILED") == -1: + raise e + try: + import certifi + import ssl + except: + print("\"certifi\" module not found. Please install it using \"python -m pip install certifi\".") + return + + with urllib.request.urlopen(req, context=ssl.create_default_context(cafile=certifi.where())) as response: + download(url, response, output) if __name__ == "__main__": main() diff --git a/tools/ninja_syntax.py b/tools/ninja_syntax.py index 7306ee1da..fdda97171 100644 --- a/tools/ninja_syntax.py +++ b/tools/ninja_syntax.py @@ -24,17 +24,10 @@ import os from io import StringIO from pathlib import Path -from typing import Dict, List, Match, Optional, Tuple, Union +from typing import Dict, Iterable, List, Match, Optional, Tuple, Union NinjaPath = Union[str, Path] -NinjaPaths = Union[ - List[str], - List[Path], - List[NinjaPath], - List[Optional[str]], - List[Optional[Path]], - List[Optional[NinjaPath]], -] +NinjaPaths = Iterable[Optional[NinjaPath]] NinjaPathOrPaths = Union[NinjaPath, NinjaPaths] @@ -118,8 +111,8 @@ def build( pool: Optional[str] = None, dyndep: Optional[NinjaPath] = None, ) -> List[str]: - outputs = serialize_paths(outputs) - out_outputs = [escape_path(x) for x in outputs] + str_outputs = serialize_paths(outputs) + out_outputs = [escape_path(x) for x in str_outputs] all_inputs = [escape_path(x) for x in serialize_paths(inputs)] if implicit: @@ -154,7 +147,7 @@ def build( for key, val in iterator: self.variable(key, val, indent=1) - return outputs + return str_outputs def include(self, path: str) -> None: self._line("include %s" % path) @@ -225,9 +218,11 @@ def serialize_path(input: Optional[NinjaPath]) -> str: def serialize_paths(input: Optional[NinjaPathOrPaths]) -> List[str]: - if isinstance(input, list): + if isinstance(input, str) or isinstance(input, Path): + return [serialize_path(input)] if input else [] + elif input is not None: return [serialize_path(path) for path in input if path] - return [serialize_path(input)] if input else [] + return [] def escape(string: str) -> str: diff --git a/tools/project.py b/tools/project.py index 044772050..f5144a223 100644 --- a/tools/project.py +++ b/tools/project.py @@ -17,7 +17,20 @@ import platform import sys from pathlib import Path -from typing import Any, Dict, List, Optional, Set, Tuple, Union +from typing import ( + Any, + Callable, + cast, + Dict, + IO, + Iterable, + List, + Optional, + Set, + Tuple, + TypedDict, + Union, +) from . import ninja_syntax from .ninja_syntax import serialize_path @@ -29,24 +42,95 @@ f"\n(Current path: {sys.executable})" ) +Library = Dict[str, Any] + class Object: def __init__(self, completed: bool, name: str, **options: Any) -> None: self.name = name - self.base_name = Path(name).with_suffix("") self.completed = completed self.options: Dict[str, Any] = { - "add_to_all": True, + "add_to_all": None, "asflags": None, - "extra_asflags": None, + "asm_dir": None, "cflags": None, - "extra_cflags": None, + "extra_asflags": [], + "extra_cflags": [], + "extra_clang_flags": [], + "host": None, + "lib": None, "mw_version": None, + "progress_category": None, + "scratch_preset_id": None, "shift_jis": None, "source": name, + "src_dir": None, } self.options.update(options) + # Internal + self.src_path: Optional[Path] = None + self.asm_path: Optional[Path] = None + self.src_obj_path: Optional[Path] = None + self.asm_obj_path: Optional[Path] = None + self.host_obj_path: Optional[Path] = None + self.ctx_path: Optional[Path] = None + + def resolve(self, config: "ProjectConfig", lib: Library) -> "Object": + # Use object options, then library options + obj = Object(self.completed, self.name, **lib) + for key, value in self.options.items(): + if value is not None or key not in obj.options: + obj.options[key] = value + + # Use default options from config + def set_default(key: str, value: Any) -> None: + if obj.options[key] is None: + obj.options[key] = value + + set_default("add_to_all", True) + set_default("asflags", config.asflags) + set_default("asm_dir", config.asm_dir) + set_default("host", False) + set_default("mw_version", config.linker_version) + set_default("scratch_preset_id", config.scratch_preset_id) + set_default("shift_jis", config.shift_jis) + set_default("src_dir", config.src_dir) + + # Validate progress categories + def check_category(category: str): + if not any(category == c.id for c in config.progress_categories): + sys.exit( + f"Progress category '{category}' missing from config.progress_categories" + ) + + progress_category = obj.options["progress_category"] + if isinstance(progress_category, list): + for category in progress_category: + check_category(category) + elif progress_category is not None: + check_category(progress_category) + + # Resolve paths + build_dir = config.out_path() + obj.src_path = Path(obj.options["src_dir"]) / obj.options["source"] + if obj.options["asm_dir"] is not None: + obj.asm_path = ( + Path(obj.options["asm_dir"]) / obj.options["source"] + ).with_suffix(".s") + base_name = Path(self.name).with_suffix("") + obj.src_obj_path = build_dir / "src" / f"{base_name}.o" + obj.asm_obj_path = build_dir / "mod" / f"{base_name}.o" + obj.host_obj_path = build_dir / "host" / f"{base_name}.o" + obj.ctx_path = build_dir / "src" / f"{base_name}.ctx" + return obj + + +class ProgressCategory: + def __init__(self, id: str, name: str) -> None: + self.id = id + self.name = name + class ProjectConfig: def __init__(self) -> None: @@ -69,37 +153,59 @@ def __init__(self) -> None: self.wrapper: Optional[Path] = None # If None, download wibo on Linux self.sjiswrap_tag: Optional[str] = None # Git tag self.sjiswrap_path: Optional[Path] = None # If None, download + self.objdiff_tag: Optional[str] = None # Git tag + self.objdiff_path: Optional[Path] = None # If None, download # Project config self.non_matching: bool = False self.build_rels: bool = True # Build REL files self.check_sha_path: Optional[Path] = None # Path to version.sha1 self.config_path: Optional[Path] = None # Path to config.yml - self.debug: bool = False # Build with debug info self.generate_map: bool = False # Generate map file(s) self.asflags: Optional[List[str]] = None # Assembler flags self.ldflags: Optional[List[str]] = None # Linker flags - self.libs: Optional[List[Dict[str, Any]]] = None # List of libraries + self.libs: Optional[List[Library]] = None # List of libraries self.linker_version: Optional[str] = None # mwld version self.version: Optional[str] = None # Version name self.warn_missing_config: bool = False # Warn on missing unit configuration self.warn_missing_source: bool = False # Warn on missing source file self.rel_strip_partial: bool = True # Generate PLFs with -strip_partial - self.rel_empty_file: Optional[ - str - ] = None # Object name for generating empty RELs + self.rel_empty_file: Optional[str] = ( + None # Object name for generating empty RELs + ) self.shift_jis = ( True # Convert source files from UTF-8 to Shift JIS automatically ) self.reconfig_deps: Optional[List[Path]] = ( None # Additional re-configuration dependency files ) + self.custom_build_rules: Optional[List[Dict[str, Any]]] = ( + None # Custom ninja build rules + ) + self.custom_build_steps: Optional[Dict[str, List[Dict[str, Any]]]] = ( + None # Custom build steps, types are ["pre-compile", "post-compile", "post-link", "post-build"] + ) + self.generate_compile_commands: bool = ( + True # Generate compile_commands.json for clangd + ) + self.extra_clang_flags: List[str] = [] # Extra flags for clangd + self.scratch_preset_id: Optional[int] = ( + None # Default decomp.me preset ID for scratches + ) + self.link_order_callback: Optional[Callable[[int, List[str]], List[str]]] = ( + None # Callback to add/remove/reorder units within a module + ) - # Progress output and progress.json config + # Progress output, progress.json and report.json config + self.progress = True # Enable report.json generation and CLI progress output self.progress_all: bool = False # Include combined "all" category self.progress_modules: bool = True # Include combined "modules" category self.progress_each_module: bool = ( - True # Include individual modules, disable for large numbers of modules + False # Include individual modules, disable for large numbers of modules + ) + self.progress_categories: List[ProgressCategory] = [] # Additional categories + self.print_progress_categories: Union[bool, List[str]] = ( + True # Print additional progress categories in the CLI progress output ) # Progress fancy printing @@ -125,16 +231,52 @@ def validate(self) -> None: if getattr(self, attr) is None: sys.exit(f"ProjectConfig.{attr} missing") - def find_object(self, name: str) -> Optional[Tuple[Dict[str, Any], Object]]: + # Creates a map of object names to Object instances + # Options are fully resolved from the library and object + def objects(self) -> Dict[str, Object]: + out = {} for lib in self.libs or {}: - for obj in lib["objects"]: - if obj.name == name: - return lib, obj - return None - + objects: List[Object] = lib["objects"] + for obj in objects: + if obj.name in out: + sys.exit(f"Duplicate object name {obj.name}") + out[obj.name] = obj.resolve(self, lib) + return out + + # Gets the output path for build-related files. def out_path(self) -> Path: return self.build_dir / str(self.version) + # Gets the path to the compilers directory. + # Exits the program if neither `compilers_path` nor `compilers_tag` is provided. + def compilers(self) -> Path: + if self.compilers_path: + return self.compilers_path + elif self.compilers_tag: + return self.build_dir / "compilers" + else: + sys.exit("ProjectConfig.compilers_tag missing") + + # Gets the wrapper to use for compiler commands, if set. + def compiler_wrapper(self) -> Optional[Path]: + wrapper = self.wrapper + + if self.use_wibo(): + wrapper = self.build_dir / "tools" / "wibo" + if not is_windows() and wrapper is None: + wrapper = Path("wine") + + return wrapper + + # Determines whether or not to use wibo as the compiler wrapper. + def use_wibo(self) -> bool: + return ( + self.wibo_tag is not None + and sys.platform == "linux" + and platform.machine() in ("i386", "x86_64") + and self.wrapper is None + ) + def is_windows() -> bool: return os.name == "nt" @@ -146,17 +288,60 @@ def is_windows() -> bool: EXE = ".exe" if is_windows() else "" -def make_flags_str(cflags: Union[str, List[str]]) -> str: - if isinstance(cflags, list): - return " ".join(cflags) - else: - return cflags +def file_is_asm(path: Path) -> bool: + return path.suffix.lower() == ".s" + + +def file_is_c(path: Path) -> bool: + return path.suffix.lower() == ".c" + + +def file_is_cpp(path: Path) -> bool: + return path.suffix.lower() in (".cc", ".cp", ".cpp", ".cxx") + + +def file_is_c_cpp(path: Path) -> bool: + return file_is_c(path) or file_is_cpp(path) + + +def make_flags_str(flags: Optional[List[str]]) -> str: + if flags is None: + return "" + return " ".join(flags) + + +# Unit configuration +class BuildConfigUnit(TypedDict): + object: Optional[str] + name: str + autogenerated: bool + + +# Module configuration +class BuildConfigModule(TypedDict): + name: str + module_id: int + ldscript: str + entry: str + units: List[BuildConfigUnit] + + +# Module link configuration +class BuildConfigLink(TypedDict): + modules: List[str] + + +# Build configuration generated by decomp-toolkit +class BuildConfig(BuildConfigModule): + version: str + modules: List[BuildConfigModule] + links: List[BuildConfigLink] # Load decomp-toolkit generated config.json def load_build_config( config: ProjectConfig, build_config_path: Path -) -> Optional[Dict[str, Any]]: +) -> Optional[BuildConfig]: if not build_config_path.is_file(): return None @@ -164,38 +349,59 @@ def versiontuple(v: str) -> Tuple[int, ...]: return tuple(map(int, (v.split(".")))) f = open(build_config_path, "r", encoding="utf-8") - build_config: Dict[str, Any] = json.load(f) + build_config: BuildConfig = json.load(f) config_version = build_config.get("version") - if not config_version: - # Invalid config.json + if config_version is None: + print("Invalid config.json, regenerating...") f.close() os.remove(build_config_path) return None dtk_version = str(config.dtk_tag)[1:] # Strip v if versiontuple(config_version) < versiontuple(dtk_version): - # Outdated config.json + print("Outdated config.json, regenerating...") f.close() os.remove(build_config_path) return None f.close() + + # Apply link order callback + if config.link_order_callback: + modules: List[BuildConfigModule] = [build_config, *build_config["modules"]] + for module in modules: + unit_names = list(map(lambda u: u["name"], module["units"])) + unit_names = config.link_order_callback(module["module_id"], unit_names) + units: List[BuildConfigUnit] = [] + for unit_name in unit_names: + units.append( + # Find existing unit or create a new one + next( + (u for u in module["units"] if u["name"] == unit_name), + {"object": None, "name": unit_name, "autogenerated": False}, + ) + ) + module["units"] = units + return build_config -# Generate build.ninja and objdiff.json +# Generate build.ninja, objdiff.json and compile_commands.json def generate_build(config: ProjectConfig) -> None: + config.validate() + objects = config.objects() build_config = load_build_config(config, config.out_path() / "config.json") - generate_build_ninja(config, build_config) - generate_objdiff_config(config, build_config) + generate_build_ninja(config, objects, build_config) + generate_objdiff_config(config, objects, build_config) + generate_compile_commands(config, objects, build_config) # Generate build.ninja def generate_build_ninja( - config: ProjectConfig, build_config: Optional[Dict[str, Any]] + config: ProjectConfig, + objects: Dict[str, Object], + build_config: Optional[BuildConfig], ) -> None: - config.validate() - out = io.StringIO() n = ninja_syntax.Writer(out) n.variable("ninja_required_version", "1.3") @@ -213,13 +419,8 @@ def generate_build_ninja( # Variables ### n.comment("Variables") - ldflags = " ".join(config.ldflags or []) - if config.generate_map: - ldflags += " -mapunused" - if config.debug: - ldflags += " -g" - n.variable("ldflags", ldflags) - if not config.linker_version: + n.variable("ldflags", make_flags_str(config.ldflags)) + if config.linker_version is None: sys.exit("ProjectConfig.linker_version missing") n.variable("mw_version", Path(config.linker_version)) n.newline() @@ -231,6 +432,7 @@ def generate_build_ninja( build_path = config.out_path() progress_path = build_path / "progress.json" + report_path = build_path / "report.json" build_tools_path = config.build_dir / "tools" download_tool = config.tools_dir / "download_tool.py" n.rule( @@ -242,23 +444,33 @@ def generate_build_ninja( decompctx = config.tools_dir / "decompctx.py" n.rule( name="decompctx", - command=f"$python {decompctx} $in -o $out -d $out.d", + command=f"$python {decompctx} $in -o $out -d $out.d $includes", description="CTX $in", depfile="$out.d", deps="gcc", ) + cargo_rule_written = False + + def write_cargo_rule(): + nonlocal cargo_rule_written + if not cargo_rule_written: + n.pool("cargo", 1) + n.rule( + name="cargo", + command="cargo build --release --manifest-path $in --bin $bin --target-dir $target", + description="CARGO $bin", + pool="cargo", + depfile=Path("$target") / "release" / "$bin.d", + deps="gcc", + ) + cargo_rule_written = True + if config.dtk_path is not None and config.dtk_path.is_file(): dtk = config.dtk_path elif config.dtk_path is not None: dtk = build_tools_path / "release" / f"dtk{EXE}" - n.rule( - name="cargo", - command="cargo build --release --manifest-path $in --bin $bin --target-dir $target", - description="CARGO $bin", - depfile=Path("$target") / "release" / "$bin.d", - deps="gcc", - ) + write_cargo_rule() n.build( outputs=dtk, rule="cargo", @@ -283,6 +495,35 @@ def generate_build_ninja( else: sys.exit("ProjectConfig.dtk_tag missing") + if config.objdiff_path is not None and config.objdiff_path.is_file(): + objdiff = config.objdiff_path + elif config.objdiff_path is not None: + objdiff = build_tools_path / "release" / f"objdiff-cli{EXE}" + write_cargo_rule() + n.build( + outputs=objdiff, + rule="cargo", + inputs=config.objdiff_path / "Cargo.toml", + implicit=config.objdiff_path / "Cargo.lock", + variables={ + "bin": "objdiff-cli", + "target": build_tools_path, + }, + ) + elif config.objdiff_tag: + objdiff = build_tools_path / f"objdiff-cli{EXE}" + n.build( + outputs=objdiff, + rule="download_tool", + implicit=download_tool, + variables={ + "tool": "objdiff-cli", + "tag": config.objdiff_tag, + }, + ) + else: + sys.exit("ProjectConfig.objdiff_tag missing") + if config.sjiswrap_path: sjiswrap = config.sjiswrap_path elif config.sjiswrap_tag: @@ -299,16 +540,10 @@ def generate_build_ninja( else: sys.exit("ProjectConfig.sjiswrap_tag missing") + wrapper = config.compiler_wrapper() # Only add an implicit dependency on wibo if we download it - wrapper = config.wrapper wrapper_implicit: Optional[Path] = None - if ( - config.wibo_tag is not None - and sys.platform == "linux" - and platform.machine() in ("i386", "x86_64") - and config.wrapper is None - ): - wrapper = build_tools_path / "wibo" + if wrapper is not None and config.use_wibo(): wrapper_implicit = wrapper n.build( outputs=wrapper, @@ -319,15 +554,11 @@ def generate_build_ninja( "tag": config.wibo_tag, }, ) - if not is_windows() and wrapper is None: - wrapper = Path("wine") wrapper_cmd = f"{wrapper} " if wrapper else "" + compilers = config.compilers() compilers_implicit: Optional[Path] = None - if config.compilers_path: - compilers = config.compilers_path - elif config.compilers_tag: - compilers = config.build_dir / "compilers" + if config.compilers_path is None and config.compilers_tag is not None: compilers_implicit = compilers n.build( outputs=compilers, @@ -338,8 +569,6 @@ def generate_build_ninja( "tag": config.compilers_tag, }, ) - else: - sys.exit("ProjectConfig.compilers_tag missing") binutils_implicit = None if config.binutils_path: @@ -361,6 +590,17 @@ def generate_build_ninja( n.newline() + ### + # Helper rule for downloading all tools + ### + n.comment("Download all tools") + n.build( + outputs="tools", + rule="phony", + inputs=[dtk, sjiswrap, wrapper, compilers, binutils, objdiff], + ) + n.newline() + ### # Build rules ### @@ -443,6 +683,54 @@ def generate_build_ninja( ) n.newline() + if len(config.custom_build_rules or {}) > 0: + n.comment("Custom project build rules (pre/post-processing)") + for rule in config.custom_build_rules or {}: + n.rule( + name=cast(str, rule.get("name")), + command=cast(str, rule.get("command")), + description=rule.get("description", None), + depfile=rule.get("depfile", None), + generator=rule.get("generator", False), + pool=rule.get("pool", None), + restat=rule.get("restat", False), + rspfile=rule.get("rspfile", None), + rspfile_content=rule.get("rspfile_content", None), + deps=rule.get("deps", None), + ) + n.newline() + + def write_custom_step(step: str, prev_step: Optional[str] = None) -> None: + implicit: List[str | Path] = [] + if config.custom_build_steps and step in config.custom_build_steps: + n.comment(f"Custom build steps ({step})") + for custom_step in config.custom_build_steps[step]: + outputs = cast(List[str | Path], custom_step.get("outputs")) + + if isinstance(outputs, list): + implicit.extend(outputs) + else: + implicit.append(outputs) + + n.build( + outputs=outputs, + rule=cast(str, custom_step.get("rule")), + inputs=custom_step.get("inputs", None), + implicit=custom_step.get("implicit", None), + order_only=custom_step.get("order_only", None), + variables=custom_step.get("variables", None), + implicit_outputs=custom_step.get("implicit_outputs", None), + pool=custom_step.get("pool", None), + dyndep=custom_step.get("dyndep", None), + ) + n.newline() + n.build( + outputs=step, + rule="phony", + inputs=implicit, + order_only=prev_step, + ) + n.comment("Host build") n.variable("host_cflags", "-I include -Wno-trigraphs") n.variable( @@ -461,22 +749,21 @@ def generate_build_ninja( ) n.newline() + # Add all build steps needed before we compile (e.g. processing assets) + write_custom_step("pre-compile") + ### # Source files ### n.comment("Source files") - build_asm_path = build_path / "mod" - build_src_path = build_path / "src" - build_host_path = build_path / "host" - build_config_path = build_path / "config.json" def map_path(path: Path) -> Path: return path.parent / (path.name + ".MAP") class LinkStep: - def __init__(self, config: Dict[str, Any]) -> None: - self.name: str = config["name"] - self.module_id: int = config["module_id"] + def __init__(self, config: BuildConfigModule) -> None: + self.name = config["name"] + self.module_id = config["module_id"] self.ldscript: Optional[Path] = Path(config["ldscript"]) self.entry = config["entry"] self.inputs: List[str] = [] @@ -500,7 +787,6 @@ def write(self, n: ninja_syntax.Writer) -> None: n.comment(f"Link {self.name}") if self.module_id == 0: elf_path = build_path / f"{self.name}.elf" - dol_path = build_path / f"{self.name}.dol" elf_ldflags = f"$ldflags -lcf {serialize_path(self.ldscript)}" if config.generate_map: elf_map = map_path(elf_path) @@ -511,15 +797,13 @@ def write(self, n: ninja_syntax.Writer) -> None: outputs=elf_path, rule="link", inputs=self.inputs, - implicit=[self.ldscript, *mwld_implicit], + implicit=[ + self.ldscript, + *mwld_implicit, + ], implicit_outputs=elf_map, variables={"ldflags": elf_ldflags}, - ) - n.build( - outputs=dol_path, - rule="elf2dol", - inputs=elf_path, - implicit=dtk, + order_only="post-compile", ) else: preplf_path = build_path / self.name / f"{self.name}.preplf" @@ -546,6 +830,7 @@ def write(self, n: ninja_syntax.Writer) -> None: implicit=mwld_implicit, implicit_outputs=preplf_map, variables={"ldflags": preplf_ldflags}, + order_only="post-compile", ) n.build( outputs=plf_path, @@ -554,6 +839,7 @@ def write(self, n: ninja_syntax.Writer) -> None: implicit=[self.ldscript, preplf_path, *mwld_implicit], implicit_outputs=plf_map, variables={"ldflags": plf_ldflags}, + order_only="post-compile", ) n.newline() @@ -565,162 +851,161 @@ def write(self, n: ninja_syntax.Writer) -> None: host_source_inputs: List[Path] = [] source_added: Set[Path] = set() - def c_build( - obj: Object, options: Dict[str, Any], lib_name: str, src_path: Path - ) -> Optional[Path]: - cflags_str = make_flags_str(options["cflags"]) - if options["extra_cflags"] is not None: - extra_cflags_str = make_flags_str(options["extra_cflags"]) - cflags_str += " " + extra_cflags_str - used_compiler_versions.add(options["mw_version"]) - - src_obj_path = build_src_path / f"{obj.base_name}.o" - src_base_path = build_src_path / obj.base_name - + def c_build(obj: Object, src_path: Path) -> Optional[Path]: # Avoid creating duplicate build rules - if src_obj_path in source_added: - return src_obj_path - source_added.add(src_obj_path) + if obj.src_obj_path is None or obj.src_obj_path in source_added: + return obj.src_obj_path + source_added.add(obj.src_obj_path) + + cflags = obj.options["cflags"] + extra_cflags = obj.options["extra_cflags"] + + # Add appropriate language flag if it doesn't exist already + # Added directly to the source so it flows to other generation tasks + if not any(flag.startswith("-lang") for flag in cflags) and not any( + flag.startswith("-lang") for flag in extra_cflags + ): + # Ensure extra_cflags is a unique instance, + # and insert into there to avoid modifying shared sets of flags + extra_cflags = obj.options["extra_cflags"] = list(extra_cflags) + if file_is_cpp(src_path): + extra_cflags.insert(0, "-lang=c++") + else: + extra_cflags.insert(0, "-lang=c") - shift_jis = options["shift_jis"] - if shift_jis is None: - shift_jis = config.shift_jis + all_cflags = cflags + extra_cflags + cflags_str = make_flags_str(all_cflags) + used_compiler_versions.add(obj.options["mw_version"]) # Add MWCC build rule + lib_name = obj.options["lib"] n.comment(f"{obj.name}: {lib_name} (linked {obj.completed})") n.build( - outputs=src_obj_path, - rule="mwcc_sjis" if shift_jis else "mwcc", + outputs=obj.src_obj_path, + rule="mwcc_sjis" if obj.options["shift_jis"] else "mwcc", inputs=src_path, variables={ - "mw_version": Path(options["mw_version"]), + "mw_version": Path(obj.options["mw_version"]), "cflags": cflags_str, - "basedir": os.path.dirname(src_base_path), - "basefile": src_base_path, + "basedir": os.path.dirname(obj.src_obj_path), + "basefile": obj.src_obj_path.with_suffix(""), }, - implicit=mwcc_sjis_implicit if shift_jis else mwcc_implicit, + implicit=( + mwcc_sjis_implicit if obj.options["shift_jis"] else mwcc_implicit + ), + order_only="pre-compile", ) # Add ctx build rule - ctx_path = build_src_path / f"{obj.base_name}.ctx" - n.build( - outputs=ctx_path, - rule="decompctx", - inputs=src_path, - implicit=decompctx, - ) + if obj.ctx_path is not None: + include_dirs = [] + for flag in all_cflags: + if ( + flag.startswith("-i ") + or flag.startswith("-I ") + or flag.startswith("-I+") + ): + include_dirs.append(flag[3:]) + includes = " ".join([f"-I {d}" for d in include_dirs]) + n.build( + outputs=obj.ctx_path, + rule="decompctx", + inputs=src_path, + implicit=decompctx, + variables={"includes": includes}, + ) # Add host build rule - if options.get("host", False): - host_obj_path = build_host_path / f"{obj.base_name}.o" - host_base_path = build_host_path / obj.base_name + if obj.options["host"] and obj.host_obj_path is not None: n.build( - outputs=host_obj_path, - rule="host_cc" if src_path.suffix == ".c" else "host_cpp", + outputs=obj.host_obj_path, + rule="host_cc" if file_is_c(src_path) else "host_cpp", inputs=src_path, variables={ - "basedir": os.path.dirname(host_base_path), - "basefile": host_base_path, + "basedir": os.path.dirname(obj.host_obj_path), + "basefile": obj.host_obj_path.with_suffix(""), }, + order_only="pre-compile", ) - if options["add_to_all"]: - host_source_inputs.append(host_obj_path) + if obj.options["add_to_all"]: + host_source_inputs.append(obj.host_obj_path) n.newline() - if options["add_to_all"]: - source_inputs.append(src_obj_path) + if obj.options["add_to_all"]: + source_inputs.append(obj.src_obj_path) - return src_obj_path + return obj.src_obj_path def asm_build( - obj: Object, options: Dict[str, Any], lib_name: str, src_path: Path + obj: Object, src_path: Path, obj_path: Optional[Path] ) -> Optional[Path]: - asflags = options["asflags"] or config.asflags - if asflags is None: + if obj.options["asflags"] is None: sys.exit("ProjectConfig.asflags missing") - asflags_str = make_flags_str(asflags) - if options["extra_asflags"] is not None: - extra_asflags_str = make_flags_str(options["extra_asflags"]) + asflags_str = make_flags_str(obj.options["asflags"]) + if len(obj.options["extra_asflags"]) > 0: + extra_asflags_str = make_flags_str(obj.options["extra_asflags"]) asflags_str += " " + extra_asflags_str - asm_obj_path = build_asm_path / f"{obj.base_name}.o" - # Avoid creating duplicate build rules - if asm_obj_path in source_added: - return asm_obj_path - source_added.add(asm_obj_path) + if obj_path is None or obj_path in source_added: + return obj_path + source_added.add(obj_path) # Add assembler build rule + lib_name = obj.options["lib"] n.comment(f"{obj.name}: {lib_name} (linked {obj.completed})") n.build( - outputs=asm_obj_path, + outputs=obj_path, rule="as", inputs=src_path, variables={"asflags": asflags_str}, implicit=gnu_as_implicit, + order_only="pre-compile", ) n.newline() - if options["add_to_all"]: - source_inputs.append(asm_obj_path) + if obj.options["add_to_all"]: + source_inputs.append(obj_path) - return asm_obj_path + return obj_path - def add_unit(build_obj, link_step: LinkStep): + def add_unit(build_obj: BuildConfigUnit, link_step: LinkStep): obj_path, obj_name = build_obj["object"], build_obj["name"] - result = config.find_object(obj_name) - if not result: + obj = objects.get(obj_name) + if obj is None: if config.warn_missing_config and not build_obj["autogenerated"]: print(f"Missing configuration for {obj_name}") - link_step.add(obj_path) + if obj_path is not None: + link_step.add(Path(obj_path)) return - lib, obj = result - lib_name = lib["lib"] - - # Use object options, then library options - options = lib.copy() - for key, value in obj.options.items(): - if value is not None or key not in options: - options[key] = value - - unit_src_path = Path(lib.get("src_dir", config.src_dir)) / options["source"] - - unit_asm_path: Optional[Path] = None - if config.asm_dir is not None: - unit_asm_path = ( - Path(lib.get("asm_dir", config.asm_dir)) / options["source"] - ).with_suffix(".s") - link_built_obj = obj.completed built_obj_path: Optional[Path] = None - if unit_src_path.exists(): - if unit_src_path.suffix in (".c", ".cp", ".cpp"): + if obj.src_path is not None and obj.src_path.exists(): + if file_is_c_cpp(obj.src_path): # Add MWCC & host build rules - built_obj_path = c_build(obj, options, lib_name, unit_src_path) - elif unit_src_path.suffix == ".s": + built_obj_path = c_build(obj, obj.src_path) + elif file_is_asm(obj.src_path): # Add assembler build rule - built_obj_path = asm_build(obj, options, lib_name, unit_src_path) + built_obj_path = asm_build(obj, obj.src_path, obj.src_obj_path) else: - sys.exit(f"Unknown source file type {unit_src_path}") + sys.exit(f"Unknown source file type {obj.src_path}") else: if config.warn_missing_source or obj.completed: - print(f"Missing source file {unit_src_path}") + print(f"Missing source file {obj.src_path}") link_built_obj = False # Assembly overrides - if unit_asm_path is not None and unit_asm_path.exists(): + if obj.asm_path is not None and obj.asm_path.exists(): link_built_obj = True - built_obj_path = asm_build(obj, options, lib_name, unit_asm_path) + built_obj_path = asm_build(obj, obj.asm_path, obj.asm_obj_path) if link_built_obj and built_obj_path is not None: # Use the source-built object link_step.add(built_obj_path) elif obj_path is not None: # Use the original (extracted) object - link_step.add(obj_path) - else: - sys.exit(f"Missing object for {obj_name}: {unit_src_path} {lib} {obj}") + link_step.add(Path(obj_path)) # Add DOL link step link_step = LinkStep(build_config) @@ -736,7 +1021,7 @@ def add_unit(build_obj, link_step: LinkStep): add_unit(unit, module_link_step) # Add empty object to empty RELs if len(module_link_step.inputs) == 0: - if not config.rel_empty_file: + if config.rel_empty_file is None: sys.exit("ProjectConfig.rel_empty_file missing") add_unit( { @@ -760,6 +1045,9 @@ def add_unit(build_obj, link_step: LinkStep): if config.compilers_path and not os.path.exists(mw_path): sys.exit(f"Linker {mw_path} does not exist") + # Add all build steps needed before we link and after compiling objects + write_custom_step("post-compile", "pre-compile") + ### # Link ### @@ -768,6 +1056,20 @@ def add_unit(build_obj, link_step: LinkStep): link_outputs.append(step.output()) n.newline() + # Add all build steps needed after linking and before GC/Wii native format generation + write_custom_step("post-link", "post-compile") + + ### + # Generate DOL + ### + n.build( + outputs=link_steps[0].output(), + rule="elf2dol", + inputs=link_steps[0].partial_output(), + implicit=dtk, + order_only="post-link", + ) + ### # Generate RELs ### @@ -782,7 +1084,7 @@ def add_unit(build_obj, link_step: LinkStep): rspfile="$rspfile", rspfile_content="$in_newline", ) - generated_rels = [] + generated_rels: List[str] = [] for idx, link in enumerate(build_config["links"]): # Map module names to link steps link_steps_local = list( @@ -827,9 +1129,13 @@ def add_unit(build_obj, link_step: LinkStep): "rspfile": config.out_path() / f"rel{idx}.rsp", "names": rel_names_arg, }, + order_only="post-link", ) n.newline() + # Add all build steps needed post-build (re-building archives and such) + write_custom_step("post-build", "post-link") + ### # Helper rule for building all source files ### @@ -868,6 +1174,7 @@ def add_unit(build_obj, link_step: LinkStep): rule="check", inputs=config.check_sha_path, implicit=[dtk, *link_outputs], + order_only="post-build", ) n.newline() @@ -883,7 +1190,29 @@ def add_unit(build_obj, link_step: LinkStep): n.build( outputs=progress_path, rule="progress", - implicit=[ok_path, configure_script, python_lib, config.config_path], + implicit=[ + ok_path, + configure_script, + python_lib, + report_path, + ], + order_only="post-build", + ) + + ### + # Generate progress report + ### + n.comment("Generate progress report") + n.rule( + name="report", + command=f"{objdiff} report generate -o $out", + description="REPORT", + ) + n.build( + outputs=report_path, + rule="report", + implicit=[objdiff, "all_source"], + order_only="post-build", ) ### @@ -932,6 +1261,7 @@ def add_unit(build_obj, link_step: LinkStep): ### # Split DOL ### + build_config_path = build_path / "config.json" n.comment("Split DOL into relocatable objects") n.rule( name="split", @@ -967,7 +1297,7 @@ def add_unit(build_obj, link_step: LinkStep): configure_script, python_lib, python_lib_dir / "ninja_syntax.py", - *(config.reconfig_deps or []) + *(config.reconfig_deps or []), ], ) n.newline() @@ -979,8 +1309,10 @@ def add_unit(build_obj, link_step: LinkStep): if build_config: if config.non_matching: n.default(link_outputs) - else: + elif config.progress: n.default(progress_path) + else: + n.default(ok_path) else: n.default(build_config_path) @@ -992,13 +1324,22 @@ def add_unit(build_obj, link_step: LinkStep): # Generate objdiff.json def generate_objdiff_config( - config: ProjectConfig, build_config: Optional[Dict[str, Any]] + config: ProjectConfig, + objects: Dict[str, Object], + build_config: Optional[BuildConfig], ) -> None: - if not build_config: + if build_config is None: return + # Load existing objdiff.json + existing_units = {} + if Path("objdiff.json").is_file(): + with open("objdiff.json", "r", encoding="utf-8") as r: + existing_config = json.load(r) + existing_units = {unit["name"]: unit for unit in existing_config["units"]} + objdiff_config: Dict[str, Any] = { - "min_version": "1.0.0", + "min_version": "2.0.0-beta.5", "custom_make": "ninja", "build_target": False, "watch_patterns": [ @@ -1014,16 +1355,17 @@ def generate_objdiff_config( "*.json", ], "units": [], + "progress_categories": [], } # decomp.me compiler name mapping - # Commented out versions have not been added to decomp.me yet COMPILER_MAP = { "GC/1.0": "mwcc_233_144", "GC/1.1": "mwcc_233_159", "GC/1.2.5": "mwcc_233_163", "GC/1.2.5e": "mwcc_233_163e", "GC/1.2.5n": "mwcc_233_163n", + "GC/1.3": "mwcc_242_53", "GC/1.3.2": "mwcc_242_81", "GC/1.3.2r": "mwcc_242_81r", "GC/2.0": "mwcc_247_92", @@ -1049,91 +1391,142 @@ def generate_objdiff_config( "Wii/1.7": "mwcc_43_213", } - build_path = config.out_path() - - def add_unit(build_obj: Dict[str, Any], module_name: str) -> None: - if build_obj["autogenerated"]: - # Skip autogenerated objects - return - + def add_unit( + build_obj: BuildConfigUnit, module_name: str, progress_categories: List[str] + ) -> None: obj_path, obj_name = build_obj["object"], build_obj["name"] base_object = Path(obj_name).with_suffix("") + name = str(Path(module_name) / base_object).replace(os.sep, "/") unit_config: Dict[str, Any] = { - "name": Path(module_name) / base_object, + "name": name, "target_path": obj_path, + "base_path": None, + "scratch": None, + "metadata": { + "complete": None, + "reverse_fn_order": None, + "source_path": None, + "progress_categories": progress_categories, + "auto_generated": build_obj["autogenerated"], + }, + "symbol_mappings": None, } - result = config.find_object(obj_name) - if not result: - objdiff_config["units"].append(unit_config) - return + # Preserve existing symbol mappings + existing_unit = existing_units.get(name) + if existing_unit is not None: + unit_config["symbol_mappings"] = existing_unit.get("symbol_mappings") - lib, obj = result - src_dir = Path(lib.get("src_dir", config.src_dir)) - - # Use object options, then library options - options = lib.copy() - for key, value in obj.options.items(): - if value is not None or key not in options: - options[key] = value - - unit_src_path = src_dir / str(options["source"]) - - if not unit_src_path.exists(): + obj = objects.get(obj_name) + if obj is None: objdiff_config["units"].append(unit_config) return - cflags = options["cflags"] - src_obj_path = build_path / "src" / f"{obj.base_name}.o" - src_ctx_path = build_path / "src" / f"{obj.base_name}.ctx" + src_exists = obj.src_path is not None and obj.src_path.exists() + if src_exists: + unit_config["base_path"] = obj.src_obj_path + unit_config["metadata"]["source_path"] = obj.src_path + + # Filter out include directories + def keep_flag(flag): + return ( + not flag.startswith("-i ") + and not flag.startswith("-i-") + and not flag.startswith("-I ") + and not flag.startswith("-I+") + and not flag.startswith("-I-") + ) + all_cflags = list( + filter(keep_flag, obj.options["cflags"] + obj.options["extra_cflags"]) + ) reverse_fn_order = False - if type(cflags) is list: - for flag in cflags: - if not flag.startswith("-inline "): - continue - for value in flag.split(" ")[1].split(","): - if value == "deferred": - reverse_fn_order = True - elif value == "nodeferred": - reverse_fn_order = False - - # Filter out include directories - def keep_flag(flag): - return not flag.startswith("-i ") and not flag.startswith("-I ") - - cflags = list(filter(keep_flag, cflags)) - - # Add appropriate lang flag - if unit_src_path.suffix in (".cp", ".cpp"): - cflags.insert(0, "-lang=c++") - else: - cflags.insert(0, "-lang=c") + for flag in all_cflags: + if not flag.startswith("-inline "): + continue + for value in flag.split(" ")[1].split(","): + if value == "deferred": + reverse_fn_order = True + elif value == "nodeferred": + reverse_fn_order = False - unit_config["base_path"] = src_obj_path - unit_config["reverse_fn_order"] = reverse_fn_order - unit_config["complete"] = obj.completed - compiler_version = COMPILER_MAP.get(options["mw_version"]) + compiler_version = COMPILER_MAP.get(obj.options["mw_version"]) if compiler_version is None: - print(f"Missing scratch compiler mapping for {options['mw_version']}") + print(f"Missing scratch compiler mapping for {obj.options['mw_version']}") else: + cflags_str = make_flags_str(all_cflags) unit_config["scratch"] = { "platform": "gc_wii", "compiler": compiler_version, - "c_flags": make_flags_str(cflags), - "ctx_path": src_ctx_path, - "build_ctx": True, + "c_flags": cflags_str, + "preset_id": obj.options["scratch_preset_id"], } + if src_exists: + unit_config["scratch"].update( + { + "ctx_path": obj.ctx_path, + "build_ctx": True, + } + ) + category_opt: List[str] | str = obj.options["progress_category"] + if isinstance(category_opt, list): + progress_categories.extend(category_opt) + elif category_opt is not None: + progress_categories.append(category_opt) + unit_config["metadata"].update( + { + "complete": obj.completed if src_exists else None, + "reverse_fn_order": reverse_fn_order, + "progress_categories": progress_categories, + } + ) objdiff_config["units"].append(unit_config) # Add DOL units for unit in build_config["units"]: - add_unit(unit, build_config["name"]) + progress_categories = [] + # Only include a "dol" category if there are any modules + # Otherwise it's redundant with the global report measures + if len(build_config["modules"]) > 0: + progress_categories.append("dol") + add_unit(unit, build_config["name"], progress_categories) # Add REL units for module in build_config["modules"]: for unit in module["units"]: - add_unit(unit, module["name"]) + progress_categories = [] + if config.progress_modules: + progress_categories.append("modules") + if config.progress_each_module: + progress_categories.append(module["name"]) + add_unit(unit, module["name"], progress_categories) + + # Add progress categories + def add_category(id: str, name: str): + objdiff_config["progress_categories"].append( + { + "id": id, + "name": name, + } + ) + + if len(build_config["modules"]) > 0: + add_category("dol", "DOL") + if config.progress_modules: + add_category("modules", "Modules") + if config.progress_each_module: + for module in build_config["modules"]: + add_category(module["name"], module["name"]) + for category in config.progress_categories: + add_category(category.id, category.name) + + def cleandict(d): + if isinstance(d, dict): + return {k: cleandict(v) for k, v in d.items() if v is not None} + elif isinstance(d, list): + return [cleandict(v) for v in d] + else: + return d # Write objdiff.json with open("objdiff.json", "w", encoding="utf-8") as w: @@ -1141,140 +1534,343 @@ def keep_flag(flag): def unix_path(input: Any) -> str: return str(input).replace(os.sep, "/") if input else "" - json.dump(objdiff_config, w, indent=4, default=unix_path) + json.dump(cleandict(objdiff_config), w, indent=2, default=unix_path) -# Calculate, print and write progress to progress.json -def calculate_progress(config: ProjectConfig) -> None: - out_path = config.out_path() - build_config = load_build_config(config, out_path / "config.json") - if not build_config: +def generate_compile_commands( + config: ProjectConfig, + objects: Dict[str, Object], + build_config: Optional[BuildConfig], +) -> None: + if build_config is None or not config.generate_compile_commands: return - class ProgressUnit: - def __init__(self, name: str) -> None: - self.name: str = name - self.code_total: int = 0 - self.code_fancy_frac: int = config.progress_code_fancy_frac - self.code_fancy_item: str = config.progress_code_fancy_item - self.code_progress: int = 0 - self.data_total: int = 0 - self.data_fancy_frac: int = config.progress_data_fancy_frac - self.data_fancy_item: str = config.progress_data_fancy_item - self.data_progress: int = 0 - self.objects_progress: int = 0 - self.objects_total: int = 0 - self.objects: Set[Object] = set() - - def add(self, build_obj: Dict[str, Any]) -> None: - self.code_total += build_obj["code_size"] - self.data_total += build_obj["data_size"] - - # Avoid counting the same object in different modules twice - include_object = build_obj["name"] not in self.objects - if include_object: - self.objects.add(build_obj["name"]) - self.objects_total += 1 - - if build_obj["autogenerated"]: - # Skip autogenerated objects - return + # The following code attempts to convert mwcc flags to clang flags + # for use with clangd. - result = config.find_object(build_obj["name"]) - if not result: - return + # Flags to ignore explicitly + CFLAG_IGNORE: Set[str] = { + # Search order modifier + # Has a different meaning to Clang, and would otherwise + # be picked up by the include passthrough prefix + "-I-", + "-i-", + } + CFLAG_IGNORE_PREFIX: Tuple[str, ...] = ( + # Recursive includes are not supported by modern compilers + "-ir ", + ) - _, obj = result - if not obj.completed: - return + # Flags to replace + CFLAG_REPLACE: Dict[str, str] = {} + CFLAG_REPLACE_PREFIX: Tuple[Tuple[str, str], ...] = ( + # Includes + ("-i ", "-I"), + ("-I ", "-I"), + ("-I+", "-I"), + # Defines + ("-d ", "-D"), + ("-D ", "-D"), + ("-D+", "-D"), + ) - self.code_progress += build_obj["code_size"] - self.data_progress += build_obj["data_size"] - if include_object: - self.objects_progress += 1 + # Flags with a finite set of options + CFLAG_REPLACE_OPTIONS: Tuple[Tuple[str, Dict[str, Tuple[str, ...]]], ...] = ( + # Exceptions + ( + "-Cpp_exceptions", + { + "off": ("-fno-cxx-exceptions",), + "on": ("-fcxx-exceptions",), + }, + ), + # RTTI + ( + "-RTTI", + { + "off": ("-fno-rtti",), + "on": ("-frtti",), + }, + ), + # Language configuration + ( + "-lang", + { + "c": ("--language=c", "--std=c99"), + "c99": ("--language=c", "--std=c99"), + "c++": ("--language=c++", "--std=c++98"), + "cplus": ("--language=c++", "--std=c++98"), + }, + ), + # Enum size + ( + "-enum", + { + "min": ("-fshort-enums",), + "int": ("-fno-short-enums",), + }, + ), + # Common BSS + ( + "-common", + { + "off": ("-fno-common",), + "on": ("-fcommon",), + }, + ), + ) + + # Flags to pass through + CFLAG_PASSTHROUGH: Set[str] = set() + CFLAG_PASSTHROUGH_PREFIX: Tuple[str, ...] = ( + "-I", # includes + "-D", # defines + ) + + clangd_config = [] + + def add_unit(build_obj: BuildConfigUnit) -> None: + obj = objects.get(build_obj["name"]) + if obj is None: + return + + # Skip unresolved objects + if ( + obj.src_path is None + or obj.src_obj_path is None + or not file_is_c_cpp(obj.src_path) + ): + return + + # Gather cflags for source file + cflags: list[str] = [] + + def append_cflags(flags: Iterable[str]) -> None: + # Match a flag against either a set of concrete flags, or a set of prefixes. + def flag_match( + flag: str, concrete: Set[str], prefixes: Tuple[str, ...] + ) -> bool: + if flag in concrete: + return True + + for prefix in prefixes: + if flag.startswith(prefix): + return True + + return False + + # Determine whether a flag should be ignored. + def should_ignore(flag: str) -> bool: + return flag_match(flag, CFLAG_IGNORE, CFLAG_IGNORE_PREFIX) + + # Determine whether a flag should be passed through. + def should_passthrough(flag: str) -> bool: + return flag_match(flag, CFLAG_PASSTHROUGH, CFLAG_PASSTHROUGH_PREFIX) + + # Attempts replacement for the given flag. + def try_replace(flag: str) -> bool: + replacement = CFLAG_REPLACE.get(flag) + if replacement is not None: + cflags.append(replacement) + return True + + for prefix, replacement in CFLAG_REPLACE_PREFIX: + if flag.startswith(prefix): + cflags.append(flag.replace(prefix, replacement, 1)) + return True + + for prefix, options in CFLAG_REPLACE_OPTIONS: + if not flag.startswith(prefix): + continue + + # "-lang c99" and "-lang=c99" are both generally valid option forms + option = flag.removeprefix(prefix).removeprefix("=").lstrip() + replacements = options.get(option) + if replacements is not None: + cflags.extend(replacements) + + return True + + return False + + for flag in flags: + # Ignore flags first + if should_ignore(flag): + continue - def code_frac(self) -> float: - return self.code_progress / self.code_total + # Then find replacements + if try_replace(flag): + continue - def data_frac(self) -> float: - return self.data_progress / self.data_total + # Pass flags through last + if should_passthrough(flag): + cflags.append(flag) + continue + + append_cflags(obj.options["cflags"]) + append_cflags(obj.options["extra_cflags"]) + cflags.extend(config.extra_clang_flags) + cflags.extend(obj.options["extra_clang_flags"]) + + unit_config = { + "directory": Path.cwd(), + "file": obj.src_path, + "output": obj.src_obj_path, + "arguments": [ + "clang", + "-nostdinc", + "-fno-builtin", + "--target=powerpc-eabi", + *cflags, + "-c", + obj.src_path, + "-o", + obj.src_obj_path, + ], + } + clangd_config.append(unit_config) # Add DOL units - all_progress = ProgressUnit("All") if config.progress_all else None - dol_progress = ProgressUnit("DOL") for unit in build_config["units"]: - if all_progress: - all_progress.add(unit) - dol_progress.add(unit) + add_unit(unit) # Add REL units - rels_progress = ProgressUnit("Modules") if config.progress_modules else None - modules_progress: List[ProgressUnit] = [] for module in build_config["modules"]: - progress = ProgressUnit(module["name"]) - modules_progress.append(progress) for unit in module["units"]: - if all_progress: - all_progress.add(unit) - if rels_progress: - rels_progress.add(unit) - progress.add(unit) + add_unit(unit) - # Print human-readable progress - print("Progress:") + # Write compile_commands.json + with open("compile_commands.json", "w", encoding="utf-8") as w: - def print_category(unit: Optional[ProgressUnit]) -> None: - if unit is None: - return + def default_format(o): + if isinstance(o, Path): + return o.resolve().as_posix() + return str(o) + + json.dump(clangd_config, w, indent=2, default=default_format) + + +# Calculate, print and write progress to progress.json +def calculate_progress(config: ProjectConfig) -> None: + config.validate() + out_path = config.out_path() + report_path = out_path / "report.json" + if not report_path.is_file(): + sys.exit(f"Report file {report_path} does not exist") + + report_data: Dict[str, Any] = {} + with open(report_path, "r", encoding="utf-8") as f: + report_data = json.load(f) + + # Convert string numbers (u64) to int + def convert_numbers(data: Dict[str, Any]) -> None: + for key, value in data.items(): + if isinstance(value, str) and value.isdigit(): + data[key] = int(value) + + convert_numbers(report_data["measures"]) + for category in report_data.get("categories", []): + convert_numbers(category["measures"]) + + # Output to GitHub Actions job summary, if available + summary_path = os.getenv("GITHUB_STEP_SUMMARY") + summary_file: Optional[IO[str]] = None + if summary_path: + summary_file = open(summary_path, "a", encoding="utf-8") + summary_file.write("```\n") + + def progress_print(s: str) -> None: + print(s) + if summary_file: + summary_file.write(s + "\n") - code_frac = unit.code_frac() - data_frac = unit.data_frac() - print( - f" {unit.name}: {code_frac:.2%} code, {data_frac:.2%} data ({unit.objects_progress} / {unit.objects_total} files)" + # Print human-readable progress + progress_print("Progress:") + + def print_category(name: str, measures: Dict[str, Any]) -> None: + total_code = measures.get("total_code", 0) + matched_code = measures.get("matched_code", 0) + matched_code_percent = measures.get("matched_code_percent", 0) + total_data = measures.get("total_data", 0) + matched_data = measures.get("matched_data", 0) + matched_data_percent = measures.get("matched_data_percent", 0) + total_functions = measures.get("total_functions", 0) + matched_functions = measures.get("matched_functions", 0) + complete_code_percent = measures.get("complete_code_percent", 0) + total_units = measures.get("total_units", 0) + complete_units = measures.get("complete_units", 0) + + progress_print( + f" {name}: {matched_code_percent:.2f}% matched, {complete_code_percent:.2f}% linked ({complete_units} / {total_units} files)" ) - print(f" Code: {unit.code_progress} / {unit.code_total} bytes") - print(f" Data: {unit.data_progress} / {unit.data_total} bytes") - if config.progress_use_fancy: - print( - "\nYou have {} out of {} {} and {} out of {} {}.".format( - math.floor(code_frac * unit.code_fancy_frac), - unit.code_fancy_frac, - unit.code_fancy_item, - math.floor(data_frac * unit.data_fancy_frac), - unit.data_fancy_frac, - unit.data_fancy_item, - ) + progress_print( + f" Code: {matched_code} / {total_code} bytes ({matched_functions} / {total_functions} functions)" + ) + progress_print( + f" Data: {matched_data} / {total_data} bytes ({matched_data_percent:.2f}%)" + ) + + print_category("All", report_data["measures"]) + for category in report_data.get("categories", []): + if config.print_progress_categories is True or ( + isinstance(config.print_progress_categories, list) + and category["id"] in config.print_progress_categories + ): + print_category(category["name"], category["measures"]) + + if config.progress_use_fancy: + measures = report_data["measures"] + total_code = measures.get("total_code", 0) + total_data = measures.get("total_data", 0) + if total_code == 0 or total_data == 0: + return + code_frac = measures.get("complete_code", 0) / total_code + data_frac = measures.get("complete_data", 0) / total_data + + progress_print( + "\nYou have {} out of {} {} and {} out of {} {}.".format( + math.floor(code_frac * config.progress_code_fancy_frac), + config.progress_code_fancy_frac, + config.progress_code_fancy_item, + math.floor(data_frac * config.progress_data_fancy_frac), + config.progress_data_fancy_frac, + config.progress_data_fancy_item, ) + ) - if all_progress: - print_category(all_progress) - print_category(dol_progress) - module_count = len(build_config["modules"]) - if module_count > 0: - print_category(rels_progress) - if config.progress_each_module: - for progress in modules_progress: - print_category(progress) + # Finalize GitHub Actions job summary + if summary_file: + summary_file.write("```\n") + summary_file.close() # Generate and write progress.json progress_json: Dict[str, Any] = {} - def add_category(category: str, unit: ProgressUnit) -> None: - progress_json[category] = { - "code": unit.code_progress, - "code/total": unit.code_total, - "data": unit.data_progress, - "data/total": unit.data_total, + def add_category(id: str, measures: Dict[str, Any]) -> None: + progress_json[id] = { + "code": measures.get("complete_code", 0), + "code/total": measures.get("total_code", 0), + "data": measures.get("complete_data", 0), + "data/total": measures.get("total_data", 0), + "matched_code": measures.get("matched_code", 0), + "matched_code/total": measures.get("total_code", 0), + "matched_data": measures.get("matched_data", 0), + "matched_data/total": measures.get("total_data", 0), + "matched_functions": measures.get("matched_functions", 0), + "matched_functions/total": measures.get("total_functions", 0), + "fuzzy_match": int(measures.get("fuzzy_match_percent", 0) * 100), + "fuzzy_match/total": 10000, + "units": measures.get("complete_units", 0), + "units/total": measures.get("total_units", 0), } - if all_progress: - add_category("all", all_progress) - add_category("dol", dol_progress) - if len(build_config["modules"]) > 0: - if rels_progress: - add_category("modules", rels_progress) - if config.progress_each_module: - for progress in modules_progress: - add_category(progress.name, progress) + if config.progress_all: + add_category("all", report_data["measures"]) + else: + # Support for old behavior where "dol" was the main category + add_category("dol", report_data["measures"]) + for category in report_data.get("categories", []): + add_category(category["id"], category["measures"]) + with open(out_path / "progress.json", "w", encoding="utf-8") as w: - json.dump(progress_json, w, indent=4) + json.dump(progress_json, w, indent=2)