From 7447abdd3f9513c5cd88b4098c6f93b5431daaed Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 29 Mar 2024 13:58:50 -0700 Subject: [PATCH 1/3] tests: move PYTHONPATH manipulation into the python file This allows executing the unittests directly with `python test_fypp.py`, which allowing easily passing options into the test harness. It also avoids having to know if user has set PYTHONPATH, since it will be evaluated after PYTHONPATH. --- test/runtests.sh | 5 ----- test/test_fypp.py | 6 ++++++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/test/runtests.sh b/test/runtests.sh index 422e171..9f62c6a 100755 --- a/test/runtests.sh +++ b/test/runtests.sh @@ -6,11 +6,6 @@ else pythons="python3" fi root=".." -if [ -z "$PYTHONPATH" ]; then - export PYTHONPATH="$root/src" -else - export PYTHONPATH="$root/src:$PYTHONPATH" -fi cd $testdir failed="0" failing_pythons="" diff --git a/test/test_fypp.py b/test/test_fypp.py index 6617051..3e66378 100644 --- a/test/test_fypp.py +++ b/test/test_fypp.py @@ -1,7 +1,13 @@ '''Unit tests for testing Fypp.''' from pathlib import Path +import os import platform +import sys import unittest + +# Allow for importing fypp +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) + import fypp From 77db1629577d493dd49c46a4df5ebfb7b95ff0ee Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 29 Mar 2024 14:35:37 -0700 Subject: [PATCH 2/3] Add support for writing a Make compatible depfile This allows build systems that can consume them (Make and Ninja, in particular) to figure out at build time which files are implicitly included in the dependency graph of a fypp output. This means for example if you have a target that includes two other files, an incremental rebuild can be triggered automatically if one of the included files are modified. --- src/fypp.py | 37 ++++++++++++++ test/include/escaped_includes.inc | 1 + test/include/multi_includes.inc | 2 + test/include/subfolder/need$ #escape.inc | 1 + test/test_fypp.py | 62 ++++++++++++++++++++++++ 5 files changed, 103 insertions(+) create mode 100644 test/include/escaped_includes.inc create mode 100644 test/include/multi_includes.inc create mode 100644 test/include/subfolder/need$ #escape.inc diff --git a/src/fypp.py b/src/fypp.py index f6bcbfc..faa1a58 100755 --- a/src/fypp.py +++ b/src/fypp.py @@ -233,6 +233,13 @@ def __init__(self, includedirs=None, encoding='utf-8'): # Directory of current file self._curdir = None + # All files that have been included + self._included_files = [] + + + def get_dependencies(self): + return self._included_files + def parsefile(self, fobj): '''Parses file or a file like object. @@ -252,6 +259,9 @@ def parsefile(self, fobj): def _includefile(self, span, fobj, fname, curdir): + # Don't add the root file, only later includes + if self._curfile: + self._included_files.append(fname) oldfile = self._curfile olddir = self._curdir self._curfile = fname @@ -2413,6 +2423,10 @@ def process_text(self, txt): return self._render() + def get_dependencies(self): + return self._parser.get_dependencies() + + def _render(self): output = self._renderer.render(self._builder.tree) self._builder.reset() @@ -2579,6 +2593,16 @@ def process_file(self, infile, outfile=None): return None + def write_dependencies(self, outfile, depfile): + def quote(text): + return text.replace('$', '$$').replace(' ', '\\ ').replace('#', '\\#') + + dependencies = [quote(d) for d in self._preprocessor.get_dependencies()] + + with open(depfile, 'w', encoding='utf-8') as f: + f.write('{}: {}'.format(quote(outfile), ' '.join(dependencies))) + + def process_text(self, txt): '''Processes a string. @@ -2674,6 +2698,8 @@ class FyppOptions(optparse.Values): setting. create_parent_folder (bool): Whether the parent folder for the output file should be created if it does not exist. Default: False. + depfile (str | None): If set, where to write a Makefile compatible + dependency file. Default: None. ''' def __init__(self): @@ -2696,6 +2722,7 @@ def __init__(self): self.encoding = 'utf-8' self.create_parent_folder = False self.file_var_root = None + self.depfile = None class FortranLineFolder: @@ -2939,6 +2966,10 @@ def get_option_parser(): parser.add_option('--file-var-root', metavar='DIR', dest='file_var_root', default=defs.file_var_root, help=msg) + msg = 'Write a Make-compatible dependency file to this location' + parser.add_option('--depfile', metavar='DEPFILE', dest='depfile', + default=defs.depfile, help=msg) + return parser @@ -2949,9 +2980,15 @@ def run_fypp(): opts, leftover = optparser.parse_args(values=options) infile = leftover[0] if len(leftover) > 0 else '-' outfile = leftover[1] if len(leftover) > 1 else '-' + + if outfile == '-' and opts.depfile: + raise optparse.OptParseError("--depfile cannot be used when writing to stdout") + try: tool = Fypp(opts) tool.process_file(infile, outfile) + if opts.depfile: + tool.write_dependencies(outfile, opts.depfile) except FyppStopRequest as exc: sys.stderr.write(_formatted_exception(exc)) sys.exit(USER_ERROR_EXIT_CODE) diff --git a/test/include/escaped_includes.inc b/test/include/escaped_includes.inc new file mode 100644 index 0000000..7a2b63c --- /dev/null +++ b/test/include/escaped_includes.inc @@ -0,0 +1 @@ +#:include 'need$ #escape.inc' diff --git a/test/include/multi_includes.inc b/test/include/multi_includes.inc new file mode 100644 index 0000000..7ff8ea0 --- /dev/null +++ b/test/include/multi_includes.inc @@ -0,0 +1,2 @@ +#:include 'fypp1.inc' +#:include 'fypp2.inc' diff --git a/test/include/subfolder/need$ #escape.inc b/test/include/subfolder/need$ #escape.inc new file mode 100644 index 0000000..d0128e8 --- /dev/null +++ b/test/include/subfolder/need$ #escape.inc @@ -0,0 +1 @@ +#:include 'fypp2.inc' diff --git a/test/test_fypp.py b/test/test_fypp.py index 3e66378..5f82dfe 100644 --- a/test/test_fypp.py +++ b/test/test_fypp.py @@ -3,6 +3,7 @@ import os import platform import sys +import tempfile import unittest # Allow for importing fypp @@ -3061,6 +3062,27 @@ def _importmodule(module): ), ] +DEPFILE_TESTS = [ + ('basic', + ([_incdir('include')], + 'include/subfolder/include_fypp1.inc', + '{output}: include/fypp1.inc', + ) + ), + ('multiple includes', + ([_incdir('include/subfolder')], + 'include/multi_includes.inc', + '{output}: include/fypp1.inc include/subfolder/fypp2.inc', + ) + ), + ('escapes', + ([_incdir('include'), _incdir('include/subfolder')], + 'include/escaped_includes.inc', + '{output}: include/subfolder/need$$\\ \\#escape.inc include/subfolder/fypp2.inc', + ) + ), +] + def _get_test_output_method(args, inp, out): '''Returns a test method for checking correctness of Fypp output. @@ -3108,6 +3130,36 @@ def test_output_from_file_input(self): return test_output_from_file_input +def _get_test_depfile_method(args, inputfile, expected): + '''Returns a test method for checking correctness of depfile. + + Args: + args (list of str): Command-line arguments to pass to Fypp. + inputfile (str): Input file with Fypp directives. + out (str): Expected output. + + Returns: + method: Method to test equality of depfile with result delivered by Fypp. + ''' + + def test_depfile(self): + '''Tests whether Fypp result matches expected output when input is in a file.''' + output = self._get_tempfile() + depfile = self._get_tempfile() + + optparser = fypp.get_option_parser() + options, leftover = optparser.parse_args(args + ['--depfile', depfile]) + self.assertEqual(len(leftover), 0) + tool = fypp.Fypp(options) + tool.process_file(inputfile, output) + tool.write_dependencies(output, depfile) + + with open(depfile, 'r', encoding='utf-8') as f: + got = f.read().strip() + self.assertEqual(got, expected.format(output=output)) + return test_depfile + + def _get_test_exception_method(args, inp, exceptions): '''Returns a test method for checking correctness of thrown exception. @@ -3205,6 +3257,16 @@ class ExceptionTest(_TestContainer): pass class ImportTest(_TestContainer): pass ImportTest.add_test_methods(IMPORT_TESTS, _get_test_output_method) +class DepfileTest(_TestContainer): + + def _get_tempfile(self): + _fd, output = tempfile.mkstemp() + os.close(_fd) + self.addCleanup(os.unlink, output) + return output + +DepfileTest.add_test_methods(DEPFILE_TESTS, _get_test_depfile_method) + if __name__ == '__main__': unittest.main() From e78b3380c3dcd6e0dcd6970c86f761dc09c0371a Mon Sep 17 00:00:00 2001 From: Dylan Baker Date: Fri, 29 Mar 2024 15:21:28 -0700 Subject: [PATCH 3/3] docs: Add use of depfile to CMake instructions --- docs/integration.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/integration.rst b/docs/integration.rst index ebab1d4..a3eea35 100644 --- a/docs/integration.rst +++ b/docs/integration.rst @@ -32,11 +32,15 @@ very first version of this example):: # Generate input file name set(infile "${CMAKE_CURRENT_SOURCE_DIR}/${infileName}") + # Create the dependency file + set(depfile "${CMAKE_CURRENT_BINARY_DIR/${outfileName}.d") + # Custom command to do the processing add_custom_command( OUTPUT "${outfile}" - COMMAND fypp "${infile}" "${outfile}" + COMMAND fypp "${infile}" "${outfile}" --depfile "${depfile}" MAIN_DEPENDENCY "${infile}" + DEPFILE "${depfile}" VERBATIM) # Finally add output file to a list