pipcl.py

'''
Python packaging operations, including PEP-517 support, for use by a `setup.py`
script.

Overview:

    The intention is to take care of as many packaging details as possible so
    that setup.py contains only project-specific information, while also giving
    as much flexibility as possible.

    For example we provide a function `build_extension()` that can be used
    to build a SWIG extension, but we also give access to the located
    compiler/linker so that a `setup.py` script can take over the details
    itself.

Doctests:
    Doctest strings are provided in some comments.

    Test in the usual way with:
        python -m doctest pipcl.py

    Test specific functions/classes with:
        python pipcl.py --doctest run_if ...

        If no functions or classes are specified, this tests everything.

Graal:
    For Graal we require that PIPCL_GRAAL_PYTHON is set to non-graal Python (we
    build for non-graal except with Graal Python's include paths and library
    directory).
'''

import base64
import codecs
import difflib
import glob
import hashlib
import inspect
import io
import os
import pickle
import platform
import re
import shlex
import shutil
import site
import subprocess
import sys
import sysconfig
import tarfile
import textwrap
import time
import zipfile

import wdev


class Package:
    '''
    Our constructor takes a definition of a Python package similar to that
    passed to `distutils.core.setup()` or `setuptools.setup()` (name, version,
    summary etc) plus callbacks for building, getting a list of sdist
    filenames, and cleaning.

    We provide methods that can be used to implement a Python package's
    `setup.py` supporting PEP-517.

    We also support basic command line handling for use
    with a legacy (pre-PEP-517) pip, as implemented
    by legacy distutils/setuptools and described in:
    https://pip.pypa.io/en/stable/reference/build-system/setup-py/

    The file pyproject.toml must exist; this is checked if/when fn_build() is
    called.

    Here is a `doctest` example of using pipcl to create a SWIG extension
    module. Requires `swig`.

    Create an empty test directory:

        >>> import os
        >>> import shutil
        >>> shutil.rmtree('pipcl_test', ignore_errors=1)
        >>> os.mkdir('pipcl_test')

    Create a `setup.py` which uses `pipcl` to define an extension module.

        >>> import textwrap
        >>> with open('pipcl_test/setup.py', 'w') as f:
        ...     _ = f.write(textwrap.dedent("""
        ...             import sys
        ...             import pipcl
        ...
        ...             def build():
        ...                 so_leaf = pipcl.build_extension(
        ...                         name = 'foo',
        ...                         path_i = 'foo.i',
        ...                         outdir = 'build',
        ...                         source_extra = 'wibble.c',
        ...                         )
        ...                 return [
        ...                         ('build/foo.py', 'foo/__init__.py'),
        ...                         ('cli.py', 'foo/__main__.py'),
        ...                         (f'build/{so_leaf}', f'foo/'),
        ...                         ('README', '$dist-info/'),
        ...                         (b'Hello world', 'foo/hw.txt'),
        ...                         ]
        ...
        ...             def sdist():
        ...                 return [
        ...                         'pyproject.toml',
        ...                         'foo.i',
        ...                         'bar.i',
        ...                         'wibble.c',
        ...                         'setup.py',
        ...                         'pipcl.py',
        ...                         'wdev.py',
        ...                         'README',
        ...                         (b'Hello word2', 'hw2.txt'),
        ...                         ]
        ...
        ...             p = pipcl.Package(
        ...                     name = 'foo',
        ...                     version = '1.2.3',
        ...                     fn_build = build,
        ...                     fn_sdist = sdist,
        ...                     entry_points = (
        ...                         { 'console_scripts': [
        ...                             'foo_cli = foo.__main__:main',
        ...                             ],
        ...                         }),
        ...                     )
        ...
        ...             build_wheel = p.build_wheel
        ...             build_sdist = p.build_sdist
        ...
        ...             # Handle old-style setup.py command-line usage:
        ...             if __name__ == '__main__':
        ...                 p.handle_argv(sys.argv)
        ...             """))

    Create the files required by the above `setup.py` - the SWIG `.i` input
    file, the README file, and copies of `pipcl.py` and `wdev.py`.

        >>> with open('pipcl_test/foo.i', 'w') as f:
        ...     _ = f.write(textwrap.dedent("""
        ...             %include bar.i
        ...             %{
        ...             #include <stdio.h>
        ...             #include <string.h>
        ...             int bar(const char* text)
        ...             {
        ...                 printf("bar(): text: %s\\\\n", text);
        ...                 int len = (int) strlen(text);
        ...                 printf("bar(): len=%i\\\\n", len);
        ...                 fflush(stdout);
        ...                 return len;
        ...             }
        ...             %}
        ...             int bar(const char* text);
        ...             """))

        >>> with open('pipcl_test/bar.i', 'w') as f:
        ...     _ = f.write( '\\n')

        >>> with open('pipcl_test/wibble.c', 'w') as f:
        ...     _ = f.write( '\\n')

        >>> with open('pipcl_test/pyproject.toml', 'w') as f:
        ...     pass

        >>> with open('pipcl_test/README', 'w') as f:
        ...     _ = f.write(textwrap.dedent("""
        ...             This is Foo.
        ...             """))

        >>> with open('pipcl_test/cli.py', 'w') as f:
        ...     _ = f.write(textwrap.dedent("""
        ...             def main():
        ...                 print('pipcl_test:main().')
        ...             if __name__ == '__main__':
        ...                 main()
        ...             """))

        >>> root = os.path.dirname(__file__)
        >>> _ = shutil.copy2(f'{root}/pipcl.py', 'pipcl_test/pipcl.py')
        >>> _ = shutil.copy2(f'{root}/wdev.py', 'pipcl_test/wdev.py')

    Use `setup.py`'s command-line interface to build and install the extension
    module into root `pipcl_test/install`.

        >>> _ = subprocess.run(
        ...         f'cd pipcl_test && {sys.executable} setup.py --root install install',
        ...         shell=1, check=1)

    The actual install directory depends on `sysconfig.get_path('platlib')`:

        >>> if windows():
        ...     install_dir = 'pipcl_test/install'
        ... else:
        ...     install_dir = f'pipcl_test/install/{sysconfig.get_path("platlib").lstrip(os.sep)}'
        >>> assert os.path.isfile( f'{install_dir}/foo/__init__.py')

    Create a test script which asserts that Python function call `foo.bar(s)`
    returns the length of `s`, and run it with `PYTHONPATH` set to the install
    directory:

        >>> with open('pipcl_test/test.py', 'w') as f:
        ...     _ = f.write(textwrap.dedent("""
        ...             import sys
        ...             import foo
        ...             text = 'hello'
        ...             print(f'test.py: calling foo.bar() with text={text!r}')
        ...             sys.stdout.flush()
        ...             l = foo.bar(text)
        ...             print(f'test.py: foo.bar() returned: {l}')
        ...             assert l == len(text)
        ...             """))
        >>> r = subprocess.run(
        ...         f'{sys.executable} pipcl_test/test.py',
        ...         shell=1, check=1, text=1,
        ...         stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
        ...         env=os.environ | dict(PYTHONPATH=install_dir),
        ...         )
        >>> print(r.stdout)
        test.py: calling foo.bar() with text='hello'
        bar(): text: hello
        bar(): len=5
        test.py: foo.bar() returned: 5
        <BLANKLINE>

    Check that building sdist and wheel succeeds. For now we don't attempt to
    check that the sdist and wheel actually work.

        >>> _ = subprocess.run(
        ...         f'cd pipcl_test && {sys.executable} setup.py sdist',
        ...         shell=1, check=1)

        >>> _ = subprocess.run(
        ...         f'cd pipcl_test && {sys.executable} setup.py bdist_wheel',
        ...         shell=1, check=1)

    Check that rebuild does nothing.

        >>> t0 = os.path.getmtime('pipcl_test/build/foo.py')
        >>> _ = subprocess.run(
        ...         f'cd pipcl_test && {sys.executable} setup.py bdist_wheel',
        ...         shell=1, check=1)
        >>> t = os.path.getmtime('pipcl_test/build/foo.py')
        >>> assert t == t0

    Check that touching bar.i forces rebuild.

        >>> os.utime('pipcl_test/bar.i')
        >>> _ = subprocess.run(
        ...         f'cd pipcl_test && {sys.executable} setup.py bdist_wheel',
        ...         shell=1, check=1)
        >>> t = os.path.getmtime('pipcl_test/build/foo.py')
        >>> assert t > t0

    Check that touching foo.i.cpp does not run swig, but does recompile/link.

        >>> t0 = time.time()
        >>> os.utime('pipcl_test/build/foo.i.cpp')
        >>> _ = subprocess.run(
        ...         f'cd pipcl_test && {sys.executable} setup.py bdist_wheel',
        ...         shell=1, check=1)
        >>> assert os.path.getmtime('pipcl_test/build/foo.py') <= t0
        >>> so = glob.glob('pipcl_test/build/*.so')
        >>> assert len(so) == 1
        >>> so = so[0]
        >>> assert os.path.getmtime(so) > t0

    Check that touching wibble.c does not run swig, but does recompile/link.

        >>> t0 = time.time()
        >>> os.utime('pipcl_test/wibble.c')
        >>> _ = subprocess.run(
        ...         f'cd pipcl_test && {sys.executable} setup.py bdist_wheel',
        ...         shell=1, check=1)
        >>> assert os.path.getmtime('pipcl_test/build/foo.py') <= t0
        >>> so = glob.glob('pipcl_test/build/*.so')
        >>> assert len(so) == 1
        >>> so = so[0]
        >>> assert os.path.getmtime(so) > t0

    Check `entry_points` causes creation of command `foo_cli` when we install
    from our wheel using pip. [As of 2024-02-24 using pipcl's CLI interface
    directly with `setup.py install` does not support entry points.]

        >>> print('Creating venv.', file=sys.stderr)
        >>> _ = subprocess.run(
        ...         f'cd pipcl_test && {sys.executable} -m venv pylocal',
        ...         shell=1, check=1)

        >>> print('Installing from wheel into venv using pip.', file=sys.stderr)
        >>> _ = subprocess.run(
        ...         f'. pipcl_test/pylocal/bin/activate && pip install pipcl_test/dist/*.whl',
        ...         shell=1, check=1)

        >>> print('Running foo_cli.', file=sys.stderr)
        >>> _ = subprocess.run(
        ...         f'. pipcl_test/pylocal/bin/activate && foo_cli',
        ...         shell=1, check=1)

    Wheels and sdists

        Wheels:
            We generate wheels according to:
            https://packaging.python.org/specifications/binary-distribution-format/

            * `{name}-{version}.dist-info/RECORD` uses sha256 hashes.
            * We do not generate other `RECORD*` files such as
              `RECORD.jws` or `RECORD.p7s`.
            * `{name}-{version}.dist-info/WHEEL` has:

              * `Wheel-Version: 1.0`
              * `Root-Is-Purelib: false`
            * No support for signed wheels.

        Sdists:
            We generate sdist's according to:
            https://packaging.python.org/specifications/source-distribution-format/
    '''
    def __init__(self,
            name,
            version,
            *,
            platform = None,
            supported_platform = None,
            summary = None,
            description = None,
            description_content_type = None,
            keywords = None,
            home_page = None,
            download_url = None,
            author = None,
            author_email = None,
            maintainer = None,
            maintainer_email = None,
            license = None,
            classifier = None,
            requires_dist = None,
            requires_python = None,
            requires_external = None,
            project_url = None,
            provides_extra = None,

            entry_points = None,

            root = None,
            fn_build = None,
            fn_clean = None,
            fn_sdist = None,
            tag_python = None,
            tag_abi = None,
            tag_platform = None,
            py_limited_api = None,

            wheel_compression = zipfile.ZIP_DEFLATED,
            wheel_compresslevel = None,
            ):
        '''
        The initial args before `entry_points` define the
        package metadata and closely follow the definitions in:
        https://packaging.python.org/specifications/core-metadata/

        Args:

            name:
                Used for metadata `Name`.
                A string, the name of the Python package.
            version:
                Used for metadata `Version`.
                A string, the version of the Python package. Also see PEP-440
                `Version Identification and Dependency Specification`.
            platform:
                Used for metadata `Platform`.
                A string or list of strings.
            supported_platform:
                Used for metadata `Supported-Platform`.
                A string or list of strings.
            summary:
                Used for metadata `Summary`.
                A string, short description of the package.
            description:
                Used for metadata `Description`.
                A string. If contains newlines, a detailed description of the
                package. Otherwise the path of a file containing the detailed
                description of the package.
            description_content_type:
                Used for metadata `Description-Content-Type`.
                A string describing markup of `description` arg. For example
                `text/markdown; variant=GFM`.
            keywords:
                Used for metadata `Keywords`.
                A string containing comma-separated keywords.
            home_page:
                Used for metadata `Home-page`.
                URL of home page.
            download_url:
                Used for metadata `Download-URL`.
                Where this version can be downloaded from.
            author:
                Used for metadata `Author`.
                Author.
            author_email:
                Used for metadata `Author-email`.
                Author email.
            maintainer:
                Used for metadata `Maintainer`.
                Maintainer.
            maintainer_email:
                Used for metadata `Maintainer-email`.
                Maintainer email.
            license:
                Used for metadata `License`.
                A string containing the license text. Written into metadata
                file `COPYING`. Is also written into metadata itself if not
                multi-line.
            classifier:
                Used for metadata `Classifier`.
                A string or list of strings. Also see:

                * https://pypi.org/pypi?%3Aaction=list_classifiers
                * https://pypi.org/classifiers/

            requires_dist:
                Used for metadata `Requires-Dist`.
                A string or list of strings, Python packages required
                at runtime. None items are ignored.
            requires_python:
                Used for metadata `Requires-Python`.
                A string or list of strings.
            requires_external:
                Used for metadata `Requires-External`.
                A string or list of strings.
            project_url:
                Used for metadata `Project-URL`.
                A string or list of strings, each of the form: `{name},
                {url}`.
            provides_extra:
                Used for metadata `Provides-Extra`.
                A string or list of strings.

            entry_points:
                String or dict specifying *.dist-info/entry_points.txt, for
                example:

                    ```
                    [console_scripts]
                    foo_cli = foo.__main__:main
                    ```

                or:

                    { 'console_scripts': [
                        'foo_cli = foo.__main__:main',
                        ],
                    }

                See: https://packaging.python.org/en/latest/specifications/entry-points/

            root:
                Root of package, defaults to current directory.

            fn_build:
                A function taking no args, or a single `config_settings` dict
                arg (as described in PEP-517), that builds the package.

                Should return a list of items; each item should be a tuple
                `(from_, to_)`, or a single string `path` which is treated as
                the tuple `(path, path)`.

                `from_` can be a string or a `bytes`. If a string it should
                be the path to a file; a relative path is treated as relative
                to `root`. If a `bytes` it is the contents of the file to be
                added.

                `to_` identifies what the file should be called within a wheel
                or when installing. If `to_` is empty or `/` we set it to the
                leaf of `from_` (`from_` must not be a `bytes`) - i.e. we place
                the file in the root directory of the wheel; otherwise if
                `to_` ends with `/` the leaf of `from_` is appended to it (and
                `from_` must not be a `bytes`).

                Initial `$dist-info/` in `_to` is replaced by
                `{name}-{version}.dist-info/`; this is useful for license files
                etc.

                Initial `$data/` in `_to` is replaced by
                `{name}-{version}.data/`. We do not enforce particular
                subdirectories, instead it is up to `fn_build()` to specify
                specific subdirectories such as `purelib`, `headers`,
                `scripts`, `data` etc.

                If we are building a wheel (e.g. `python setup.py bdist_wheel`,
                or PEP-517 pip calls `self.build_wheel()`), we add file `from_`
                to the wheel archive with name `to_`.

                If we are installing (e.g. `install` command in
                the argv passed to `self.handle_argv()`), then
                we copy `from_` to `{sitepackages}/{to_}`, where
                `sitepackages` is the installation directory, the
                default being `sysconfig.get_path('platlib')` e.g.
                `myvenv/lib/python3.9/site-packages/`.

                When calling this function, we assert that the file
                pyproject.toml exists in the current directory. (We do this
                here rather than in pipcl.Package's constructor, as otherwise
                importing setup.py from non-package-related code could fail.)

            fn_clean:
                A function taking a single arg `all_` that cleans generated
                files. `all_` is true iff `--all` is in argv.

                For safety and convenience, can also returns a list of
                files/directory paths to be deleted. Relative paths are
                interpreted as relative to `root`. All paths are asserted to be
                within `root`.

            fn_sdist:
                A function taking no args, or a single `config_settings` dict
                arg (as described in PEP517), that returns a list of items to
                be copied into the sdist. The list should be in the same format
                as returned by `fn_build`.

                It can be convenient to use `pipcl.git_items()`.

                The specification for sdists requires that the list contains
                `pyproject.toml`; we enforce this with a Python assert.

            tag_python:
                First element of wheel tag defined in PEP-425. If None we use
                `cp{version}`.

                For example if code works with any Python version, one can use
                'py3'.

            tag_abi:
                Second element of wheel tag defined in PEP-425. If None we use
                `none`.

            tag_platform:
                Third element of wheel tag defined in PEP-425. Default
                is `os.environ('AUDITWHEEL_PLAT')` if set, otherwise
                derived from `sysconfig.get_platform()` (was
                `setuptools.distutils.util.get_platform(), before that
                `distutils.util.get_platform()` as specified in the PEP), e.g.
                `openbsd_7_0_amd64`.

                For pure python packages use: `tag_platform=any`

            py_limited_api:
                If true we build wheels that use the Python Limited API. We use
                the version of `sys.executable` to define `Py_LIMITED_API` when
                compiling extensions, and use ABI tag `abi3` in the wheel name
                if argument `tag_abi` is None.

            wheel_compression:
                Used as `zipfile.ZipFile()`'s `compression` parameter when
                creating wheels.

            wheel_compresslevel:
                Used as `zipfile.ZipFile()`'s `compresslevel` parameter when
                creating wheels.

        Occurrences of `None` in lists are ignored.
        '''
        assert name
        assert version

        def assert_str( v):
            if v is not None:
                assert isinstance( v, str), f'Not a string: {v!r}'
        def assert_str_or_multi( v):
            if v is not None:
                assert isinstance( v, (str, tuple, list)), f'Not a string, tuple or list: {v!r}'

        assert_str( name)
        assert_str( version)
        assert_str_or_multi( platform)
        assert_str_or_multi( supported_platform)
        assert_str( summary)
        assert_str( description)
        assert_str( description_content_type)
        assert_str( keywords)
        assert_str( home_page)
        assert_str( download_url)
        assert_str( author)
        assert_str( author_email)
        assert_str( maintainer)
        assert_str( maintainer_email)
        assert_str( license)
        assert_str_or_multi( classifier)
        assert_str_or_multi( requires_dist)
        assert_str( requires_python)
        assert_str_or_multi( requires_external)
        assert_str_or_multi( project_url)
        assert_str_or_multi( provides_extra)
        
        assert re.match('^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])\\Z', name, re.IGNORECASE), (
                f'Invalid package name'
                f' (https://packaging.python.org/en/latest/specifications/name-normalization/)'
                f': {name!r}'
                )

        # https://packaging.python.org/en/latest/specifications/core-metadata/.
        assert re.match('([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$', name, re.IGNORECASE), \
                f'Bad name: {name!r}'

        _assert_version_pep_440(version)

        # https://packaging.python.org/en/latest/specifications/binary-distribution-format/
        if tag_python:
            assert '-' not in tag_python
        if tag_abi:
            assert '-' not in tag_abi
        if tag_platform:
            assert '-' not in tag_platform

        self.name = name
        self.version = version
        self.platform = platform
        self.supported_platform = supported_platform
        self.summary = summary
        self.description = description
        self.description_content_type = description_content_type
        self.keywords = keywords
        self.home_page = home_page
        self.download_url = download_url
        self.author = author
        self.author_email  = author_email
        self.maintainer = maintainer
        self.maintainer_email = maintainer_email
        self.license = license
        self.classifier = classifier
        self.requires_dist = requires_dist
        self.requires_python = requires_python
        self.requires_external = requires_external
        self.project_url = project_url
        self.provides_extra = provides_extra
        self.entry_points = entry_points

        self.root = os.path.abspath(root if root else os.getcwd())
        self.fn_build = fn_build
        self.fn_clean = fn_clean
        self.fn_sdist = fn_sdist
        self.tag_python_ = tag_python
        self.tag_abi_ = tag_abi
        self.tag_platform_ = tag_platform
        self.py_limited_api = py_limited_api

        self.wheel_compression = wheel_compression
        self.wheel_compresslevel = wheel_compresslevel
        
        # If true and we are building for graal, we set PIPCL_PYTHON_CONFIG to
        # a command that will print includes/libs from graal_py's sysconfig.
        #
        self.graal_legacy_python_config = True


    def build_wheel(self,
            wheel_directory,
            config_settings=None,
            metadata_directory=None,
            ):
        '''
        A PEP-517 `build_wheel()` function.

        Also called by `handle_argv()` to handle the `bdist_wheel` command.

        Returns leafname of generated wheel within `wheel_directory`.
        '''
        log2(
                f' wheel_directory={wheel_directory!r}'
                f' config_settings={config_settings!r}'
                f' metadata_directory={metadata_directory!r}'
                )

        if os.environ.get('CIBUILDWHEEL') == '1':
            # Don't special-case graal builds when running under cibuildwheel.
            pass
        elif sys.implementation.name == 'graalpy':
            # We build for Graal by building a native Python wheel with Graal
            # Python's include paths and library directory. We then rename the
            # wheel to contain graal's tag etc.
            #
            log0(f'### Graal build: deferring to cpython.')
            python_native = os.environ.get('PIPCL_GRAAL_PYTHON')
            assert python_native, f'Graal build requires that PIPCL_GRAAL_PYTHON is set.'
            env_extra = dict(
                    PIPCL_SYSCONFIG_PATH_include = sysconfig.get_path('include'),
                    PIPCL_SYSCONFIG_PATH_platinclude = sysconfig.get_path('platinclude'),
                    PIPCL_SYSCONFIG_CONFIG_VAR_LIBDIR = sysconfig.get_config_var('LIBDIR'),
                    )
            # Tell native build to run pipcl.py itself to get python-config
            # information about include paths etc.
            if self.graal_legacy_python_config:
                env_extra['PIPCL_PYTHON_CONFIG'] = f'{python_native} {os.path.abspath(__file__)} --graal-legacy-python-config'
            
            # Create venv.
            venv_name = os.environ.get('PIPCL_GRAAL_NATIVE_VENV')
            if venv_name:
                log1(f'Graal using pre-existing {venv_name=}')
            else:
                venv_name = 'venv-pipcl-graal-native'
                run(f'{shlex.quote(python_native)} -m venv {venv_name}')
                log1(f'Graal using {venv_name=}')
            
            newfiles = NewFiles(f'{wheel_directory}/*.whl')
            run(
                    f'. {venv_name}/bin/activate && python setup.py --dist-dir {shlex.quote(wheel_directory)} bdist_wheel',
                    env_extra = env_extra,
                    prefix = f'pipcl.py graal {python_native}: ',
                    )
            wheel = newfiles.get_one()
            wheel_leaf = os.path.basename(wheel)
            python_major_minor = run(f'{shlex.quote(python_native)} -c "import platform; import sys; sys.stdout.write(str().join(platform.python_version_tuple()[:2]))"', capture=1)
            cpabi = f'cp{python_major_minor}-abi3'
            assert cpabi in wheel_leaf, f'Expected wheel to be for {cpabi=}, but {wheel=}.'
            graalpy_ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
            log1(f'{graalpy_ext_suffix=}')
            m = re.match(r'\.graalpy(\d+[^\-]*)-(\d+)', graalpy_ext_suffix)
            gpver = m[1]
            cpver = m[2]
            graalpy_wheel_tag = f'graalpy{cpver}-graalpy{gpver}_{cpver}_native'
            name = wheel_leaf.replace(cpabi, graalpy_wheel_tag)
            destination = f'{wheel_directory}/{name}'
            log0(f'### Graal build: copying {wheel=} to {destination=}')
            # Copying results in two wheels which appears to confuse pip, showing:
            #   Found multiple .whl files; unspecified behaviour. Will call build_wheel.
            os.rename(wheel, destination)
            log1(f'Returning {name=}.')
            return name

        wheel_name = self.wheel_name()
        path = f'{wheel_directory}/{wheel_name}'

        # Do a build and get list of files to copy into the wheel.
        #
        items = list()
        if self.fn_build:
            items = self._call_fn_build(config_settings)

        log2(f'Creating wheel: {path}')
        os.makedirs(wheel_directory, exist_ok=True)
        record = _Record()
        with zipfile.ZipFile(path, 'w', self.wheel_compression, self.wheel_compresslevel) as z:

            def add(from_, to_):
                if isinstance(from_, str):
                    z.write(from_, to_)
                    record.add_file(from_, to_)
                elif isinstance(from_, bytes):
                    z.writestr(to_, from_)
                    record.add_content(from_, to_)
                else:
                    assert 0

            def add_str(content, to_):
                add(content.encode('utf8'), to_)

            dist_info_dir = self._dist_info_dir()

            # Add the files returned by fn_build().
            #
            for item in items:
                from_, (to_abs, to_rel) = self._fromto(item)
                add(from_, to_rel)

            # Add <name>-<version>.dist-info/WHEEL.
            #
            add_str(
                    f'Wheel-Version: 1.0\n'
                    f'Generator: pipcl\n'
                    f'Root-Is-Purelib: false\n'
                    f'Tag: {self.wheel_tag_string()}\n'
                    ,
                    f'{dist_info_dir}/WHEEL',
                    )
            # Add <name>-<version>.dist-info/METADATA.
            #
            add_str(self._metainfo(), f'{dist_info_dir}/METADATA')

            # Add <name>-<version>.dist-info/COPYING.
            if self.license:
                add_str(self.license, f'{dist_info_dir}/COPYING')

            # Add <name>-<version>.dist-info/entry_points.txt.
            entry_points_text = self._entry_points_text()
            if entry_points_text:
                add_str(entry_points_text, f'{dist_info_dir}/entry_points.txt')

            # Update <name>-<version>.dist-info/RECORD. This must be last.
            #
            z.writestr(f'{dist_info_dir}/RECORD', record.get(f'{dist_info_dir}/RECORD'))

        st = os.stat(path)
        log1( f'Have created wheel size={st.st_size:,}: {path}')
        if g_verbose >= 2:
            with zipfile.ZipFile(path, compression=self.wheel_compression) as z:
                log2(f'Contents are:')
                for zi in sorted(z.infolist(), key=lambda z: z.filename):
                    log2(f'    {zi.file_size: 10,d} {zi.filename}')

        return os.path.basename(path)


    def build_sdist(self,
            sdist_directory,
            formats,
            config_settings=None,
            ):
        '''
        A PEP-517 `build_sdist()` function.

        Also called by `handle_argv()` to handle the `sdist` command.

        Returns leafname of generated archive within `sdist_directory`.
        '''
        assert self.fn_sdist, f'fn_sdist() not provided.'
        log2(
                f' sdist_directory={sdist_directory!r}'
                f' formats={formats!r}'
                f' config_settings={config_settings!r}'
                )
        if formats and formats != 'gztar':
            raise Exception( f'Unsupported: formats={formats}')
        items = list()
        if inspect.signature(self.fn_sdist).parameters:
            items = self.fn_sdist(config_settings)
        else:
            items = self.fn_sdist()

        prefix = f'{_normalise2(self.name)}-{self.version}'
        os.makedirs(sdist_directory, exist_ok=True)
        tarpath = f'{sdist_directory}/{prefix}.tar.gz'
        log2(f'Creating sdist: {tarpath}')

        with tarfile.open(tarpath, 'w:gz') as tar:

            names_in_tar = list()
            def check_name(name):
                if name in names_in_tar:
                    raise Exception(f'Name specified twice: {name}')
                names_in_tar.append(name)

            def add(from_, name):
                check_name(name)
                if isinstance(from_, str):
                    log2( f'Adding file: {os.path.relpath(from_)} => {name}')
                    tar.add( from_, f'{prefix}/{name}', recursive=False)
                elif isinstance(from_, bytes):
                    log2( f'Adding: {name}')
                    ti = tarfile.TarInfo(f'{prefix}/{name}')
                    ti.size = len(from_)
                    ti.mtime = time.time()
                    tar.addfile(ti, io.BytesIO(from_))
                else:
                    assert 0

            def add_string(text, name):
                textb = text.encode('utf8')
                return add(textb, name)

            found_pyproject_toml = False
            for item in items:
                from_, (to_abs, to_rel) = self._fromto(item)
                if isinstance(from_, bytes):
                    add(from_, to_rel)
                else:
                    if from_.startswith(f'{os.path.abspath(sdist_directory)}/'):
                        # Source files should not be inside <sdist_directory>.
                        assert 0, f'Path is inside sdist_directory={sdist_directory}: {from_!r}'
                    assert os.path.exists(from_), f'Path does not exist: {from_!r}'
                    assert os.path.isfile(from_), f'Path is not a file: {from_!r}'
                    add(from_, to_rel)
                if to_rel == 'pyproject.toml':
                    found_pyproject_toml = True

            assert found_pyproject_toml, f'Cannot create sdist because file not specified: pyproject.toml'

            # Always add a PKG-INFO file.
            add_string(self._metainfo(), 'PKG-INFO')

            if self.license:
                if 'COPYING' in names_in_tar:
                    log2(f'Not writing .license because file already in sdist: COPYING')
                else:
                    add_string(self.license, 'COPYING')

        log1( f'Have created sdist: {tarpath}')
        return os.path.basename(tarpath)

    def wheel_tag_string(self):
        '''
        Returns <tag_python>-<tag_abi>-<tag_platform>.
        '''
        return f'{self.tag_python()}-{self.tag_abi()}-{self.tag_platform()}'

    def tag_python(self):
        '''
        Get two-digit python version, e.g. 'cp3.8' for python-3.8.6.
        '''
        if self.tag_python_:
            ret = self.tag_python_
        else:
            ret = 'cp' + ''.join(platform.python_version().split('.')[:2])
        assert '-' not in ret
        return ret

    def tag_abi(self):
        '''
        ABI tag.
        '''
        if self.tag_abi_:
            return self.tag_abi_
        elif self.py_limited_api:
            return 'abi3'
        else:
            return 'none'

    def tag_platform(self):
        '''
        Find platform tag used in wheel filename.
        '''
        ret = self.tag_platform_
        log0(f'From self.tag_platform_: {ret=}.')
        
        if not ret:
            # Prefer this to PEP-425. Appears to be undocumented,
            # but set in manylinux docker images and appears
            # to be used by cibuildwheel and auditwheel, e.g.
            # https://github.com/rapidsai/shared-action-workflows/issues/80
            ret = os.environ.get( 'AUDITWHEEL_PLAT')
            log0(f'From AUDITWHEEL_PLAT: {ret=}.')

        if not ret:
            # Notes:
            #
            # PEP-425. On Linux gives `linux_x86_64` which is rejected by
            # pypi.org.
            #
            # On local MacOS/arm64 mac-mini have seen sysconfig.get_platform()
            # unhelpfully return `macosx-10.9-universal2` if `python3` is the
            # system Python /usr/bin/python3; this happens if we source `.
            # /etc/profile`.
            #
            ret = sysconfig.get_platform()
            ret = ret.replace('-', '_').replace('.', '_').lower()
            log0(f'From sysconfig.get_platform(): {ret=}.')

            ret = _macos_fixup_platform_tag(ret)

        log0( f'tag_platform(): returning {ret=}.')
        assert '-' not in ret
        return ret

    def wheel_name(self):
        ret = f'{_normalise2(self.name)}-{self.version}-{self.tag_python()}-{self.tag_abi()}-{self.tag_platform()}.whl'
        assert ret.count('-') == 4, f'Expected 4 dash characters in {ret=}.'
        return ret

    def wheel_name_match(self, wheel):
        '''
        Returns true if `wheel` matches our wheel. We basically require the
        name to be the same, except that we accept platform tags that contain
        extra items (see pep-0600/), for example we return true with:

            self:   foo-cp38-none-manylinux2014_x86_64.whl
            wheel:  foo-cp38-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl
        '''
        log2(f'{wheel=}')
        assert wheel.endswith('.whl')
        wheel2 = wheel[:-len('.whl')]
        name, version, tag_python, tag_abi, tag_platform = wheel2.split('-')
        
        py_limited_api_compatible = False
        if self.py_limited_api and tag_abi == 'abi3':
            # Allow lower tag_python number.
            m = re.match('cp([0-9]+)', tag_python)
            tag_python_int = int(m.group(1))
            m = re.match('cp([0-9]+)', self.tag_python())
            tag_python_int_self = int(m.group(1))
            if tag_python_int <= tag_python_int_self:
                # This wheel uses Python stable ABI same or older than ours, so
                # we can use it.
                log2(f'py_limited_api; {tag_python=} compatible with {self.tag_python()=}.')
                py_limited_api_compatible = True
            
        log2(f'{_normalise2(self.name) == name=}')
        log2(f'{self.version == version=}')
        log2(f'{self.tag_python() == tag_python=} {self.tag_python()=} {tag_python=}')
        log2(f'{py_limited_api_compatible=}')
        log2(f'{self.tag_abi() == tag_abi=}')
        log2(f'{self.tag_platform() in tag_platform.split(".")=}')
        log2(f'{self.tag_platform()=}')
        log2(f'{tag_platform.split(".")=}')
        ret = (1
                and _normalise2(self.name) == name
                and self.version == version
                and (self.tag_python() == tag_python or py_limited_api_compatible)
                and self.tag_abi() == tag_abi
                and self.tag_platform() in tag_platform.split('.')
                )
        log2(f'Returning {ret=}.')
        return ret
    
    def _entry_points_text(self):
        if self.entry_points:
            if isinstance(self.entry_points, str):
                return self.entry_points
            ret = ''
            for key, values in self.entry_points.items():
                ret += f'[{key}]\n'
                for value in values:
                    ret += f'{value}\n'
            return ret

    def _call_fn_build( self, config_settings=None):
        assert self.fn_build
        assert os.path.isfile('pyproject.toml'), (
                'Cannot create package because file does not exist: pyproject.toml'
                )
        log2(f'calling self.fn_build={self.fn_build}')
        if inspect.signature(self.fn_build).parameters:
            ret = self.fn_build(config_settings)
        else:
            ret = self.fn_build()
        assert isinstance( ret, (list, tuple)), \
                f'Expected list/tuple from {self.fn_build} but got: {ret!r}'

        # Check that any extensions that we have built, have same
        # py_limited_api value. If package is marked with py_limited_api=True
        # then non-py_limited_api extensions seem to fail at runtime on
        # Windows.
        #
        # (We could possibly allow package py_limited_api=False and extensions
        # py_limited_api=True, but haven't tested this, and it seems simpler to
        # be strict.)
        for item in ret:
            from_, (to_abs, to_rel) = self._fromto(item)
            from_abs = os.path.abspath(from_)
            is_py_limited_api = _extensions_to_py_limited_api.get(from_abs)
            if is_py_limited_api is not None:
                assert bool(self.py_limited_api) == bool(is_py_limited_api), (
                        f'Extension was built with'
                        f' py_limited_api={is_py_limited_api} but pipcl.Package'
                        f' name={self.name!r} has'
                        f' py_limited_api={self.py_limited_api}:'
                        f' {from_abs!r}'
                        )
        
        return ret


    def _argv_clean(self, all_):
        '''
        Called by `handle_argv()`.
        '''
        if not self.fn_clean:
            return
        paths = self.fn_clean(all_)
        if paths:
            if isinstance(paths, str):
                paths = paths,
            for path in paths:
                if not os.path.isabs(path):
                    path = ps.path.join(self.root, path)
                path = os.path.abspath(path)
                assert path.startswith(self.root+os.sep), \
                        f'path={path!r} does not start with root={self.root+os.sep!r}'
                log2(f'Removing: {path}')
                shutil.rmtree(path, ignore_errors=True)


    def install(self, record_path=None, root=None):
        '''
        Called by `handle_argv()` to handle `install` command..
        '''
        log2( f'{record_path=} {root=}')

        # Do a build and get list of files to install.
        #
        items = list()
        if self.fn_build:
            items = self._call_fn_build( dict())

        root2 = install_dir(root)
        log2( f'{root2=}')

        log1( f'Installing into: {root2!r}')
        dist_info_dir = self._dist_info_dir()

        if not record_path:
            record_path = f'{root2}/{dist_info_dir}/RECORD'
        record = _Record()

        def add_file(from_, to_abs, to_rel):
            os.makedirs( os.path.dirname( to_abs), exist_ok=True)
            if isinstance(from_, bytes):
                log2(f'Copying content into {to_abs}.')
                with open(to_abs, 'wb') as f:
                    f.write(from_)
                record.add_content(from_, to_rel)
            else:
                log0(f'{from_=}')
                log2(f'Copying from {os.path.relpath(from_, self.root)} to {to_abs}')
                shutil.copy2( from_, to_abs)
                record.add_file(from_, to_rel)

        def add_str(content, to_abs, to_rel):
            log2( f'Writing to: {to_abs}')
            os.makedirs( os.path.dirname( to_abs), exist_ok=True)
            with open( to_abs, 'w') as f:
                f.write( content)
            record.add_content(content, to_rel)

        for item in items:
            from_, (to_abs, to_rel) = self._fromto(item)
            log0(f'{from_=} {to_abs=} {to_rel=}')
            to_abs2 = f'{root2}/{to_rel}'
            add_file( from_, to_abs2, to_rel)

        add_str( self._metainfo(), f'{root2}/{dist_info_dir}/METADATA', f'{dist_info_dir}/METADATA')

        if self.license:
            add_str( self.license, f'{root2}/{dist_info_dir}/COPYING', f'{dist_info_dir}/COPYING')

        entry_points_text = self._entry_points_text()
        if entry_points_text:
            add_str(
                    entry_points_text,
                    f'{root2}/{dist_info_dir}/entry_points.txt',
                    f'{dist_info_dir}/entry_points.txt',
                    )

        log2( f'Writing to: {record_path}')
        with open(record_path, 'w') as f:
            f.write(record.get())

        log2(f'Finished.')


    def _argv_dist_info(self, root):
        '''
        Called by `handle_argv()`. There doesn't seem to be any documentation
        for `setup.py dist_info`, but it appears to be like `egg_info` except
        it writes to a slightly different directory.
        '''
        if root is None:
            root = f'{normalise2(self.name)}-{self.version}.dist-info'
        self._write_info(f'{root}/METADATA')
        if self.license:
            with open( f'{root}/COPYING', 'w') as f:
                f.write( self.license)


    def _argv_egg_info(self, egg_base):
        '''
        Called by `handle_argv()`.
        '''
        if egg_base is None:
            egg_base = '.'
        self._write_info(f'{egg_base}/.egg-info')


    def _write_info(self, dirpath=None):
        '''
        Writes egg/dist info to files in directory `dirpath` or `self.root` if
        `None`.
        '''
        if dirpath is None:
            dirpath = self.root
        log2(f'Creating files in directory {dirpath}')
        os.makedirs(dirpath, exist_ok=True)
        with open(os.path.join(dirpath, 'PKG-INFO'), 'w') as f:
            f.write(self._metainfo())

        # These don't seem to be required?
        #
        #with open(os.path.join(dirpath, 'SOURCES.txt', 'w') as f:
        #    pass
        #with open(os.path.join(dirpath, 'dependency_links.txt', 'w') as f:
        #    pass
        #with open(os.path.join(dirpath, 'top_level.txt', 'w') as f:
        #    f.write(f'{self.name}\n')
        #with open(os.path.join(dirpath, 'METADATA', 'w') as f:
        #    f.write(self._metainfo())


    def handle_argv(self, argv):
        '''
        Attempt to handles old-style (pre PEP-517) command line passed by
        old releases of pip to a `setup.py` script, and manual running of
        `setup.py`.

        This is partial support at best.
        '''
        global g_verbose
        #log2(f'argv: {argv}')

        class ArgsRaise:
            pass

        class Args:
            '''
            Iterates over argv items.
            '''
            def __init__( self, argv):
                self.items = iter( argv)
            def next( self, eof=ArgsRaise):
                '''
                Returns next arg. If no more args, we return <eof> or raise an
                exception if <eof> is ArgsRaise.
                '''
                try:
                    return next( self.items)
                except StopIteration:
                    if eof is ArgsRaise:
                        raise Exception('Not enough args')
                    return eof

        command = None
        opt_all = None
        opt_dist_dir = 'dist'
        opt_egg_base = None
        opt_formats = None
        opt_install_headers = None
        opt_record = None
        opt_root = None

        args = Args(argv[1:])

        while 1:
            arg = args.next(None)
            if arg is None:
                break

            elif arg in ('-h', '--help', '--help-commands'):
                log0(textwrap.dedent('''
                        Usage:
                            [<options>...] <command> [<options>...]
                        Commands:
                            bdist_wheel
                                Creates a wheel called
                                <dist-dir>/<name>-<version>-<details>.whl, where
                                <dist-dir> is "dist" or as specified by --dist-dir,
                                and <details> encodes ABI and platform etc.
                            clean
                                Cleans build files.
                            dist_info
                                Creates files in <name>-<version>.dist-info/ or
                                directory specified by --egg-base.
                            egg_info
                                Creates files in .egg-info/ or directory
                                directory specified by --egg-base.
                            install
                                Builds and installs. Writes installation
                                information to <record> if --record was
                                specified.
                            sdist
                                Make a source distribution:
                                    <dist-dir>/<name>-<version>.tar.gz
                        Options:
                            --all
                                Used by "clean".
                            --compile
                                Ignored.
                            --dist-dir | -d <dist-dir>
                                Default is "dist".
                            --egg-base <egg-base>
                                Used by "egg_info".
                            --formats <formats>
                                Used by "sdist".
                            --install-headers <directory>
                                Ignored.
                            --python-tag <python-tag>
                                Ignored.
                            --record <record>
                                Used by "install".
                            --root <path>
                                Used by "install".
                            --single-version-externally-managed
                                Ignored.
                            --verbose -v
                                Extra diagnostics.
                        Other:
                            windows-vs [-y <year>] [-v <version>] [-g <grade] [--verbose]
                                Windows only; looks for matching Visual Studio.
                            windows-python [-v <version>] [--verbose]
                                Windows only; looks for matching Python.
                        '''))
                return

            elif arg in ('bdist_wheel', 'clean', 'dist_info', 'egg_info', 'install', 'sdist'):
                assert command is None, 'Two commands specified: {command} and {arg}.'
                command = arg

            elif arg in ('windows-vs', 'windows-python', 'show-sysconfig'):
                assert command is None, 'Two commands specified: {command} and {arg}.'
                command = arg

            elif arg == '--all':                                opt_all = True
            elif arg == '--compile':                            pass
            elif arg == '--dist-dir' or arg == '-d':            opt_dist_dir = args.next()
            elif arg == '--egg-base':                           opt_egg_base = args.next()
            elif arg == '--formats':                            opt_formats = args.next()
            elif arg == '--install-headers':                    opt_install_headers = args.next()
            elif arg == '--python-tag':                         pass
            elif arg == '--record':                             opt_record = args.next()
            elif arg == '--root':                               opt_root = args.next()
            elif arg == '--single-version-externally-managed':  pass
            elif arg == '--verbose' or arg == '-v':             g_verbose += 1

            else:
               raise Exception(f'Unrecognised arg: {arg}')

        assert command, 'No command specified'

        log1(f'Handling command={command}')
        if 0:   pass
        elif command == 'bdist_wheel':  self.build_wheel(opt_dist_dir)
        elif command == 'clean':        self._argv_clean(opt_all)
        elif command == 'dist_info':    self._argv_dist_info(opt_egg_base)
        elif command == 'egg_info':     self._argv_egg_info(opt_egg_base)
        elif command == 'install':      self.install(opt_record, opt_root)
        elif command == 'sdist':        self.build_sdist(opt_dist_dir, opt_formats)

        elif command == 'windows-python':
            version = None
            while 1:
                arg = args.next(None)
                if arg is None:
                    break
                elif arg == '-v':
                    version = args.next()
                elif arg == '--verbose':
                    g_verbose += 1
                else:
                    assert 0, f'Unrecognised {arg=}'
            python = wdev.WindowsPython(version=version)
            print(f'Python is:\n{python.description_ml("    ")}')

        elif command == 'windows-vs':
            grade = None
            version = None
            year = None
            while 1:
                arg = args.next(None)
                if arg is None:
                    break
                elif arg == '-g':
                    grade = args.next()
                elif arg == '-v':
                    version = args.next()
                elif arg == '-y':
                    year = args.next()
                elif arg == '--verbose':
                    g_verbose += 1
                else:
                    assert 0, f'Unrecognised {arg=}'
            vs = wdev.WindowsVS(year=year, grade=grade, version=version)
            print(f'Visual Studio is:\n{vs.description_ml("    ")}')

        elif command == 'show-sysconfig':
            show_sysconfig()
            for mod in platform, sys:
                log0(f'{mod.__name__}:')
                for n in dir(mod):
                    if n.startswith('_'):
                        continue
                    log0(f'{mod.__name__}.{n}')
                    if mod is platform and n == 'uname':
                        continue
                    if mod is platform and n == 'pdb':
                        continue
                    if mod is sys and n in ('breakpointhook', 'exit'):
                        # We don't want to call these.
                        continue
                    v = getattr(mod, n)
                    if callable(v):
                        try:
                            v = v()
                        except Exception:
                            pass
                        else:
                            #print(f'{n=}', flush=1)
                            try:
                                print(f'    {mod.__name__}.{n}()={v!r}')
                            except Exception:
                                print(f'    Failed to print value of {mod.__name__}.{n}().')
                    else:
                        try:
                            print(f'    {mod.__name__}.{n}={v!r}')
                        except Exception:
                            print(f'    Failed to print value of {mod.__name__}.{n}.')
        
        else:
            assert 0, f'Unrecognised command: {command}'

        log2(f'Finished handling command: {command}')


    def __str__(self):
        return ('{'
            f'name={self.name!r}'
            f' version={self.version!r}'
            f' platform={self.platform!r}'
            f' supported_platform={self.supported_platform!r}'
            f' summary={self.summary!r}'
            f' description={self.description!r}'
            f' description_content_type={self.description_content_type!r}'
            f' keywords={self.keywords!r}'
            f' home_page={self.home_page!r}'
            f' download_url={self.download_url!r}'
            f' author={self.author!r}'
            f' author_email={self.author_email!r}'
            f' maintainer={self.maintainer!r}'
            f' maintainer_email={self.maintainer_email!r}'
            f' license={self.license!r}'
            f' classifier={self.classifier!r}'
            f' requires_dist={self.requires_dist!r}'
            f' requires_python={self.requires_python!r}'
            f' requires_external={self.requires_external!r}'
            f' project_url={self.project_url!r}'
            f' provides_extra={self.provides_extra!r}'

            f' root={self.root!r}'
            f' fn_build={self.fn_build!r}'
            f' fn_sdist={self.fn_sdist!r}'
            f' fn_clean={self.fn_clean!r}'
            f' tag_python={self.tag_python_!r}'
            f' tag_abi={self.tag_abi_!r}'
            f' tag_platform={self.tag_platform_!r}'
            '}'
            )

    def _dist_info_dir( self):
        return f'{_normalise2(self.name)}-{self.version}.dist-info'

    def _metainfo(self):
        '''
        Returns text for `.egg-info/PKG-INFO` file, or `PKG-INFO` in an sdist
        `.tar.gz` file, or `...dist-info/METADATA` in a wheel.
        '''
        # 2021-04-30: Have been unable to get multiline content working on
        # test.pypi.org so we currently put the description as the body after
        # all the other headers.
        #
        ret = ['']
        def add(key, value):
            if value is None:
                return
            if isinstance( value, (tuple, list)):
                for v in value:
                    if v is not None:
                        add( key, v)
                return
            if key == 'License' and '\n' in value:
                # This is ok because we write `self.license` into
                # *.dist-info/COPYING.
                #
                log1( f'Omitting license because contains newline(s).')
                return
            assert '\n' not in value, f'key={key} value contains newline: {value!r}'
            if key == 'Project-URL':
                assert value.count(',') == 1, f'For {key=}, should have one comma in {value!r}.'
            ret[0] += f'{key}: {value}\n'
        #add('Description', self.description)
        add('Metadata-Version', '2.1')

        # These names are from:
        # https://packaging.python.org/specifications/core-metadata/
        #
        for name in (
                'Name',
                'Version',
                'Platform',
                'Supported-Platform',
                'Summary',
                'Description-Content-Type',
                'Keywords',
                'Home-page',
                'Download-URL',
                'Author',
                'Author-email',
                'Maintainer',
                'Maintainer-email',
                'License',
                'Classifier',
                'Requires-Dist',
                'Requires-Python',
                'Requires-External',
                'Project-URL',
                'Provides-Extra',
                ):
            identifier = name.lower().replace( '-', '_')
            add( name, getattr( self, identifier))

        ret = ret[0]

        # Append description as the body
        if self.description:
            if '\n' in self.description:
                description_text = self.description.strip()
            else:
                with open(self.description) as f:
                    description_text = f.read()
            ret += '\n' # Empty line separates headers from body.
            ret += description_text
            ret += '\n'
        return ret

    def _path_relative_to_root(self, path, assert_within_root=True):
        '''
        Returns `(path_abs, path_rel)`, where `path_abs` is absolute path and
        `path_rel` is relative to `self.root`.

        Interprets `path` as relative to `self.root` if not absolute.

        We use `os.path.realpath()` to resolve any links.

        if `assert_within_root` is true, assert-fails if `path` is not within
        `self.root`.
        '''
        if os.path.isabs(path):
            p = path
        else:
            p = os.path.join(self.root, path)
        p = os.path.realpath(os.path.abspath(p))
        if assert_within_root:
            assert p.startswith(self.root+os.sep) or p == self.root, \
                    f'Path not within root={self.root+os.sep!r}: {path=} {p=}'
        p_rel = os.path.relpath(p, self.root)
        return p, p_rel

    def _fromto(self, p):
        '''
        Returns `(from_, (to_abs, to_rel))`.

        If `p` is a string we convert to `(p, p)`. Otherwise we assert that
        `p` is a tuple `(from_, to_)` where `from_` is str/bytes and `to_` is
        str. If `from_` is a bytes it is contents of file to add, otherwise the
        path of an existing file; non-absolute paths are assumed to be relative
        to `self.root`.

        If `to_` is empty or `/` we set it to the leaf of `from_` (which must
        be a str) - i.e. we place the file in the root directory of the wheel;
        otherwise if `to_` ends with `/` we append the leaf of `from_` (which
        must be a str).

        If `to_` starts with `$dist-info/`, we replace this with
        `self._dist_info_dir()`.

        If `to_` starts with `$data/`, we replace this with
        `{self.name}-{self.version}.data/`.

        We assert that `to_abs` is `within self.root`.

        `to_rel` is derived from the `to_abs` and is relative to self.root`.
        '''
        ret = None
        if isinstance(p, str):
            p = p, p
        assert isinstance(p, tuple) and len(p) == 2

        from_, to_ = p
        assert isinstance(from_, (str, bytes))
        assert isinstance(to_, str)
        if to_ == '/' or to_ == '':
            to_ = os.path.basename(from_)
        elif to_.endswith('/'):
            to_ += os.path.basename(from_)
        prefix = '$dist-info/'
        if to_.startswith( prefix):
            to_ = f'{self._dist_info_dir()}/{to_[ len(prefix):]}'
        prefix = '$data/'
        if to_.startswith( prefix):
            to_ = f'{_normalise2(self.name)}-{self.version}.data/{to_[ len(prefix):]}'
        if isinstance(from_, str):
            from_, _ = self._path_relative_to_root( from_, assert_within_root=False)
        to_ = self._path_relative_to_root(to_)
        assert isinstance(from_, (str, bytes))
        log2(f'returning {from_=} {to_=}')
        return from_, to_

_extensions_to_py_limited_api = dict()

def build_extension(
        name,
        path_i,
        outdir,
        *,
        builddir=None,
        includes=None,
        defines=None,
        libpaths=None,
        libs=None,
        optimise=True,
        debug=False,
        compiler_extra='',
        linker_extra='',
        swig=None,
        cpp=True,
        source_extra=None,
        prerequisites_swig=None,
        prerequisites_compile=None,
        prerequisites_link=None,
        infer_swig_includes=True,
        py_limited_api=False,
        ):
    '''
    Builds a Python extension module using SWIG. Works on Windows, Linux, MacOS
    and OpenBSD.

    On Unix, sets rpath when linking shared libraries.

    Args:
        name:
            Name of generated extension module.
        path_i:
            Path of input SWIG `.i` file. Internally we use swig to generate a
            corresponding `.c` or `.cpp` file.
        outdir:
            Output directory for generated files:

                * `{outdir}/{name}.py`
                * `{outdir}/_{name}.so`     # Unix
                * `{outdir}/_{name}.*.pyd`  # Windows
            We return the leafname of the `.so` or `.pyd` file.
        builddir:
            Where to put intermediate files, for example the .cpp file
            generated by swig and `.d` dependency files. Default is `outdir`.
        includes:
            A string, or a sequence of extra include directories to be prefixed
            with `-I`.
        defines:
            A string, or a sequence of extra preprocessor defines to be
            prefixed with `-D`.
        libpaths
            A string, or a sequence of library paths to be prefixed with
            `/LIBPATH:` on Windows or `-L` on Unix.
        libs
            A string, or a sequence of library names. Each item is prefixed
            with `-l` on non-Windows.
        optimise:
            Whether to use compiler optimisations and define NDEBUG.
        debug:
            Whether to build with debug symbols.
        compiler_extra:
            Extra compiler flags. Can be None.
        linker_extra:
            Extra linker flags. Can be None.
        swig:
            Swig command; if false we use 'swig'.
        cpp:
            If true we tell SWIG to generate C++ code instead of C.
        source_extra:
            Extra source files to build into the shared library,
        prerequisites_swig:
        prerequisites_compile:
        prerequisites_link:

            [These are mainly for use on Windows. On other systems we
            automatically generate dynamic dependencies using swig/compile/link
            commands' `-MD` and `-MF` args.]

            Sequences of extra input files/directories that should force
            running of swig, compile or link commands if they are newer than
            any existing generated SWIG `.i` file, compiled object file or
            shared library file.

            If present, the first occurrence of `True` or `False` forces re-run
            or no re-run. Any occurrence of None is ignored. If an item is a
            directory path we look for newest file within the directory tree.

            If not a sequence, we convert into a single-item list.

            prerequisites_swig

                We use swig's -MD and -MF args to generate dynamic dependencies
                automatically, so this is not usually required.

            prerequisites_compile
            prerequisites_link

                On non-Windows we use cc's -MF and -MF args to generate dynamic
                dependencies so this is not usually required.
        infer_swig_includes:
            If true, we extract `-I<path>` and `-I <path>` args from
            `compile_extra` (also `/I` on windows) and use them with swig so
            that it can see the same header files as C/C++. This is useful
            when using enviromment variables such as `CC` and `CXX` to set
            `compile_extra`.
        py_limited_api:
            If true we build for current Python's limited API / stable ABI.

            Note that we will assert false if this extension is added to a
            pipcl.Package that has a different <py_limited_api>, because
            on Windows importing a non-py_limited_api extension inside a
            py_limited=True package fails.

    Returns the leafname of the generated library file within `outdir`, e.g.
    `_{name}.so` on Unix or `_{name}.cp311-win_amd64.pyd` on Windows.
    '''
    if compiler_extra is None:
        compiler_extra = ''
    if linker_extra is None:
        linker_extra = ''
    if builddir is None:
        builddir = outdir
    if not swig:
        swig = 'swig'
        
    if source_extra is None:
        source_extra = list()
    if isinstance(source_extra, str):
        source_extra = [source_extra]
    
    includes_text = _flags( includes, '-I')
    defines_text = _flags( defines, '-D')
    libpaths_text = _flags( libpaths, '/LIBPATH:', '"') if windows() else _flags( libpaths, '-L')
    libs_text = _flags( libs, '' if windows() else '-l')
    path_cpp = f'{builddir}/{os.path.basename(path_i)}'
    path_cpp += '.cpp' if cpp else '.c'
    os.makedirs( outdir, exist_ok=True)

    # Run SWIG.
    #
    if infer_swig_includes:
        # Extract include flags from `compiler_extra`.
        swig_includes_extra = ''
        compiler_extra_items = shlex.split(compiler_extra)
        i = 0
        while i < len(compiler_extra_items):
            item = compiler_extra_items[i]
            # Swig doesn't seem to like a space after `I`.
            if item == '-I' or (windows() and item == '/I'):
                swig_includes_extra += f' -I{compiler_extra_items[i+1]}'
                i += 1
            elif item.startswith('-I') or (windows() and item.startswith('/I')):
                swig_includes_extra += f' -I{compiler_extra_items[i][2:]}'
            i += 1
        swig_includes_extra = swig_includes_extra.strip()
    deps_path = f'{path_cpp}.d'
    prerequisites_swig2 = _get_prerequisites( deps_path)
    run_if(
            f'''
            {swig}
                -Wall
                {"-c++" if cpp else ""}
                -python
                -module {name}
                -outdir {outdir}
                -o {path_cpp}
                -MD -MF {deps_path}
                {includes_text}
                {swig_includes_extra}
                {path_i}
            '''
            ,
            path_cpp,
            path_i,
            prerequisites_swig,
            prerequisites_swig2,
            )

    if pyodide():
        so_suffix = '.so'
        log0(f'pyodide: PEP-3149 suffix untested, so omitting. {_so_suffix()=}.')
    else:
        so_suffix = _so_suffix(use_so_versioning = not py_limited_api)
    path_so_leaf = f'_{name}{so_suffix}'
    path_so = f'{outdir}/{path_so_leaf}'

    py_limited_api2 = current_py_limited_api() if py_limited_api else None

    compiler_command, pythonflags = base_compiler(cpp=cpp)
    linker_command, _ = base_linker(cpp=cpp)
    # setuptools on Linux seems to use slightly different compile flags:
    #
    # -fwrapv -O3 -Wall -O2 -g0 -DPY_CALL_TRAMPOLINE
    #

    general_flags = ''
    if windows():
        permissive = '/permissive-'
        EHsc = '/EHsc'
        T = '/Tp' if cpp else '/Tc'
        optimise2 = '/DNDEBUG /O2' if optimise else '/D_DEBUG'
        debug2 = '/Zi' if debug else ''
        py_limited_api3 = f'/DPy_LIMITED_API={py_limited_api2}' if py_limited_api2 else ''

    else:
        if debug:
            general_flags += '/Zi' if windows() else ' -g'
        if optimise:
            general_flags += ' /DNDEBUG /O2' if windows() else ' -O2 -DNDEBUG'

        py_limited_api3 = f'-DPy_LIMITED_API={py_limited_api2}' if py_limited_api2 else ''

    if windows():
        pass
    elif darwin():
        # MacOS's linker does not like `-z origin`.
        rpath_flag = "-Wl,-rpath,@loader_path/"
        # Avoid `Undefined symbols for ... "_PyArg_UnpackTuple" ...'.
        general_flags += ' -undefined dynamic_lookup'
    elif pyodide():
        # Setting `-Wl,-rpath,'$ORIGIN',-z,origin` gives:
        #   emcc: warning: ignoring unsupported linker flag: `-rpath` [-Wlinkflags]
        #   wasm-ld: error: unknown -z value: origin
        #
        rpath_flag = "-Wl,-rpath,'$ORIGIN'"
    else:
        rpath_flag = "-Wl,-rpath,'$ORIGIN',-z,origin"
    
    # Fun fact - on Linux, if the -L and -l options are before '{path_cpp}'
    # they seem to be ignored...
    #
    path_os = list()

    for path_source in [path_cpp] + source_extra:
        path_o = f'{path_source}.obj' if windows() else f'{path_source}.o'
        path_os.append(path_o)

        prerequisites_path = f'{path_o}.d'

        if windows():
            compiler_command2 = f'''
                    {compiler_command}
                        # General:
                        /c                          # Compiles without linking.
                        {EHsc}                      # Enable "Standard C++ exception handling".

                        #/MD                         # Creates a multithreaded DLL using MSVCRT.lib.
                        {'/MDd' if debug else '/MD'}

                        # Input/output files:
                        {T}{path_source}            # /Tp specifies C++ source file.
                        /Fo{path_o}                 # Output file. codespell:ignore

                        # Include paths:
                        {includes_text}
                        {pythonflags.includes}      # Include path for Python headers.

                        # Code generation:
                        {optimise2}
                        {debug2}
                        {permissive}                # Set standard-conformance mode.

                        # Diagnostics:
                        #/FC                         # Display full path of source code files passed to cl.exe in diagnostic text.
                        /W3                         # Sets which warning level to output. /W3 is IDE default.
                        /diagnostics:caret          # Controls the format of diagnostic messages.
                        /nologo                     #

                        {defines_text}
                        {compiler_extra}

                        {py_limited_api3}
                    '''

        else:
            compiler_command2 = f'''
                    {compiler_command}
                        -fPIC
                        {general_flags.strip()}
                        {pythonflags.includes}
                        {includes_text}
                        {defines_text}
                        -MD -MF {prerequisites_path}
                        -c {path_source}
                        -o {path_o}
                        {compiler_extra}
                        {py_limited_api3}
                    '''
        run_if(
                compiler_command2,
                path_o,
                path_source,
                [path_source] + _get_prerequisites(prerequisites_path),
                )

    # Link
    prerequisites_path = f'{path_so}.d'
    if windows():
        debug2 = '/DEBUG' if debug else ''
        base, _ = os.path.splitext(path_so_leaf)
        command2 = f'''
                {linker_command}
                    /DLL                    # Builds a DLL.
                    /EXPORT:PyInit__{name}  # Exports a function.
                    /IMPLIB:{base}.lib      # Overrides the default import library name.
                    {libpaths_text}
                    {pythonflags.ldflags}
                    /OUT:{path_so}          # Specifies the output file name.
                    {debug2}
                    /nologo
                    {libs_text}
                    {' '.join(path_os)}
                    {linker_extra}
                '''
    elif pyodide():
        command2 = f'''
                {linker_command}
                    -MD -MF {prerequisites_path}
                    -o {path_so}
                    {' '.join(path_os)}
                    {libpaths_text}
                    {libs_text}
                    {linker_extra}
                    {pythonflags.ldflags}
                    {rpath_flag}
                '''
    else:
        command2 = f'''
                {linker_command}
                    -shared
                    {general_flags.strip()}
                    -MD -MF {prerequisites_path}
                    -o {path_so}
                    {' '.join(path_os)}
                    {libpaths_text}
                    {libs_text}
                    {linker_extra}
                    {pythonflags.ldflags}
                    {rpath_flag}
                    {py_limited_api3}
                '''
    link_was_run = run_if(
            command2,
            path_so,
            path_cpp,
            *path_os,
            *_get_prerequisites(f'{path_so}.d'),
            )

    if link_was_run and darwin():
        # We need to patch up references to shared libraries in `libs`.
        sublibraries = list()
        for lib in () if libs is None else libs:
            for libpath in libpaths:
                found = list()
                for suffix in '.so', '.dylib':
                    path = f'{libpath}/lib{os.path.basename(lib)}{suffix}'
                    if os.path.exists( path):
                        found.append( path)
                if found:
                    assert len(found) == 1, f'More than one file matches lib={lib!r}: {found}'
                    sublibraries.append( found[0])
                    break
            else:
                log2(f'Warning: can not find path of lib={lib!r} in libpaths={libpaths}')
        macos_patch( path_so, *sublibraries)

        #run(f'ls -l {path_so}', check=0)
        #run(f'file {path_so}', check=0)

    _extensions_to_py_limited_api[os.path.abspath(path_so)] = py_limited_api
    
    return path_so_leaf


# Functions that might be useful.
#


def base_compiler(vs=None, pythonflags=None, cpp=False, use_env=True):
    '''
    Returns basic compiler command and PythonFlags.

    Args:
        vs:
            Windows only. A `wdev.WindowsVS` instance or None to use default
            `wdev.WindowsVS` instance.
        pythonflags:
            A `pipcl.PythonFlags` instance or None to use default
            `pipcl.PythonFlags` instance.
        cpp:
            If true we return C++ compiler command instead of C. On Windows
            this has no effect - we always return `cl.exe`.
        use_env:
            If true we return '$CC' or '$CXX' if the corresponding
            environmental variable is set (without evaluating with `getenv()`
            or `os.environ`).

    Returns `(cc, pythonflags)`:
        cc:
            C or C++ command. On Windows this is of the form
            `{vs.vcvars}&&{vs.cl}`; otherwise it is typically `cc` or `c++`.
        pythonflags:
            The `pythonflags` arg or a new `pipcl.PythonFlags` instance.
    '''
    if not pythonflags:
        pythonflags = PythonFlags()
    cc = None
    if use_env:
        if cpp:
            if os.environ.get( 'CXX'):
                cc = '$CXX'
        else:
            if os.environ.get( 'CC'):
                cc = '$CC'
    if cc:
        pass
    elif windows():
        if not vs:
            vs = wdev.WindowsVS()
        cc = f'"{vs.vcvars}"&&"{vs.cl}"'
    elif wasm():
        cc = 'em++' if cpp else 'emcc'
    else:
        cc = 'c++' if cpp else 'cc'
    cc = macos_add_cross_flags( cc)
    return cc, pythonflags


def base_linker(vs=None, pythonflags=None, cpp=False, use_env=True):
    '''
    Returns basic linker command.

    Args:
        vs:
            Windows only. A `wdev.WindowsVS` instance or None to use default
            `wdev.WindowsVS` instance.
        pythonflags:
            A `pipcl.PythonFlags` instance or None to use default
            `pipcl.PythonFlags` instance.
        cpp:
            If true we return C++ linker command instead of C. On Windows this
            has no effect - we always return `link.exe`.
        use_env:
            If true we use `os.environ['LD']` if set.

    Returns `(linker, pythonflags)`:
        linker:
            Linker command. On Windows this is of the form
            `{vs.vcvars}&&{vs.link}`; otherwise it is typically `cc` or `c++`.
        pythonflags:
            The `pythonflags` arg or a new `pipcl.PythonFlags` instance.
    '''
    if not pythonflags:
        pythonflags = PythonFlags()
    linker = None
    if use_env:
        if os.environ.get( 'LD'):
            linker = '$LD'
    if linker:
        pass
    elif windows():
        if not vs:
            vs = wdev.WindowsVS()
        linker = f'"{vs.vcvars}"&&"{vs.link}"'
    elif wasm():
        linker = 'em++' if cpp else 'emcc'
    else:
        linker = 'c++' if cpp else 'cc'
    linker = macos_add_cross_flags( linker)
    return linker, pythonflags


def git_info( directory):
    '''
    Returns `(sha, comment, diff, branch)`, all items are str or None if not
    available.

    directory:
        Root of git checkout.
    '''
    sha, comment, diff, branch = None, None, None, None
    e, out = run(
            f'cd {directory} && (PAGER= git show --pretty=oneline|head -n 1 && git diff)',
            capture=1,
            check=0
            )
    if not e:
        sha, _ = out.split(' ', 1)
        comment, diff = _.split('\n', 1)
    e, out = run(
            f'cd {directory} && git rev-parse --abbrev-ref HEAD',
            capture=1,
            check=0
            )
    if not e:
        branch = out.strip()
    log1(f'git_info(): directory={directory!r} returning branch={branch!r} sha={sha!r} comment={comment!r}')
    return sha, comment, diff, branch


def git_items( directory, submodules=False):
    '''
    Returns list of paths for all files known to git within a `directory`.

    Args:
        directory:
            Must be somewhere within a git checkout.
        submodules:
            If true we also include git submodules.

    Returns:
        A list of paths for all files known to git within `directory`. Each
        path is relative to `directory`. `directory` must be somewhere within a
        git checkout.

    We run a `git ls-files` command internally.

    This function can be useful for the `fn_sdist()` callback.
    '''
    command = 'cd ' + directory + ' && git ls-files'
    if submodules:
        command += ' --recurse-submodules'
    log1(f'Running {command=}')
    text = subprocess.check_output( command, shell=True)
    ret = []
    for path in text.decode('utf8').strip().split( '\n'):
        path2 = os.path.join(directory, path)
        # Sometimes git ls-files seems to list empty/non-existent directories
        # within submodules.
        #
        if not os.path.exists(path2):
            log2(f'Ignoring git ls-files item that does not exist: {path2}')
        elif os.path.isdir(path2):
            log2(f'Ignoring git ls-files item that is actually a directory: {path2}')
        else:
            ret.append(path)
    return ret


def git_get(
        local,
        *,
        remote=None,
        branch=None,
        tag=None,
        text=None,
        depth=1,
        env_extra=None,
        update=True,
        submodules=True,
        ):
    '''
    Creates/updates local checkout <local> of remote repository and returns
    absolute path of <local>.

    If <text> is set but does not start with 'git:', it is assumed to be an up
    to date local checkout, and we return absolute path of <text> without doing
    any git operations.
    
    Args:
        local:
            Local directory. Created and/or updated using `git clone` and `git
            fetch` etc.
        remote:
            Remote git repostitory, for example
            'https://github.com/ArtifexSoftware/mupdf.git'. Can be overridden
            by <text>.
        branch:
            Branch to use; can be overridden by <text>.
        tag:
            Tag to use; can be overridden by <text>.
        text:
            If None or empty:
                Ignored.
            
            If starts with 'git:':
                The remaining text should be a command-line
                style string containing some or all of these args:
                    --branch <branch>
                    --tag <tag>
                    <remote>
                These overrides <branch>, <tag> and <remote>.
            Otherwise:
                <text> is assumed to be a local directory, and we simply return
                it as an absolute path without doing any git operations.
            
            For example these all clone/update/branch master of https://foo.bar/qwerty.git to local
            checkout 'foo-local':
            
                git_get('foo-local', remote='https://foo.bar/qwerty.git', branch='master')
                git_get('foo-local', text='git:--branch master https://foo.bar/qwerty.git')
                git_get('foo-local', text='git:--branch master', remote='https://foo.bar/qwerty.git')
                git_get('foo-local', text='git:', branch='master', remote='https://foo.bar/qwerty.git')
        depth:
            Depth of local checkout when cloning and fetching, or None.
        env_extra:
            Dict of extra name=value environment variables to use whenever we
            run git.
        update:
            If false we do not update existing repository. Might be useful if
            testing without network access.
        submodules:
            If true, we clone with `--recursive --shallow-submodules` and run
            `git submodule update --init --recursive` before returning.
    '''
    log0(f'{remote=} {local=} {branch=} {tag=} {text=}')
    
    if text:
        if text.startswith('git:'):
            args = iter(shlex.split(text[len('git:'):]))
            while 1:
                try:
                    arg = next(args)
                except StopIteration:
                    break
                if arg == '--branch':
                    branch = next(args)
                    tag = None
                elif arg == '--tag':
                    tag = next(args)
                    branch = None
                else:
                    remote = arg
            assert remote, f'<remote> unset and no remote specified in {text=}.'
            assert branch or tag, f'<branch> and <tag> unset and no branch/tag specified in {text=}.'
        else:
            log0(f'Using local directory {text!r}.')
            return os.path.abspath(text)
        
    assert (branch and not tag) or (not branch and tag), f'Must specify exactly one of <branch> and <tag>; {branch=} {tag=}.'
    
    depth_arg = f' --depth {depth}' if depth else ''
    
    def do_update():
        # This seems to pull in the entire repository.
        log0(f'do_update(): attempting to update {local=}.')
        # Remove any local changes.
        run(f'cd {local} && git reset --hard', env_extra=env_extra)
        if tag:
            # `-u` avoids `fatal: Refusing to fetch into current branch`.
            # Using '+' and `revs/tags/` prefix seems to avoid errors like:
            #   error: cannot update ref 'refs/heads/v3.16.44':
            #   trying to write non-commit object
            #   06c4ae5fe39a03b37a25a8b95214d9f8f8a867b8 to branch
            #   'refs/heads/v3.16.44'
            #
            run(f'cd {local} && git fetch -fuv{depth_arg} {remote} +refs/tags/{tag}:refs/tags/{tag}', env_extra=env_extra)
            run(f'cd {local} && git checkout {tag}', env_extra=env_extra)
        if branch:
            # `-u` avoids `fatal: Refusing to fetch into current branch`.
            run(f'cd {local} && git fetch -fuv{depth_arg} {remote} {branch}:{branch}', env_extra=env_extra)
            run(f'cd {local} && git checkout {branch}', env_extra=env_extra)
    
    do_clone = True
    if os.path.isdir(f'{local}/.git'):
        if update:
            # Try to update existing checkout.
            try:
                do_update()
                do_clone = False
            except Exception as e:
                log0(f'Failed to update existing checkout {local}: {e}')
        else:
            do_clone = False
    
    if do_clone:
        # No existing git checkout, so do a fresh clone.
        #_fs_remove(local)
        log0(f'Cloning to: {local}')
        command = f'git clone --config core.longpaths=true{depth_arg}'
        if submodules:
            command += f' --recursive --shallow-submodules'
        if branch:
            command += f' -b {branch}'
        if tag:
            command += f' -b {tag}'
        command += f' {remote} {local}'
        run(command, env_extra=env_extra)
        do_update()
    
    if submodules:
        run(f'cd {local} && git submodule update --init --recursive', env_extra=env_extra)

    # Show sha of checkout.
    run( f'cd {local} && git show --pretty=oneline|head -n 1', check=False)
    return os.path.abspath(local)
    

def run(
        command,
        *,
        capture=False,
        check=1,
        verbose=1,
        env=None,
        env_extra=None,
        timeout=None,
        caller=1,
        prefix=None,
        encoding=None,  # System default.
        errors='backslashreplace',
        ):
    '''
    Runs a command using `subprocess.run()`.

    Args:
        command:
            A string, the command to run.

            Multiple lines in `command` are treated as a single command.

            * If a line starts with `#` it is discarded.
            * If a line contains ` #`, the trailing text is discarded.

            When running the command on Windows, newlines are replaced by
            spaces; otherwise each line is terminated by a backslash character.
        capture:
            If true, we include the command's output in our return value.
        check:
            If true we raise an exception on error; otherwise we include the
            command's returncode in our return value.
        verbose:
            If true we show the command.
        env:
            None or dict to use instead of <os.environ>.
        env_extra:
            None or dict to add to <os.environ> or <env>.
        timeout:
            If not None, timeout in seconds; passed directly to
            subprocess.run(). Note that on MacOS subprocess.run() seems to
            leave processes running if timeout expires.
        prefix:
            String prefix for each line of output.

            If true:
            * We run command with stdout=subprocess.PIPE and
              stderr=subprocess.STDOUT, repetaedly reading the command's output
              and writing it to stdout with <prefix>.
            * We do not support <timeout>, which must be None.
    Returns:
        check capture   Return
        --------------------------
          false false   returncode
          false  true   (returncode, output)
          true  false   None or raise exception
          true   true   output or raise exception
    '''
    if env is None:
        env = os.environ
        if env_extra:
            env = env.copy()
    if env_extra:
        env.update(env_extra)
    lines = _command_lines( command)
    if verbose:
        text = f'Running:'
        nl = '\n    '
        text += f' {nl.join(lines)}'
        if env_extra:
            text += f'\nwith:\n'
            for k in sorted(env_extra.keys()):
                text += f'    {k}={shlex.quote(env_extra[k])}\n'
        log1(text, caller=caller+1)
    sep = ' ' if windows() else ' \\\n'
    command2 = sep.join( lines)
    
    if prefix:
        assert not timeout, f'Timeout not supported with prefix.'
        child = subprocess.Popen(
                command2,
                shell=True,
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                encoding=encoding,
                errors=errors,
                env=env,
                )
        if capture:
            capture_text = ''
        decoder = codecs.getincrementaldecoder(child.stdout.encoding)(errors)
        line_start = True
        
        while 1:
            raw = os.read( child.stdout.fileno(), 10000)
            text = decoder.decode(raw, final=not raw)
            if capture:
                capture_text += text
            lines = text.split('\n')
            for i, line in enumerate(lines):
                if line_start:
                    sys.stdout.write(prefix)
                    line_start = False
                sys.stdout.write(line)
                if i < len(lines) - 1:
                    sys.stdout.write('\n')
                    line_start = True
            sys.stdout.flush()
            if not raw:
                break
        if not line_start:
            sys.stdout.write('\n')
        e = child.wait()
        if check and e:
            raise subprocess.CalledProcessError(e, command2, capture_text if capture else None)
        if check:
            return capture_text if capture else None
        else:
            return (e, capture_text) if capture else e
    else:
        cp = subprocess.run(
                command2,
                shell=True,
                stdout=subprocess.PIPE if capture else None,
                stderr=subprocess.STDOUT if capture else None,
                check=check,
                encoding=encoding,
                errors=errors,
                env=env,
                timeout=timeout,
                )
    if check:
        return cp.stdout if capture else None
    else:
        return (cp.returncode, cp.stdout) if capture else cp.returncode


def darwin():
    return sys.platform.startswith( 'darwin')

def windows():
    return platform.system() == 'Windows'

def wasm():
    return os.environ.get( 'OS') in ('wasm', 'wasm-mt')

def pyodide():
    return os.environ.get( 'PYODIDE') == '1'

def linux():
    return platform.system() == 'Linux'

def openbsd():
    return platform.system() == 'OpenBSD'


def show_system():
    '''
    Show useful information about the system plus argv and environ.
    
    Omits os.environ if $PIPCL_SHOW_ENV is '0'.
    '''
    def log(text):
        log0(text, caller=3)
    
    #log(f'{__file__=}')
    #log(f'{__name__=}')
    log(f'{os.getcwd()=}')
    log(f'{platform.machine()=}')
    log(f'{platform.platform()=}')
    log(f'{platform.python_implementation()=}')
    log(f'{platform.python_version()=}')
    log(f'{platform.system()=}')
    if sys.implementation.name != 'graalpy':
        log(f'{platform.uname()=}')
    log(f'{sys.executable=}')
    log(f'{sys.version=}')
    log(f'{sys.version_info=}')
    log(f'{list(sys.version_info)=}')
    
    log(f'{sysconfig.get_config_var("Py_GIL_DISABLED")=}')
    try:
        log(f'{sys._is_gil_enabled()=}')
    except AttributeError:
        log(f'sys._is_gil_enabled() => AttributeError')
    
    log(f'CPU bits: {cpu_bits()}')
    
    log(f'sys.argv ({len(sys.argv)}):')
    for i, arg in enumerate(sys.argv):
        log(f'    {i}: {arg!r}')
    
    PIPCL_SHOW_ENV = os.environ.get('PIPCL_SHOW_ENV')
    if PIPCL_SHOW_ENV == '0':
        log(f'[Not showing os.environ because {PIPCL_SHOW_ENV=}.]')
    else:
        log(f'os.environ ({len(os.environ)}):')
        for k in sorted( os.environ.keys()):
            v = os.environ[ k]
            if 'BEGIN OPENSSH PRIVATE KEY' in v:
                # Don't show private keys.
                log(f'    {k} ****')
            else:
                log( f'    {k}: {v!r}')


class PythonFlags:
    '''
    Compile/link flags for the current python, for example the include path
    needed to get `Python.h`.

    The 'PIPCL_PYTHON_CONFIG' environment variable allows to override
    the location of the python-config executable.

    Members:
        .includes:
            String containing compiler flags for include paths.
        .ldflags:
            String containing linker flags for library paths.
    '''
    def __init__(self):
        
        # Experimental detection of python flags from sysconfig.*() instead of
        # python-config command.
        includes_, ldflags_ = sysconfig_python_flags()
      
        if pyodide():
            _include_dir = os.environ[ 'PYO3_CROSS_INCLUDE_DIR']
            _lib_dir = os.environ[ 'PYO3_CROSS_LIB_DIR']
            self.includes = f'-I {_include_dir}'
            self.ldflags = f'-L {_lib_dir}'

        elif 0:

            self.includes = includes_
            self.ldflags = ldflags_
        
        elif windows():
            wp = wdev.WindowsPython()
            self.includes = f'/I"{wp.include}"'
            self.ldflags = f'/LIBPATH:"{wp.libs}"'

        elif pyodide():
            _include_dir = os.environ[ 'PYO3_CROSS_INCLUDE_DIR']
            _lib_dir = os.environ[ 'PYO3_CROSS_LIB_DIR']
            self.includes = f'-I {_include_dir}'
            self.ldflags = f'-L {_lib_dir}'

        else:
            python_config = os.environ.get("PIPCL_PYTHON_CONFIG")
            if not python_config:
                # We use python-config which appears to work better than pkg-config
                # because it copes with multiple installed python's, e.g.
                # manylinux_2014's /opt/python/cp*-cp*/bin/python*.
                #
                # But... on non-macos it seems that we should not attempt to specify
                # libpython on the link command. The manylinux docker containers
                # don't actually contain libpython.so, and it seems that this
                # deliberate. And the link command runs ok.
                #
                python_exe = os.path.realpath( sys.executable)
                if darwin():
                    # Basic install of dev tools with `xcode-select --install` doesn't
                    # seem to provide a `python3-config` or similar, but there is a
                    # `python-config.py` accessible via sysconfig.
                    #
                    # We try different possibilities and use the last one that
                    # works.
                    #
                    python_config = None
                    for pc in (
                            f'python3-config',
                            f'{sys.executable} {sysconfig.get_config_var("srcdir")}/python-config.py',
                            f'{python_exe}-config',
                            ):
                        e = subprocess.run(
                                f'{pc} --includes',
                                shell=1,
                                stdout=subprocess.DEVNULL,
                                stderr=subprocess.DEVNULL,
                                check=0,
                                ).returncode
                        log2(f'{e=} from {pc!r}.')
                        if e == 0:
                            python_config = pc
                    assert python_config, f'Cannot find python-config'
                else:
                    python_config = f'{python_exe}-config'
            log2(f'Using {python_config=}.')
            try:
                self.includes = run( f'{python_config} --includes', capture=1, verbose=0).strip()
            except Exception as e:
                raise Exception('We require python development tools to be installed.') from e
            self.ldflags = run( f'{python_config} --ldflags', capture=1, verbose=0).strip()
            if linux():
                # It seems that with python-3.10 on Linux, we can get an
                # incorrect -lcrypt flag that on some systems (e.g. WSL)
                # causes:
                #
                #   ImportError: libcrypt.so.2: cannot open shared object file: No such file or directory
                #
                ldflags2 = self.ldflags.replace(' -lcrypt ', ' ')
                if ldflags2 != self.ldflags:
                    log2(f'### Have removed `-lcrypt` from ldflags: {self.ldflags!r} -> {ldflags2!r}')
                    self.ldflags = ldflags2

        if 0:
            log1(f'{self.includes=}')
            log1(f'    {includes_=}')
            log1(f'{self.ldflags=}')
            log1(f'    {ldflags_=}')


def macos_add_cross_flags(command):
    '''
    If running on MacOS and environment variables ARCHFLAGS is set
    (indicating we are cross-building, e.g. for arm64), returns
    `command` with extra flags appended. Otherwise returns unchanged
    `command`.
    '''
    if darwin():
        archflags = os.environ.get( 'ARCHFLAGS')
        if archflags:
            command = f'{command} {archflags}'
            log2(f'Appending ARCHFLAGS to command: {command}')
            return command
    return command


def macos_patch( library, *sublibraries):
    '''
    If running on MacOS, patches `library` so that all references to items in
    `sublibraries` are changed to `@rpath/{leafname}`. Does nothing on other
    platforms.

    library:
        Path of shared library.
    sublibraries:
        List of paths of shared libraries; these have typically been
        specified with `-l` when `library` was created.
    '''
    log2( f'macos_patch(): library={library}  sublibraries={sublibraries}')
    if not darwin():
        return
    if not sublibraries:
        return
    subprocess.run( f'otool -L {library}', shell=1, check=1)
    command = 'install_name_tool'
    names = []
    for sublibrary in sublibraries:
        name = subprocess.run(
                f'otool -D {sublibrary}',
                shell=1,
                check=1,
                capture_output=1,
                encoding='utf8',
                ).stdout.strip()
        name = name.split('\n')
        assert len(name) == 2 and name[0] == f'{sublibrary}:', f'{name=}'
        name = name[1]
        # strip trailing so_name.
        leaf = os.path.basename(name)
        m = re.match('^(.+[.]((so)|(dylib)))[0-9.]*$', leaf)
        assert m
        log2(f'Changing {leaf=} to {m.group(1)}')
        leaf = m.group(1)
        command += f' -change {name} @rpath/{leaf}'
    command += f' {library}'
    log2( f'Running: {command}')
    subprocess.run( command, shell=1, check=1)
    subprocess.run( f'otool -L {library}', shell=1, check=1)


def _macos_fixup_platform_tag(tag):
    '''
    Patch up platform tag on MacOS.

    E.g. `foo-1.2.3-cp311-none-macosx_13_x86_64.whl` causes `pip` to fail with:
    `not a supported wheel on this platform`. We seem to need to add `_0` to
    the OS version. (This is documented at
    https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#macos).

    And with graal we need to replace trailing `universal2` with x86_64
    or arm64. On non-graal this causes problems because non-universal
    platform tags seem more restricted than platform tags from
    sysconfig.get_platform(). For example:
    
        pip install ...-macosx_10_13_arm64.whl
            ERROR: ...-macosx_10_13_arm64.whl is not a supported wheel on this platform.
        pip install ...-macosx_10_13_universal2.whl
            Ok.
    '''
    m = re.match( '^macosx_([0-9_]+)_([^0-9].+)$', tag)
    if not m:
        return tag
    a = m.group(1)
    if '_' not in a:
        a += '_0'
    b = m.group(2)
    if sys.implementation.name == 'graalpy' and b == 'universal2':
        # Replace 'universal2' with x86_64 or arm64.
        b = platform.machine()
    ret = f'macosx_{a}_{b}'
    #log0(f'Changing from {tag=} to {ret=}.')
    return ret
    

# Internal helpers.
#

def _command_lines( command):
    '''
    Process multiline command by running through `textwrap.dedent()`, removes
    comments (lines starting with `#` or ` #` until end of line), removes
    entirely blank lines.

    Returns list of lines.
    '''
    command = textwrap.dedent( command)
    lines = []
    for line in command.split( '\n'):
        if line.startswith( '#'):
            h = 0
        else:
            h = line.find( ' #')
        if h >= 0:
            line = line[:h]
        if line.strip():
            lines.append(line.rstrip())
    return lines


def cpu_bits():
    return int.bit_length(sys.maxsize+1)


def _cpu_name():
    '''
    Returns `x32` or `x64` depending on Python build.
    '''
    #log(f'sys.maxsize={hex(sys.maxsize)}')
    return f'x{32 if sys.maxsize == 2**31 - 1 else 64}'


def run_if( command, out, *prerequisites, caller=1):
    '''
    Runs a command only if the output file is not up to date.

    Args:
        command:
            The command to run. We write this and a hash of argv[0] into a file
            <out>.cmd so that we know to run a command if the command itself
            has changed.
        out:
            Path of the output file.

        prerequisites:
            List of prerequisite paths or true/false/None items. If an item
            is None it is ignored, otherwise if an item is not a string we
            immediately return it cast to a bool. We recurse into directories,
            effectively using the newest file in the directory.

    Returns:
        True if we ran the command, otherwise None.


    If the output file does not exist, the command is run:

        >>> verbose(1)
        1
        >>> log_line_numbers(0)
        >>> out = 'run_if_test_out'
        >>> if os.path.exists( out):
        ...     os.remove( out)
        >>> if os.path.exists( f'{out}.cmd'):
        ...     os.remove( f'{out}.cmd')
        >>> run_if( f'touch {out}', out, caller=0)
        pipcl.py:run_if(): Running command because: File does not exist: 'run_if_test_out'
        pipcl.py:run_if(): Running: touch run_if_test_out
        True

    If we repeat, the output file will be up to date so the command is not run:

        >>> run_if( f'touch {out}', out, caller=0)
        pipcl.py:run_if(): Not running command because up to date: 'run_if_test_out'

    If we change the command, the command is run:

        >>> run_if( f'touch {out};', out, caller=0)
        pipcl.py:run_if(): Running command because: Command has changed:
        pipcl.py:run_if():     @@ -1,2 +1,2 @@
        pipcl.py:run_if():      touch
        pipcl.py:run_if():     -run_if_test_out
        pipcl.py:run_if():     +run_if_test_out;
        pipcl.py:run_if(): 
        pipcl.py:run_if(): Running: touch run_if_test_out;
        True

    If we add a prerequisite that is newer than the output, the command is run:

        >>> time.sleep(1)
        >>> prerequisite = 'run_if_test_prerequisite'
        >>> run( f'touch {prerequisite}', caller=0)
        pipcl.py:run(): Running: touch run_if_test_prerequisite
        >>> run_if( f'touch  {out}', out, prerequisite, caller=0)
        pipcl.py:run_if(): Running command because: Command has changed:
        pipcl.py:run_if():     @@ -1,2 +1,2 @@
        pipcl.py:run_if():      touch
        pipcl.py:run_if():     -run_if_test_out;
        pipcl.py:run_if():     +run_if_test_out
        pipcl.py:run_if(): 
        pipcl.py:run_if(): Running: touch  run_if_test_out
        True

    If we repeat, the output will be newer than the prerequisite, so the
    command is not run:

        >>> run_if( f'touch  {out}', out, prerequisite, caller=0)
        pipcl.py:run_if(): Not running command because up to date: 'run_if_test_out'
    
    We detect changes to the contents of argv[0]:
    
    Create a shell script and run it:
    
    >>> _ = subprocess.run('rm run_if_test_argv0.* 1>/dev/null 2>/dev/null || true', shell=1)
    >>> with open('run_if_test_argv0.sh', 'w') as f:
    ...     print('#! /bin/sh', file=f)
    ...     print('echo hello world > run_if_test_argv0.out', file=f)
    >>> _ = subprocess.run(f'chmod u+x run_if_test_argv0.sh', shell=1)
    >>> run_if( f'./run_if_test_argv0.sh', f'run_if_test_argv0.out', caller=0)
    pipcl.py:run_if(): Running command because: File does not exist: 'run_if_test_argv0.out'
    pipcl.py:run_if(): Running: ./run_if_test_argv0.sh
    True
    
    Running it a second time does nothing:
    
    >>> run_if( f'./run_if_test_argv0.sh', f'run_if_test_argv0.out', caller=0)
    pipcl.py:run_if(): Not running command because up to date: 'run_if_test_argv0.out'
    
    Modify the script.
    
    >>> with open('run_if_test_argv0.sh', 'a') as f:
    ...     print('\\necho hello >> run_if_test_argv0.out', file=f)
    
    And now it is run because the hash of argv[0] has changed:
    
    >>> run_if( f'./run_if_test_argv0.sh', f'run_if_test_argv0.out', caller=0)
    pipcl.py:run_if(): Running command because: arg0 hash has changed.
    pipcl.py:run_if(): Running: ./run_if_test_argv0.sh
    True
    '''
    doit = False
    
    # Path of file containing pickle data for command and hash of command's
    # first arg.
    cmd_path = f'{out}.cmd'
    
    def hash_get(path):
        try:
            with open(path, 'rb') as f:
                return hashlib.md5(f.read()).hexdigest()
        except Exception as e:
            #log(f'Failed to get hash of {path=}: {e}')
            return None
    
    command_args = shlex.split(command or '')
    command_arg0_path = fs_find_in_paths(command_args[0])
    command_arg0_hash = hash_get(command_arg0_path)
    
    cmd_args, cmd_arg0_hash = (None, None)
    if os.path.isfile(cmd_path):
        with open(cmd_path, 'rb') as f:
            try:
                cmd_args, cmd_arg0_hash = pickle.load(f)
            except Exception as e:
                #log(f'pickle.load() failed with {cmd_path=}: {e}')
                pass

    if not doit:
        # Set doit if outfile does not exist.
        out_mtime = _fs_mtime( out)
        if out_mtime == 0:
            doit = f'File does not exist: {out!r}'

    if not doit:
        # Set doit if command has changed.
        if command_args != cmd_args:
            if cmd_args is None:
                doit = 'No previous command stored'
            else:
                doit = f'Command has changed'
                if 0:
                    doit += f':\n    {cmd!r}\n    {command!r}'
                if 0:
                    doit += f'\nbefore:\n'
                    doit += textwrap.indent(cmd, '    ')
                    doit += f'\nafter:\n'
                    doit += textwrap.indent(command, '    ')
                if 1:
                    # Show diff based on commands split into pseudo lines by
                    # shlex.split().
                    doit += ':\n'
                    lines = difflib.unified_diff(
                            cmd_args,
                            command_args,
                            lineterm='',
                            )
                    # Skip initial lines.
                    assert next(lines) == '--- '
                    assert next(lines) == '+++ '
                    for line in lines:
                        doit += f'    {line}\n'

    if not doit:
        # Set doit if argv[0] hash has changed.
        #print(f'{cmd_arg0_hash=} {command_arg0_hash=}', file=sys.stderr)
        if command_arg0_hash != cmd_arg0_hash:
            doit = f'arg0 hash has changed.'
            #doit = f'arg0 hash has changed from {cmd_arg0_hash=} to {command_arg0_hash=}..'
    
    if not doit:
        # See whether any prerequisites are newer than target.
        def _make_prerequisites(p):
            if isinstance( p, (list, tuple)):
                return list(p)
            else:
                return [p]
        prerequisites_all = list()
        for p in prerequisites:
            prerequisites_all += _make_prerequisites( p)
        if 0:
            log2( 'prerequisites_all:', caller=caller+1)
            for i in  prerequisites_all:
                log2( f'    {i!r}', caller=caller+1)
        pre_mtime = 0
        pre_path = None
        for prerequisite in prerequisites_all:
            if isinstance( prerequisite, str):
                mtime = _fs_mtime_newest( prerequisite)
                if mtime >= pre_mtime:
                    pre_mtime = mtime
                    pre_path = prerequisite
            elif prerequisite is None:
                pass
            elif prerequisite:
                doit = str(prerequisite)
                break
        if not doit:
            if pre_mtime > out_mtime:
                doit = f'Prerequisite is new: {os.path.abspath(pre_path)!r}'

    if doit:
        # Remove `cmd_path` before we run the command, so any failure
        # will force rerun next time.
        #
        try:
            os.remove( cmd_path)
        except Exception:
            pass
        log1( f'Running command because: {doit}', caller=caller+1)

        run( command, caller=caller+1)

        # Write the command we ran, into `cmd_path`.
        
        with open(cmd_path, 'wb') as f:
            pickle.dump((command_args, command_arg0_hash), f)
        return True
    else:
        log1( f'Not running command because up to date: {out!r}', caller=caller+1)

    if 0:
        log2( f'out_mtime={time.ctime(out_mtime)} pre_mtime={time.ctime(pre_mtime)}.'
                f' pre_path={pre_path!r}: returning {ret!r}.'
                )


def fs_find_in_paths( name, paths=None, verbose=False):
    '''
    Looks for `name` in paths and returns complete path. `paths` is list/tuple
    or `os.pathsep`-separated string; if `None` we use `$PATH`. If `name`
    contains `/`, we return `name` itself if it is a file or None, regardless
    of <paths>.
    '''
    if '/' in name:
        return name if os.path.isfile( name) else None
    if paths is None:
        paths = os.environ.get( 'PATH', '')
        if verbose:
            log('From os.environ["PATH"]: {paths=}')
    if isinstance( paths, str):
        paths = paths.split( os.pathsep)
        if verbose:
            log('After split: {paths=}')
    for path in paths:
        p = os.path.join( path, name)
        if verbose:
            log('Checking {p=}')
        if os.path.isfile( p):
            if verbose:
                log('Returning because is file: {p!r}')
            return p
    if verbose:
        log('Returning None because not found: {name!r}')


def _get_prerequisites(path):
    '''
    Returns list of prerequisites from Makefile-style dependency file, e.g.
    created by `cc -MD -MF <path>`.
    '''
    ret = list()
    if os.path.isfile(path):
        with open(path) as f:
            for line in f:
                for item in line.split():
                    if item.endswith( (':', '\\')):
                        continue
                    ret.append( item)
    return ret


def _fs_mtime_newest( path):
    '''
    path:
        If a file, returns mtime of the file. If a directory, returns mtime of
        newest file anywhere within directory tree. Otherwise returns 0.
    '''
    ret = 0
    if os.path.isdir( path):
        for dirpath, dirnames, filenames in os.walk( path):
            for filename in filenames:
                path = os.path.join( dirpath, filename)
                ret = max( ret, _fs_mtime( path))
    else:
        ret = _fs_mtime( path)
    return ret


def _flags( items, prefix='', quote=''):
    '''
    Turns sequence into string, prefixing/quoting each item.
    '''
    if not items:
        return ''
    if isinstance( items, str):
        items = items,
    ret = ''
    for item in items:
        if ret:
            ret += ' '
        ret += f'{prefix}{quote}{item}{quote}'
    return ret.strip()


def _fs_mtime( filename, default=0):
    '''
    Returns mtime of file, or `default` if error - e.g. doesn't exist.
    '''
    try:
        return os.path.getmtime( filename)
    except OSError:
        return default


def _normalise(name):
    # https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization
    return re.sub(r"[-_.]+", "-", name).lower()


def _normalise2(name):
    # https://packaging.python.org/en/latest/specifications/binary-distribution-format/
    return _normalise(name).replace('-', '_')


def _assert_version_pep_440(version):
    assert re.match(
                r'^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$',
                version,
            ), \
            f'Bad version: {version!r}.'


g_verbose = int(os.environ.get('PIPCL_VERBOSE', '1'))

def verbose(level=None):
    '''
    Sets verbose level if `level` is not None.
    Returns verbose level.
    '''
    global g_verbose
    if level is not None:
        g_verbose = level
    return g_verbose

g_log_line_numbers = True

def log_line_numbers(yes):
    '''
    Sets whether to include line numbers; helps with doctest.
    '''
    global g_log_line_numbers
    g_log_line_numbers = bool(yes)

def log(text='', caller=1):
    _log(text, 0, caller+1)

def log0(text='', caller=1):
    _log(text, 0, caller+1)

def log1(text='', caller=1):
    _log(text, 1, caller+1)

def log2(text='', caller=1):
    _log(text, 2, caller+1)

def _log(text, level, caller):
    '''
    Logs lines with prefix, if <level> is lower or equal to <g_verbose>.
    '''
    if level <= g_verbose:
        fr = inspect.stack(context=0)[caller]
        filename = relpath(fr.filename)
        for line in text.split('\n'):
            if g_log_line_numbers:
                print(f'{filename}:{fr.lineno}:{fr.function}(): {line}', file=sys.stdout, flush=1)
            else:
                print(f'{filename}:{fr.function}(): {line}', file=sys.stdout, flush=1)


def relpath(path, start=None, allow_up=True):
    '''
    A safe alternative to os.path.relpath(), avoiding an exception on Windows
    if the drive needs to change - in this case we use os.path.abspath().
    
    Args:
        path:
            Path to be processed.
        start:
            Start directory or current directory if None.
        allow_up:
            If false we return absolute path is <path> is not within <start>.
    '''
    if windows():
        try:
            ret = os.path.relpath(path, start)
        except ValueError:
            # os.path.relpath() fails if trying to change drives.
            ret = os.path.abspath(path)
    else:
        ret = os.path.relpath(path, start)
    if not allow_up and ret.startswith('../') or ret.startswith('..\\'):
        ret = os.path.abspath(path)
    return ret


def _so_suffix(use_so_versioning=True):
    '''
    Filename suffix for shared libraries is defined in pep-3149.  The
    pep claims to only address posix systems, but the recommended
    sysconfig.get_config_var('EXT_SUFFIX') also seems to give the
    right string on Windows.
    
    If use_so_versioning is false, we return only the last component of
    the suffix, which removes any version number, for example changing
    `.cp312-win_amd64.pyd` to `.pyd`.
    '''
    # Example values:
    #   linux:      .cpython-311-x86_64-linux-gnu.so
    #   macos:      .cpython-311-darwin.so
    #   openbsd:    .cpython-310.so
    #   windows     .cp311-win_amd64.pyd
    #
    # Only Linux and Windows seem to identify the cpu. For example shared
    # libraries in numpy-1.25.2-cp311-cp311-macosx_11_0_arm64.whl are called
    # things like `numpy/core/_simd.cpython-311-darwin.so`.
    #
    ret = sysconfig.get_config_var('EXT_SUFFIX')
    if not use_so_versioning:
        # Use last component only.
        ret = os.path.splitext(ret)[1]
    return ret


def get_soname(path):
    '''
    If we are on Linux and `path` is softlink and points to a shared library
    for which `objdump -p` contains 'SONAME', return the pointee. Otherwise
    return `path`. Useful if Linux shared libraries have been created with
    `-Wl,-soname,...`, where we need to embed the versioned library.
    '''
    if linux() and os.path.islink(path):
        path2 = os.path.realpath(path)
        if subprocess.run(f'objdump -p {path2}|grep SONAME', shell=1, check=0).returncode == 0:
            return path2
    elif openbsd():
        # Return newest .so with version suffix.
        sos = glob.glob(f'{path}.*')
        log1(f'{sos=}')
        sos2 = list()
        for so in sos:
            suffix = so[len(path):]
            if not suffix or re.match('^[.][0-9.]*[0-9]$', suffix):
                sos2.append(so)
        sos2.sort(key=lambda p: os.path.getmtime(p))
        log1(f'{sos2=}')
        return sos2[-1]
    return path


def current_py_limited_api():
    '''
    Returns value of PyLIMITED_API to build for current Python.
    '''
    a, b = map(int, platform.python_version().split('.')[:2])
    return f'0x{a:02x}{b:02x}0000'
    

def install_dir(root=None):
    '''
    Returns install directory used by `install()`.

    This will be `sysconfig.get_path('platlib')`, modified by `root` if not
    None.
    '''
    # todo: for pure-python we should use sysconfig.get_path('purelib') ?
    root2 = sysconfig.get_path('platlib')
    if root:
        if windows():
            # If we are in a venv, `sysconfig.get_path('platlib')`
            # can be absolute, e.g.
            # `C:\\...\\venv-pypackage-3.11.1-64\\Lib\\site-packages`, so it's
            # not clear how to append it to `root`. So we just use `root`.
            return root
        else:
            # E.g. if `root` is `install' and `sysconfig.get_path('platlib')`
            # is `/usr/local/lib/python3.9/site-packages`, we set `root2` to
            # `install/usr/local/lib/python3.9/site-packages`.
            #
            return os.path.join( root, root2.lstrip( os.sep))
    else:
        return root2


class _Record:
    '''
    Internal - builds up text suitable for writing to a RECORD item, e.g.
    within a wheel.
    '''
    def __init__(self):
        self.text = ''

    def add_content(self, content, to_, verbose=True):
        if isinstance(content, str):
            content = content.encode('utf8')

        # Specification for the line we write is supposed to be in
        # https://packaging.python.org/en/latest/specifications/binary-distribution-format
        # but it's not very clear.
        #
        h = hashlib.sha256(content)
        digest = h.digest()
        digest = base64.urlsafe_b64encode(digest)
        digest = digest.rstrip(b'=')
        digest = digest.decode('utf8')

        self.text += f'{to_},sha256={digest},{len(content)}\n'
        if verbose:
            log2(f'Adding {to_}')

    def add_file(self, from_, to_):
        log1(f'Adding file: {os.path.relpath(from_)} => {to_}')
        with open(from_, 'rb') as f:
            content = f.read()
        self.add_content(content, to_, verbose=False)

    def get(self, record_path=None):
        '''
        Returns contents of the RECORD file. If `record_path` is
        specified we append a final line `<record_path>,,`; this can be
        used to include the RECORD file itself in the contents, with
        empty hash and size fields.
        '''
        ret = self.text
        if record_path:
            ret += f'{record_path},,\n'
        return ret


class NewFiles:
    '''
    Detects new/modified/updated files matching a glob pattern. Useful for
    detecting wheels created by pip or cubuildwheel etc.
    '''
    def __init__(self, glob_pattern):
        # Find current matches of <glob_pattern>.
        self.glob_pattern = glob_pattern
        self.items0 = self._items()
    def get(self):
        '''
        Returns list of new matches of <glob_pattern> - paths of files that
        were not present previously, or have different mtimes or have different
        contents.
        '''
        ret = list()
        items = self._items()
        for path, id_ in items.items():
            id0 = self.items0.get(path)
            if id0 != id_:
                ret.append(path)
        return ret
    def get_n(self, n):
        '''
        Returns new files matching <glob_pattern>, asserting that there are
        exactly <n>.
        '''
        ret = self.get()
        assert len(ret) == n, f'{len(ret)=}: {ret}'
        return ret
    def get_one(self):
        '''
        Returns new match of <glob_pattern>, asserting that there is exactly
        one.
        '''
        return self.get_n(1)[0]
    def _file_id(self, path):
        mtime = os.stat(path).st_mtime
        with open(path, 'rb') as f:
            content = f.read()
            hash_ = hashlib.md5(content).digest()
            # With python >= 3.11 we can do:
            #hash_ = hashlib.file_digest(f, hashlib.md5).digest()
        return mtime, hash_
    def _items(self):
        ret = dict()
        for path in glob.glob(self.glob_pattern):
            if os.path.isfile(path):
                ret[path] = self._file_id(path)
        return ret


def swig_get(swig, quick, swig_local='pipcl-swig-git'):
    '''
    Returns <swig> or a new swig binary.
    
    If <swig> is true and starts with 'git:' (not Windows), the remaining text
    is passed to git_get() and we clone/update/build swig, and return the built
    binary. We default to the main swig repository, branch master, so for
    example 'git:' will return the latest swig from branch master.
    
    Otherwise we simply return <swig>.
    
    Args:
        swig:
            If starts with 'git:', passed as <text> arg to git_get().
        quick:
            If true, we do not update/build local checkout if the binary is
            already present.
        swig_local:
            path to use for checkout.
    '''
    if swig and swig.startswith('git:'):
        assert platform.system() != 'Windows', f'Cannot build swig on Windows.'
        # Note that {swig_local}/install/bin/swig doesn't work on MacOS because
        # {swig_local}/INSTALL is a file and the fs is case-insensitive.
        swig_binary = f'{swig_local}/install-dir/bin/swig'
        if quick and os.path.isfile(swig_binary):
            log1(f'{quick=} and {swig_binary=} already exists, so not downloading/building.')
        else:
            # Clone swig.
            swig_env_extra = None
            swig_local = git_get(
                    swig_local,
                    text=swig,
                    remote='https://github.com/swig/swig.git',
                    branch='master',
                    )
            if darwin():
                run(f'brew install automake')
                run(f'brew install pcre2')
                run(f'brew install bison')
                # Default bison doesn't work, and Brew's bison is not added to $PATH.
                #
                # > bison is keg-only, which means it was not symlinked into /opt/homebrew,
                # > because macOS already provides this software and installing another version in
                # > parallel can cause all kinds of trouble.
                # > 
                # > If you need to have bison first in your PATH, run:
                # >   echo 'export PATH="/opt/homebrew/opt/bison/bin:$PATH"' >> ~/.zshrc
                #
                swig_env_extra = dict()
                macos_add_brew_path('bison', swig_env_extra)
                run(f'which bison')
                run(f'which bison', env_extra=swig_env_extra)
            # Build swig.
            run(f'cd {swig_local} && ./autogen.sh', env_extra=swig_env_extra)
            run(f'cd {swig_local} && ./configure --prefix={swig_local}/install-dir', env_extra=swig_env_extra)
            run(f'cd {swig_local} && make', env_extra=swig_env_extra)
            run(f'cd {swig_local} && make install', env_extra=swig_env_extra)
        assert os.path.isfile(swig_binary)
        return swig_binary
    else:
        return swig


def macos_add_brew_path(package, env=None, gnubin=True):
    '''
    Adds path(s) for Brew <package>'s binaries to env['PATH'].
    
    We assert-fail if the relevant directory does no exist.
    
    Args:
        package:
            Name of package. We get <package_root> of installed package by
            running `brew --prefix <package>`.
        env:
            The environment dict to modify. If None we use os.environ. If PATH
            is not in <env>, we first copy os.environ['PATH'] into <env>.
        gnubin:
            If true, we also add path to gnu binaries if it exists,
            <package_root>/libexe/gnubin.
    '''
    if not darwin():
        return
    if env is None:
        env = os.environ
    if 'PATH' not in env:
        env['PATH'] = os.environ['PATH']
    package_root = run(f'brew --prefix {package}', capture=1).strip()
    log(f'{package=} {package_root=}')
    def add(path):
        log(f'{path=}')
        if os.path.isdir(path):
            log(f'Prepending to $PATH: {path}')
            PATH = env['PATH']
            env['PATH'] = f'{path}:{PATH}'
            return 1
        else:
            log(f'Not a directory: {path=}')
            return 0
    n = 0
    n += add(f'{package_root}/bin')
    if gnubin:
        n += add(f'{package_root}/libexec/gnubin')
    assert n, f'Failed to add to $PATH, {package=} {gnubin=}.'


def _show_dict(d):
    ret = ''
    for n in sorted(d.keys()):
        v = d[n]
        ret += f'    {n}: {v!r}\n'
    return ret

def show_sysconfig():
    '''
    Shows contents of sysconfig.get_paths() and sysconfig.get_config_vars() dicts.
    '''
    import sysconfig
    paths = sysconfig.get_paths()
    log0(f'show_sysconfig().')
    log0(f'sysconfig.get_paths():\n{_show_dict(sysconfig.get_paths())}')
    log0(f'sysconfig.get_config_vars():\n{_show_dict(sysconfig.get_config_vars())}')


def sysconfig_python_flags():
    '''
    Returns include paths and library directory for Python.
    
    Uses sysconfig.*(), overridden by environment variables
    PIPCL_SYSCONFIG_PATH_include, PIPCL_SYSCONFIG_PATH_platinclude and
    PIPCL_SYSCONFIG_CONFIG_VAR_LIBDIR if set.
    '''
    include1_ = os.environ.get('PIPCL_SYSCONFIG_PATH_include') or sysconfig.get_path('include')
    include2_ = os.environ.get('PIPCL_SYSCONFIG_PATH_platinclude') or sysconfig.get_path('platinclude')
    ldflags_ = os.environ.get('PIPCL_SYSCONFIG_CONFIG_VAR_LIBDIR') or sysconfig.get_config_var('LIBDIR')

    includes_ = [include1_]
    if include2_ != include1_:
        includes_.append(include2_)
    if windows():
        includes_ = [f'/I"{i}"' for i in includes_]
        ldflags_ = f'/LIBPATH:"{ldflags_}"'
    else:
        includes_ = [f'-I {i}' for i in includes_]
        ldflags_ = f'-L {ldflags_}'
    includes_ = ' '.join(includes_)
    return includes_, ldflags_


def venv_in(path=None):
    '''
    If path is None, returns true if we are in a venv. Otherwise returns true
    only if we are in venv <path>.
    '''
    if path:
        return os.path.abspath(sys.prefix) == os.path.abspath(path)
    else:
        return sys.prefix != sys.base_prefix


def venv_run(args, path, recreate=True, clean=False):
    '''
    Runs Python command inside venv and returns termination code.
    
    Args:
        args:
            List of args or string command.
        path:
            Path of venv directory.
        recreate:
            If false we do not run `<sys.executable> -m venv <path>` if <path>
            already exists. This avoids a delay in the common case where <path>
            is already set up, but fails if <path> exists but does not contain
            a valid venv.
        clean:
            If true we first delete <path>.
    '''
    if clean:
        log(f'Removing any existing venv {path}.')
        assert path.startswith('venv-')
        shutil.rmtree(path, ignore_errors=1)
    if recreate or not os.path.isdir(path):
        run(f'{sys.executable} -m venv {path}')
    
    if isinstance(args, str):
        args_string = args
    elif platform.system() == 'Windows':
        # shlex not reliable on Windows so we use Use crude quoting with "...".
        args_string = ''
        for i, arg in enumerate(args):
            assert '"' not in arg
            if i:
                args_string += ' '
            args_string += f'"{arg}"'
    else:
        args_string = shlex.join(args)
    
    if platform.system() == 'Windows':
        command = f'{path}\\Scripts\\activate && python {args_string}'
    else:
        command = f'. {path}/bin/activate && python {args_string}'
    e = run(command, check=0)
    return e


if __name__ == '__main__':
    # Internal-only limited command line support, used if
    # graal_legacy_python_config is true.
    #
    includes, ldflags = sysconfig_python_flags()
    if sys.argv[1] == '--doctest':
        import doctest
        if sys.argv[2:]:
            for f in sys.argv[2:]:
                ff = globals()[f]
                doctest.run_docstring_examples(ff, globals())
        else:
            doctest.testmod(None)
    elif sys.argv[1:] == ['--graal-legacy-python-config', '--includes']:
        print(includes)
    elif sys.argv[1:] == ['--graal-legacy-python-config', '--ldflags']:
        print(ldflags)
    else:
        assert 0, f'Expected `--graal-legacy-python-config --includes|--ldflags` but {sys.argv=}'