diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 5cb0c05..0000000 Binary files a/.DS_Store and /dev/null differ diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 0000000..bdaab28 --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,39 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package + +on: + release: + types: [published] + +permissions: + contents: read + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.gitignore b/.gitignore index d0899e8..7702d49 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,7 @@ studentfiles -checkpy/storage/ checkpy/tests/ -readme.md *.json +.DS_Store # Byte-compiled / optimized / DLL files __pycache__/ @@ -21,7 +20,6 @@ dist/ downloads/ eggs/ .eggs/ -lib/ lib64/ parts/ sdist/ @@ -93,4 +91,4 @@ ENV/ .spyderproject # Rope project settings -.ropeproject \ No newline at end of file +.ropeproject diff --git a/README.md b/README.md new file mode 100644 index 0000000..eeca4b6 --- /dev/null +++ b/README.md @@ -0,0 +1,242 @@ +## checkpy + +A Python tool for running tests on Python source files. Intended to be +used by students whom are taking courses from [Programming Lab at the UvA](https://www.proglab.nl). + +Check out for examples of tests. + +### Installation + + pip install checkpy + +To download tests, run checkPy with ``-d`` as follows: + + checkpy -d YOUR_GITHUB_TESTS_URL + +For instance: + + checkpy -d spcourse/tests + +Here spcourse/tests points to . You can also use the full url. This tests repository contains a test for `hello.py`. Here is how to run it: + + $ echo 'print("hello world")' > hello.py + $ checkpy hello + Testing: hello.py + :( prints "Hello, world!" + assert 'hello world\n' == 'Hello, world!\n' + - hello world + ? ^ + + Hello, world! + ? ^ + + + :) prints exactly 1 line of output + +### Usage + + usage: checkpy [-h] [-module MODULE] [-download GITHUBLINK] [-register LOCALLINK] [-update] [-list] [-clean] [--dev] + [--silent] [--json] [--gh-auth GH_AUTH] + [files ...] + + checkPy: a python testing framework for education. You are running Python version 3.10.6 and checkpy version 2.0.0. + + positional arguments: + files names of files to be tested + + options: + -h, --help show this help message and exit + -module MODULE provide a module name or path to run all tests from the module, or target a module for a + specific test + -download GITHUBLINK download tests from a Github repository and exit + -register LOCALLINK register a local folder that contains tests and exit + -update update all downloaded tests and exit + -list list all download locations and exit + -clean remove all tests from the tests folder and exit + --dev get extra information to support the development of tests + --silent do not print test results to stdout + --json return output as json, implies silent + --gh-auth GH_AUTH username:personal_access_token for authentication with GitHub. + --output-limit OUTPUTLIMIT + limit the number of characters stored for each test's output field. Default is 1000. Set to 0 to disable this limit. + +To test a single file call: + + checkpy YOUR_FILE_NAME + +### An example + +Tests in checkpy are functions with assertions. For instance: + +```Py +from checkpy import * + +@test() +def printsHello(): + """prints Hello, world!""" + assert outputOf() == "Hello, world!\n" +``` + +checkpy's `test` decorator marks the function below as a test. The docstring is a short description of the test for the student. This test does just one thing, assert that the output of the student's code matches the expected output exactly. checkpy leverages pytest's assertion rewriting to autmatically create assertion messages. For instance, a student might see the following when running this test: + + $ checkpy hello + Testing: hello.py + :( prints Hello, world! + assert 'hello world\n' == 'Hello, world!\n' + - hello world + ? ^ + + Hello, world! + ? ^ + + + +### Writing tests + +Tests are discovered by filename. If you want to test a file called ``hello.py``, the corresponding test must be named ``helloTest.py``. These tests must be placed in a folder called `tests`. For instance: `tests/helloTest.py`. Tests are distributed via GitHub repositories, but for development purposes tests can also be registered locally via the `-r` flag. For instance: + + mkdir tests + touch tests/helloTest.py + checkpy -r tests/helloTest.py + +Once registered, checkpy will start looking in that directory for tests. Now we need a test. A test minimally consists of the following: + +```Py +from checkpy import * + +@test() +def printsHello(): + """prints Hello, world!""" + assert outputOf() == "Hello, world!\n" +``` + +A function marked as a test through checkpy's test decorator. The docstring is a short, generally one-line, description of the test for the student. Then at least one assert. + +> Quick tip, use only binary expressions in assertions and keep them relatively simple for students to understand. If a binary expression is not possible, or you do not want to spoil the output, raise your own assertionError instead: ```raise AssertionError("Your program did not output the answer to the ultimate question of life, the universe, and everything")```. + +While developing, you can run checkpy with the `--dev` flag to get verbose error messages and full tracebacks. So here we might do: + + $ checkpy --dev hello + Testing: hello.py + :( prints "Hello, world!" + assert 'hello world\n' == 'Hello, world!\n' + - hello world + ? ^ + + Hello, world! + ? ^ + + + :) prints exactly 1 line of output + +Check out for many examples of checkpy tests. + +### Short examples + +#### Dependencies between tests + +```Py +@test() +def exactHello(): + """prints \"Hello, world!\"""" + assert outputOf() == "Hello, world!\n" + +@failed(exactHello) +def oneLine(): + """prints exactly 1 line of output""" + assert outputOf().count("\n") == 1 + +@passed(exactHello) +def allGood(): + """Good job, everything is correct! You are ready to hand in.""" +``` + +#### Test functions + +```Py +@test() +def testSquare(): + """square(2) returns 4""" + assert getFunction("square")(4) == 4 +``` + +#### Give hints + +```Py +@test() +def testSquare(): + """square(2) returns 4""" + assert getFunction("square")(4) == 4, "did you remember to round your output?" +``` + +#### Handle randomness with pytest's `approx` + +```Py +@test() +def testThrowDice(): + """throw() returns 7 on average""" + throw = getFunction("throw") + avg = sum(throw() for i in range(1000)) / 1000 + assert avg == approx(7, abs=0.5) +``` + +#### Ban language constructs + +```Py +import ast + +@test() +def testSquare(): + """square(2) returns 4""" + assert ast.While not in static.AbstractSyntaxTree() + assert getFunction("square")(4) == 4 +``` + +#### Check types + +```Py +@test() +def testFibonacci(): + """fibonacci(10) returns the first ten fibonacci numbers""" + fibonacci = getFunction("fibonacci") + assert fibonacci(10) == Type(list[int]) + assert fibonacci(10) == [0, 1, 1, 2, 3, 5, 8, 13, 21, 34] +``` + +#### Configure which files should be present + +> Calling `only`, `include`, `exclude`, `require` or `download` has checkpy create a temporary directory to which the specified files are copied/downloaded. The tests then run in that directory. + +```Py +only("sentiment.py") +download("pos_words.txt", "https://github.com/spcourse/text/raw/main/en/sentiment/pos_words.txt") +download("neg_words.txt", "https://github.com/spcourse/text/raw/main/en/sentiment/neg_words.txt") + +@test() +def testPositiveSentiment(): + """recognises a positive sentence""" + ... +``` + +#### Change the timeout + +```Py +@test(timeout=60) +def exactHello(): + """prints \"Hello, world!\"""" + assert outputOf() == "Hello, world!\n" +``` + +#### Short declarative tests + +> This is a new style of tests for simple repetative use cases. Be sure to check out for many more examples. For example [sentimentTest.py](https://github.com/spcourse/tests/blob/676cf5f0d2b0fbc82c7580a76b4359af273b0ca7/tests/text/sentimentTest.py) + +```Py +correctForPos = test()(declarative + .function("sentiment_of_text") + .params("text") + .returnType(int) + .call("Coronet has the best lines of all day cruisers.") + .returns(1) + .description("recognises a positive sentence") +) +``` + +### Distributing tests + +checkpy downloads tests directly from Github repos. The requirement is that a folder called ``tests`` exists within the repo that contains only tests and folders (which checkpy treats as modules). checkpy will pull from the default branch. To download tests call checkpy with the optional ``-d`` argument and pass your github repo url. checkpy will automatically keep tests up to date by checking for any new commits on GitHub. + +### Testing checkpy + + python3 run_tests.py diff --git a/README.rst b/README.rst deleted file mode 100644 index 5211387..0000000 --- a/README.rst +++ /dev/null @@ -1,169 +0,0 @@ -CheckPy -======= - -A Python tool for running tests on Python source files. Intended to be -used by students whom are taking courses in the `Minor -Programming `__ at the -`UvA `__. - -Installation ------------- - -:: - - pip install checkpy - -Besides installing checkPy, you might want to download some tests along with it. Simply run checkPy with the ``-d`` arg as follows: - -:: - - checkpy -d YOUR_GITHUB_TESTS_URL - -Usage ------ - -:: - - usage: checkpy [-h] [-m MODULE] [-d GITHUBLINK] [-clean] [file] - - checkPy: a simple python testing framework - - positional arguments: - file name of file to be tested - - optional arguments: - -h, --help show this help message and exit - -m MODULE provide a module name or path to run all tests from the - module, or target a module for a specific test - -d GITHUBLINK download tests from a Github repository and exit - -clean remove all tests from the tests folder and exit - - -To simply test a single file, call: - -:: - - checkpy YOUR_FILE_NAME - -If you are unsure whether multiple tests exist with the same name, you can target a specific test by specifying its module: - -:: - - checkpy YOUR_FILE_NAME -m YOUR_MODULE_NAME - -If you want to test all files from a module within your current working directory, then this is the command for you: - -:: - - checkpy -m YOUR_MODULE_NAME - -Features --------- - -- Support for ordering of tests -- Execution of tests can be made dependable on the outcome of other - tests -- The test designer need not concern herself with exception handling - and printing -- The full scope of Python is available when designing tests -- Full control over displayed information -- Support for importing modules without executing scripts that are not - wrapped by ``if __name__ == "__main__"`` -- Support for overriding functions from imports in order to for - instance prevent blocking function calls -- Support for grouping tests in modules, - allowing the user to target tests from a specific module or run all tests in a module with a single command. - -An example ----------- - -Tests in checkPy are collections of abstract methods that you as a test -designer need to implement. A test may look something like the -following: - -.. code-block:: python - - 0| @t.failed(exact) - 1| @t.test(1) - 2| def contains(test): - 3| test.test = lambda : assertlib.contains(lib.outputOf(_fileName), "100") - 4| test.description = lambda : "contains 100 in the output" - 5| test.fail = lambda info : "the correct answer (100) cannot be found in the output" - -From top to bottom: - -- The decorator ``failed`` on line 0 defines a precondition. The test - ``exact`` must have failed for the following tests to execute. -- The decorator ``test`` on line 1 prescribes that the following method - creates a test with order number ``1``. Tests are executed in order, - lowest first. -- The method definition on line 2 describes the name of the test - (``contains``), and takes in an instance of ``Test`` found in - ``test.py``. This instance is provided by the decorator ``test`` on - the previous line. -- On line 3 the ``test`` method is bound to a lambda which describes - the test that is to be executed. In this case asserting that the - print output of ``_fileName`` contains the number ``100``. - ``_fileName`` is a magic variable that refers to the to be tested - source file. Besides resulting in a boolean indicating passing or - failing the test, the test method may also return a message. This - message can be used in other methods to provide valuable information - to the user. In this case however, no message is provided. -- On line 4 the ``description`` method is bound to a lambda which when - called produces a string message describing the intent of the test. -- On line 5 the ``fail`` method is bound to a lambda. This method is - used to provide information that should be shown to the user in case - the test fails. The method takes in a - message (``info``) which comes from the second returned value of the - ``test`` method. This message can be used to relay information found during - execution of the test to the user. - -Writing tests -------------- - -Test methods are discovered in checkPy by filename. If one wants to test -a file ``foo.py``, the corresponding test must be named ``fooTest.py``. -checkPy assumes that all methods in the test file are tests, as such one -should not use the ``from ... import ...`` statement when importing -modules. - -A test minimally consists of the following: - -.. code-block:: python - - import check.test as t - @t.test(0) - def someTest(test): - test.test = lambda : False - test.description = lambda : "some description" - -Here the method ``someTest`` is marked as test by the decorator -``test``. The abstract methods ``test`` and ``description`` are -implemented as these are the only methods that necessarily require -implementation. For more information on tests and their abstract methods -you should refer to ``test.py``. Note that besides defining the ``Test`` -class and its abstract methods, ``test.py`` also provides several -decorators for introducing test dependencies such as ``failed``. - -When providing a concrete implementation for the test method one should -take a closer look at ``lib.py`` and ``assertlib.py``. ``lib.py`` -provides a collection of useful functions to help implement tests. Most -notably ``getFunction`` and ``outputOf``. These provide the tester with -a function from the source file and the complete print output -respectively. Calling ``getFunction`` makes checkPy evaluate only import -statements and code inside definitions of the to be tested file. -Effectively all other parts of code are wrapped by -``if __name__ == "__main__"`` and thus ignored. ``assertlib.py`` -provides a collection of assertions that one may find usefull when -implementing tests. - -For inspiration inspect some existing collections of tests like the tests for `progNS2016 `__. - - -Distributing tests ------------------- - -CheckPy can download tests directly from Github repos. -The only requirement is that a folder called ``tests`` exists within the repo that contains only tests and folders (which checkpy treats as modules). -Simply call checkPy with the optional ``-d`` argument and pass your github repo url. -Tests will then be automatically downloaded and installed. diff --git a/checkpy/__init__.py b/checkpy/__init__.py index 7ffcd8e..9da967c 100644 --- a/checkpy/__init__.py +++ b/checkpy/__init__.py @@ -1,37 +1,69 @@ -def testModule(moduleName): - """ - Test all files from module - """ - import caches - caches.clearAllCaches() - import tester - import downloader - downloader.updateSilently() - results = tester.testModule(moduleName) - try: - if __IPYTHON__: - import matplotlib.pyplot - matplotlib.pyplot.close("all") - except: - pass - return results - -def test(fileName): - """ - Run tests for a single file - """ - import caches - caches.clearAllCaches() - import tester - import downloader - downloader.updateSilently() - result = tester.test(fileName) - try: - if __IPYTHON__: - import matplotlib.pyplot - matplotlib.pyplot.close("all") - except: - pass - return result - -from downloader import download, update \ No newline at end of file +import pathlib as _pathlib +import typing as _typing + +# Path to the directory checkpy was called from +USERPATH: _pathlib.Path = _pathlib.Path.cwd() + +# Path to the directory of checkpy +CHECKPYPATH: _pathlib.Path = _pathlib.Path(__file__).parent + +# TODO rm me once below is fixed: +# https://github.com/pytest-dev/pytest/issues/9174 +# importing requests before dessert/pytest assert rewrite prevents +# a ValueError on python3.10 +import requests as _requests + +import dessert as _dessert +with _dessert.rewrite_assertions_context(): + from checkpy.lib import declarative + +from checkpy.tests import test, failed, passed +from checkpy.lib.basic import outputOf, getModule, getFunction +from checkpy.lib.sandbox import only, include, includeFromTests, exclude, require, download +from checkpy.lib import static +from checkpy.lib import monkeypatch +from checkpy.lib.type import Type +from pytest import approx + + +__all__ = [ + "test", + "failed", + "passed", + "outputOf", + "getModule", + "getFunction", + "Type", + "static", + "monkeypatch", + "declarative", + "only", + "include", + "includeFromTests", + "exclude", + "require", + "download", + "file", + "approx" +] + +# To be tested file +file: _typing.Optional[_pathlib.Path] = None + +# Path to the tests directory +testPath: _typing.Optional[_pathlib.Path] = None + +class _Context: + def __init__(self, debug=False, json=False, silent=False, outputLimit=1000): + self.debug = debug + self.json = json + self.silent = silent + self.outputLimit = outputLimit + + def __reduce__(self): + return ( + _Context, + (self.debug, self.json, self.silent, self.outputLimit) + ) + +context = _Context() \ No newline at end of file diff --git a/checkpy/__main__.py b/checkpy/__main__.py index f343c01..eca9567 100644 --- a/checkpy/__main__.py +++ b/checkpy/__main__.py @@ -1,53 +1,112 @@ import sys import os import argparse -import downloader -import tester -import shutil -import time +from checkpy import context +from checkpy import downloader +from checkpy import tester +from checkpy import printer +from checkpy.tester import TesterResult +import json +import importlib.metadata +import warnings + def main(): - parser = argparse.ArgumentParser(description="checkPy: a simple python testing framework") - parser.add_argument("-module", action="store", dest="module", help="provide a module name or path to run all tests from the module, or target a module for a specific test") - parser.add_argument("-download", action="store", dest="githubLink", help="download tests from a Github repository and exit") - parser.add_argument("-update", action="store_true", help="update all downloaded tests and exit") - parser.add_argument("-list", action="store_true", help="list all download locations and exit") - parser.add_argument("-clean", action="store_true", help="remove all tests from the tests folder and exit") - parser.add_argument("file", action="store", nargs="?", help="name of file to be tested") - args = parser.parse_args() - - rootPath = os.sep.join(os.path.abspath(os.path.dirname(__file__)).split(os.sep)[:-1]) - if rootPath not in sys.path: - sys.path.append(rootPath) - - if args.githubLink: - downloader.download(args.githubLink) - return - - if args.update: - downloader.update() - return - - if args.list: - downloader.list() - return - - if args.clean: - downloader.clean() - return - - if args.file and args.module: - downloader.updateSilently() - tester.test(args.file, module = args.module) - elif args.file and not args.module: - downloader.updateSilently() - tester.test(args.file) - elif not args.file and args.module: - downloader.updateSilently() - tester.testModule(args.module) - else: - parser.print_help() - return + warnings.filterwarnings("ignore") + + parser = argparse.ArgumentParser( + description = + """ + checkPy: a python testing framework for education. + You are running Python version {}.{}.{} and checkpy version {}. + """ + .format(sys.version_info[0], sys.version_info[1], sys.version_info[2], importlib.metadata.version("checkpy")) + ) + + parser.add_argument("-module", "--module", action="store", dest="module", help="provide a module name or path to run all tests from the module, or target a module for a specific test") + parser.add_argument("-download", "--download", action="store", dest="githubLink", help="download tests from a Github repository and exit") + parser.add_argument("-register", "--register", action="store", dest="localLink", help="register a local folder that contains tests and exit") + parser.add_argument("-update", "--update", action="store_true", help="update all downloaded tests and exit") + parser.add_argument("-list", "--list", action="store_true", help="list all download locations and exit") + parser.add_argument("-clean", "--clean", action="store_true", help="remove all tests from the tests folder and exit") + parser.add_argument("--dev", action="store_true", help="get extra information to support the development of tests") + parser.add_argument("--silent", action="store_true", help="do not print test results to stdout") + parser.add_argument("--json", action="store_true", help="return output as json, implies silent") + parser.add_argument("--gh-auth", action="store", help="username:personal_access_token for authentication with GitHub.") + parser.add_argument("--output-limit", action="store", type=int, default=1000, dest="outputLimit", help="limit the number of characters stored for each test's output field. Default is 1000. Set to 0 to disable this limit.") + parser.add_argument("files", action="store", nargs="*", help="names of files to be tested") + args = parser.parse_args() + + rootPath = os.sep.join(os.path.abspath(os.path.dirname(__file__)).split(os.sep)[:-1]) + if rootPath not in sys.path: + sys.path.append(rootPath) + + context.outputLimit = args.outputLimit + + if args.gh_auth: + split_auth = args.gh_auth.split(":") + + if len(split_auth) != 2: + printer.displayError("Invalid --gh-auth option. {} is not of the form username:personal_access_token. Note the :".format(args.gh_auth)) + return + + downloader.set_gh_auth(*split_auth) + + if args.githubLink: + downloader.download(args.githubLink) + return + + if args.localLink: + downloader.register(args.localLink) + return + + if args.update: + downloader.update() + return + + if args.list: + downloader.list() + return + + if args.clean: + downloader.clean() + return + + if args.json: + args.silent = True + context.silent = True + context.json = True + + if args.dev: + context.debug = True + + if args.files: + downloader.updateSilently() + + results: list[TesterResult] = [] + for f in args.files: + if args.module: + result = tester.test(f, module=args.module) + else: + result = tester.test(f) + results.append(result) + + if args.json: + print(json.dumps([r.asDict() for r in results], indent=4)) + return + + if args.module: + downloader.updateSilently() + moduleResults = tester.testModule(args.module) + + if args.json: + if moduleResults is None: + print("[]") + else: + print(json.dumps([r.asDict() for r in moduleResults], indent=4)) + return + + parser.print_help() if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/checkpy/assertlib.py b/checkpy/assertlib.py deleted file mode 100644 index d56e418..0000000 --- a/checkpy/assertlib.py +++ /dev/null @@ -1,46 +0,0 @@ -import lib -import re -import os - -def exact(actual, expected): - return actual == expected - -def exactAndSameType(actual, expected): - return exact(actual, expected) and sameType(actual, expected) - -def between(actual, lower, upper): - return lower <= actual <= upper - -def ignoreWhiteSpace(actual, expected): - return exact(lib.removeWhiteSpace(actual), lib.removeWhiteSpace(expected)) - -def contains(actual, expectedElement): - return expectedElement in actual - -def containsOnly(actual, expectedElements): - return len([el for el in actual if el not in expectedElements]) == 0 - -def sameType(actual, expected): - return type(actual) is type(expected) - -def match(actual, expectedRegEx): - return True if re.match(expectedRegEx, actual) else False - -def sameLength(actual, expected): - return len(actual) == len(expected) - -def fileExists(fileName): - return os.path.isfile(fileName) - -def numberOnLine(number, line, deviation = 0): - return any(between(n, number - deviation, number + deviation) for n in lib.getNumbersFromString(line)) - -def fileContainsFunctionCalls(fileName, *functionNames): - source = lib.removeComments(lib.source(fileName)) - fCallInSrc = lambda fName, src : re.match(re.compile(".*{}[ \\t]*\(.*?\).*".format(fName), re.DOTALL), src) - return all(fCallInSrc(fName, source) for fName in functionNames) - -def fileContainsFunctionDefinitions(fileName, *functionNames): - source = lib.removeComments(lib.source(fileName)) - fDefInSrc = lambda fName, src : re.match(re.compile(".*def[ \\t]+{}[ \\t]*\(.*?\).*".format(fName), re.DOTALL), src) - return all(fDefInSrc(fName, source) for fName in functionNames) diff --git a/checkpy/assertlib/__init__.py b/checkpy/assertlib/__init__.py new file mode 100644 index 0000000..20ce75d --- /dev/null +++ b/checkpy/assertlib/__init__.py @@ -0,0 +1 @@ +from checkpy.assertlib.basic import * diff --git a/checkpy/assertlib/basic.py b/checkpy/assertlib/basic.py new file mode 100644 index 0000000..a5a0688 --- /dev/null +++ b/checkpy/assertlib/basic.py @@ -0,0 +1,53 @@ +from checkpy import lib +import re +import os +import warnings + +warnings.warn( + """checkpy.assertlib is deprecated. Use `assert` statements instead.""", + DeprecationWarning, + stacklevel=2 +) + +def exact(actual, expected): + return actual == expected + +def exactAndSameType(actual, expected): + return exact(actual, expected) and sameType(actual, expected) + +def between(actual, lower, upper): + return lower <= actual <= upper + +def ignoreWhiteSpace(actual, expected): + return exact(lib.removeWhiteSpace(actual), lib.removeWhiteSpace(expected)) + +def contains(actual, expectedElement): + return expectedElement in actual + +def containsOnly(actual, expectedElements): + return len([el for el in actual if el not in expectedElements]) == 0 + +def sameType(actual, expected): + return type(actual) is type(expected) + +def match(actual, expectedRegEx): + return True if re.match(expectedRegEx, actual) else False + +def sameLength(actual, expected): + return len(actual) == len(expected) + +def fileExists(fileName): + return os.path.isfile(fileName) + +def numberOnLine(number, line, deviation = 0): + return any(between(n, number - deviation, number + deviation) for n in lib.getNumbersFromString(line)) + +def fileContainsFunctionCalls(fileName, *functionNames): + source = lib.removeComments(lib.source(fileName)) + fCallInSrc = lambda fName, src : re.match(re.compile(r".*{}[ \t]*(.*?).*".format(fName), re.DOTALL), src) + return all(fCallInSrc(fName, source) for fName in functionNames) + +def fileContainsFunctionDefinitions(fileName, *functionNames): + source = lib.removeComments(lib.source(fileName)) + fDefInSrc = lambda fName, src : re.match(re.compile(r".*def[ \t]+{}[ \t]*(.*?).*".format(fName), re.DOTALL), src) + return all(fDefInSrc(fName, source) for fName in functionNames) diff --git a/checkpy/caches.py b/checkpy/caches.py index 39db273..99f1ffc 100644 --- a/checkpy/caches.py +++ b/checkpy/caches.py @@ -1,51 +1,68 @@ import sys +from functools import wraps _caches = [] -class _Cache(object): - def __init__(self): - self._cache = {} - _caches.append(self) +class _Cache(dict): + """A dict() subclass that appends a self-reference to _caches""" + def __init__(self, *args, **kwargs): + super(_Cache, self).__init__(*args, **kwargs) + _caches.append(self) - def __setitem__(self, key, value): - self._cache[key] = value +_testCache = _Cache() - def __getitem__(self, key): - return self._cache.get(key, None) - def __contains__(self, key): - return key in self._cache +def cache(*keys): + """cache decorator - def delete(self, key): - if key not in self._cache: - return False - del self._cache[key] - return True + Caches input and output of a function. If arguments are passed to + the decorator, take those as key for the cache. Otherwise use the + function arguments and sys.argv as key. - def clear(self): - self._cache.clear() + sys.argv is used here because of user-written code like this: + + import sys + my_variable = sys.argv[1] + def my_function(): + print(my_variable) + + Depending on the state of sys.argv during execution of the module, + the outcome of my_function() changes. + """ + def cacheWrapper(func): + localCache = _Cache() + + @wraps(func) + def cachedFuncWrapper(*args, **kwargs): + if keys: + key = str(keys) + else: + key = str(args) + str(kwargs) + str(sys.argv) + if key not in localCache: + localCache[key] = func(*args, **kwargs) + + return localCache[key] + return cachedFuncWrapper + + return cacheWrapper + + +def cacheTestResult(testFunction): + def wrapper(runFunction): + @wraps(runFunction) + def runFunctionWrapper(*args, **kwargs): + result = runFunction(*args, **kwargs) + _testCache[testFunction.__name__] = result + return result + return runFunctionWrapper + return wrapper -""" -cache decorator -Caches input and output of a function. If arguments are passed to -the decorator, take those as key for the cache, otherwise the -function arguments. -""" -def cache(*keys): - def cacheWrapper(func, localCache = _Cache()): - def cachedFuncWrapper(*args, **kwargs): - if keys: - key = keys - else: - key = args + tuple(kwargs.values()) + tuple(sys.argv) - if key not in localCache: - localCache[key] = func(*args, **kwargs) +def getCachedTestResult(testFunction): + return _testCache[testFunction.__name__] - return localCache[key] - return cachedFuncWrapper - return cacheWrapper def clearAllCaches(): - for cache in _caches: - cache.clear() + for cache in _caches: + cache.clear() + _testCache.clear() diff --git a/checkpy/database/__init__.py b/checkpy/database/__init__.py new file mode 100644 index 0000000..54e9ad7 --- /dev/null +++ b/checkpy/database/__init__.py @@ -0,0 +1 @@ +from checkpy.database.database import * diff --git a/checkpy/database/database.py b/checkpy/database/database.py new file mode 100644 index 0000000..fece0c5 --- /dev/null +++ b/checkpy/database/database.py @@ -0,0 +1,131 @@ +import tinydb +from tinydb.table import Table +import time +import contextlib +import checkpy +import pathlib +from typing import Generator, Iterable, Tuple + +_DBPATH = checkpy.CHECKPYPATH / "database" / "db.json" + +@contextlib.contextmanager +def database() -> Generator[tinydb.TinyDB, None, None]: + _DBPATH.touch() + try: + db = tinydb.TinyDB(str(_DBPATH)) + yield db + finally: + db.close() + +@contextlib.contextmanager +def githubTable()-> Generator[Table, None, None]: + with database() as db: + yield db.table("github") + +@contextlib.contextmanager +def localTable() -> Generator[Table, None, None]: + with database() as db: + yield db.table("local") + +def clean(): + with database() as db: + db.drop_tables() + +def forEachTestsPath() -> Iterable[pathlib.Path]: + for path in forEachGithubPath(): + yield path + + for path in forEachLocalPath(): + yield path + +def forEachUserAndRepo() -> Iterable[Tuple[str, str]]: + with githubTable() as table: + return [(entry["user"], entry["repo"]) for entry in table.all()] + +def forEachGithubPath() -> Iterable[pathlib.Path]: + with githubTable() as table: + for entry in table.all(): + yield pathlib.Path(entry["path"]) + +def forEachLocalPath() -> Iterable[pathlib.Path]: + with localTable() as table: + for entry in table.all(): + yield pathlib.Path(entry["path"]) + +def isKnownGithub(username: str, repoName: str) -> bool: + query = tinydb.Query() + with githubTable() as table: + return table.contains((query.user == username) & (query.repo == repoName)) + +def addToGithubTable( + username: str, + repoName: str, + commitMessage: str, + commitSha: str + ): + if not isKnownGithub(username, repoName): + path = str(checkpy.CHECKPYPATH / "tests" / repoName) + + with githubTable() as table: + table.insert({ + "user" : username, + "repo" : repoName, + "path" : path, + "message" : commitMessage, + "sha" : commitSha, + "timestamp" : time.time() + }) + +def addToLocalTable(localPath: pathlib.Path): + query = tinydb.Query() + with localTable() as table: + if not table.search(query.path == str(localPath)): + table.insert({ + "path" : str(localPath) + }) + +def updateGithubTable( + username: str, + repoName: str, + commitMessage: str, + commitSha: str + ): + query = tinydb.Query() + path = str(checkpy.CHECKPYPATH / "tests" / repoName) + with githubTable() as table: + table.update({ + "user" : username, + "repo" : repoName, + "path" : path, + "message" : commitMessage, + "sha" : commitSha, + "timestamp" : time.time() + }, query.user == username and query.repo == repoName) + +def timestampGithub(username: str, repoName: str) -> float: + query = tinydb.Query() + with githubTable() as table: + return table.search(query.user == username and query.repo == repoName)[0]["timestamp"] + +def setTimestampGithub(username: str, repoName: str): + query = tinydb.Query() + with githubTable() as table: + table.update( + {"timestamp" : time.time()}, + query.user == username and query.repo == repoName + ) + +def githubPath(username: str, repoName: str) -> pathlib.Path: + query = tinydb.Query() + with githubTable() as table: + return pathlib.Path(table.search(query.user == username and query.repo == repoName)[0]["path"]) + +def commitSha(username: str, repoName: str) -> str: + query = tinydb.Query() + with githubTable() as table: + return table.search(query.user == username and query.repo == repoName)[0]["sha"] + +def commitMessage(username: str, repoName: str) -> str: + query = tinydb.Query() + with githubTable() as table: + return table.search(query.user == username and query.repo == repoName)[0]["message"] diff --git a/checkpy/downloader.py b/checkpy/downloader.py deleted file mode 100644 index 8b82094..0000000 --- a/checkpy/downloader.py +++ /dev/null @@ -1,340 +0,0 @@ -import requests -import zipfile as zf -import os -import shutil -import tinydb -import time -import caches -import printer -import exception - -class Folder(object): - def __init__(self, name, path): - self.name = name - self.path = path - - def pathAsString(self): - return self.path.asString() - -class File(object): - def __init__(self, name, path): - self.name = name - self.path = path - - def pathAsString(self): - return self.path.asString() - -class Path(object): - def __init__(self, path): - self._path = os.path.normpath(path) - - @property - def fileName(self): - return os.path.basename(self.asString()) - - @property - def folderName(self): - _, name = os.path.split(os.path.dirname(self.asString())) - return name - - def asString(self): - return self._path - - def isPythonFile(self): - return self.fileName.endswith(".py") - - def exists(self): - return os.path.exists(self.asString()) - - def walk(self): - for path, subdirs, files in os.walk(self.asString()): - yield Path(path), [Path(sd) for sd in subdirs], [Path(f) for f in files] - - def pathFromFolder(self, folderName): - path = "" - seen = False - for item in self: - if seen: - path = os.path.join(path, item) - if item == folderName: - seen = True - return Path(path) - - def __add__(self, other): - try: - # Python 3 - if isinstance(other, bytes) or isinstance(other, str): - return Path(os.path.join(self.asString(), other)) - except NameError: - # Python 2 - if isinstance(other, str) or isinstance(other, unicode): - return Path(os.path.join(self.asString(), other)) - return Path(os.path.join(self.asString(), other.asString())) - - def __sub__(self, other): - my_items = [item for item in self] - other_items = [item for item in other] - total = "" - for item in my_items[len(other_items):]: - total = os.path.join(total, item) - return Path(total) - - def __iter__(self): - for item in self.asString().split(os.path.sep): - yield item - - def __repr__(self): - return self.asString() - - def __hash__(self): - return hash(repr(self)) - - def __eq__(self, other): - return isinstance(other, type(self)) and repr(self) == repr(other) - - def __contains__(self, item): - return item in [item for item in self] - - def __nonzero__ (self): - return len(self.asString()) != 0 - - -HERE = Path(os.path.abspath(os.path.dirname(__file__))) -HEREFOLDER = Folder(HERE.folderName, HERE) -TESTSFOLDER = Folder("tests", HERE + "tests") -DBFOLDER = Folder("storage", HERE + "storage") -DBFILE = File("downloadLocations.json", DBFOLDER.path + "downloadLocations.json") - - -def download(githubLink): - if githubLink.endswith("/"): - githubLink = githubLink[:-1] - - if "/" not in githubLink: - printer.displayError("{} is not a valid download location".format(githubLink)) - return - - username = githubLink.split("/")[-2].lower() - repoName = githubLink.split("/")[-1].lower() - - try: - _syncRelease(username, repoName) - _download(username, repoName) - except exception.DownloadError as e: - printer.displayError(str(e)) - -def update(): - for username, repoName in _forEachUserAndRepo(): - try: - _syncRelease(username, repoName) - _download(username, repoName) - except exception.DownloadError as e: - printer.displayError(str(e)) - -def list(): - for username, repoName in _forEachUserAndRepo(): - printer.displayCustom("{} from {}".format(repoName, username)) - -def clean(): - shutil.rmtree(TESTSFOLDER.pathAsString(), ignore_errors=True) - if (DBFILE.path.exists()): - os.remove(DBFILE.pathAsString()) - printer.displayCustom("Removed all tests") - return - -def updateSilently(): - for username, repoName in _forEachUserAndRepo(): - # only attempt update if 300 sec have passed - if time.time() - _timestamp(username, repoName) < 300: - continue - - _setTimestamp(username, repoName) - try: - if _newReleaseAvailable(username, repoName): - _download(username, repoName) - except exception.DownloadError as e: - pass - -def _newReleaseAvailable(githubUserName, githubRepoName): - # unknown/new download - if not _isKnownDownloadLocation(githubUserName, githubRepoName): - return True - - releaseJson = _getReleaseJson(githubUserName, githubRepoName) - - # new release id found - if releaseJson["id"] != _releaseId(githubUserName, githubRepoName): - _updateDownloadLocations(githubUserName, githubRepoName, releaseJson["id"], releaseJson["tag_name"]) - return True - - # no new release found - return False - -def _syncRelease(githubUserName, githubRepoName): - releaseJson = _getReleaseJson(githubUserName, githubRepoName) - - if _isKnownDownloadLocation(githubUserName, githubRepoName): - _updateDownloadLocations(githubUserName, githubRepoName, releaseJson["id"], releaseJson["tag_name"]) - else: - _addToDownloadLocations(githubUserName, githubRepoName, releaseJson["id"], releaseJson["tag_name"]) - -# this performs one api call, beware of rate limit!!! -# returns a dictionary representing the json returned by github -# incase of an error, raises an exception.DownloadError -def _getReleaseJson(githubUserName, githubRepoName): - apiReleaseLink = "https://api.github.com/repos/{}/{}/releases/latest".format(githubUserName, githubRepoName) - - try: - r = requests.get(apiReleaseLink) - except requests.exceptions.ConnectionError as e: - raise exception.DownloadError(message = "Oh no! It seems like there is no internet connection available?!") - - # exceeded rate limit, - if r.status_code == 403: - raise exception.DownloadError(message = "Tried finding new releases from {}/{} but exceeded the rate limit, try again within an hour!".format(githubUserName, githubRepoName)) - - # no releases found or page not found - if r.status_code == 404: - raise exception.DownloadError(message = "Failed to check for new tests from {}/{} because: no releases found (404)".format(githubUserName, githubRepoName)) - - # random error - if not r.ok: - raise exception.DownloadError(message = "Failed to sync releases from {}/{} because: {}".format(githubUserName, githubRepoName, r.reason)) - - return r.json() - -# download tests for githubUserName and githubRepoName from what is known in downloadlocations.json -# use _syncRelease() to force an update in downloadLocations.json -def _download(githubUserName, githubRepoName): - githubLink = "https://github.com/{}/{}".format(githubUserName, githubRepoName) - zipLink = githubLink + "/archive/{}.zip".format(_releaseTag(githubUserName, githubRepoName)) - - try: - r = requests.get(zipLink) - except requests.exceptions.ConnectionError as e: - raise exception.DownloadError(message = "Oh no! It seems like there is no internet connection available?!") - - if not r.ok: - raise exception.DownloadError(message = "Failed to download {} because: {}".format(githubLink, r.reason)) - - try: - # Python 2 - import StringIO - f = StringIO.StringIO(r.content) - except ModuleNotFoundError: - # Python 3 - import io - f = io.BytesIO(r.content) - - with zf.ZipFile(f) as z: - destFolder = Folder(githubRepoName, TESTSFOLDER.path + githubRepoName) - - existingTests = set() - for path, subdirs, files in destFolder.path.walk(): - for f in files: - existingTests.add((path + f) - destFolder.path) - - newTests = set() - for path in [Path(name) for name in z.namelist()]: - if path.isPythonFile(): - newTests.add(path.pathFromFolder("tests")) - - for filePath in [fp for fp in existingTests - newTests if fp.isPythonFile()]: - printer.displayRemoved(filePath.asString()) - - for filePath in [fp for fp in newTests - existingTests if fp.isPythonFile()]: - printer.displayAdded(filePath.asString()) - - for filePath in existingTests - newTests: - os.remove((destFolder.path + filePath).asString()) - - _extractTests(z, destFolder) - - printer.displayCustom("Finished downloading: {}".format(githubLink)) - -@caches.cache() -def _downloadLocationsDatabase(): - if not DBFOLDER.path.exists(): - os.makedirs(DBFOLDER.pathAsString()) - if not os.path.isfile(DBFILE.pathAsString()): - with open(DBFILE.pathAsString(), 'w') as f: - pass - return tinydb.TinyDB(DBFILE.pathAsString()) - -def _forEachUserAndRepo(): - for username, repoName in ((entry["user"], entry["repo"]) for entry in _downloadLocationsDatabase().all()): - yield username, repoName - -def _isKnownDownloadLocation(username, repoName): - query = tinydb.Query() - return _downloadLocationsDatabase().contains((query.user == username) & (query.repo == repoName)) - -def _addToDownloadLocations(username, repoName, releaseId, releaseTag): - if not _isKnownDownloadLocation(username, repoName): - _downloadLocationsDatabase().insert(\ - { - "user" : username, - "repo" : repoName, - "release" : releaseId, - "tag" : releaseTag, - "timestamp" : time.time() - }) - -def _updateDownloadLocations(username, repoName, releaseId, releaseTag): - query = tinydb.Query() - _downloadLocationsDatabase().update(\ - { - "user" : username, - "repo" : repoName, - "release" : releaseId, - "tag" : releaseTag, - "timestamp" : time.time() - }, query.user == username and query.repo == repoName) - -def _timestamp(username, repoName): - query = tinydb.Query() - return _downloadLocationsDatabase().search(query.user == username and query.repo == repoName)[0]["timestamp"] - -def _setTimestamp(username, repoName): - query = tinydb.Query() - _downloadLocationsDatabase().update(\ - { - "timestamp" : time.time() - }, query.user == username and query.repo == repoName) - -def _releaseId(username, repoName): - query = tinydb.Query() - return _downloadLocationsDatabase().search(query.user == username and query.repo == repoName)[0]["release"] - -def _releaseTag(username, repoName): - query = tinydb.Query() - return _downloadLocationsDatabase().search(query.user == username and query.repo == repoName)[0]["tag"] - -def _extractTests(zipfile, destFolder): - if not destFolder.path.exists(): - os.makedirs(destFolder.pathAsString()) - - for path in [Path(name) for name in zipfile.namelist()]: - _extractTest(zipfile, path, destFolder) - -def _extractTest(zipfile, path, destFolder): - if "tests" not in path: - return - - subfolderPath = path.pathFromFolder("tests") - filePath = destFolder.path + subfolderPath - - if path.isPythonFile(): - _extractFile(zipfile, path, filePath) - elif subfolderPath and not os.path.exists(filePath.asString()): - os.makedirs(filePath.asString()) - -def _extractFile(zipfile, path, filePath): - zipPathString = path.asString().replace("\\", "/") - if os.path.isfile(filePath.asString()): - with zipfile.open(zipPathString) as new, open(filePath.asString(), "r") as existing: - if new.read().strip() != existing.read().strip(): - printer.displayUpdate(path.asString()) - - with zipfile.open(zipPathString) as source, open(filePath.asString(), "wb+") as target: - shutil.copyfileobj(source, target) diff --git a/checkpy/downloader/__init__.py b/checkpy/downloader/__init__.py new file mode 100644 index 0000000..5b65241 --- /dev/null +++ b/checkpy/downloader/__init__.py @@ -0,0 +1 @@ +from .downloader import * diff --git a/checkpy/downloader/downloader.py b/checkpy/downloader/downloader.py new file mode 100644 index 0000000..fff296f --- /dev/null +++ b/checkpy/downloader/downloader.py @@ -0,0 +1,243 @@ +import requests +import zipfile as zf +import os +import io +import pathlib +import shutil +import time + +from typing import Dict, Optional, Set, Union + +from checkpy import database +from checkpy import printer +from checkpy.entities import exception + +user: Optional[str] = None +personal_access_token: Optional[str] = None + +def set_gh_auth(username: str, pat: str): + global user, personal_access_token + user = username + personal_access_token = pat + +def download(githubLink: str): + if githubLink.endswith("/"): + githubLink = githubLink[:-1] + + if "/" not in githubLink: + printer.displayError(f"{githubLink} is not a valid download location") + return + + username = githubLink.split("/")[-2].lower() + repoName = githubLink.split("/")[-1].lower() + + try: + _syncCommit(username, repoName) + _download(username, repoName) + except exception.DownloadError as e: + printer.displayError(str(e)) + +def register(localLink: Union[str, pathlib.Path]): + path = pathlib.Path(localLink) + + if not path.exists(): + printer.displayError("{} does not exist") + return + + database.addToLocalTable(path) + +def update(): + for username, repoName in database.forEachUserAndRepo(): + try: + _syncCommit(username, repoName) + _download(username, repoName) + except exception.DownloadError as e: + printer.displayError(str(e)) + +def list(): + for username, repoName in database.forEachUserAndRepo(): + printer.displayCustom(f"Github: {repoName} from {username}") + for path in database.forEachLocalPath(): + printer.displayCustom(f"Local: {path}") + +def clean(): + for path in database.forEachGithubPath(): + shutil.rmtree(str(path), ignore_errors=True) + database.clean() + printer.displayCustom("Removed all tests") + return + +def updateSilently(): + for username, repoName in database.forEachUserAndRepo(): + # only attempt update if 300 sec have passed + if time.time() - database.timestampGithub(username, repoName) < 300: + continue + + database.setTimestampGithub(username, repoName) + try: + if _newCommitAvailable(username, repoName): + _download(username, repoName) + except exception.DownloadError: + pass + +def _newCommitAvailable(githubUserName: str, githubRepoName: str) -> bool: + # unknown/new download + if not database.isKnownGithub(githubUserName, githubRepoName): + return True + commitJson = _getLatestCommitJson(githubUserName, githubRepoName) + + # new commit found + if commitJson["sha"] != database.commitSha(githubUserName, githubRepoName): + database.updateGithubTable( + githubUserName, + githubRepoName, + commitJson["commit"]["message"], + commitJson["sha"], + ) + return True + + # no new commit found + return False + +def _syncCommit(githubUserName: str, githubRepoName: str): + commitJson = _getLatestCommitJson(githubUserName, githubRepoName) + + if database.isKnownGithub(githubUserName, githubRepoName): + database.updateGithubTable( + githubUserName, + githubRepoName, + commitJson["commit"]["message"], + commitJson["sha"], + ) + else: + database.addToGithubTable( + githubUserName, + githubRepoName, + commitJson["commit"]["message"], + commitJson["sha"], + ) + +def _get_with_auth(url: str) -> requests.Response: + """ + Get a url with authentication if available. + Returns a requests.Response object. + """ + global user + global personal_access_token + if user and personal_access_token: + return requests.get(url, auth=(user, personal_access_token)) + else: + return requests.get(url) + +def _getLatestCommitJson(githubUserName: str, githubRepoName: str) -> Dict: + """ + Get the latest commit from the default branch of the given repository. + This performs one api call, beware of rate limit!!! + Returns a dictionary representing the json returned by github + In case of an error, raises an exception.DownloadError + """ + apiCommitLink = f"https://api.github.com/repos/{githubUserName}/{githubRepoName}/commits" + + try: + r = _get_with_auth(apiCommitLink) + except requests.exceptions.ConnectionError as e: + raise exception.DownloadError(message="Oh no! It seems like there is no internet connection available?!") + + # exceeded rate limit, + if r.status_code == 403: + raise exception.DownloadError(message=f"Tried finding new commits from {githubUserName}/{githubRepoName} but exceeded the rate limit, try again within an hour!") + + # no commits found or page not found + if r.status_code == 404: + raise exception.DownloadError(message=f"Failed to check for new commits from {githubUserName}/{githubRepoName} because: no commits found (404)") + + # random error + if not r.ok: + raise exception.DownloadError(message=f"Failed to get commits from {githubUserName}/{githubRepoName} because: {r.reason}") + + return r.json()[0] + +# download tests for githubUserName and githubRepoName from what is known in db +# use _syncCommit() to force an update in db +def _download(githubUserName: str, githubRepoName: str): + sha = database.commitSha(githubUserName, githubRepoName) + zipUrl = f'https://api.github.com/repos/{githubUserName}/{githubRepoName}/zipball/{sha}' + + try: + r = _get_with_auth(zipUrl) + except requests.exceptions.ConnectionError as e: + raise exception.DownloadError(message = "Oh no! It seems like there is no internet connection available?!") + + gitHubUrl = f'https://github.com/{githubUserName}/{githubRepoName}' # just for feedback + + if not r.ok: + raise exception.DownloadError(message = f"Failed to download {gitHubUrl} because: {r.reason}") + + f = io.BytesIO(r.content) + + with zf.ZipFile(f) as z: + destPath = database.githubPath(githubUserName, githubRepoName) + + existingFiles: Set[pathlib.Path] = set() + for path, subdirs, files in os.walk(destPath): + for fil in files: + existingFiles.add((pathlib.Path(path) / fil).relative_to(destPath)) + + newFiles: Set[pathlib.Path] = set() + for name in z.namelist(): + if name: + path: str = pathlib.Path(name).as_posix() + if "tests/" in path: + newFiles.add(pathlib.Path(path.split("tests/")[1])) + + for filePath in [fp for fp in existingFiles - newFiles if fp.suffix == ".py"]: + printer.displayRemoved(str(filePath)) + + for filePath in [fp for fp in newFiles - existingFiles if fp.suffix == ".py"]: + printer.displayAdded(str(filePath)) + + for filePath in existingFiles - newFiles: + (destPath / filePath).unlink() # remove file + + _extractTests(z, destPath) + + printer.displayCustom(f"Finished downloading: {gitHubUrl}") + +def _extractTests(zipfile: zf.ZipFile, destPath: pathlib.Path): + if not destPath.exists(): + os.makedirs(str(destPath)) + + for path in [pathlib.Path(name) for name in zipfile.namelist()]: + _extractTest(zipfile, path, destPath) + +def _extractTest(zipfile: zf.ZipFile, path: pathlib.Path, destPath: pathlib.Path): + if not "tests/" in path.as_posix(): + return + + subfolderPath = pathlib.Path(path.as_posix().split("tests/")[1]) + filePath = destPath / subfolderPath + + if path.suffix: + _extractFile(zipfile, path, filePath) + elif subfolderPath and not filePath.exists(): + os.makedirs(str(filePath)) + +def _extractFile(zipfile: zf.ZipFile, path: pathlib.Path, filePath: pathlib.Path): + zipPathString = path.as_posix() + if filePath.is_file(): + with zipfile.open(zipPathString) as new, open(str(filePath), "r") as existing: + # read file and decode + try: + newText = new.read().decode('utf-8') + except UnicodeDecodeError: + # Skip any non utf-8 file + return + + # strip trailing whitespace, remove carrier return + newText = ''.join(newText.strip().splitlines()) + existingText = ''.join(existing.read().strip().splitlines()) + if newText != existingText: + printer.displayUpdate(str(path)) + + with zipfile.open(zipPathString) as source, open(str(filePath), "wb+") as target: + shutil.copyfileobj(source, target) diff --git a/checkpy/entities/__init__.py b/checkpy/entities/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/checkpy/entities/exception.py b/checkpy/entities/exception.py new file mode 100644 index 0000000..1f93d31 --- /dev/null +++ b/checkpy/entities/exception.py @@ -0,0 +1,55 @@ +import typing as _typing + +class CheckpyError(Exception): + def __init__( + self, + exception: + _typing.Optional[Exception]=None, + message: str="", + output: str="", + stacktrace: str="" + ): + self._exception = exception + self._message = message + self._output = output + self._stacktrace = stacktrace + + def output(self) -> str: + return self._output + + def stacktrace(self) -> str: + return self._stacktrace + + def __str__(self): + if self._exception: + return "\"{}\" occured {}".format(repr(self._exception), self._message) + return "{} -> {}".format(self.__class__.__name__, self._message) + + def __repr__(self): + return self.__str__() + +class SourceException(CheckpyError): + pass + +class InputError(CheckpyError): + pass + +class TestError(CheckpyError): + pass + +class DownloadError(CheckpyError): + pass + +class ExitError(CheckpyError): + pass + +class PathError(CheckpyError): + pass + +class TooManyFilesError(CheckpyError): + pass + +class MissingRequiredFiles(CheckpyError): + def __init__(self, missingFiles: _typing.List[str]): + super().__init__(message=f"Missing the following required file{'s' if len(missingFiles) != 1 else ''}: {', '.join(missingFiles)}") + self.missingFiles = tuple(missingFiles) \ No newline at end of file diff --git a/checkpy/entities/function.py b/checkpy/entities/function.py new file mode 100644 index 0000000..e263ffb --- /dev/null +++ b/checkpy/entities/function.py @@ -0,0 +1,71 @@ +import os +import sys +import re +import inspect +import io +import typing + +import checkpy.entities.exception as exception +import checkpy.lib +import checkpy.lib.io + + +class Function: + def __init__(self, function: typing.Callable): + self._function = function + self._printOutput = "" + + def __call__(self, *args, **kwargs) -> typing.Any: + try: + with checkpy.lib.io.captureStdout() as _outStreamListener: + outcome = self._function(*args, **kwargs) + + self._printOutput = _outStreamListener.content + checkpy.lib.addOutput(self._printOutput) + + return outcome + except Exception as e: + if isinstance(e,TypeError): + no_arguments = re.search(r"(\w+\(\)) takes (\d+) positional arguments but (\d+) were given", str(e)) + if no_arguments: + raise exception.SourceException( + exception=None, + message=f"{no_arguments.group(1)} should take {no_arguments.group(3)} arguments but takes {no_arguments.group(2)} instead" + ) + argumentNames = self.arguments + nArgs = len(args) + len(kwargs) + + message = "while trying to execute {}()".format(self.name) + if nArgs > 0: + if len(argumentNames) == len(args): + argsRepr = ", ".join("{}={}".format(argumentNames[i], args[i]) for i in range(len(args))) + kwargsRepr = ", ".join("{}={}".format(kwargName, kwargs[kwargName]) for kwargName in argumentNames[len(args):nArgs]) + representation = ", ".join(s for s in [argsRepr, kwargsRepr] if s) + message = "while trying to execute {}({})".format(self.name, representation) + else: + argsRepr = ','.join(str(arg) for arg in args) + message = f"while trying to execute {self.name}({argsRepr})" + raise exception.SourceException(exception = e, message = message) + + @property + def name(self) -> str: + """gives the name of the function""" + return self._function.__name__ + + @property + def arguments(self) -> typing.List[str]: + """gives the parameter names of the function""" + return self.parameters + + @property + def parameters(self) -> typing.List[str]: + """gives the parameter names of the function""" + return inspect.getfullargspec(self._function)[0] + + @property + def printOutput(self) -> str: + """stateful function that returns the print (stdout) output of the latest function call as a string""" + return self._printOutput + + def __repr__(self): + return self._function.__name__ diff --git a/checkpy/entities/path.py b/checkpy/entities/path.py new file mode 100644 index 0000000..c0a51a8 --- /dev/null +++ b/checkpy/entities/path.py @@ -0,0 +1,146 @@ +import os +import sys +import shutil +import checkpy.entities.exception as exception +import warnings + +warnings.warn( + """checkpy.entities.path is deprecated. Use pathlib.Path instead.""", + DeprecationWarning, + stacklevel=2 +) + +class Path(object): + def __init__(self, path): + path = os.path.normpath(path) + self._drive, path = os.path.splitdrive(path) + + items = str(path).split(os.path.sep) + + if len(items) > 0: + # if path started with root, add root + if items[0] == "": + items[0] = os.path.sep + + # remove any empty items (for instance because of "/") + self._items = [item for item in items if item] + + @property + def fileName(self): + return list(self)[-1] + + @property + def folderName(self): + return list(self)[-2] + + def containingFolder(self): + return Path(self._join(self._drive, list(self)[:-1])) + + def isPythonFile(self): + return self.fileName.endswith(".py") + + def absolutePath(self): + return Path(os.path.abspath(str(self))) + + def exists(self): + return os.path.exists(str(self)) + + def walk(self): + for path, subdirs, files in os.walk(str(self)): + yield Path(path), subdirs, files + + def copyTo(self, destination): + shutil.copyfile(str(self), str(destination)) + + def pathFromFolder(self, folderName): + path = "" + seen = False + items = [] + for item in self: + if seen: + items.append(item) + if item == folderName: + seen = True + + if not seen: + raise exception.PathError(message = "folder {} does not exist in {}".format(folderName, self)) + return Path(self._join(self._drive, items)) + + def __add__(self, other): + if sys.version_info >= (3,0): + supportedTypes = [str, bytes, Path] + else: + supportedTypes = [str, unicode, Path] + + if not any(isinstance(other, t) for t in supportedTypes): + raise exception.PathError(message = "can't add {} to Path only {}".format(type(other), supportedTypes)) + + if not isinstance(other, Path): + other = Path(other) + + # if other path starts with root, throw error + if list(other)[0] == os.path.sep: + raise exception.PathError(message = "can't add {} to Path because it starts at root") + + return Path(self._join(self._drive, list(self) + list(other))) + + def __sub__(self, other): + if sys.version_info >= (3,0): + supportedTypes = [str, bytes, Path] + else: + supportedTypes = [str, unicode, Path] + + if not any(isinstance(other, t) for t in supportedTypes): + raise exception.PathError(message = "can't subtract {} from Path only {}".format(type(other), supportedTypes)) + + if not isinstance(other, Path): + other = Path(other) + + myItems = list(self) + otherItems = list(other) + + for items in (myItems, otherItems): + if len(items) >= 1 and items[0] != os.path.sep and items[0] != ".": + items.insert(0, ".") + + for i in range(min(len(myItems), len(otherItems))): + if myItems[i] != otherItems[i]: + raise exception.PathError(message = "tried subtracting, but subdirs do not match: {} and {}".format(self, other)) + + return Path(self._join(self._drive, myItems[len(otherItems):])) + + def __iter__(self): + for item in self._items: + yield item + + def __hash__(self): + return hash(repr(self)) + + def __eq__(self, other): + return isinstance(other, type(self)) and repr(self) == repr(other) + + def __contains__(self, item): + return str(item) in list(self) + + def __len__(self): + return len(self._items) + + def __str__(self): + return self._join(self._drive, list(self)) + + def __repr__(self): + return "/".join([item for item in self]) + + def _join(self, drive, items): + result = drive + for item in items: + result = os.path.join(result, item) + return result + + +def current(): + return Path(os.getcwd()) + +userPath = Path(os.getcwd()) + +CHECKPYPATH = Path(os.path.abspath(os.path.dirname(__file__))[:-len("/entities")]) diff --git a/checkpy/exception.py b/checkpy/exception.py deleted file mode 100644 index 6201011..0000000 --- a/checkpy/exception.py +++ /dev/null @@ -1,20 +0,0 @@ -import traceback - -class CheckpyError(Exception): - def __init__(self, exception = None, message = ""): - self._exception = exception - self._message = message - - def __str__(self): - if self._exception: - return "\"{}\" occured {}".format(repr(self._exception), self._message) - return "{} -> {}".format(self.__class__.__name__, self._message) - - def __repr__(self): - return self.__str__() - -class SourceException(CheckpyError): - pass - -class DownloadError(CheckpyError): - pass \ No newline at end of file diff --git a/checkpy/interactive.py b/checkpy/interactive.py new file mode 100644 index 0000000..c2b0ea2 --- /dev/null +++ b/checkpy/interactive.py @@ -0,0 +1,100 @@ +from checkpy.tester import TesterResult +from checkpy.tester import runTests as _runTests +from checkpy.tester import runTestsSynchronously as _runTestsSynchronously +from checkpy import caches as _caches +import checkpy +import copy +import checkpy.tester.discovery as _discovery +import pathlib as _pathlib +from typing import List, Optional + +__all__ = ["testModule", "test", "testOffline"] + + +def testModule(moduleName: str, debugMode=False, silentMode=False) -> Optional[List[TesterResult]]: + """ + Test all files from module + """ + _caches.clearAllCaches() + + from . import tester + from . import downloader + downloader.updateSilently() + + try: + oldContext = copy.copy(checkpy.context) + checkpy.context.silent = silentMode + checkpy.context.debug = debugMode + + results = tester.testModule(moduleName) + finally: + checkpy.context = oldContext + + _closeAllMatplotlib() + + return results + +def test(fileName: str, debugMode=False, silentMode=False) -> TesterResult: + """ + Run tests for a single file + """ + _caches.clearAllCaches() + + from . import tester + from . import downloader + downloader.updateSilently() + + try: + oldContext = copy.copy(checkpy.context) + checkpy.context.silent = silentMode + checkpy.context.debug = debugMode + + result = tester.test(fileName) + finally: + checkpy.context = oldContext + + _closeAllMatplotlib() + + return result + +def testOffline(fileName: str, testPath: str | _pathlib.Path, multiprocessing=True, debugMode=False, silentMode=False) -> TesterResult: + """ + Run a test offline. + Takes in the name of file to be tested and an absolute path to the tests directory. + If multiprocessing is True (by default), runs all tests in a seperate process. All tests run in the same process otherwise. + """ + _caches.clearAllCaches() + + fileStem = fileName.split(".")[0] + filePath = _discovery.getPath(fileStem) + + testModuleName = f"{fileStem}Test" + testFileName = f"{fileStem}Test.py" + testPath = _discovery.getTestPathsFrom(testFileName, _pathlib.Path(testPath))[0] + + try: + oldContext = copy.copy(checkpy.context) + checkpy.context.silent = silentMode + checkpy.context.debug = debugMode + + if multiprocessing: + result = _runTests(testModuleName, testPath, filePath) + else: + result = _runTestsSynchronously(testModuleName, testPath, filePath) + finally: + checkpy.context = oldContext + + _closeAllMatplotlib() + + return result + +def _closeAllMatplotlib(): + try: + if __IPYTHON__: # type: ignore [name-defined] + try: + import matplotlib.pyplot + matplotlib.pyplot.close("all") + except: + pass + except: + pass diff --git a/checkpy/lib.py b/checkpy/lib.py deleted file mode 100644 index ebe1b2f..0000000 --- a/checkpy/lib.py +++ /dev/null @@ -1,198 +0,0 @@ -import sys -import re -try: - # Python 2 - import StringIO -except: - # Python 3 - import io as StringIO -import contextlib -import importlib -import imp -import tokenize -import exception as excep -import caches - -@contextlib.contextmanager -def _stdoutIO(stdout=None): - old = sys.stdout - if stdout is None: - stdout = StringIO.StringIO() - sys.stdout = stdout - yield stdout - sys.stdout = old - -@contextlib.contextmanager -def _stdinIO(stdin=None): - old_input = input - def new_input(prompt = None): - return old_input() - __builtins__["input"] = new_input - - old = sys.stdin - if stdin is None: - stdin = StringIO.StringIO() - sys.stdin = stdin - - yield stdin - - __builtins__["input"] = old_input - sys.stdin = old - -def getFunction(functionName, fileName): - return getattr(module(fileName), functionName) - -def outputOf(fileName, stdinArgs = ()): - _, output = moduleAndOutputFromSource(fileName, source(fileName), stdinArgs = tuple(stdinArgs)) - return output - -def outputOfSource(fileName, source): - _, output = moduleAndOutputFromSource(fileName, source) - return output - -def source(fileName): - source = "" - with open(fileName) as f: - source = f.read() - return source - -def sourceOfDefinitions(fileName): - newSource = "" - - with open(fileName) as f: - insideDefinition = False - for line in removeComments(f.read()).split("\n"): - line += "\n" - if not line.strip(): - continue - - if (line.startswith(" ") or line.startswith("\t")) and insideDefinition: - newSource += line - elif line.startswith("def ") or line.startswith("class "): - newSource += line - insideDefinition = True - elif line.startswith("import ") or line.startswith("from "): - newSource += line - else: - insideDefinition = False - return newSource - -def module(fileName, src = None): - if not src: - src = source(fileName) - mod, _ = moduleAndOutputFromSource(fileName, src) - return mod - -@caches.cache() -def moduleAndOutputFromSource(fileName, source, stdinArgs = None): - mod = None - output = "" - exception = None - - with _stdoutIO() as stdout, _stdinIO() as stdin: - if stdinArgs: - for arg in stdinArgs: - stdin.write(str(arg) + "\n") - stdin.seek(0) - - moduleName = fileName[:-3] if fileName.endswith(".py") else fileName - try: - mod = imp.new_module(moduleName) - exec(source) in mod.__dict__ - sys.modules[moduleName] = mod - - except Exception as e: - exception = excep.SourceException(e, "while trying to import the code") - - for name, func in [(name, f) for name, f in mod.__dict__.items() if callable(f)]: - if func.__module__ == moduleName: - setattr(mod, name, wrapFunctionWithExceptionHandler(func)) - output = stdout.getvalue() - if exception: - raise exception - - return mod, output - -def neutralizeFunction(function): - def dummy(*args, **kwargs): - pass - setattr(function, "__code__", dummy.__code__) - -def neutralizeFunctionFromImport(mod, functionName, importedModuleName): - for attr in [getattr(mod, name) for name in dir(mod)]: - if getattr(attr, "__name__", None) == importedModuleName: - if hasattr(attr, functionName): - neutralizeFunction(getattr(attr, functionName)) - if getattr(attr, "__name__", None) == functionName and getattr(attr, "__module__", None) == importedModuleName: - if hasattr(mod, functionName): - neutralizeFunction(getattr(mod, functionName)) - -def wrapFunctionWithExceptionHandler(func): - def exceptionWrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except Exception as e: - argListRepr = "" - if args: - for i in range(len(args)): - argListRepr += ", " + "{}={}".format(func.__code__.co_varnames[i], args[i]) - for kwargName in func.__code__.co_varnames[len(args):func.func_code.co_argcount]: - argListRepr += ", {}={}".format(kwargName, kwargs[kwargName]) - - if not argListRepr: - raise excep.SourceException(e, "while trying to execute the function {}".format(func.__name__)) - raise excep.SourceException(e, "while trying to execute the function {} with arguments ({})".format(func.__name__, argListRepr)) - return exceptionWrapper - -def removeWhiteSpace(s): - return re.sub(r"\s+", "", s, flags=re.UNICODE) - -def getPositiveIntegersFromString(s): - return [int(i) for i in re.findall(r"\d+", s)] - -def getNumbersFromString(s): - return [eval(n) for n in re.findall(r"[-+]?\d*\.\d+|\d+", s)] - -def getLine(text, lineNumber): - lines = text.split("\n") - try: - return lines[lineNumber] - except IndexError: - raise IndexError("Expected to have atleast {} lines in:\n{}".format(lineNumber + 1, text)) - -# inspiration from http://stackoverflow.com/questions/1769332/script-to-remove-python-comments-docstrings -def removeComments(source): - io_obj = StringIO.StringIO(source) - out = "" - prev_toktype = tokenize.INDENT - last_lineno = -1 - last_col = 0 - indentation = "\t" - for token_type, token_string, (start_line, start_col), (end_line, end_col), ltext in tokenize.generate_tokens(io_obj.readline): - if start_line > last_lineno: - last_col = 0 - - # figure out type of indentation used - if token_type == tokenize.INDENT: - indentation = "\t" if "\t" in token_string else " " - - # write indentation - if start_col > last_col and last_col == 0: - out += indentation * (start_col - last_col) - # write other whitespace - elif start_col > last_col: - out += " " * (start_col - last_col) - - # ignore comments - if token_type == tokenize.COMMENT: - pass - # put all docstrings on a single line - elif token_type == tokenize.STRING: - out += re.sub("\n", " ", token_string) - else: - out += token_string - - prev_toktype = token_type - last_col = end_col - last_lineno = end_line - return out diff --git a/checkpy/lib/__init__.py b/checkpy/lib/__init__.py new file mode 100644 index 0000000..cca6163 --- /dev/null +++ b/checkpy/lib/__init__.py @@ -0,0 +1,61 @@ +from checkpy.lib.basic import * +from checkpy.lib.io import * +from checkpy.lib.static import getSource +from checkpy.lib.static import getSourceOfDefinitions +from checkpy.lib.static import removeComments +from checkpy.lib.static import getFunctionDefinitions +from checkpy.lib.static import getFunctionCalls +from checkpy.lib.static import getNumbersFrom +from checkpy.lib.monkeypatch import documentFunction +from checkpy.lib.monkeypatch import neutralizeFunction + +# backward-compatible imports (v2 -> v1) +from checkpy.lib.basic import removeWhiteSpace +from checkpy.lib.basic import getPositiveIntegersFromString +from checkpy.lib.basic import getNumbersFromString +from checkpy.lib.basic import getLine +from checkpy.lib.basic import fileExists +from checkpy.lib.basic import addOutput +from checkpy.lib.sandbox import download +from checkpy.lib.basic import require + + +# backward-compatible renames (v2 -> v1) +def source(fileName) -> str: + import warnings + warnings.warn( + """source() is deprecated. + Use getSource() instead.""", + DeprecationWarning, stacklevel=2 + ) + return getSource(fileName) + + +def sourceOfDefinitions(fileName) -> str: + import warnings + warnings.warn( + """sourceOfDefinitions() is deprecated. + Use getSourceOfDefinitions() instead.""", + DeprecationWarning, stacklevel=2 + ) + return getSourceOfDefinitions(fileName) + + +def module(*args, **kwargs): + import warnings + warnings.warn( + """module() is deprecated. + Use getModule() instead.""", + DeprecationWarning, stacklevel=2 + ) + return getModule(*args, **kwargs) + + +def moduleAndOutputOf(*args, **kwargs): + import warnings + warnings.warn( + """moduleAndOutputOf() is deprecated.q + Use getModuleAndOutputOf() instead.""", + DeprecationWarning, stacklevel=2 + ) + return getModuleAndOutputOf(*args, **kwargs) \ No newline at end of file diff --git a/checkpy/lib/basic.py b/checkpy/lib/basic.py new file mode 100644 index 0000000..64a2c84 --- /dev/null +++ b/checkpy/lib/basic.py @@ -0,0 +1,298 @@ +import contextlib +import os +import pathlib +import re +import shutil +import sys +import traceback + +from pathlib import Path +from types import ModuleType +from typing import Any, Iterable, List, Optional, Tuple, Union +from warnings import warn + +import checkpy +import checkpy.tester +from checkpy.entities import path, exception, function +from checkpy import caches +from checkpy.lib.static import getSource +import checkpy.lib.io + +__all__ = [ + "getFunction", + "getModule", + "outputOf", + "getModuleAndOutputOf", +] + +def getFunction( + functionName: str, + fileName: Optional[Union[str, Path]]=None, + src: Optional[str]=None, + argv: Optional[List[str]]=None, + stdinArgs: Optional[List[str]]=None, + ignoreExceptions: Iterable[Exception]=(), + overwriteAttributes: Iterable[Tuple[str, Any]]=() +) -> function.Function: + """Run the file, ignore any side effects, then get the function with functionName""" + module = _getModuleAndOutputOf( + fileName=fileName, + src=src, + argv=argv, + stdinArgs=stdinArgs, + ignoreExceptions=ignoreExceptions, + overwriteAttributes=overwriteAttributes + )[0] + + if functionName not in module.__dict__: + raise AssertionError( + f"Function '{functionName}' not found in module '{module.__name__}'" + ) + + return getattr(module, functionName) + + +def outputOf( + fileName: Optional[Union[str, Path]]=None, + src: Optional[str]=None, + argv: Optional[List[str]]=None, + stdinArgs: Optional[List[str]]=None, + ignoreExceptions: Iterable[Exception]=(), + overwriteAttributes: Iterable[Tuple[str, Any]]=() +) -> str: + """Get the output after running the file.""" + _, output = _getModuleAndOutputOf( + fileName=fileName, + src=src, + argv=argv, + stdinArgs=stdinArgs, + ignoreExceptions=ignoreExceptions, + overwriteAttributes=overwriteAttributes + ) + checkpy.lib.addOutput(output) + return output + + +def getModule( + fileName: Optional[Union[str, Path]]=None, + src: Optional[str]=None, + argv: Optional[List[str]]=None, + stdinArgs: Optional[List[str]]=None, + ignoreExceptions: Iterable[Exception]=(), + overwriteAttributes: Iterable[Tuple[str, Any]]=() +) -> ModuleType: + """Get the python Module after running the file.""" + mod, output = _getModuleAndOutputOf( + fileName=fileName, + src=src, + argv=argv, + stdinArgs=stdinArgs, + ignoreExceptions=ignoreExceptions, + overwriteAttributes=overwriteAttributes + ) + checkpy.lib.addOutput(output) + return mod + + +def getModuleAndOutputOf( + fileName: Optional[Union[str, Path]]=None, + src: Optional[str]=None, + argv: Optional[List[str]]=None, + stdinArgs: Optional[List[str]]=None, + ignoreExceptions: Iterable[Exception]=(), + overwriteAttributes: Iterable[Tuple[str, Any]]=() + ) -> Tuple[ModuleType, str]: + """ + This function handles most of checkpy's under the hood functionality + + fileName (optional): the name of the file to run + src (optional): the source code to run + argv (optional): set sys.argv to argv before importing, + stdinArgs (optional): arguments passed to stdin + ignoreExceptions (optional): exceptions that will silently pass while importing + overwriteAttributes (optional): attributes to overwrite in the imported module + """ + mod, output = _getModuleAndOutputOf( + fileName=fileName, + src=src, + argv=argv, + stdinArgs=stdinArgs, + ignoreExceptions=ignoreExceptions, + overwriteAttributes=overwriteAttributes + ) + checkpy.lib.addOutput(output) + return mod, output + +@caches.cache() +def _getModuleAndOutputOf( + fileName: Optional[Union[str, Path]]=None, + src: Optional[str]=None, + argv: Optional[List[str]]=None, + stdinArgs: Optional[List[str]]=None, + ignoreExceptions: Iterable[Exception]=(), + overwriteAttributes: Iterable[Tuple[str, Any]]=() + ) -> Tuple[ModuleType, str]: + """ + This function handles most of checkpy's under the hood functionality + + fileName (optional): the name of the file to run + src (optional): the source code to run + argv (optional): set sys.argv to argv before importing, + stdinArgs (optional): arguments passed to stdin + ignoreExceptions (optional): exceptions that will silently pass while importing + overwriteAttributes (optional): attributes to overwrite in the imported module + """ + if fileName is None: + if checkpy.file is None: + raise checkpy.entities.exception.CheckpyError( + message=f"Cannot call getModuleAndOutputOf() without passing fileName as argument if not test is running." + ) + fileName = checkpy.file.name + + if src is None: + src = getSource(fileName) + + mod = None + output = "" + excep = None + + with checkpy.lib.io.captureStdout() as stdoutListener: + + # flush stdin + sys.stdin.seek(0) + sys.stdin.flush() + + # fill stdin with args + if stdinArgs: + for arg in stdinArgs: + sys.stdin.write(str(arg) + "\n") + sys.stdin.seek(0) + + # if argv given, overwrite sys.argv + if argv: + sys.argv, argv = argv, sys.argv + + if any(name == "__name__" and attr == "__main__" for name, attr in overwriteAttributes): + moduleName = "__main__" + else: + moduleName = str(fileName).split(".")[0] + + mod = ModuleType(moduleName) + # overwrite attributes + for attr, value in overwriteAttributes: + setattr(mod, attr, value) + + try: + # execute code in mod + exec(src, mod.__dict__) + + # add resulting module to sys + sys.modules[moduleName] = mod + except tuple(ignoreExceptions) as e: # type: ignore + pass + except exception.CheckpyError as e: + excep = e + except Exception as e: + checkpy.lib.addOutput(stdoutListener.content) + excep = exception.SourceException( + exception = e, + message = "while trying to import the code", + output = stdoutListener.content, + stacktrace = traceback.format_exc()) + except SystemExit as e: + checkpy.lib.addOutput(stdoutListener.content) + excep = exception.ExitError( + message = "exit({}) while trying to import the code".format(int(e.args[0])), + output = stdoutListener.content, + stacktrace = traceback.format_exc()) + + # wrap every function in mod with Function + for name, func in [(name, f) for name, f in mod.__dict__.items() if callable(f)]: + if func.__module__ == moduleName: + setattr(mod, name, function.Function(func)) + + # reset sys.argv + if argv: + sys.argv = argv + + output = stdoutListener.content + if excep: + raise excep + + return mod, output + + +def addOutput(output: str): + """ + Add output to the active test's output. + If no active test is found, this function does nothing. + """ + test = checkpy.tester.getActiveTest() + if test is not None: + test.addOutput(output) + + +def removeWhiteSpace(s): + warn("""checkpy.lib.removeWhiteSpace() is deprecated. Instead use: + import re + re.sub(r"\\s+", "", text) + """, DeprecationWarning, stacklevel=2) + return re.sub(r"\s+", "", s, flags=re.UNICODE) + + +def getPositiveIntegersFromString(s): + warn("""checkpy.lib.getPositiveIntegersFromString() is deprecated. Instead use: + import re + [int(i) for i in re.findall(r"\\d+", text)] + """, DeprecationWarning, stacklevel=2) + return [int(i) for i in re.findall(r"\d+", s)] + + +def getNumbersFromString(s): + warn("""checkpy.lib.getNumbersFromString() is deprecated. Instead use: + lib.static.getNumbersFrom(s) + """, DeprecationWarning, stacklevel=2) + return [eval(n) for n in re.findall(r"[-+]?\d*\.\d+|[-+]?\d+", s)] + + +def getLine(text, lineNumber): + warn("""checkpy.lib.getLine() is deprecated. Instead try: + lines = text.split("\\n") + assert len(lines) >= lineNumber + 1 + line = lines[lineNumber] + """, DeprecationWarning, stacklevel=2) + lines = text.split("\n") + try: + return lines[lineNumber] + except IndexError: + raise IndexError("Expected to have atleast {} lines in:\n{}".format(lineNumber + 1, text)) + + +def fileExists(fileName): + warn("""checkpy.lib.fileExists() is deprecated. Use pathlib.Path instead: + from pathlib import Path + Path(filename).exists() + """, DeprecationWarning, stacklevel=2) + return path.Path(fileName).exists() + + +def require(fileName, source=None): + warn("""checkpy.lib.require() is deprecated. Use requests to download files: + import requests + url = 'http://google.com/favicon.ico' + r = requests.get(url, allow_redirects=True) + with open('google.ico', 'wb') as f: + f.write(r.content) + """, DeprecationWarning, stacklevel=2) + from checkpy.lib import download + + if source: + download(fileName, source) + return + + filePath = checkpy.USERPATH / fileName + + if not fileExists(str(filePath)): + raise exception.CheckpyError(message="Required file {} does not exist".format(fileName)) + + shutil.copyfile(filePath, pathlib.Path.cwd() / fileName) diff --git a/checkpy/lib/declarative.py b/checkpy/lib/declarative.py new file mode 100644 index 0000000..c2ddee0 --- /dev/null +++ b/checkpy/lib/declarative.py @@ -0,0 +1,472 @@ +""" +A declarative approach to writing checks through method chaining. For example: + +``` +testSquare = test(timeout=3)(declarative + .function("square") # assert function square() is defined + .params("x") # assert that square() accepts one parameter called x + .returnType("int") # assert that the function always returns an integer + .call(2) # call the function with argument 2 + .returns(4) # assert that the function returns 4 + .call(3) # now call the function with argument 3 + .returns(9) # assert that the function returns 9 +) +""" + +import re + +from copy import deepcopy +from typing import Any, Callable, Dict, Iterable, Optional, List, Union +from typing_extensions import Self +from uuid import uuid4 + +import checkpy.tests +import checkpy.tester +import checkpy.entities.function +import checkpy.entities.exception +import checkpy + + +__all__ = ["function", "FunctionState"] + + +class function: + """ + A declarative approach to writing checks through method chaining. + Each method adds a part of a test on a stack. + + For example: + + ``` + from checkpy import * + + testSquare = test(timeout=3)(declarative + .function("square") # assert function square() is defined + .params("x") # assert that square() accepts one parameter called x + .returnType("int") # assert that the function always returns an integer + .call(2) # call the function with argument 2 + .returns(4) # assert that the function returns 4 + .call(3) # now call the function with argument 3 + .returns(9) # assert that the function returns 9 + ) + + # This `function` object can be reused for multiple tests. For example: + + square = (declarative + .function("square") + .params("x") + .returnType("int") + ) + + testSquare2 = test()(square.call(2).returns(4)) # A test is only created after calling checkpy's test decorator + testSquare3 = test()(square.call(3).returns(9)) + + # Tests created by this approach can depend and be depended on by other tests like normal: + + testSquare4 = passed(testSquare2, testSquare3)( + square.call(4).returns(16) # testSquare4 will only run if both testSquare2 and 3 pass + ) + + @passed(testSquare2) + def testSquareError(): # testSquareError will only run if testSquare2 passes. + \"\"\"square("foo") raises a ValueError\"\"\" + try: + square("foo") + except ValueError: + return + return False + ``` + """ + def __init__(self, functionName: str, fileName: Optional[str]=None): + self._initialState: FunctionState = FunctionState(functionName, fileName=fileName) + self._stack: List[Callable[["FunctionState"], None]] = [] + self._description: Optional[str] = None + + name = self.name(functionName) + self._stack = name._stack + self.__name__ = name.__name__ + self.__doc__ = name.__doc__ + + def name(self, functionName: str) -> Self: + """Assert that a function with functionName is defined.""" + def testName(state: FunctionState): + state.name = functionName + state.description = f"defines the function {functionName}()" + + source = checkpy.static.getSource(state.fileName) + funcDefs = checkpy.static.getFunctionDefinitions(source) + assert functionName in funcDefs,\ + f'no function found with name {functionName}()' + + state.description = "" + + return self.do(testName) + + def params(self, *params: str) -> Self: + """Assert that the function accepts exactly these parameters.""" + def testParams(state: FunctionState): + state.params = list(params) + state.description = f"defines the function as {state.name}({', '.join(params)})" + + real = state.function.parameters + expected = state.params + + assert len(expected) == len(real),\ + f"expected {len(expected)} parameter(s), your function {state.name}() takes"\ + f" {len(real)} parameter(s)" + + assert expected == real,\ + f"parameters should exactly match the requested function definition" + + state.description = "" + + return self.do(testParams) + + def returnType(self, type_: type) -> Self: + """ + From now on, assert that the function always returns values of type_ when called. + Note that type_ can be any typehint. For instance: + + `function("square").returnType(Optional[int]).call(2) # assert that square returns an int or None` + """ + def testType(state: FunctionState): + state.returnType = type_ + + return self.do(testType) + + def returns(self, expected: Any) -> Self: + """Assert that the last call returns expected.""" + def testReturned(state: FunctionState): + state.description = f"{state.getFunctionCallRepr()} should return {expected}" + assert expected == state.returned, f"{state.getFunctionCallRepr()} returned: {state.returned}" + state.description = "" + + return self.do(testReturned) + + def stdout(self, expected: Any) -> Self: + """Assert that the last call printed expected.""" + def testStdout(state: FunctionState): + nonlocal expected + expected = str(expected) + descrStr = expected.replace("\n", "\\n") + if len(descrStr) > 40: + descrStr = descrStr[:20] + " ... " + descrStr[-20:] + state.description = f"{state.getFunctionCallRepr()} should print {descrStr}" + + actual = state.function.printOutput + assert expected == actual + state.description = "" + + return self.do(testStdout) + + def stdoutRegex(self, regex: Union[str, re.Pattern], readable: Optional[str]=None) -> Self: + """ + Assert that the last call printed output matching regex. + If readable is passed, show that instead of the regex in the test's output. + Uses built-in re.search underneath the hood to find the first match. + """ + def testStdoutRegex(state: FunctionState): + nonlocal regex + if isinstance(regex, str): + regex = re.compile(regex) + + if readable: + state.description = f"{state.getFunctionCallRepr()} should print {readable}" + else: + state.description = f"{state.getFunctionCallRepr()} should print output matching regular expression: {regex}" + + actual = state.function.printOutput + + match = regex.search(actual) + if not match: + if readable: + raise AssertionError(f"The printed output does not match the expected output. This is expected:\n" + f"{readable}\n" + f"This is what {state.getFunctionCallRepr()} printed:\n" + f"{actual}") + raise AssertionError(f"The printed output does not match regular expression: {regex}.\n" + f"This is what {state.getFunctionCallRepr()} printed:\n" + f"{actual}") + state.description = "" + + return self.do(testStdoutRegex) + + def call(self, *args: Any, **kwargs: Any) -> Self: + """Call the function with args and kwargs.""" + def testCall(state: FunctionState): + state.args = list(args) + state.kwargs = kwargs + state.description = f"calling function {state.getFunctionCallRepr()}" + state.returned = state.function(*args, **kwargs) + + state.description = f"{state.getFunctionCallRepr()} returns a value of type {checkpy.Type(state.returnType)}" + type_ = state.returnType + returned = state.returned + assert checkpy.Type(type_) == returned, f"{state.getFunctionCallRepr()} returned: {returned}" == returned + state.description = f"calling function {state.getFunctionCallRepr()}" + + return self.do(testCall) + + def timeout(self, time: int) -> Self: + """Reset the timeout on the check to time.""" + def setTimeout(state: FunctionState): + state.timeout = time + + return self.do(setTimeout) + + def description(self, description: str) -> Self: + """ + Fixate the test's description on this description. + The test's description will not change after a call to this method, + and can only change by calling this method again. + """ + def setDecription(state: FunctionState): + state.setDescriptionFormatter(lambda descr, state: descr) + state.isDescriptionMutable = True + state.description = description + state.isDescriptionMutable = False + + self = self.do(setDecription) + + # If the description step is the only step (after the mandatory name step), put it first + if len(self._stack) == 2: + self._stack.reverse() + + if self._description is None: + self._description = description + + return self + + def do(self, function: Callable[["FunctionState"], None]) -> Self: + """ + Put function on the internal stack and call it after all previous calls have resolved. + .do serves as an entry point for extensibility. Allowing you, the test writer, to insert + specific and custom asserts, hints, and the like. For example: + + ``` + def checkDataFileIsUnchanged(state: "FunctionState"): + with open("data.txt") as f: + assert f.read() == "42\\n", "make sure not to change the file data.txt" + + testDataUnchanged = test()(function("process_data").call("data.txt").do(checkDataFileIsUnchanged)) + ``` + """ + self = deepcopy(self) + self._stack.append(function) + + self.__name__ = f"declarative_function_test_{self._initialState.name}()_{uuid4()}" + self.__doc__ = self._description if self._description is not None else self._initialState.description + + return self + + def __call__(self, test: Optional[checkpy.tests.Test]=None) -> "FunctionState": + """Run the test.""" + if test is None: + test = checkpy.tester.getActiveTest() + + initialDescription = "" + if test is not None\ + and test.description != test.PLACEHOLDER_DESCRIPTION\ + and test.description != self._initialState.description: + initialDescription = test.description + + stack = list(self._stack) + state = deepcopy(self._initialState) + + for step in stack: + step(state) + + if initialDescription: + state.setDescriptionFormatter(lambda descr, state: descr) + state.description = initialDescription + elif state.wasCalled: + state.description = f"{state.getFunctionCallRepr()} works as expected" + else: + state.description = f"{state.name} is correctly defined" + + return state + + +class FunctionState: + """ + The state of the current test. + This object serves as the "single source of truth" for each method in `function`. + """ + def __init__(self, functionName: str, fileName: Optional[str]=None): + self._description: str = f"defines the function {functionName}()" + self._name: str = functionName + self._fileName: Optional[str] = fileName + self._params: Optional[List[str]] = None + self._wasCalled: bool = False + self._returned: Any = None + self._returnType: Any = Any + self._args: List[Any] = [] + self._kwargs: Dict[str, Any] = {} + self._timeout: int = 10 + self._isDescriptionMutable: bool = True + + @staticmethod + def _descriptionFormatter(descr: str, state: "FunctionState") -> str: + return f"testing {state.name}()" + (f" >> {descr}" if descr else "") + + @property + def name(self) -> str: + """The name of the function to be tested.""" + return self._name + + @name.setter + def name(self, newName: str): + self._name = str(newName) + + @property + def fileName(self) -> Optional[str]: + """ + The name of the Python file to run and import. + If this is not set (`None`), the default file (`checkpy.file.name`) is used. + """ + return self._fileName + + @fileName.setter + def fileName(self, newFileName: Optional[str]): + self._fileName = newFileName + + @property + def params(self) -> List[str]: + """The exact parameter names and order that the function accepts.""" + if self._params is None: + raise checkpy.entities.exception.CheckpyError( + message=f"params are not set for declarative function test {self._name}()" + ) + return self._params + + @params.setter + def params(self, parameters: Iterable[str]): + self._params = list(parameters) + + @property + def function(self) -> checkpy.entities.function.Function: + """The executable function.""" + return checkpy.getFunction(self.name, fileName=self.fileName) + + @property + def wasCalled(self) -> bool: + """Has the function been called yet?""" + return self._wasCalled + + @property + def returned(self) -> Any: + """What the last function call returned.""" + if not self.wasCalled: + raise checkpy.entities.exception.CheckpyError( + message=f"function was never called for declarative function test {self._name}" + ) + return self._returned + + @returned.setter + def returned(self, newReturned: Any): + self._wasCalled = True + self._returned = newReturned + + @property + def args(self) -> List[Any]: + """The args that were given to the last function call (excluding keyword args)""" + return self._args + + @args.setter + def args(self, newArgs: Iterable[Any]): + self._args = list(newArgs) + + @property + def kwargs(self) -> Dict[str, Any]: + """The keyword args that were given to the last function call (excluding normal args)""" + return self._kwargs + + @kwargs.setter + def kwargs(self, newKwargs: Dict[str, Any]): + self._kwargs = newKwargs + + @property + def returnType(self) -> type: + """ + The typehint of what the function should return according to the test. + This is not the typehint of the function itself! + """ + return self._returnType + + @returnType.setter + def returnType(self, newReturnType: type): + self._returnType = newReturnType + + @property + def timeout(self) -> int: + """ + The timeout of the test in seconds. + This is not the time left, just the total time available for this test. + """ + return self._timeout + + @timeout.setter + def timeout(self, newTimeout: int): + self._timeout = newTimeout + + test = checkpy.tester.getActiveTest() + if test is None: + raise checkpy.entities.exception.CheckpyError( + message=f"Cannot change timeout while there is no test running." + ) + test.timeout = self.timeout + + @property + def description(self) -> str: + """The description of the test, what is ultimately shown on the screen.""" + return self._descriptionFormatter(self._description, self) + + @description.setter + def description(self, newDescription: str): + if not self.isDescriptionMutable: + return + self._description = newDescription + + test = checkpy.tester.getActiveTest() + if test is None: + raise checkpy.entities.exception.CheckpyError( + message=f"Cannot change description while there is no test running." + ) + test.description = self.description + + @property + def isDescriptionMutable(self): + """Can the description be changed (mutated)?""" + return self._isDescriptionMutable + + @isDescriptionMutable.setter + def isDescriptionMutable(self, newIsDescriptionMutable: bool): + self._isDescriptionMutable = newIsDescriptionMutable + + def getFunctionCallRepr(self): + """ + Helper method to get a formatted string of the function call. + For instance: foo(2, bar=3) + Note this method can only be called after a call to the tested function. + Do be sure to check state.wasCalled! + """ + argsRepr = ", ".join(f'"{a}"' if isinstance(a, str) else str(a) for a in self.args) + kwargsRepr = ", ".join(f'{k}="{v}"' if isinstance(v, str) else f'{k}={v}' for k, v in self.kwargs.items()) + repr = ', '.join([a for a in (argsRepr, kwargsRepr) if a]) + return f"{self.name}({repr})" + + def setDescriptionFormatter(self, formatter: Callable[[str, "FunctionState"], str]): + """ + The test's description is formatted by a function accepting the new description and the state. + This method allows you to overwrite that function, for instance: + + `state.setDescriptionFormatter(lambda descr, state: f"Testing your function {state.name}: {descr}")` + """ + self._descriptionFormatter = formatter # type:ignore [method-assign, assignment] + + test = checkpy.tester.getActiveTest() + if test is None: + raise checkpy.entities.exception.CheckpyError( + message=f"Cannot change descriptionFormatter while there is no test running." + ) + test.description = self.description diff --git a/checkpy/lib/explanation.py b/checkpy/lib/explanation.py new file mode 100644 index 0000000..41fe550 --- /dev/null +++ b/checkpy/lib/explanation.py @@ -0,0 +1,165 @@ +import re + +from types import ModuleType +from typing import Any, Callable, List, Optional, Union + +from dessert.util import assertrepr_compare + +import checkpy + +__all__ = ["addExplainer"] + + +_explainers: List[Callable[[str, Any, Any], Optional[str]]] = [] + + +def addExplainer(explainer: Callable[[str, Any, Any], Optional[str]]) -> None: + _explainers.append(explainer) + + +def explainCompare(op: str, left: Any, right: Any) -> Optional[str]: + for explainer in _explainers: + rep = explainer(op, left, right) + if rep: + return rep + + # Custom Type message + if isinstance(left, checkpy.Type) or isinstance(right, checkpy.Type): + if isinstance(left, checkpy.Type): + left, right = right, left + if isinstance(left, str): + left = f'"{left}"' + return f"{left} is of type {right}" + + # Custom AbstractSyntaxTree message + if isinstance(right, checkpy.static.AbstractSyntaxTree): + if op == "in": + return f"'{left.__name__}' is used in the source code" + + prefix = f"'{left.__name__}' is not used in the source code\n~" + allLines = right.source.split("\n") + lineNoWidth = len(str(max(n.lineno for n in right.foundNodes))) + lines = [] + for node in right.foundNodes: + lines.append(f"On line {str(node.lineno).rjust(lineNoWidth)}: {allLines[node.lineno - 1]}") + return prefix + "\n~".join(lines) + + # Fall back on pytest (dessert) explanations + rep = assertrepr_compare(MockConfig(), op, left, right) + if rep: + # On how to introduce newlines see: + # https://github.com/vmalloc/dessert/blob/97616513a9ea600d50d53e9499044b51aeaf037a/dessert/util.py#L32 + return "\n~".join(rep) + + return rep + + +def simplifyAssertionMessage(assertion: Union[str, AssertionError]) -> str: + message = str(assertion) + + # Filter out pytest's "Use -v to get the full diff" message + lines = message.split("\n") + lines = [line for line in lines if "Use -v to get the full diff" not in line] + message = "\n".join(lines) + + # Find any substitution lines of the form where ... = ... from pytest + whereRegex = re.compile(r"\n[\s]*\+(\s*)(where|and)[\s]*(.*) = (.*)") + whereLines = whereRegex.findall(message) + + # If there are none, nothing to do + if not whereLines: + return message + + # Find the line containing assert ..., this is what will be substituted + match = re.compile(r".*assert .*").search(message) + + # If there is no line starting with "assert ", nothing to do + if match is None: + return message + + assertLine = match.group(0) + + # Always include any lines before the assert line (a custom message) + result = message[:match.start()] + + substitutionRegex = re.compile(r"(.*) = (.*)") + oldIndent = 0 + oldSub = "" + skipping = False + + # For each where line, apply the substitution on the first match + for indent, _, left, right in whereLines: + newIndent = len(indent) + + # If the previous step was skipped, and the next step is more indented, keep skipping + if skipping and newIndent > oldIndent: + continue + + # If the new indentation is smaller, there is a new substitution, cut off the old part + # This prevents previous substitutions from interfering with new substitutions + # For instance (2 == 1) + where 2 = foo(1) => (foo(1) == 1) where 1 = ... + if newIndent <= oldIndent: + cutttingMatch = re.search(re.escape(oldSub), assertLine) + if cutttingMatch is None: + raise checkpy.entities.exception.CheckpyError( + message=f"parsing the assertion '{message}' failed." + f" Please create an issue over at https://github.com/Jelleas/CheckPy/issues" + f" and copy-paste this entire message." + ) + end = cutttingMatch.end() + result += assertLine[:end] + assertLine = assertLine[end:] + oldSub = "" + + # Otherwise, no longer skipping + oldIndent = newIndent + skipping = False + + # If the right contains any checkpy function or module, skip + if _shouldSkip(right): + skipping = True + continue + + # Substitute the first match in assertLine + oldAssertLine = assertLine + assertLine = re.sub( + r"([^\w])" + re.escape(left) + r"([^\w\.])", + r"\1" + right + r"\2", + assertLine, + count=1, + flags=re.S + ) + + # If substitution succeeds, keep track of the sub + if oldAssertLine != assertLine: + oldSub = right + # Else substitution failed, start skipping. + else: + skipping = True + + # Ensure all newlines are escaped + assertLine = assertLine.replace("\n", "\\n") + return result + assertLine + + +def _shouldSkip(content): + modules = [checkpy] + for elem in checkpy.__all__: + attr = getattr(checkpy, elem) + if isinstance(attr, ModuleType): + modules.append(attr) + + skippedFunctionNames = [] + for module in modules: + for elem in module.__all__: + attr = getattr(module, elem) + if callable(attr): + skippedFunctionNames.append(elem) + + return any(elem in content for elem in skippedFunctionNames) + + +class MockConfig: + """This config is only used for config.getoption('verbose')""" + def getoption(*args, **kwargs): + return 0 diff --git a/checkpy/lib/io.py b/checkpy/lib/io.py new file mode 100644 index 0000000..a925429 --- /dev/null +++ b/checkpy/lib/io.py @@ -0,0 +1,118 @@ +import contextlib +import io +import os +import sys +import traceback +import typing + +import checkpy.entities.exception as exception + +__all__ = [ + # "captureStdin", + "captureStdout" +] + +@contextlib.contextmanager +def captureStdout() -> typing.Generator["_StreamListener", None, None]: + listener = _StreamListener(sys.stdout) + try: + listener.start() + yield listener + except: + raise + finally: + listener.stop() + + +@contextlib.contextmanager +def replaceStdout() -> typing.Generator["_Stream", None, None]: + old_stdout = sys.stdout + old_stderr = sys.stderr + + stdout = _Stream() + + try: + sys.stdout = stdout + sys.stderr = open(os.devnull) + yield stdout + except: + raise + finally: + sys.stderr.close() + sys.stdout = old_stdout + sys.stderr = old_stderr + + +@contextlib.contextmanager +def replaceStdin() -> typing.Generator[typing.TextIO, None, None]: + def newInput(oldInput): + def input(prompt=None): + try: + return oldInput() + except EOFError: + e = exception.InputError( + message="You requested too much user input", + stacktrace=traceback.format_exc()) + raise e + return input + + oldInput = input + old = sys.stdin + + stdin = io.StringIO() + + try: + __builtins__["input"] = newInput(oldInput) + sys.stdin = stdin + yield stdin + except: + raise + finally: + sys.stdin = old + __builtins__["input"] = oldInput + + +class _Stream(io.StringIO): + def __init__(self, *args, **kwargs): + io.StringIO.__init__(self, *args, **kwargs) + self._listeners: typing.List["_StreamListener"] = [] + + def register(self, listener: "_StreamListener"): + self._listeners.append(listener) + + def unregister(self, listener: "_StreamListener"): + self._listeners.remove(listener) + + def write(self, text: str): + """Overwrites StringIO.write to update all listeners""" + io.StringIO.write(self, text) + self._onUpdate(text) + + def writelines(self, lines: typing.Iterable): + """Overwrites StringIO.writelines to update all listeners""" + io.StringIO.writelines(self, lines) + for line in lines: + self._onUpdate(line) + + def _onUpdate(self, content: str): + for listener in self._listeners: + listener.update(content) + + +class _StreamListener: + def __init__(self, stream: _Stream): + self._stream = stream + self.content = "" + + def start(self): + self.stream.register(self) + + def stop(self): + self.stream.unregister(self) + + def update(self, content: str): + self.content += content + + @property + def stream(self) -> _Stream: + return self._stream \ No newline at end of file diff --git a/checkpy/lib/monkeypatch.py b/checkpy/lib/monkeypatch.py new file mode 100644 index 0000000..a23b98d --- /dev/null +++ b/checkpy/lib/monkeypatch.py @@ -0,0 +1,62 @@ +from typing import Callable as _Callable + +from checkpy.entities.function import Function as _Function + + +__all__ = [ + "documentFunction", + "neutralizeFunction", + "patchMatplotlib", + "patchNumpy" +] + + +def documentFunction(func: _Callable, documentation: str) -> "_PrintableFunction": + """Creates a function that shows documentation when its printed / shows up in an error.""" + return _PrintableFunction(func, documentation) + + +def neutralizeFunction(function: _Callable): + """ + Patches the function to do nothing (no op). + Useful for unblocking blocking functions like time.sleep() or plt.slow() + """ + def dummy(*args, **kwargs): + pass + setattr(function, "__code__", dummy.__code__) + + +def patchMatplotlib(): + """ + Patches matplotlib's blocking functions to do nothing. + Make sure this is called before any matplotlib.pyplot import. + """ + try: + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + plt.switch_backend("Agg") + neutralizeFunction(plt.pause) + neutralizeFunction(plt.show) + except ImportError: + pass + + +def patchNumpy(): + """ + Always have numpy raise floating-point errors as an error, no warnings. + """ + try: + import numpy + numpy.seterr('raise') + except ImportError: + pass + + +class _PrintableFunction(_Function): + def __init__(self, function: _Callable, docs: str): + super().__init__(function) + self._docs = docs + + def __repr__(self): + return self._docs diff --git a/checkpy/lib/sandbox.py b/checkpy/lib/sandbox.py new file mode 100644 index 0000000..28c59f5 --- /dev/null +++ b/checkpy/lib/sandbox.py @@ -0,0 +1,340 @@ +import contextlib +import glob +import os +import shutil +import tempfile +from pathlib import Path +from typing import List, Set, Optional, Union + +import requests + +import checkpy + +from checkpy.entities.exception import TooManyFilesError, MissingRequiredFiles, DownloadError + + +__all__ = ["exclude", "include", "includeFromTests", "only", "require", "download"] + + +DEFAULT_FILE_LIMIT = 10000 + + +def exclude(*patterns: Union[str, Path]): + """ + Exclude all files matching patterns from the check's sandbox. + + If this is the first call to only/include/exclude/require/download initialize the sandbox: + * Create a temp dir + * Copy over all files from current dir (except those files excluded through exclude()) + + Patterns are shell globs in the same style as .gitignore. + """ + config.exclude(*patterns) + +def include(*patterns: Union[str, Path]): + """ + Include all files matching patterns to the check's sandbox. + + If this is the first call to only/include/exclude/require/download initialize the sandbox: + * Create a temp dir + * Copy over all files from current dir (except those files excluded through exclude()) + + Patterns are shell globs in the same style as .gitignore. + """ + config.include(*patterns) + +def includeFromTests(*patterns: Union[str, Path]): + """ + Include all files matching patterns from the tests directory to the check's sandbox. + + If this is the first call to only/include/exclude/require/download initialize the sandbox: + * Create a temp dir + * Copy over all files from current dir (except those files excluded through exclude()) + + Patterns are shell globs in the same style as .gitignore. + """ + config.includeFromTests(*patterns) + +def only(*patterns: Union[str, Path]): + """ + Only files matching patterns will be in the check's sandbox. + + If this is the first call to only/include/exclude/require/download initialize the sandbox: + * Create a temp dir + * Copy over all files from current dir (except those files excluded through exclude()) + + Patterns are shell globs in the same style as .gitignore. + """ + config.only(*patterns) + +def require(*filePaths: Union[str, Path]): + """ + Include all files in the check's sandbox. + Raises checkpy.entities.exception.MissingRequiredFiles if any required file is missing. + Note that this function does not accept patterns (globs), but concrete filenames or paths. + + If this is the first call to only/include/exclude/require/download initialize the sandbox: + * Create a temp dir + * Copy over all files from current dir (except those files excluded through exclude()) + + Patterns are shell globs in the same style as .gitignore. + """ + config.require(*filePaths) + +def download(fileName: str, source: str): + """ + Download a file from source and store it in fileName. + + If this is the first call to only/include/exclude/require/download initialize the sandbox: + * Create a temp dir + * Copy over all files from current dir (except those files excluded through exclude()) + """ + config.download(fileName, source) + + +class Config: + def __init__(self, onUpdate=lambda config: None): + self.includedFiles: Set[str] = set() + self.excludedFiles: Set[str] = set() + self.missingRequiredFiles: List[str] = [] + self.downloads: List[Download] = [] + self.includedFroms: List[IncludedFrom] = [] + self.isSandboxed = False + self.root = Path.cwd() + self.onUpdate = onUpdate + + def _initSandbox(self): + if self.isSandboxed: + return + + self.includedFiles = _glob("*", root=self.root) + self.isSandboxed = True + + def exclude(self, *patterns: Union[str, Path]): + self._initSandbox() + + newExcluded: Set[str] = set() + + for pattern in patterns: + newExcluded |= _glob(pattern, root=self.root) + + self.includedFiles -= newExcluded + self.excludedFiles.update(newExcluded) + + self.onUpdate(self) + + def include(self, *patterns: Union[str, Path]): + self._initSandbox() + + newIncluded: Set[str] = set() + + for pattern in patterns: + newIncluded |= _glob(pattern, root=self.root) + + self.excludedFiles -= newIncluded + self.includedFiles.update(newIncluded) + + self.onUpdate(self) + + def includeFromTests(self, *patterns: Union[str, Path]): + self._initSandbox() + + included: Set[str] = set() + for pattern in patterns: + included |= _glob(pattern, root=checkpy.testPath) + self.includedFroms.extend( + IncludedFrom((checkpy.testPath / source).resolve(), checkpy.testPath) for source in included) + self.onUpdate(self) + + def only(self, *patterns: Union[str, Path]): + self._initSandbox() + + allFiles = self.includedFiles | self.excludedFiles + self.includedFiles = set.union(*[_glob(p, root=self.root) for p in patterns]) + self.excludedFiles = allFiles - self.includedFiles + + self.onUpdate(self) + + def require(self, *filePaths: Union[str, Path]): + self._initSandbox() + + with cd(self.root): + for fp in filePaths: + fp = str(fp) + if not Path(fp).exists(): + self.missingRequiredFiles.append(fp) + else: + try: + self.excludedFiles.remove(fp) + except KeyError: + pass + else: + self.includedFiles.add(fp) + + self.onUpdate(self) + + def download(self, fileName: str, source: str): + self._initSandbox() + + self.downloads.append(Download(fileName, source)) + self.onUpdate(self) + +config = Config() + + +class IncludedFrom: + def __init__(self, source: Path, root: Path): + self.source = source + self.root = root + self._isIncluded = False + + def include(self): + if self._isIncluded: + return + + if self.source.is_relative_to(self.root): + dest = (Path.cwd() / self.source.relative_to(self.root)).absolute() + dest.parent.mkdir(parents=True, exist_ok=True) + else: + dest = (Path.cwd() / self.source.name).absolute() + + origin = self.source.absolute() + shutil.copy(origin, dest) + self._isIncluded = True + +class Download: + def __init__(self, fileName: str, source: str): + self.fileName: str = str(fileName) + self.source: str = str(source) + self._isDownloaded = False + + def download(self): + if self._isDownloaded: + return + + try: + r = requests.get(self.source, allow_redirects=True) + except requests.exceptions.ConnectionError: + raise DownloadError(message="Oh no! It seems like there is no internet connection available.") + + if not r.ok: + raise DownloadError(message=f"Failed to download {self.source} because: {r.reason}") + + with open(self.fileName, "wb+") as target: + target.write(r.content) + + self._isDownloaded = True + + +@contextlib.contextmanager +def sandbox(name: Union[str, Path]=""): + tempDir = None + dir = None + + oldIncluded: Set[str] = set() + oldExcluded: Set[str] = set() + + def sync(config: Config, sandboxDir: Path): + for dl in config.downloads: + dl.download() + + for includedFrom in config.includedFroms: + includedFrom.include() + + nonlocal oldIncluded, oldExcluded + for f in config.excludedFiles - oldExcluded: + dest = (sandboxDir / f).absolute() + try: + os.remove(dest) + except FileNotFoundError: + pass + + for f in config.includedFiles - oldIncluded: + dest = (sandboxDir / f).absolute() + dest.parent.mkdir(parents=True, exist_ok=True) + origin = (config.root / f).absolute() + shutil.copy(origin, dest) + + oldIncluded = set(config.includedFiles) + oldExcluded = set(config.excludedFiles) + + def onUpdate(config: Config): + if config.missingRequiredFiles: + raise MissingRequiredFiles(config.missingRequiredFiles) + + nonlocal tempDir + nonlocal dir + if dir is None or tempDir is None: + tempDir = tempfile.TemporaryDirectory() + dir = Path(Path(tempDir.name) / name) + dir.mkdir(exist_ok=True) + os.chdir(dir) + + sync(config, dir) + + with sandboxConfig(onUpdate=onUpdate): + try: + yield + finally: + os.chdir(config.root) + if tempDir: + tempDir.cleanup() + + +@contextlib.contextmanager +def sandboxConfig(onUpdate=lambda config: None): + global config + oldConfig = config + try: + config = Config(onUpdate=onUpdate) + yield config + finally: + config = oldConfig + + +@contextlib.contextmanager +def cd(dest: Union[str, Path]): + origin = Path.cwd() + try: + os.chdir(dest) + yield dest + finally: + os.chdir(origin) + + +def _glob( + pattern: Union[str, Path], + root: Union[str, Path, None]=None, + skip_dirs: bool=False, + limit: int=DEFAULT_FILE_LIMIT + ) -> Set[str]: + with cd(root) if root else contextlib.nullcontext(): + pattern = str(pattern) + + # Implicit recursive iff no / in pattern and starts with * + if "/" not in pattern and pattern.startswith("*"): + pattern = f"**/{pattern}" + + files = glob.iglob(pattern, recursive=True) + + all_files = set() + + def add_file(f): + fname = str(Path(f)) + all_files.add(fname) + if len(all_files) > limit: + raise TooManyFilesError( + message=f"found {len(all_files)} files but checkpy only accepts up to {limit} number of files" + ) + + # Expand dirs + for file in files: + if os.path.isdir(file) and not skip_dirs: + for f in _glob(f"{file}/**/*", skip_dirs=True): + if not os.path.isdir(f): + add_file(f) + else: + add_file(file) + + return all_files + diff --git a/checkpy/lib/static.py b/checkpy/lib/static.py new file mode 100644 index 0000000..08b303c --- /dev/null +++ b/checkpy/lib/static.py @@ -0,0 +1,264 @@ +import ast as _ast +import io as _io +import re as _re +import tokenize as _tokenize + +from pathlib import Path as _Path +from typing import Optional as _Optional +from typing import Union as _Union +from typing import List as _List + +import checkpy as _checkpy +import checkpy.entities.exception as _exception + + +__all__ = [ + "getSource", + "getSourceOfDefinitions", + "removeComments", + "getNumbersFrom", + "getFunctionCalls", + "getFunctionDefinitions", + "getAstNodes", + "AbstractSyntaxTree" +] + + +def getSource(fileName: _Optional[_Union[str, _Path]]=None) -> str: + """Get the contents of the file.""" + if fileName is None: + if _checkpy.file is None: + raise _exception.CheckpyError( + message=f"Cannot call getSource() without passing fileName as argument if no test is running." + ) + fileName = _checkpy.file.name + + with open(fileName) as f: + return f.read() + + +def getSourceOfDefinitions(fileName: _Optional[_Union[str, _Path]]=None) -> str: + """Get just the source code inside definitions (def / class) and any imports.""" + if fileName is None: + if _checkpy.file is None: + raise _exception.CheckpyError( + message=f"Cannot call getSourceOfDefinitions() without passing fileName as argument if not test is running." + ) + fileName = _checkpy.file.name + + source = getSource(fileName) + + class Visitor(_ast.NodeVisitor): + def __init__(self): + self.lineNumbers = set() + + def visit_ClassDef(self, node: _ast.ClassDef): + if node.end_lineno is None: + self.lineNumbers.add(node.lineno - 1) + else: + self.lineNumbers |= set(range(node.lineno - 1, node.end_lineno)) + super().generic_visit(node) + + def visit_FunctionDef(self, node: _ast.FunctionDef): + if node.end_lineno is None: + self.lineNumbers.add(node.lineno - 1) + else: + self.lineNumbers |= set(range(node.lineno - 1, node.end_lineno)) + super().generic_visit(node) + + def visit_Import(self, node: _ast.Import): + self.lineNumbers.add(node.lineno - 1) + super().generic_visit(node) + + def visit_ImportFrom(self, node: _ast.ImportFrom): + self.lineNumbers.add(node.lineno - 1) + super().generic_visit(node) + + tree = _ast.parse(source) + visitor = Visitor() + visitor.visit(tree) + + lines = source.split("\n") + return "\n".join(lines[n] for n in sorted(visitor.lineNumbers)) + + +def getNumbersFrom(text: str) -> _List[_Union[int, float]]: + """ + Get all numbers from a string. + Only the first dot in a number is used: + 7.3.4 produces float(7.3). The 4 is discarded. + 'e' is unsupported and considered a normal seperator: + 1e7 produces int(1) and int(7) + Whitespace (or generally non-digits) matters: + 7-8 produces int(7) and int(8) + 7 -8 produces int(7) and int(-8) + 7 - 8 produces int(7) and int(8) + """ + numbers: _List[_Union[int, float]] = [] + + n = "" + for c in text + " ": + if c.isdigit() or c == "." or (c in "+-" and not n): + n += c + elif n: + # drop everything after the second '.' + if n.count(".") > 1: + n = ".".join(n.split(".")[:2]) + + try: + numbers.append(float(n) if "." in n else int(n)) + except ValueError: + pass + + n = "" + + return numbers + + +# inspiration from http://stackoverflow.com/questions/1769332/script-to-remove-python-comments-docstrings +def removeComments(source: str) -> str: + """Remove comments from a string containing Python source code.""" + io_obj = _io.StringIO(source) + out = "" + last_lineno = -1 + last_col = 0 + indentation = "\t" + for token_type, token_string, (start_line, start_col), (end_line, end_col), ltext in _tokenize.generate_tokens(io_obj.readline): + if start_line > last_lineno: + last_col = 0 + + # figure out type of indentation used + if token_type == _tokenize.INDENT: + indentation = "\t" if "\t" in token_string else " " + + # write indentation + if start_col > last_col and last_col == 0: + out += indentation * (start_col - last_col) + # write other whitespace + elif start_col > last_col: + out += " " * (start_col - last_col) + + # ignore comments + if token_type == _tokenize.COMMENT: + pass + # put all docstrings on a single line + elif token_type == _tokenize.STRING: + out += _re.sub("\n", " ", token_string) + else: + out += token_string + + last_col = end_col + last_lineno = end_line + return out + + +def getFunctionCalls(source: _Optional[str]=None) -> _List[str]: + """Get all names of function called in source.""" + class CallVisitor(_ast.NodeVisitor): + def __init__(self): + self.parts = [] + + def visit_Attribute(self, node): + super().generic_visit(node) + self.parts.append(node.attr) + + def visit_Name(self, node): + self.parts.append(node.id) + + @property + def call(self): + return ".".join(self.parts) + + class FunctionsVisitor(_ast.NodeVisitor): + def __init__(self): + self.functionCalls = [] + + def visit_Call(self, node): + callVisitor = CallVisitor() + callVisitor.visit(node.func) + super().generic_visit(node) + self.functionCalls.append(callVisitor.call) + + if source is None: + source = getSource() + + tree = _ast.parse(source) + visitor = FunctionsVisitor() + visitor.visit(tree) + return visitor.functionCalls + + +def getFunctionDefinitions(source: _Optional[str]=None) -> _List[str]: + """Get all names of Function definitions from source.""" + class FunctionsVisitor(_ast.NodeVisitor): + def __init__(self): + self.functionNames = [] + + def visit_FunctionDef(self, node: _ast.FunctionDef): + self.functionNames.append(node.name) + super().generic_visit(node) + + if source is None: + source = getSource() + + tree = _ast.parse(source) + visitor = FunctionsVisitor() + visitor.visit(tree) + return visitor.functionNames + + +def getAstNodes(*types: type, source: _Optional[str]=None) -> _List[_ast.AST]: + """ + Given ast.AST types find all nodes with those types. + Every node found will have a `.lineno` attribute to get its line number. + Some examples: + + ``` + getAstNodes(ast.For) # Will find all for-loops in the source + getAstNodes(ast.Mult, ast.Add) # Will find all uses of multiplication (*) and addition (+) + ``` + """ + nodes: _List[_ast.AST] = [] + + class Visitor(_ast.NodeVisitor): + def __init__(self): + self.lineNumber = 0 + + def generic_visit(self, node: _ast.AST): + if hasattr(node, "lineno"): + self.lineNumber = node.lineno + + if any(isinstance(node, type_) for type_ in types): + node.lineno = self.lineNumber + nodes.append(node) + + super().generic_visit(node) + + if source is None: + source = getSource() + + tree = _ast.parse(source) + Visitor().visit(tree) + return nodes + + +class AbstractSyntaxTree: + """ + An 'in' and 'not in' comparible AbstractSyntaxTree for any ast.Node (any type of ast.AST). + For instance: + + ``` + assert ast.For not in AbstractSyntaxTree() # assert that a for-loop is not present + assert ast.Mult in AbstractSyntaxTree() # assert that multiplication is present + ``` + """ + def __init__(self, fileName: _Optional[str]=None): + # Keep track of any nodes found from last search for a pretty assertion message + self.foundNodes: _List[_ast.AST] = [] + + # Similarly hold on to the source code + self.source: str = getSource(fileName=fileName) + + def __contains__(self, item: type) -> bool: + self.foundNodes = getAstNodes(item, source=self.source) + return bool(self.foundNodes) \ No newline at end of file diff --git a/checkpy/lib/type.py b/checkpy/lib/type.py new file mode 100644 index 0000000..070e0f9 --- /dev/null +++ b/checkpy/lib/type.py @@ -0,0 +1,37 @@ +import typeguard + +class Type: + """ + An equals and not equals comparible type annotation. + + assert [1, 2.0, None] == Type(List[Union[int, float, None]]) + assert [1, 2, 3] == Type(Iterable[int]) + assert {1: "foo"} != Type(Dict[int, str]) + assert (1, "foo", 3) != Type(Tuple[int, str, int]) + + This is built on top of typeguard.check_type, see docs @ + https://typeguard.readthedocs.io/en/stable/api.html#typeguard.check_type + """ + def __init__(self, type_: type): + self._type = type_ + + def __eq__(self, __value: object) -> bool: + isEq = True + def callback(err: typeguard.TypeCheckError, memo: typeguard.TypeCheckMemo): + nonlocal isEq + isEq = False + typeguard.check_type(__value, self._type, typecheck_fail_callback=callback) + return isEq + + def __repr__(self) -> str: + return (str(self._type) + .replace("typing.", "") + .replace("", "int") + .replace("", "float") + .replace("", "bool") + .replace("", "str") + .replace("", "list") + .replace("", "tuple") + .replace("", "dict") + .replace("", "set") + ) diff --git a/checkpy/printer.py b/checkpy/printer.py deleted file mode 100644 index 057d245..0000000 --- a/checkpy/printer.py +++ /dev/null @@ -1,60 +0,0 @@ -import exception as excep -import os -import colorama -colorama.init() - -class _Colors: - PASS = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - NAME = '\033[96m' - ENDC = '\033[0m' - -class _Smileys: - HAPPY = ":)" - SAD = ":(" - CONFUSED = ":S" - -def display(testResult): - color, smiley = _selectColorAndSmiley(testResult) - msg = "{}{} {}{}".format(color, smiley, testResult.description, _Colors.ENDC) - if testResult.message: - msg += "\n - {}".format(testResult.message) - print(msg) - return msg - -def displayTestName(testName): - msg = "{}Testing: {}{}".format(_Colors.NAME, testName, _Colors.ENDC) - print(msg) - return msg - -def displayUpdate(fileName): - msg = "{}Updated: {}{}".format(_Colors.WARNING, os.path.basename(fileName), _Colors.ENDC) - print(msg) - return msg - -def displayRemoved(fileName): - msg = "{}Removed: {}{}".format(_Colors.WARNING, os.path.basename(fileName), _Colors.ENDC) - print(msg) - return msg - -def displayAdded(fileName): - msg = "{}Added: {}{}".format(_Colors.WARNING, os.path.basename(fileName), _Colors.ENDC) - print(msg) - return msg - -def displayCustom(message): - print(message) - return message - -def displayError(message): - msg = "{}{} {}{}".format(_Colors.WARNING, _Smileys.CONFUSED, message, _Colors.ENDC) - print(msg) - return msg - -def _selectColorAndSmiley(testResult): - if testResult.hasPassed: - return _Colors.PASS, _Smileys.HAPPY - if type(testResult.message) is excep.SourceException: - return _Colors.WARNING, _Smileys.CONFUSED - return _Colors.FAIL, _Smileys.SAD diff --git a/checkpy/printer/__init__.py b/checkpy/printer/__init__.py new file mode 100644 index 0000000..b5b8eef --- /dev/null +++ b/checkpy/printer/__init__.py @@ -0,0 +1 @@ +from checkpy.printer.printer import * diff --git a/checkpy/printer/printer.py b/checkpy/printer/printer.py new file mode 100644 index 0000000..a740497 --- /dev/null +++ b/checkpy/printer/printer.py @@ -0,0 +1,90 @@ +import os +import traceback +import typing + +import colorama +colorama.init() + +import checkpy +from checkpy.entities import exception +import checkpy.tests + +class _Colors: + PASS = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + NAME = '\033[96m' + ENDC = '\033[0m' + +class _Smileys: + HAPPY = ":)" + SAD = ":(" + CONFUSED = ":S" + NEUTRAL = ":|" + +def display(testResult: checkpy.tests.TestResult) -> str: + color, smiley = _selectColorAndSmiley(testResult) + msg = "{}{} {}{}".format(color, smiley, testResult.description, _Colors.ENDC) + if testResult.message: + msg += "\n " + "\n ".join(testResult.message.split("\n")) + + if checkpy.context.debug and testResult.exception: + exc = testResult.exception + if hasattr(exc, "stacktrace"): + stack = str(exc.stacktrace()) + else: + stack = "".join(traceback.format_tb(testResult.exception.__traceback__)) + msg += "\n" + stack + if not checkpy.context.silent: + print(msg) + return msg + +def displayTestName(testName: str) -> str: + msg = "{}Testing: {}{}".format(_Colors.NAME, testName, _Colors.ENDC) + if not checkpy.context.silent: + print(msg) + return msg + +def displayUpdate(fileName: str) -> str: + msg = "{}Updated: {}{}".format(_Colors.WARNING, os.path.basename(fileName), _Colors.ENDC) + if not checkpy.context.silent: + print(msg) + return msg + +def displayRemoved(fileName: str) -> str: + msg = "{}Removed: {}{}".format(_Colors.WARNING, os.path.basename(fileName), _Colors.ENDC) + if not checkpy.context.silent: + print(msg) + return msg + +def displayAdded(fileName: str) -> str: + msg = "{}Added: {}{}".format(_Colors.WARNING, os.path.basename(fileName), _Colors.ENDC) + if not checkpy.context.silent: + print(msg) + return msg + +def displayCustom(message: str) -> str: + if not checkpy.context.silent: + print(message) + return message + +def displayWarning(message: str) -> str: + msg = "{}Warning: {}{}".format(_Colors.WARNING, message, _Colors.ENDC) + if not checkpy.context.silent: + print(msg) + return msg + +def displayError(message: str) -> str: + msg = "{}{} {}{}".format(_Colors.WARNING, _Smileys.CONFUSED, message, _Colors.ENDC) + if not checkpy.context.silent: + print(msg) + return msg + +def _selectColorAndSmiley(testResult: checkpy.tests.TestResult) -> typing.Tuple[str, str]: + if testResult.hasPassed: + return _Colors.PASS, _Smileys.HAPPY + if type(testResult.message) is exception.SourceException: + return _Colors.WARNING, _Smileys.CONFUSED + if testResult.hasPassed is None: + return _Colors.WARNING, _Smileys.NEUTRAL + return _Colors.FAIL, _Smileys.SAD diff --git a/checkpy/tester.py b/checkpy/tester.py deleted file mode 100644 index 3de1d54..0000000 --- a/checkpy/tester.py +++ /dev/null @@ -1,185 +0,0 @@ -import printer -import caches -import os -import sys -import importlib -import re -import multiprocessing -import time -import dill - -HERE = os.path.abspath(os.path.dirname(__file__)) - -def test(testName, module = ""): - fileName = _getFileName(testName) - filePath = _getFilePath(testName) - if filePath not in sys.path: - sys.path.append(filePath) - - testFileName = fileName[:-3] + "Test.py" - testFilePath = _getTestDirPath(testFileName, module = module) - if testFilePath is None: - printer.displayError("No test found for {}".format(fileName)) - return - - if testFilePath not in sys.path: - sys.path.append(testFilePath) - - return _runTests(importlib.import_module(testFileName[:-3]), os.path.join(filePath, fileName)) - -def testModule(module): - testNames = _getTestNames(module) - - if not testNames: - printer.displayError("no tests found in module: {}".format(module)) - return - - return [test(testName, module = module) for testName in testNames] - - -def _getTestNames(moduleName): - moduleName = _backslashToForwardslash(moduleName) - for (dirPath, dirNames, fileNames) in os.walk(os.path.join(HERE, "tests")): - dirPath = _backslashToForwardslash(dirPath) - if moduleName in dirPath: - return [fileName[:-7] for fileName in fileNames if fileName.endswith(".py") and not fileName.startswith("_")] - -def _getTestDirPath(testFileName, module = ""): - module = _backslashToForwardslash(module) - testFileName = _backslashToForwardslash(testFileName) - for (dirPath, dirNames, fileNames) in os.walk(os.path.join(HERE, "tests")): - if module in _backslashToForwardslash(dirPath) and testFileName in fileNames: - return dirPath - -def _getFileName(completeFilePath): - fileName = os.path.basename(completeFilePath) - if not fileName.endswith(".py"): - fileName += ".py" - return fileName - -def _getFilePath(completeFilePath): - filePath = os.path.dirname(completeFilePath) - if not filePath: - filePath = os.path.dirname(os.path.abspath(_getFileName(completeFilePath))) - return filePath - -def _backslashToForwardslash(text): - return re.sub("\\\\", "/", text) - -def _runTests(module, fileName): - signalQueue = multiprocessing.Queue() - resultQueue = multiprocessing.Queue() - tester = _Tester(module, fileName, signalQueue, resultQueue) - p = multiprocessing.Process(target=tester.run, name="Tester") - p.start() - - start = time.time() - isTiming = False - - while p.is_alive(): - while not signalQueue.empty(): - signal = signalQueue.get() - isTiming = signal.isTiming - description = signal.description - timeout = signal.timeout - if signal.resetTimer: - start = time.time() - - if isTiming and time.time() - start > timeout: - result = TesterResult() - result.addOutput(printer.displayError("Timeout ({} seconds) reached during: {}".format(timeout, description))) - p.terminate() - p.join() - return result - - time.sleep(0.1) - - if not resultQueue.empty(): - return resultQueue.get() - -class TesterResult(object): - def __init__(self): - self.nTests = 0 - self.nPassedTests = 0 - self.nFailedTests = 0 - self.nRunTests = 0 - self.output = [] - - def addOutput(self, output): - self.output.append(output) - -class _Signal(object): - def __init__(self, isTiming = False, resetTimer = False, description = None, timeout = None): - self.isTiming = isTiming - self.resetTimer = resetTimer - self.description = description - self.timeout = timeout - -class _Tester(object): - def __init__(self, module, fileName, signalQueue, resultQueue): - self.module = module - self.fileName = fileName - self.signalQueue = signalQueue - self.resultQueue = resultQueue - - def run(self): - result = TesterResult() - - self.module._fileName = self.fileName - self._sendSignal(_Signal(isTiming = False)) - - result.addOutput(printer.displayTestName(os.path.basename(self.module._fileName))) - - if hasattr(self.module, "before"): - try: - self.module.before() - except Exception as e: - result.addOutput(printer.displayError("Something went wrong at setup:\n{}".format(e))) - return - - reservedNames = ["before", "after"] - testCreators = [method for method in self.module.__dict__.values() if callable(method) and method.__name__ not in reservedNames] - - result.nTests = len(testCreators) - - testResults = self._runTests(testCreators) - - result.nRunTests = len(testResults) - result.nPassedTests = len([tr for tr in testResults if tr.hasPassed]) - result.nFailedTests = len([tr for tr in testResults if not tr.hasPassed]) - - for testResult in testResults: - result.addOutput(printer.display(testResult)) - - if hasattr(self.module, "after"): - try: - self.module.after() - except Exception as e: - result.addOutput(printer.displayError("Something went wrong at closing:\n{}".format(e))) - - self._sendResult(result) - - def _runTests(self, testCreators): - cachedResults = {} - - # run tests in noncolliding execution order - for test in self._getTestsInExecutionOrder([tc() for tc in testCreators]): - self._sendSignal(_Signal(isTiming = True, resetTimer = True, description = test.description(), timeout = test.timeout())) - cachedResults[test] = test.run() - self._sendSignal(_Signal(isTiming = False)) - - # return test results in specified order - return [cachedResults[test] for test in sorted(cachedResults.keys()) if cachedResults[test] != None] - - def _sendResult(self, result): - self.resultQueue.put(result) - - def _sendSignal(self, signal): - self.signalQueue.put(signal) - - def _getTestsInExecutionOrder(self, tests): - testsInExecutionOrder = [] - for i, test in enumerate(tests): - dependencies = self._getTestsInExecutionOrder([tc() for tc in test.dependencies()]) + [test] - testsInExecutionOrder.extend([t for t in dependencies if t not in testsInExecutionOrder]) - return testsInExecutionOrder \ No newline at end of file diff --git a/checkpy/tester/__init__.py b/checkpy/tester/__init__.py new file mode 100644 index 0000000..2e8fc28 --- /dev/null +++ b/checkpy/tester/__init__.py @@ -0,0 +1,3 @@ +from checkpy.tester.tester import * + +__all__ = ["test", "testModule", "getActiveTest", "only", "include", "exclude", "require"] \ No newline at end of file diff --git a/checkpy/tester/discovery.py b/checkpy/tester/discovery.py new file mode 100644 index 0000000..2c89740 --- /dev/null +++ b/checkpy/tester/discovery.py @@ -0,0 +1,48 @@ +import os +import checkpy.database as database +import pathlib +from typing import Optional, List, Union + +def testExists(testName: str, module: str="") -> bool: + testFileName = testName.split(".")[0] + "Test.py" + testPaths = getTestPaths(testFileName, module=module) + return len(testPaths) > 0 + +def getPath(path: Union[str, pathlib.Path]) -> Optional[pathlib.Path]: + filePath = os.path.dirname(path) + if not filePath: + filePath = os.path.dirname(os.path.abspath(path)) + + fileName = os.path.basename(path) + + if "." in fileName: + path = pathlib.Path(os.path.join(filePath, fileName)) + return path if path.exists() else None + + for extension in [".py", ".ipynb"]: + path = pathlib.Path(os.path.join(filePath, fileName + extension)) + if path.exists(): + return path + + return None + +def getTestNames(moduleName: str) -> Optional[List[str]]: + for testsPath in database.forEachTestsPath(): + for (dirPath, subdirs, files) in os.walk(testsPath): + if moduleName in dirPath: + return [f[:-len("test.py")] for f in files if f.lower().endswith("test.py")] + return None + +def getTestPaths(testFileName: str, module: str="") -> List[pathlib.Path]: + testFilePaths: List[pathlib.Path] = [] + for testPath in database.forEachTestsPath(): + testFilePaths.extend(getTestPathsFrom(testFileName, testPath, module=module)) + return testFilePaths + +def getTestPathsFrom(testFileName: str, path: pathlib.Path, module: str="") -> List[pathlib.Path]: + """Get all testPaths from a tests folder (path).""" + testFilePaths: List[pathlib.Path] = [] + for (dirPath, _, fileNames) in os.walk(path): + if testFileName in fileNames and (not module or module in dirPath): + testFilePaths.append(pathlib.Path(dirPath)) + return testFilePaths \ No newline at end of file diff --git a/checkpy/tester/tester.py b/checkpy/tester/tester.py new file mode 100644 index 0000000..f42cdcd --- /dev/null +++ b/checkpy/tester/tester.py @@ -0,0 +1,391 @@ +import checkpy +from checkpy import printer +from checkpy.entities import exception +from checkpy.tester import discovery +from checkpy.lib.sandbox import sandbox +from checkpy.lib.explanation import explainCompare +from checkpy.tests import Test, TestResult, TestFunction +import checkpy.lib.io + +from types import ModuleType +from typing import Dict, Iterable, List, Optional, Union + +import copy +import contextlib +import os +import pathlib +import queue +import subprocess +import sys +import importlib +import time +import warnings + +import dessert +import multiprocessing as mp + + +__all__ = ["getActiveTest", "test", "testModule", "TesterResult", "runTests", "runTestsSynchronously"] + + +_activeTest: Optional[Test] = None + + +def getActiveTest() -> Optional[Test]: + return _activeTest + + +def test( + testName: str, + module="", + debugMode: Union[bool, None]=None, + silentMode: Union[bool, None]=None + ) -> "TesterResult": + if debugMode is not None: + checkpy.context.debug = debugMode + + if silentMode is not None: + checkpy.context.silent = silentMode + + result = TesterResult(testName) + + discoveredPath = discovery.getPath(testName) + if discoveredPath is None: + result.addOutput(printer.displayError("File not found: {}".format(testName))) + return result + path = str(discoveredPath) + + fileName = os.path.basename(path) + filePath = os.path.dirname(path) + + testFileName = fileName.split(".")[0] + "Test.py" + testPaths = discovery.getTestPaths(testFileName, module=module) + + if not testPaths: + result.addOutput(printer.displayError("No test found for {}".format(fileName))) + return result + + if len(testPaths) > 1: + result.addOutput(printer.displayWarning("Found {} tests: {}, using: {}".format(len(testPaths), testPaths, testPaths[0]))) + + testPath = testPaths[0] + + if path.endswith(".ipynb"): + if subprocess.call(['jupyter', 'nbconvert', '--to', 'script', path]) != 0: + result.addOutput(printer.displayError("Failed to convert Jupyter notebook to .py")) + return result + + path = path.replace(".ipynb", ".py") + + # remove all magic lines from notebook + with open(path, "r") as f: + lines = f.readlines() + with open(path, "w") as f: + f.write("".join([l for l in lines if "get_ipython" not in l])) + + with _addToSysPath(filePath): + testerResult = runTests( + testFileName.split(".")[0], + testPath, + path + ) + + if path.endswith(".ipynb"): + os.remove(path) + + testerResult.output = result.output + testerResult.output + return testerResult + + +def testModule( + module: str, + debugMode: Union[bool, None]=None, + silentMode: Union[bool, None]=None + ) -> Optional[List["TesterResult"]]: + if debugMode is not None: + checkpy.context.debug = debugMode + + if silentMode is not None: + checkpy.context.silent = silentMode + + testNames = discovery.getTestNames(module) + + if not testNames: + printer.displayError("no tests found in module: {}".format(module)) + return None + + return [test(testName, module=module) for testName in testNames] + +def runTests(moduleName: str, testPath: pathlib.Path, fileName: str) -> "TesterResult": + result: Optional[TesterResult] = None + + with _addToSysPath(testPath): + ctx = mp.get_context("spawn") + + signalQueue: "mp.Queue[_Signal]" = ctx.Queue() + resultQueue: "mp.Queue[TesterResult]" = ctx.Queue() + tester = _Tester(moduleName, testPath, pathlib.Path(fileName), signalQueue, resultQueue) + p = ctx.Process(target=tester.run, name="Tester") + p.start() + + start = time.time() + isTiming = False + + while p.is_alive(): + while not signalQueue.empty(): + signal = signalQueue.get() + + if signal.description is not None: + description = signal.description + if signal.isTiming is not None: + isTiming = signal.isTiming + if signal.timeout is not None: + timeout = signal.timeout + if signal.resetTimer: + start = time.time() + + if isTiming and time.time() - start > timeout: + result = TesterResult(pathlib.Path(fileName).name) + result.addOutput(printer.displayError("Timeout ({} seconds) reached during: {}".format(timeout, description))) + p.terminate() + p.join() + return result + + if not resultQueue.empty(): + # .get before .join to prevent hanging indefinitely due to a full pipe + # https://bugs.python.org/issue8426 + result = resultQueue.get() + p.terminate() + p.join() + break + + time.sleep(0.1) + + if not resultQueue.empty(): + result = resultQueue.get() + + if result is None: + raise exception.CheckpyError(message="An error occured while testing. The testing process exited unexpectedly.") + + return result + + +def runTestsSynchronously(moduleName: str, testPath: pathlib.Path, fileName: str) -> "TesterResult": + signalQueue = queue.Queue() + resultQueue = queue.Queue() + + tester = _Tester( + moduleName=moduleName, + testPath=testPath, + filePath=pathlib.Path(fileName), + signalQueue=signalQueue, + resultQueue=resultQueue + ) + + with _addToSysPath(testPath): + try: + old_context = copy.copy(checkpy.context) + tester.run() + finally: + checkpy.context = old_context + + return resultQueue.get() + + +class TesterResult: + def __init__(self, name: str): + self.name = name + self.nTests = 0 + self.nPassedTests = 0 + self.nFailedTests = 0 + self.nRunTests = 0 + self.output: List[str] = [] + self.testResults: List[TestResult] = [] + + def addOutput(self, output: str): + self.output.append(output) + + def addResult(self, testResult: TestResult): + self.testResults.append(testResult) + + def asDict(self) -> Dict[str, Union[str, int, List]]: + return { + "name": self.name, + "nTests": self.nTests, + "nPassed": self.nPassedTests, + "nFailed": self.nFailedTests, + "nRun": self.nRunTests, + "output": self.output, + "results": [tr.asDict() for tr in self.testResults] + } + + +class _Signal: + def __init__( + self, + isTiming: Optional[bool]=None, + resetTimer: Optional[bool]=None, + description: Optional[str]=None, + timeout: Optional[int]=None + ): + self.isTiming = isTiming + self.resetTimer = resetTimer + self.description = description + self.timeout = timeout + + +class _Tester: + def __init__( + self, + moduleName: str, + testPath: pathlib.Path, + filePath: pathlib.Path, + signalQueue: "mp.Queue[_Signal]", + resultQueue: "mp.Queue[TesterResult]" + ): + self.moduleName = moduleName + self.testPath = testPath + self.filePath = filePath.absolute() + self.signalQueue = signalQueue + self.resultQueue = resultQueue + self._context = checkpy.context + + def run(self): + checkpy.context = self._context + + warnings.filterwarnings("ignore") + if checkpy.context.debug: + warnings.simplefilter('always', DeprecationWarning) + + checkpy.file = self.filePath + checkpy.testPath = self.testPath + + # overwrite argv so that it seems the file was run directly + sys.argv = [self.filePath.name] + + # have pytest (dessert) rewrite the asserts in the AST + with dessert.rewrite_assertions_context(): + + # TODO: should be a cleaner way to inject "pytest_assertrepr_compare" + dessert.util._reprcompare = explainCompare + + with sandbox(): + try: + module = importlib.import_module(self.moduleName) + except exception.MissingRequiredFiles as e: + result = TesterResult(self.filePath.name) + result.addOutput(printer.displayError(e)) + self._sendResult(result) + return + module._fileName = self.filePath.name # type: ignore [attr-defined] + + self._runTestsFromModule(module) + + def _runTestsFromModule(self, module: ModuleType): + self._sendSignal(_Signal(isTiming=False)) + + result = TesterResult(self.filePath.name) + result.addOutput(printer.displayTestName(self.filePath.name)) + + if hasattr(module, "before"): + try: + module.before() + except Exception as e: + result.addOutput(printer.displayError("Something went wrong at setup:\n{}".format(e))) + return + + testFunctions = [method for method in module.__dict__.values() if getattr(method, "isTestFunction", False)] + result.nTests = len(testFunctions) + + testResults = self._runTests(testFunctions) + + result.nRunTests = len(testResults) + result.nPassedTests = len([tr for tr in testResults if tr.hasPassed]) + result.nFailedTests = len([tr for tr in testResults if not tr.hasPassed]) + + for testResult in testResults: + result.addResult(testResult) + result.addOutput(printer.display(testResult)) + + if hasattr(module, "after"): + try: + module.after() + except Exception as e: + result.addOutput(printer.displayError("Something went wrong at closing:\n{}".format(e))) + + self._sendResult(result) + + def _runTests(self, testFunctions: Iterable[TestFunction]) -> List[TestResult]: + cachedResults: Dict[Test, Optional[TestResult]] = {} + + def handleDescriptionChange(test: Test): + self._sendSignal(_Signal( + description=test.description + )) + + def handleTimeoutChange(test: Test): + self._sendSignal(_Signal( + isTiming=True, + resetTimer=True, + timeout=test.timeout + )) + + global _activeTest + + # run tests in non-colliding execution order + for testFunction in self._getTestFunctionsInExecutionOrder(testFunctions): + test = Test( + self.filePath.name, + testFunction.priority, + timeout=testFunction.timeout, + onDescriptionChange=handleDescriptionChange, + onTimeoutChange=handleTimeoutChange + ) + + _activeTest = test + + run = testFunction(test) + + self._sendSignal(_Signal( + isTiming=True, + resetTimer=True, + description=test.description, + timeout=test.timeout + )) + + with checkpy.lib.io.replaceStdout() as stdout, checkpy.lib.io.replaceStdin() as stdin: + cachedResults[test] = run() + + _activeTest = None + + self._sendSignal(_Signal(isTiming=False)) + + # return test results in specified order + sortedResults = [cachedResults[test] for test in sorted(cachedResults)] + return [result for result in sortedResults if result is not None] + + def _sendResult(self, result: TesterResult): + self.resultQueue.put(result) + + def _sendSignal(self, signal: _Signal): + self.signalQueue.put(signal) + + def _getTestFunctionsInExecutionOrder(self, testFunctions: Iterable[TestFunction]) -> List[TestFunction]: + sortedTFs: List[TestFunction] = [] + for tf in testFunctions: + dependencies = self._getTestFunctionsInExecutionOrder(tf.dependencies) + [tf] + sortedTFs.extend([t for t in dependencies if t not in sortedTFs]) + return sortedTFs + +@contextlib.contextmanager +def _addToSysPath(path: str): + addedToPath = False + path = str(path) + try: + if path not in sys.path: + addedToPath = True + sys.path.append(path) + yield + finally: + if addedToPath and path in sys.path: + sys.path.remove(path) \ No newline at end of file diff --git a/checkpy/tests.py b/checkpy/tests.py index d9aa81f..61c0b28 100644 --- a/checkpy/tests.py +++ b/checkpy/tests.py @@ -1,111 +1,409 @@ -import caches +import inspect +import traceback -class Test(object): - def __init__(self, priority): - self._priority = priority +from typing import Any, Dict, Set, Tuple, Union, Callable, Iterable, Optional - def __lt__(self, other): - return self._priority < other._priority +from checkpy import caches +from checkpy.entities import exception +from checkpy.lib.sandbox import sandbox +from checkpy.lib.explanation import simplifyAssertionMessage - @caches.cache() - def run(self): - try: - result = self.test() - if type(result) == tuple: - hasPassed, info = result - else: - hasPassed, info = result, "" - except Exception as e: - return TestResult(False, self.description(), self.exception(e)) +__all__ = ["test", "failed", "passed"] - return TestResult(hasPassed, self.description(), self.success(info) if hasPassed else self.fail(info)) - @staticmethod - def test(): - raise NotImplementedError() +def test( + priority: Optional[int]=None, + timeout: Optional[int]=None + ) -> Callable[[Callable], "TestFunction"]: + def testDecorator(testFunction: Callable) -> TestFunction: + return TestFunction(testFunction, priority=priority, timeout=timeout) + return testDecorator - @staticmethod - def description(): - raise NotImplementedError() - @staticmethod - def success(info): - return "" +def failed( + *preconditions: "TestFunction", + priority: Optional[int]=None, + timeout: Optional[int]=None, + hide: bool=True + ) -> Callable[[Callable], "FailedTestFunction"]: + def failedDecorator(testFunction: Callable) -> FailedTestFunction: + return FailedTestFunction(testFunction, preconditions, priority=priority, timeout=timeout, hide=hide) + return failedDecorator - @staticmethod - def fail(info): - return info - @staticmethod - def exception(exception): - return exception +def passed( + *preconditions: "TestFunction", + priority: Optional[int]=None, + timeout: Optional[int]=None, + hide: bool=True + ) -> Callable[[Callable], "PassedTestFunction"]: + def passedDecorator(testFunction: Callable) -> PassedTestFunction: + return PassedTestFunction(testFunction, preconditions, priority=priority, timeout=timeout, hide=hide) + return passedDecorator - @staticmethod - def dependencies(): - return set() - @staticmethod - def timeout(): - return 10 +class Test: + DEFAULT_TIMEOUT = 10 + PLACEHOLDER_DESCRIPTION = "placeholder test description" + + def __init__(self, + fileName: str, + priority: int, + timeout: Optional[int]=None, + onDescriptionChange: Callable[["Test"], None]=lambda self: None, + onTimeoutChange: Callable[["Test"], None]=lambda self: None + ): + self._fileName = fileName + self._priority = priority + + self._onDescriptionChange = onDescriptionChange + self._onTimeoutChange = onTimeoutChange + + self._description = Test.PLACEHOLDER_DESCRIPTION + self._timeout = Test.DEFAULT_TIMEOUT if timeout is None else timeout + + self._output: list[str] = [] + + def __lt__(self, other): + return self._priority < other._priority + + @property + def fileName(self) -> str: + return self._fileName + + @staticmethod + def test() -> Union[bool, Tuple[bool, str]]: + raise NotImplementedError() + + @staticmethod + def success(info: str) -> str: + return "" + + @staticmethod + def fail(info: str) -> str: + return info + + @staticmethod + def exception(exception: Exception) -> Exception: + return exception + + @staticmethod + def dependencies() -> Set["TestFunction"]: + return set() + + @property + def description(self) -> str: + return self._description + + @description.setter + def description(self, newDescription: Union[str, Callable[[], str]]): + if callable(newDescription): + self._description = newDescription() + else: + self._description = newDescription + + self._onDescriptionChange(self) + + @property + def timeout(self) -> int: + return self._timeout + + @timeout.setter + def timeout(self, new_timeout: Union[int, Callable[[], int]]): + if callable(new_timeout): + self._timeout = new_timeout() + else: + self._timeout = new_timeout + + self._onTimeoutChange(self) + + @property + def output(self) -> str: + from checkpy import context # avoid circular import + outputLimit = context.outputLimit + + output = "\n".join(self._output) + + return self._formatOutput(output, outputLimit) + + @staticmethod + def _formatOutput(text: str, maxChars: int) -> str: + if maxChars <= 0 or len(text) < maxChars: + return text + + lines = text.split('\n') + # return str(lines) + firstPart = [] + lastPart = [] + + # Collect the first part of the text + totalChars = 0 + for line in lines: + # Accept up to maxChars // 2 for first part + if totalChars + len(line) > maxChars // 2: + if all(l == "" or l.isspace() for l in firstPart): + # If the first part is empty, show up to maxChars // 2 + firstPart.append(line[:maxChars // 2] + "<<< output truncated >>>") + + break + firstPart.append(line) + + totalChars += len(line) + 1 # +1 for the newline character + + # Collect the last part of the text + totalChars = 0 + for line in reversed(lines[len(firstPart):]): + # Accept up to maxChars // 2 for first part + if totalChars + len(line) > maxChars // 2: + if all(l == "" or l.isspace() for l in lastPart): + # If the last part is empty, show up to maxChars // 2 + lastPart.insert(0, "<<< output truncated >>>" + line[-(maxChars // 2):]) + break + lastPart.insert(0, line) + + totalChars += len(line) + 1 # +1 for the newline character + + # Combine the parts with the omitted message + nLinesOmitted = len(lines) - len(firstPart) - len(lastPart) + + if nLinesOmitted > 0: + sep = f"<<< {nLinesOmitted} lines omitted >>>" + result = '\n'.join(('\n'.join(firstPart), sep, '\n'.join(lastPart))) + else: + result = '\n'.join(('\n'.join(firstPart), '\n'.join(lastPart))) + + return result + + def addOutput(self, output: str) -> None: + self._output.append(output) + + def __setattr__(self, __name: str, __value: Any) -> None: + value = __value + if __name in ["fail", "success", "exception"]: + if not callable(__value): + value = lambda *args, **kwargs: __value + super().__setattr__(__name, value) class TestResult(object): - def __init__(self, hasPassed, description, message): - self._hasPassed = hasPassed - self._description = description - self._message = message - - @property - def description(self): - return self._description - - @property - def message(self): - return self._message - - @property - def hasPassed(self): - return self._hasPassed - -def test(priority): - def testDecorator(testCreator): - @caches.cache(testCreator) - def testWrapper(): - t = Test(priority) - testCreator(t) - return t - return testWrapper - return testDecorator - - -def failed(*precondTestCreators): - def failedDecorator(testCreator): - def testWrapper(): - test = testCreator() - dependencies = test.dependencies - test.dependencies = lambda : dependencies() | set(precondTestCreators) - run = test.run - def runMethod(): - testResults = [t().run() for t in precondTestCreators] - return run() if not any(t is None for t in testResults) and not any(t.hasPassed for t in testResults) else None - test.run = runMethod - return test - return testWrapper - return failedDecorator - - -def passed(*precondTestCreators): - def passedDecorator(testCreator): - def testWrapper(): - test = testCreator() - dependencies = test.dependencies - test.dependencies = lambda : dependencies() | set(precondTestCreators) - run = test.run - def runMethod(): - testResults = [t().run() for t in precondTestCreators] - return run() if not any(t is None for t in testResults) and all(t.hasPassed for t in testResults) else None - test.run = runMethod - return test - return testWrapper - return passedDecorator + def __init__( + self, + hasPassed: Optional[bool], + description: str, + message: str, + output: str, + exception: Optional[Exception]=None + ): + self._hasPassed = hasPassed + self._description = description + self._message = message + self._exception = exception + self._output = output + + @property + def description(self): + return self._description + + @property + def message(self): + return self._message + + @property + def hasPassed(self): + return self._hasPassed + + @property + def output(self): + return self._output + + @property + def exception(self): + return self._exception + + def asDict(self) -> Dict[str, Union[bool, None, str]]: + return { + "passed": self.hasPassed, + "description": str(self.description), + "message": str(self.message), + "exception": str(self.exception), + "output": str(self.output) + } + + +class TestFunction: + _previousPriority = -1 + + def __init__( + self, + function: Callable, + priority: Optional[int]=None, + timeout: Optional[int]=None + ): + self._function = function + self.isTestFunction = True + self.priority = self._getPriority(priority) + self.dependencies: Set[TestFunction] = getattr(self._function, "dependencies", set()) + self.timeout = self._getTimeout(timeout) + self.__name__ = function.__name__ + + def __call__(self, test: Test) -> Callable[[], Union["TestResult", None]]: + self.useDocStringDescription(test) + + @caches.cacheTestResult(self) + def runMethod(): + with sandbox(): + try: + if getattr(self._function, "isTestFunction", False): + self._function(test)() + elif (len(inspect.getfullargspec(self._function).args) > + (1 if inspect.ismethod(self._function) else 0)): + self._function(test) + else: + self._function() + + # support for old-style tests + hasPassed, info = True, "" + if test.test != Test.test: + result = test.test() + + if type(result) == tuple: + hasPassed, info = result + else: + hasPassed = result + except AssertionError as e: + # last = traceback.extract_tb(e.__traceback__)[-1] + # print(last, dir(last), last.line, last.lineno) + + assertMsg = simplifyAssertionMessage(str(e)) + failMsg = test.fail("") + if failMsg and not failMsg.endswith("\n"): + failMsg += "\n" + msg = failMsg + assertMsg + + return TestResult(False, test.description, msg, test.output) + except exception.CheckpyError as e: + return TestResult(False, test.description, str(test.exception(e)), test.output, exception=e) + except Exception as e: + e = exception.TestError( + exception = e, + message = "while testing", + stacktrace = traceback.format_exc()) + return TestResult(False, test.description, str(test.exception(e)), test.output, exception=e) + + # Ensure hasPassed is None or a boolean + # This is needed as boolean operators on np.bool_ return np.bool_ + hasPassed = hasPassed if hasPassed is None else bool(hasPassed) + + return TestResult(hasPassed, test.description, test.success(info) if hasPassed else test.fail(info), test.output) + + return runMethod + + def useDocStringDescription(self, test: Test) -> None: + if getattr(self._function, "isTestFunction", False): + self._function.useDocStringDescription(test) # type: ignore [attr-defined] + + if self._function.__doc__ is not None: + test.description = self._function.__doc__ + + def _getPriority(self, priority: Optional[int]) -> int: + if priority is not None: + TestFunction._previousPriority = priority + return priority + + if getattr(self._function, "isTestFunction", False): + inheritedPriority = getattr(self._function, "priority", None) + if inheritedPriority: + TestFunction._previousPriority = inheritedPriority + return inheritedPriority + + TestFunction._previousPriority += 1 + return TestFunction._previousPriority + + def _getTimeout(self, timeout: Optional[int]) -> int: + if timeout is not None: + return timeout + + if getattr(self._function, "isTestFunction", False): + inheritedTimeout = getattr(self._function, "timeout", None) + if inheritedTimeout: + return inheritedTimeout + + return Test.DEFAULT_TIMEOUT + + +class FailedTestFunction(TestFunction): + HIDE_MESSAGE = "can't check until another check fails" + + def __init__( + self, + function: Callable, + preconditions: Iterable[TestFunction], + priority: Optional[int]=None, + timeout: Optional[int]=None, + hide: Optional[bool]=None + ): + super().__init__(function=function, priority=priority, timeout=timeout) + + for precond in preconditions: + if not isinstance(precond, TestFunction): + raise exception.CheckpyError( + f"{precond} is not a checkpy test and cannot be used as a dependency for test {function}." + f" Did you forget to use the @test() decorator for {precond}?" + ) + + self.preconditions = preconditions + self.shouldHide = self._getHide(hide) + + def __call__(self, test: Test) -> Callable[[], Union["TestResult", None]]: + self.useDocStringDescription(test) + + @caches.cacheTestResult(self) + def runMethod(): + if getattr(self._function, "isTestFunction", False): + run = self._function(test) + else: + run = TestFunction.__call__(self, test) + + self.requireDocstringIfNotHidden(test) + + testResults = [caches.getCachedTestResult(t) for t in self.preconditions] + if self.shouldRun(testResults): + return run() + + if self.shouldHide: + return None + + return TestResult( + None, + test.description, + self.HIDE_MESSAGE, + test.output + ) + return runMethod + + def requireDocstringIfNotHidden(self, test: Test) -> None: + if not self.shouldHide and test.description == Test.PLACEHOLDER_DESCRIPTION: + raise exception.TestError(message=f"Test {self.__name__} requires a docstring description if hide=False") + + @staticmethod + def shouldRun(testResults: Iterable[TestResult]) -> bool: + return not any(t is None for t in testResults) and not any(t.hasPassed for t in testResults) + + def _getHide(self, hide: Optional[bool]) -> bool: + if hide is not None: + return hide + + inheritedHide = getattr(self._function, "hide", None) + if inheritedHide: + return inheritedHide + + return True + + +class PassedTestFunction(FailedTestFunction): + HIDE_MESSAGE = "can't check until another check passes" + + @staticmethod + def shouldRun(testResults: Iterable[TestResult]) -> bool: + return not any(t is None for t in testResults) and all(t.hasPassed for t in testResults) diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..5d774cf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +SPHINXPROJ = CheckPy +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) \ No newline at end of file diff --git a/docs/_static/fifteen.png b/docs/_static/fifteen.png new file mode 100644 index 0000000..613b147 Binary files /dev/null and b/docs/_static/fifteen.png differ diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..3ae3379 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# CheckPy documentation build configuration file, created by +# sphinx-quickstart on Wed Jan 31 14:12:52 2018. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'CheckPy' +copyright = '2018, Jelle van Assema (JelleAs)' +author = 'Jelle van Assema (JelleAs)' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.4' +# The full version, including alpha/beta/rc tags. +release = '0.4' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'alabaster' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# This is required for the alabaster theme +# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars +html_sidebars = { + '**': [ + 'relations.html', # needs 'show_related': True theme option to display + 'searchbox.html', + ] +} + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'CheckPydoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'CheckPy.tex', 'CheckPy Documentation', + 'Jelle van Assema (JelleAs)', 'manual'), +] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'checkpy', 'CheckPy Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'CheckPy', 'CheckPy Documentation', + author, 'CheckPy', 'One line description of project.', + 'Miscellaneous'), +] + + + diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..0572a76 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,59 @@ +Welcome to CheckPy! +=================================== +An education oriented testing framework for Python. +Developed for courses in the +`Minor Programming `__ at the `University of Amsterdam `__. + +.. image:: _static/fifteen.png + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + intro + + +Installation +------------ + +:: + + pip install checkpy + + +Features +-------- + + - Customizable output, you choose what the users see. + - Support for blackbox and whitebox testing. + - The full scope of Python is available when designing tests. + - Automatic test distribution, CheckPy will keep its tests up to date by periodically checking for new tests. + - No infinite loops! CheckPy kills tests after a predefined timeout. + + +Usage +----- + +:: + + usage: checkpy [-h] [-module MODULE] [-download GITHUBLINK] + [-register LOCALLINK] [-update] [-list] [-clean] [--dev] + [file] + + checkPy: a python testing framework for education. You are running Python + version 3.6.2 and checkpy version 0.4.0. + + positional arguments: + file name of file to be tested + + optional arguments: + -h, --help show this help message and exit + -module MODULE provide a module name or path to run all tests from + the module, or target a module for a specific test + -download GITHUBLINK download tests from a Github repository and exit + -register LOCALLINK register a local folder that contains tests and exit + -update update all downloaded tests and exit + -list list all download locations and exit + -clean remove all tests from the tests folder and exit + --dev get extra information to support the development of + tests diff --git a/docs/intro.rst b/docs/intro.rst new file mode 100644 index 0000000..c2b2f52 --- /dev/null +++ b/docs/intro.rst @@ -0,0 +1,289 @@ +Introduction to CheckPy +=================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +Installation +------------------- + +:: + + pip install checkpy + + +Writing and running your first test +----------------------------------- + +First create a new directory to store your tests somewhere on your computer. +Then navigate to that directory, and create a new file called ``helloTest.py``. +CheckPy discovers tests for a particular source file (i.e. ``hello.py``) by looking +for a test file starting with a corresponding name (``hello``) and ending with ``test.py``. +So CheckPy uses ``helloTest.py`` to test ``hello.py``. + +Now open up ``helloTest.py`` and insert the following code: + +.. code-block:: python + + import checkpy.tests as t + import checkpy.lib as lib + import checkpy.assertlib as asserts + + @t.test(0) + def exactlyHelloWorld(test): + def testMethod(): + output = lib.outputOf(test.fileName) + return asserts.exact(output.strip(), "Hello, world!") + + test.test = testMethod + test.description = lambda : "prints exactly: Hello, world!" + +Next, create a file called ``hello.py`` somewhere on your computer. Insert the +following snippet of code in ``hello.py``: + +.. code-block:: python + + print("Hello, world!") + +Now there's only one thing left to do. We need to tell CheckPy where the tests are +located. You can do this by calling CheckPy with the `-register` flag and by providing an +absolute path to the directory ``helloTest.py`` is located in. Say ``helloTest.py`` +is located in ``\Users\foo\bar\tests\`` then call CheckPy like so: + +:: + + checkpy -register \Users\foo\bar\tests\ + +Alright, we're there. We got a test (``helloTest.py``), a Python file we want to test (``hello.py``), +and we've told CheckPy where to look for tests. Now navigate over to the +directory that contains ``hello.py`` and call CheckPy as follows: + +:: + + checkpy hello + + +Writing simple tests in CheckPy +-------------------------------- + +Tests in CheckPy are instances of ``checkpy.tests.Test``. These ``Test`` instances have several +abstract methods that you can implement or rather, overwrite by binding a new method. +These methods are executed when CheckPy runs a test. For instance you have the +``description`` method which is called to produce a description for the test, the ``timeout`` +method which is called to determine the maximum alotted time for this test, and +ofcourse the ``test`` method which is called to actually perform the test. This +``Test`` instance is automatically provided to you when you decorate a function with +the ``checkpy.tests.test`` decorator. In our hello-world example this looked something like: + +.. code-block:: python + + @t.test(0) + def exactlyHelloWorld(test): + +Here the ``t.test`` decorator (``t`` is short for ``checkpy.tests``) decorates +the function ``exactlyHelloWorld``. This causes CheckPy to treat ``exactlyHelloWorld`` +as a `test creator` function. That when called produces an instance of ``Test``. +The ``t.test`` decorator accepts an argument that is used to determine the order +in which the result of the test is shown to the screen (lowest first). The decorator +then passes an instance of ``Test`` to the decorated function (``exactlyHelloWorld``). +It is up to ``exactlyHelloWorld`` to overwrite some or all abstract methods of that one +instance of ``Test`` that it receives. + +Lets start with the necessities. CheckPy requires you to overwrite two methods from every +``Test`` instance. These methods are ``test`` and ``description``. The ``description`` method +should produce the description, that can be just a string, for the user to see. In our +hello-world example we used this ``description`` method: + +.. code-block:: python + + test.description = lambda : "prints exactly: Hello, world!" + +Depending on whether the test fails or passes, the user sees this string in red or green +respectively. The other method we have to overwrite, the ``test`` method, should return +``True`` or ``False`` depending on whether the tests passes or fails. You are free to implement +this method in any which way you want. CheckPy just offers some useful tools to make +your testing life easier. Again, looking back at our hello-world example, we used this +``test`` method: + +.. code-block:: python + + def testMethod(): + output = lib.outputOf(test.fileName) + return asserts.exact(output.strip(), "Hello, world!") + + test.test = testMethod + +So what's going on here? Python doesn't support multi statement lambda functions. This means that +if you want to use multiple statements, you have to resort to named functions, i.e. +``testMethod()``, and then bind this named function to the respective method of the ``Test`` +instance. You can put the above test method in a single statement lambda function, but +readability will suffer from it. Especially once we move on to some more complex test methods. + +Now there are just 2 lines of code in this ``testMethod``. First we take the output of +something called ``test.fileName``. ``test.fileName`` just refers to the name of the file +the user wants to test, CheckPy will automatically set this for you. ``lib.outputOf`` is +a function that gives you all the print-output of a python file. Thus what this line of code +does is simply run the file that the user wants to test, and then return the print output as +a string. + +The line ``return asserts.exact(output.strip(), "Hello, world!")`` is equivalent to +``return output.strip() == "Hello, world!"``. The ``checkpy.assertlib`` module, that is renamed +to ``asserts`` here, simply offers a collection of functions to perform assertions. These +functions do nothing more than return ``True`` or ``False``. + +That's all there is to it. You simply write a function and decorate it with the `test` decorator. +Overwrite a couple of methods of ``Test``, and you're good to go. + +Testing functions +----------------- + +Let's make life a little more exciting. CheckPy can do a lot more besides simply running a Python file +and looking at print output. Specifically CheckPy lets you import said Python file +as a module and do all sort of things with it and to it. Lets focus on Functions for now. + +For an assignment on (biological) virus simulations, we asked students to do the following: + +Write a function ``generateVirus(length)``. +This function should accept one argument ``length``, that is an integer representing the length of the virus. +The function should return a virus, that is a string of random nucleotides (``'A'``, ``'T'``, ``'G'`` or ``'C'``). + +This is just a small part of a bigger assignment that ultimately moves towards a simulation +of viruses in a patient. We can use CheckPy to test several aspects of this assignment. +For instance to test whether only the nucleotides ATGC occurred we wrote the following: + +.. code-block:: python + + @t.test(10) + def onlyATGC(test): + def testMethod(): + generateVirus = lib.getFunction("generateVirus", test.fileName) + pairs = "".join(generateVirus(10) for _ in range(1000)) + return asserts.containsOnly(pairs, "AGTC") + + test.test = testMethod + test.description = lambda : "generateVirus produces viruses consisting only of A, T, G and C" + +To test whether the function actually exists and accepted just one argument, we wrote the following: + +.. code-block:: python + + @t.test(0) + def isDefined(test): + def testMethod(): + generateVirus = lib.getFunction("generateVirus", test.fileName) + return len(generateVirus.arguments) == 1 + + test.test = testMethod + test.description = lambda : "generateVirus is defined and accepts just one argument" + + +Testing programs with arguments +------------------------------- + +Taking a closer look at the ``checkpy.lib`` module we find three functions that +allow you to interact with dynamic components and results from the program we are +testing. All these functions (``outputOf``, ``getFunction`` and ``getModule``) take +in the same optional arguments that let you change the dynamic environment in which +the code is tested. Zooming in on ``outputOf``: + +.. code-block:: Python + + def outputOf( + fileName, + src = None, + argv = None, + stdinArgs = None, + ignoreExceptions = (), + overwriteAttributes = () + ): + """ + fileName is the file name you want the stdout output of + src can be used to ignore the source code of fileName, and instead use this string + argv is a collection of elements that are used to overwrite sys.argv + stdinArgs is a collection of arguments that are passed to stdin + ignoreExceptions is a collection of exceptions that should be ignored during execution + overwriteAttributes is a collection of tuples (attribute, value) that are overwritten + before trying to import the file + """ + +Lets see what we can do with this. +For an assignment we asked students to write a program that prints out how many +liters of water were used while showering. The program should prompt the user +for the number of minutes they shower, and then print out many liters of water +were used. We told them 1 minute of showering equaled 12 liters used. + +For this assignment we wrote the following test: + +.. code-block:: Python + + @t.test(10) + def oneLiter(test): + def testMethod(): + output = lib.outputOf( + test.fileName, + stdinArgs = [1], + overwriteAttributes = [("__name__", "__main__")] + ) + return asserts.contains(output, "12") + + test.test = testMethod + test.description = lambda : "1 minute equals 12 bottles." + +The above test runs the student's file, pushes the number 1 in stdin and sets the +attribute ``__name__`` to ``"__main__"``. It does not ignore any exceptions, +that means that CheckPy will fail the test if an exception is raised and kindly +tell the user what exception was raised. Argv is set to the default (just the program name). + +Customizing output +------------------ + +An instance of ``Test`` has a couple of methods that you can use to show the user +exactly what you want the user to see. We have already seen the ``.description()`` +method that you can overwrite with a function that should produce the description +of the test. This description then turns green or red, with a happy or sad smiley +depending on whether the test fails or passes. +Besides the ``.description()`` method you also find the ``.success(info)`` and +``.fail(info)`` methods. These methods take in an argument called ``info`` and +should produce a message for the user to read when the test succeeds or fails +respectively. This message is printed directly under the description. +Take the following test: + +.. code-block:: Python + + @t.test(0) + def failExample1(test): + def testMethod(): + output = lib.outputOf(test.fileName) + line = lib.getLine(output, 0) + return asserts.numberOnLine(42, line) + + test.test = testMethod + test.description = lambda : "demonstrating the use of .fail()!" + test.fail = lambda info : "could not find 42 in the first line of the output" + +The above test looks for the number 42 on the first line of the output. If the test +fails it will print that it could not find 42 in the output. Okay, this is a little +boring, CheckPy just prints a static description if the test fails. So let's spice +things up. Take the following test: + +.. code-block:: Python + + @t.test(0) + def failExample2(test): + def testMethod(): + output = lib.outputOf(test.fileName) + line = lib.getLine(output, 0) + return asserts.numberOnLine(42, line), lib.getNumbersFromString(line) + + test.test = testMethod + test.description = lambda : "demonstrating the use of .fail()!" + test.fail = lambda info : "could not find 42 on the first line of output, only these numbers: {}".format(info) + +This test also looks for 42 on the first line of the output. If this test fails +however it will also print what numbers it did find on that one line of output. +Here's what's happening. The ``.test()`` method can return a second value besides +simply a boolean indicating whether the passed. This value is passed to the +``.fail(info)`` and ``.success(info)`` methods. So you can use this second return +value to customize what ``.fail(info)`` and ``.success(info)`` do. Here the +implementation of ``.fail(info)`` simply prints out whatever ``info`` it receives. diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..da73edd --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,36 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build +set SPHINXPROJ=CheckPy + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% + +:end +popd diff --git a/run_tests.py b/run_tests.py new file mode 100644 index 0000000..7d68a61 --- /dev/null +++ b/run_tests.py @@ -0,0 +1,39 @@ +import unittest +import argparse + +def unittests(): + test_loader = unittest.TestLoader() + test_suite = test_loader.discover('tests/unittests', pattern='*_test.py') + return test_suite + +def integrationtests(): + test_loader = unittest.TestLoader() + test_suite = test_loader.discover('tests/integrationtests', pattern='*_test.py') + return test_suite + +def main(): + parser = argparse.ArgumentParser(description = "tests for checkpy") + parser.add_argument("-integration", action="store_true", help="run integration tests") + parser.add_argument("-unit", action="store_true", help="run unittests") + parser.add_argument("-all", action="store_true", help="run all tests") + args = parser.parse_args() + + runner = unittest.TextTestRunner(verbosity=1, buffer=True) + + if args.all: + runner.run(unittests()) + runner.run(integrationtests()) + return + + if args.unit: + runner.run(unittests()) + return + + if args.integration: + runner.run(integrationtests()) + return + + parser.print_help() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/setup.py b/setup.py index b95809e..09e4d38 100644 --- a/setup.py +++ b/setup.py @@ -5,16 +5,17 @@ here = path.abspath(path.dirname(__file__)) # Get the long description from the README file -with open(path.join(here, 'README.rst'), encoding='utf-8') as f: +with open(path.join(here, 'README.md'), encoding='utf-8') as f: long_description = f.read() setup( name='checkPy', - version='0.2.15', + version='2.1.2', description='A simple python testing framework for educational purposes', long_description=long_description, + long_description_content_type='text/markdown', url='https://github.com/Jelleas/CheckPy', @@ -25,15 +26,15 @@ # See https://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ - 'Development Status :: 3 - Alpha', + 'Development Status :: 4 - Beta', 'Intended Audience :: Education', 'Topic :: Education :: Testing', 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.7', ], keywords='new unexperienced programmers automatic testing minor programming', @@ -42,7 +43,15 @@ include_package_data=True, - install_requires=["requests", "tinydb", "dill", "colorama"], + install_requires=[ + "requests", + "tinydb", + "dill", + "colorama", + "pytest", + "dessert", + "typeguard" + ], extras_require={ 'dev': [], diff --git a/tests/integrationtests/downloader_test.py b/tests/integrationtests/downloader_test.py new file mode 100644 index 0000000..4e8bc84 --- /dev/null +++ b/tests/integrationtests/downloader_test.py @@ -0,0 +1,116 @@ +import unittest +import os +import sys +from contextlib import contextmanager +try: + # Python 2 + import StringIO +except: + # Python 3 + import io as StringIO +import checkpy +import checkpy.interactive +import checkpy.downloader as downloader +import checkpy.caches as caches +import checkpy.entities.exception as exception + +@contextmanager +def capturedOutput(): + new_out, new_err = StringIO.StringIO(), StringIO.StringIO() + old_out, old_err = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = new_out, new_err + yield sys.stdout, sys.stderr + finally: + sys.stdout, sys.stderr = old_out, old_err + +class Base(unittest.TestCase): + def setUp(self): + caches.clearAllCaches() + + def tearDown(self): + caches.clearAllCaches() + +class BaseClean(unittest.TestCase): + def setUp(self): + caches.clearAllCaches() + downloader.clean() + + def tearDown(self): + downloader.clean() + caches.clearAllCaches() + +class TestDownload(BaseClean): + def setUp(self): + super(TestDownload, self).setUp() + self.fileName = "some.py" + with open(self.fileName, "w") as f: + f.write("print(\"foo\")") + + def tearDown(self): + super(TestDownload, self).tearDown() + if os.path.isfile(self.fileName): + os.remove(self.fileName) + + def test_spelledOutLink(self): + downloader.download("https://github.com/jelleas/tests") + testerResult = checkpy.interactive.test(self.fileName) + self.assertTrue(len(testerResult.testResults) == 1) + self.assertTrue(testerResult.testResults[0].hasPassed) + + def test_incompleteLink(self): + downloader.download("jelleas/tests") + testerResult = checkpy.interactive.test(self.fileName) + self.assertTrue(len(testerResult.testResults) == 1) + self.assertTrue(testerResult.testResults[0].hasPassed) + + def test_deadLink(self): + with capturedOutput() as (out, err): + downloader.download("jelleas/doesnotexist") + self.assertTrue("DownloadError" in out.getvalue().strip()) + +class TestUpdate(BaseClean): + def test_clean(self): + downloader.update() + with capturedOutput() as (out, err): + downloader.list() + self.assertEqual(out.getvalue().strip(), "") + + def test_oneDownloaded(self): + downloader.download("jelleas/tests") + with capturedOutput() as (out, err): + downloader.update() + self.assertEqual(\ + out.getvalue().split("\n")[0].strip(), + "Finished downloading: https://github.com/jelleas/tests" + ) + +class TestList(BaseClean): + def test_clean(self): + with capturedOutput() as (out, err): + downloader.list() + self.assertEqual(out.getvalue().strip(), "") + + def test_oneDownloaded(self): + downloader.download("jelleas/tests") + with capturedOutput() as (out, err): + downloader.list() + output = out.getvalue() + self.assertTrue("tests" in output and "jelleas" in output) + +class TestClean(Base): + def test_clean(self): + downloader.clean() + with capturedOutput() as (out, err): + downloader.list() + self.assertEqual(out.getvalue().strip(), "") + + def test_cleanAfterDownload(self): + downloader.download("jelleas/tests") + downloader.clean() + with capturedOutput() as (out, err): + downloader.list() + self.assertEqual(out.getvalue().strip(), "") + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/integrationtests/tester_test.py b/tests/integrationtests/tester_test.py new file mode 100644 index 0000000..0ad5ed5 --- /dev/null +++ b/tests/integrationtests/tester_test.py @@ -0,0 +1,163 @@ +import unittest +import os +import sys +from contextlib import contextmanager +try: + # Python 2 + import StringIO +except: + # Python 3 + import io as StringIO +import checkpy +import checkpy.tester as tester +import checkpy.downloader as downloader +import checkpy.caches as caches +import checkpy.entities.exception as exception +import checkpy.tester.discovery as discovery + +@contextmanager +def capturedOutput(): + new_out, new_err = StringIO.StringIO(), StringIO.StringIO() + old_out, old_err = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = new_out, new_err + yield sys.stdout, sys.stderr + finally: + sys.stdout, sys.stderr = old_out, old_err + +class TestTest(unittest.TestCase): + def setUp(self): + caches.clearAllCaches() + self.fileName = "some.py" + self.source = "print(\"foo\")" + self.write(self.source) + if not discovery.testExists(self.fileName): + downloader.clean() + downloader.download("jelleas/tests") + + def tearDown(self): + if os.path.isfile(self.fileName): + os.remove(self.fileName) + caches.clearAllCaches() + + def write(self, source): + with open(self.fileName, "w") as f: + f.write(source) + + def test_oneTest(self): + for testerResult in [tester.test(self.fileName), tester.test(self.fileName.split(".")[0])]: + self.assertTrue(len(testerResult.testResults) == 1) + self.assertTrue(testerResult.testResults[0].hasPassed) + self.assertTrue("Testing: some.py".lower() in testerResult.output[0].lower()) + self.assertTrue(":)" in testerResult.output[1]) + self.assertTrue("prints exactly: foo".lower() in testerResult.output[1].lower()) + + def test_fileMising(self): + os.remove(self.fileName) + testerResult = tester.test(self.fileName) + self.assertTrue("file not found".lower() in testerResult.output[0].lower()) + self.assertTrue(self.fileName.lower() in testerResult.output[0].lower()) + + def test_testMissing(self): + fileName = "foo.py" + with open(fileName, "w") as f: + pass + testerResult = tester.test(fileName) + self.assertTrue("No test found for".lower() in testerResult.output[0].lower()) + self.assertTrue(fileName.lower() in testerResult.output[0].lower()) + os.remove(fileName) + + +class TestTestNotebook(unittest.TestCase): + def setUp(self): + caches.clearAllCaches() + self.fileName = "some.ipynb" + self.source = r"""{ +"cells": [ +{ +"cell_type": "code", +"execution_count": null, +"metadata": {}, +"outputs": [], +"source": [ +"print(\"foo\")" +] +} +], +"metadata": { +"kernelspec": { +"display_name": "Python 3", +"language": "python", +"name": "python3" +}, +"language_info": { +"codemirror_mode": { +"name": "ipython", +"version": 3 +}, +"file_extension": ".py", +"mimetype": "text/x-python", +"name": "python", +"nbconvert_exporter": "python", +"pygments_lexer": "ipython3", +"version": "3.6.2" +} +}, +"nbformat": 4, +"nbformat_minor": 2 +}""" + self.write(self.source) + if not discovery.testExists(self.fileName): + downloader.clean() + downloader.download("jelleas/tests") + + def tearDown(self): + if os.path.isfile(self.fileName): + os.remove(self.fileName) + caches.clearAllCaches() + + def write(self, source): + with open(self.fileName, "w") as f: + f.write(source) + + def test_oneTest(self): + for testerResult in [tester.test(self.fileName), tester.test(self.fileName.split(".")[0])]: + self.assertTrue(len(testerResult.testResults) == 1) + self.assertTrue(testerResult.testResults[0].hasPassed) + self.assertTrue("Testing: {}".format(self.fileName.split(".")[0]).lower() in testerResult.output[0].lower()) + self.assertTrue(":)" in testerResult.output[1]) + self.assertTrue("prints exactly: foo".lower() in testerResult.output[1].lower()) + + +class TestTestSandbox(unittest.TestCase): + def setUp(self): + caches.clearAllCaches() + self.fileName = "sandbox.py" + self.source = "print(\"foo\")" + self.write(self.source) + if not discovery.testExists(self.fileName): + downloader.clean() + downloader.download("jelleas/tests") + + def tearDown(self): + if os.path.isfile(self.fileName): + os.remove(self.fileName) + caches.clearAllCaches() + + def write(self, source): + with open(self.fileName, "w") as f: + f.write(source) + + def test_oneTest(self): + for testerResult in [tester.test(self.fileName), tester.test(self.fileName.split(".")[0])]: + self.assertTrue(len(testerResult.testResults) == 2) + self.assertTrue(testerResult.testResults[0].hasPassed) + self.assertTrue(testerResult.testResults[1].hasPassed) + self.assertTrue("Testing: sandbox.py".lower() in testerResult.output[0].lower()) + self.assertTrue(":)" in testerResult.output[1]) + self.assertTrue("prints exactly: foo".lower() in testerResult.output[1].lower()) + self.assertTrue(":)" in testerResult.output[2]) + self.assertTrue("sandbox.py and sandboxTest.py exist".lower() in testerResult.output[2].lower()) + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unittests/function_test.py b/tests/unittests/function_test.py new file mode 100644 index 0000000..9f62eb9 --- /dev/null +++ b/tests/unittests/function_test.py @@ -0,0 +1,124 @@ +import unittest +import checkpy.lib as lib +import checkpy.entities.exception as exception +from checkpy.entities.function import Function + +class TestFunction(unittest.TestCase): + def setUp(self): + context = lib.io.replaceStdout() + context.__enter__() + self.addCleanup(context.__exit__, None, None, None) + + +class TestFunctionName(TestFunction): + def test_name(self): + def foo(): + pass + self.assertEqual(Function(foo).name, "foo") + +class TestFunctionArguments(TestFunction): + def test_noArgs(self): + def foo(): + pass + self.assertEqual(Function(foo).arguments, []) + + def test_args(self): + def foo(bar, baz): + pass + self.assertEqual(Function(foo).arguments, ["bar", "baz"]) + + def test_kwargs(self): + def foo(bar = None, baz = None): + pass + self.assertEqual(Function(foo).arguments, ["bar", "baz"]) + + def test_argsAndKwargs(self): + def foo(bar, baz = None): + pass + self.assertEqual(Function(foo).arguments, ["bar", "baz"]) + +class TestFunctionCall(TestFunction): + def test_dummy(self): + def foo(): + return None + f = Function(foo) + self.assertEqual(f(), None) + + def test_arg(self): + def foo(bar): + return bar + 1 + f = Function(foo) + self.assertEqual(f(1), 2) + self.assertEqual(f(bar = 1), 2) + + def test_kwarg(self): + def foo(bar=0): + return bar + 1 + f = Function(foo) + self.assertEqual(f(), 1) + + self.assertEqual(f(1), 2) + + self.assertEqual(f(bar = 1), 2) + + def test_exception(self): + def foo(): + raise ValueError("baz") + f = Function(foo) + with self.assertRaises(exception.SourceException): + f() + + +class TestFunctionPrintOutput(TestFunction): + def test_noOutput(self): + def foo(): + pass + f = Function(foo) + f() + self.assertEqual(f.printOutput, "") + + def test_oneLineOutput(self): + def foo(): + print("bar") + f = Function(foo) + f() + self.assertEqual(f.printOutput, "bar\n") + + def test_twoLineOutput(self): + def foo(): + print("bar") + print("baz") + f = Function(foo) + f() + self.assertEqual(f.printOutput, "bar\nbaz\n") + + def test_indirectPrint(self): + def foo(): + Function(bar)() + def bar(): + print("baz") + foo = Function(foo) + foo() + self.assertEqual(foo.printOutput, "baz\n") + + def test_indirectPrintWithOrder(self): + def foo(): + print("foo") + Function(bar)() + print("baz") + def bar(): + print("bar") + foo = Function(foo) + foo() + self.assertEqual(foo.printOutput, "foo\nbar\nbaz\n") + + def test_multipleCalls(self): + def foo(): + print("foo") + foo = Function(foo) + foo() + foo() + self.assertEqual(foo.printOutput, "foo\n") + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unittests/lib_test.py b/tests/unittests/lib_test.py new file mode 100644 index 0000000..fc72c81 --- /dev/null +++ b/tests/unittests/lib_test.py @@ -0,0 +1,361 @@ +import unittest +from io import StringIO +import os +import shutil +import tempfile + +import checkpy.lib as lib +import checkpy.caches as caches +import checkpy.entities.exception as exception + + +class Base(unittest.TestCase): + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tempdir) + os.chdir(self.tempdir) + + self.fileName = "dummy.py" + self.source = "def f(x):" +\ + " return x * 2" + self.write(self.source) + + stdout_context = lib.io.replaceStdout() + stdout_context.__enter__() + self.addCleanup(stdout_context.__exit__, None, None, None) + + stdin_context = lib.io.replaceStdin() + stdin_context.__enter__() + self.addCleanup(stdin_context.__exit__, None, None, None) + + def tearDown(self): + caches.clearAllCaches() + + def write(self, source): + with open(self.fileName, "w") as f: + f.write(source) + + +class TestFileExists(Base): + def test_fileDoesNotExist(self): + self.assertFalse(lib.fileExists("idonotexist.random")) + + def test_fileExists(self): + self.assertTrue(lib.fileExists(self.fileName)) + + +class TestSource(Base): + def test_expectedOutput(self): + source = lib.source(self.fileName) + self.assertEqual(source, self.source) + + +class TestSourceOfDefinitions(Base): + def test_noDefinitions(self): + source = \ +""" +height = int(input("Height: ")) + +#retry while height > 23 +while height > 23: + height = int(input("Height: ")) + +#prints the # and blanks +for i in range(height): + for j in range(height - i - 1): + print(" ", end="") + for k in range(i + 2): + print("#", end="") + print("") +""" + self.write(source) + self.assertEqual(lib.sourceOfDefinitions(self.fileName), "") + + def test_oneDefinition(self): + source = \ +""" +def main(): + pass +if __name__ == "__main__": + main() +""" + expectedOutcome = \ +"""def main(): + pass""" + self.write(source) + self.assertEqual(lib.sourceOfDefinitions(self.fileName), expectedOutcome) + + def test_comments(self): + source = \ +""" +# foo +\"\"\"bar\"\"\" +""" + self.write(source) + self.assertEqual(lib.sourceOfDefinitions(self.fileName), "") + + @unittest.expectedFailure + def test_multilineString(self): + source = \ +""" +x = \"\"\"foo\"\"\" +""" + self.write(source) + self.assertEqual(lib.sourceOfDefinitions(self.fileName), source) + + def test_import(self): + source = \ +"""import os +from os import path""" + self.write(source) + self.assertEqual(lib.sourceOfDefinitions(self.fileName), source) + + +class TestGetFunction(Base): + def test_sameName(self): + func = lib.getFunction("f", self.fileName) + self.assertEqual(func.name, "f") + + def test_sameArgs(self): + func = lib.getFunction("f", self.fileName) + self.assertEqual(func.arguments, ["x"]) + + def test_expectedOutput(self): + func = lib.getFunction("f", self.fileName) + self.assertEqual(func(2), 4) + + +class TestOutputOf(Base): + def test_helloWorld(self): + source = \ +""" +print("Hello, world!") +""" + self.write(source) + self.assertEqual(lib.outputOf(self.fileName), "Hello, world!\n") + + def test_function(self): + source = \ +""" +def f(x): + print(x) +f(1) +""" + self.write(source) + self.assertEqual(lib.outputOf(self.fileName), "1\n") + + def test_input(self): + source = \ +""" +x = input("foo") +print(x) +""" + self.write(source) + output = lib.outputOf(self.fileName, stdinArgs = ["3"]) + self.assertEqual(int(output), 3) + + def test_noInput(self): + source = \ +""" +x = input("foo") +print(x) +""" + self.write(source) + with self.assertRaises(exception.InputError): + lib.outputOf(self.fileName) + + def test_argv(self): + source = \ +""" +import sys +print(sys.argv[1]) +""" + self.write(source) + output = lib.outputOf(self.fileName, argv = [self.fileName, "foo"]) + self.assertEqual(output, "foo\n") + + def test_ValueError(self): + source = \ +""" +print("bar") +raise ValueError +print("foo") +""" + self.write(source) + with self.assertRaises(exception.SourceException): + output = lib.outputOf(self.fileName, argv = [self.fileName, "foo"]) + self.assertEqual(output, "bar\n") + + def test_ignoreValueError(self): + source = \ +""" +print("foo") +raise ValueError +print("bar") +""" + self.write(source) + output = lib.outputOf(self.fileName, ignoreExceptions = [ValueError]) + self.assertEqual(output, "foo\n") + + def test_ignoreSystemExit(self): + source = \ +""" +import sys +print("foo") +sys.exit(1) +print("bar") +""" + self.write(source) + output = lib.outputOf(self.fileName, ignoreExceptions = [SystemExit]) + self.assertEqual(output, "foo\n") + + def test_src(self): + source = \ +""" +print("bar") +""" + self.write(source) + output = lib.outputOf(self.fileName, src = "print(\"foo\")") + self.assertEqual(output, "foo\n") + + +class TestModule(Base): + def test_function(self): + source = \ +""" +def f(): + pass +""" + self.write(source) + self.assertTrue(hasattr(lib.module(self.fileName), "f")) + + def test_class(self): + source = \ +""" +class C: + pass +""" + self.write(source) + self.assertTrue(hasattr(lib.module(self.fileName), "C")) + + def test_import(self): + source = \ +""" +import os +""" + self.write(source) + self.assertTrue(hasattr(lib.module(self.fileName), "os")) + + def test_global(self): + source = \ +""" +x = 3 +""" + self.write(source) + self.assertTrue(hasattr(lib.module(self.fileName), "x")) + + def test_indirectGlobal(self): + source = \ +""" +def f(): + global x + x = 3 +f() +""" + self.write(source) + self.assertTrue(hasattr(lib.module(self.fileName), "x")) + + def test_local(self): + source = \ +""" +def f(): + x = 3 +f() +""" + self.write(source) + self.assertTrue(not hasattr(lib.module(self.fileName), "x")) + + +class TestNeutralizeFunction(unittest.TestCase): + def test_dummy(self): + def dummy(): + return "foo" + lib.neutralizeFunction(dummy) + self.assertEqual(dummy(), None) + + +class TestDownload(unittest.TestCase): + def setUp(self): + self.tempdir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tempdir) + os.chdir(self.tempdir) + + def test_fileDownload(self): + fileName = "someTest.py" + + with lib.sandbox.sandbox(): + lib.download(fileName, "https://raw.githubusercontent.com/Jelleas/tests/master/tests/{}".format(fileName)) + self.assertTrue(os.path.isfile(fileName)) + + def test_fileDownloadRename(self): + fileName = "someRandomFileName.name" + + + with lib.sandbox.sandbox(): + lib.download(fileName, "https://raw.githubusercontent.com/Jelleas/tests/master/tests/someTest.py") + self.assertTrue(os.path.isfile(fileName)) + +class TestCaptureStdout(unittest.TestCase): + def test_blank(self): + with lib.io.replaceStdout(): + with lib.io.captureStdout() as stdout: + self.assertTrue(len(stdout.content) == 0) + + def test_noOutput(self): + with lib.io.replaceStdout(): + with lib.io.captureStdout() as stdout: + print("foo") + self.assertEqual("foo\n", stdout.content) + + def test_noLeakage(self): + import sys + try: + original_stdout = sys.stdout + mock_stdout = StringIO() + sys.stdout = mock_stdout + + with lib.io.replaceStdout(): + with lib.io.captureStdout(): + print("foo") + + self.assertEqual(len(mock_stdout.getvalue()), 0) + self.assertEqual(len(sys.stdout.getvalue()), 0) + finally: + sys.stdout = original_stdout + mock_stdout.close() + +class TestReplaceStdin(unittest.TestCase): + def test_noInput(self): + with lib.io.replaceStdin() as stdin: + with self.assertRaises(exception.InputError): + input() + + def test_oneInput(self): + with lib.io.replaceStdin() as stdin: + stdin.write("foo\n") + stdin.seek(0) + self.assertEqual(input(), "foo") + with self.assertRaises(exception.InputError): + input() + + def test_noLeakage(self): + with lib.io.replaceStdout(): + with lib.io.replaceStdin() as stdin, lib.io.captureStdout() as stdout: + stdin.write("foo\n") + stdin.seek(0) + self.assertEqual(input("hello!"), "foo") + self.assertTrue(len(stdout.content) == 0) + + +if __name__ == '__main__': + unittest.main()