From 648cfe96144b15b8a29c77bf2b29c2faa0632a82 Mon Sep 17 00:00:00 2001 From: Clair Mould <86794332+clmould@users.noreply.github.com> Date: Fri, 12 Dec 2025 17:45:19 +0000 Subject: [PATCH 1/2] Combine csv example into examples.ipynb and update test accordingly --- examples/csv_output.ipynb | 80 ------ examples/examples.ipynb | 494 ++++++++++++++++---------------- tests/examples/test_examples.py | 44 +-- 3 files changed, 268 insertions(+), 350 deletions(-) delete mode 100644 examples/csv_output.ipynb diff --git a/examples/csv_output.ipynb b/examples/csv_output.ipynb deleted file mode 100644 index 2825f0451c..0000000000 --- a/examples/csv_output.ipynb +++ /dev/null @@ -1,80 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "source": [ - "# Output to csv\n", - "\n", - "Routine to read from a PROCESS MFILE and write specified values into a csv.\n", - "\n", - "Input files:\n", - "- MFILE.DAT as output from PROCESS\n", - "- .json variable list as defined by user (defaults to local `mfile_to_csv_vars.json`)\n", - "\n", - "Instructions:\n", - "- from command line: `python mfile_to_csv.py -f -v `\n", - "- from this Jupyter notebook: run the cell below\n", - "\n", - "Output file:\n", - "- .csv will be saved to the directory of the input file" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "slideshow": { - "slide_type": "subslide" - } - }, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "\n", - "from process.io import mfile_to_csv\n", - "\n", - "# Project directory for example result file and default .json list;\n", - "# not needed if you replace both target filepaths below.\n", - "data_dir = Path(\"data\")\n", - "# Replace this path/to/MFILE.DAT with your target file:\n", - "mfilename = data_dir / \"large_tokamak_1_MFILE.DAT\"\n", - "\n", - "# Either replace this with your own path/to/file.json target,\n", - "# or add your required variables into the identified file:\n", - "varfilename = data_dir / \"mfile_to_csv_vars.json\"\n", - "# This routine attempts to find every variable in the given list and\n", - "# writes the variable name, description and value to the output csv.\n", - "# Any listed variable that isn't in that MFILE will be skipped.\n", - "\n", - "# call to function:\n", - "mfile_to_csv.main(args=[\"-f\", str(mfilename), \"-v\", str(varfilename)])" - ] - } - ], - "metadata": { - "celltoolbar": "Slideshow", - "kernelspec": { - "display_name": "env", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/examples/examples.ipynb b/examples/examples.ipynb index 3e9bedbb9a..986fc75e75 100644 --- a/examples/examples.ipynb +++ b/examples/examples.ipynb @@ -1,242 +1,254 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Introduction to running PROCESS\n", - "\n", - "A Jupyter notebook to demonstrate usage of the `process` package.\n", - "\n", - "## Motivation\n", - "Process is moving away from being a runnable package with a single command-line entry-point to an importable package which can be scripted. This notebook is a good way of demonstrating the functionality of the package, and could provide a better way of working for modellers, who may wish to create their own scripts or notebooks for different tasks.\n", - "\n", - "## Setup\n", - "Currently the various classes and \"utilities\" scripts in Process have different interfaces and read and write files in differing manners and in locations that can't be easily controlled. To partially avoid the headaches associated with this, the code cell below defines a function to allow each example to be run in a temporary directory, much like a test. Input files are copied to this temporary directory and outputs contained there before the directory is removed.\n", - "\n", - "This temporary directory function is only required for running the examples below and removing any modifications afterwards, not in regular use of Process where the outputs will want to be preserved. Further development work will unify these disparate ways of running Process into a common Pythonic form." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2\n", - "\n", - "from pathlib import Path\n", - "from shutil import copy\n", - "from tempfile import TemporaryDirectory\n", - "\n", - "# Define project root dir; when running a notebook, the cwd is the dir the notebook is in\n", - "PROJ_DIR = Path.cwd().parent\n", - "\n", - "\n", - "def copy_to_temp_dir(input_rel):\n", - " \"\"\"Copy an input file to a new temp dir and return its new path.\n", - "\n", - " The new TemporaryDirectory object is returned to avoid destruction of the\n", - " object, which results in deletion of the directory prematurely. This way\n", - " the cleanup() method can be used to delete the directory when required.\n", - " :param input_rel: file path relative to project root dir\n", - " :type input_rel: str\n", - " :return: temporary dir and absolute path to file in temp dir\n", - " :rtype: (TemporaryDirectory, pathlib.Path)\n", - " \"\"\"\n", - " # Create temporary dir to contain the run's outputs\n", - " temp_dir = TemporaryDirectory()\n", - " temp_dir_path = Path(temp_dir.name)\n", - "\n", - " # Define absolute path for input file\n", - " input_rel_path = Path(input_rel)\n", - " input_abs_path = PROJ_DIR / input_rel_path\n", - "\n", - " try:\n", - " assert input_abs_path.exists()\n", - " except AssertionError as err:\n", - " raise FileNotFoundError(\"Input file doesn't exist.\") from err\n", - "\n", - " # Copy input file to temp dir\n", - " copy(input_abs_path, temp_dir_path)\n", - " temp_input_path = temp_dir_path / input_abs_path.name\n", - "\n", - " return temp_dir, temp_input_path, temp_dir_path" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "## Basic run of Process\n", - "Run Process on an input file using the `SingleRun` class. This outputs an `MFILE.DAT` and an `OUT.DAT`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from process.main import SingleRun, setup_loggers\n", - "\n", - "# setup the loggers so that the output is not spammed with model errors/warnings\n", - "setup_loggers()\n", - "\n", - "# Define input file name relative to project dir, then copy to temp dir\n", - "script_dir = Path(\"__file__\").parent.resolve()\n", - "input_rel = script_dir / \"data/large_tokamak_IN.DAT\"\n", - "\n", - "temp_dir, temp_input_path, temp_dir_path = copy_to_temp_dir(input_rel)\n", - "\n", - "# Run process on an input file in a temporary directory\n", - "single_run = SingleRun(temp_input_path.as_posix())\n", - "single_run.run()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Plot summary\n", - "Create a summary of the generated `MFILE.DAT` using `plot_proc`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from process.io import plot_proc\n", - "\n", - "# plot_proc uses command line arguments of the current process. Jupyter adds command line arguments under the hood causing plot_proc to fail. running plot proc in its own process isolates it from the jupyter command line arguments\n", - "# Use the --output-format argument \"none\" to not save outputs.\n", - "# Pdf and png output are also available\n", - "plot_proc.main(\n", - " args=[\"-f\", single_run.mfile_path.as_posix(), \"--output-format\", \"none\", \"--show\"]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Delete temp dir\n", - "temp_dir.cleanup()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## View key output variables\n", - "Run the large tokamak scenario using `SingleRun` to set some values on the `CostModel` instance and then print them." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Define input file name relative to project dir\n", - "input_rel = script_dir / \"data/large_tokamak_IN.DAT\"\n", - "print(input_rel)\n", - "temp_dir, temp_input_path, _ = copy_to_temp_dir(input_rel)\n", - "\n", - "# Run process on an input file\n", - "single_run = SingleRun(temp_input_path.as_posix())\n", - "single_run.run()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import process.data_structure\n", - "\n", - "# Print some values on the CostModel instance\n", - "print(f\"Heat transport system: {process.data_structure.cost_variables.c226:.3e} M$\")\n", - "print(f\"Electrical plant equipment: {process.data_structure.cost_variables.c24:.3e} M$\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Clean up\n", - "temp_dir.cleanup()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## VaryRun\n", - "Vary iteration parameters until a feasible solution is found, using the `VaryRun` class." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "from process.main import VaryRun\n", - "\n", - "input_rel = script_dir / \"data/run_process.conf\"\n", - "temp_dir, temp_input_path, _ = copy_to_temp_dir(input_rel)\n", - "\n", - "# .conf file relies on a separate input file too; copy this as well\n", - "# TODO This double input file requirement needs to be removed\n", - "input_rel_2 = script_dir / \"data/large_tokamak_IN.DAT\"\n", - "copy(PROJ_DIR / input_rel_2, temp_dir.name)\n", - "\n", - "# VaryRun uses process_config.py, which changes the current working directory\n", - "# via os.chdir() to the temporary dir. Apart from being bad practice, once the\n", - "# temp dir is removed, this causes Path.cwd() (as used in plot_scans.py) to\n", - "# throw an exception when trying to return the (now deleted) CWD. Hence it\n", - "# needs to be set back after VaryRun()\n", - "# TODO Remove the os.chdir() from VaryRun\n", - "cwd = Path.cwd()\n", - "\n", - "vary_run = VaryRun(temp_input_path.as_posix())\n", - "vary_run.run()\n", - "os.chdir(cwd)\n", - "\n", - "temp_dir.cleanup()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "env", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Introduction to running PROCESS\n", + "\n", + "A Jupyter notebook to demonstrate usage of the `process` package.\n", + "\n", + "## Motivation\n", + "Process is moving away from being a runnable package with a single command-line entry-point to an importable package which can be scripted. This notebook is a good way of demonstrating the functionality of the package, and could provide a better way of working for modellers, who may wish to create their own scripts or notebooks for different tasks.\n", + "\n", + "## Setup\n", + "Currently the various classes and \"utilities\" scripts in Process have different interfaces and read and write files in differing manners and in locations that can't be easily controlled. To partially avoid the headaches associated with this, the code cell below defines a function to allow each example to be run in a temporary directory, much like a test. Input files are copied to this temporary directory and outputs contained there before the directory is removed.\n", + "\n", + "This temporary directory function is only required for running the examples below and removing any modifications afterwards, not in regular use of Process where the outputs will want to be preserved. Further development work will unify these disparate ways of running Process into a common Pythonic form." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n", + "\n", + "from pathlib import Path\n", + "from shutil import copy\n", + "from tempfile import TemporaryDirectory\n", + "\n", + "# Define project root dir; when running a notebook, the cwd is the dir the notebook is in\n", + "PROJ_DIR = Path.cwd().parent\n", + "\n", + "\n", + "def copy_to_temp_dir(input_rel):\n", + " \"\"\"Copy an input file to a new temp dir and return its new path.\n", + "\n", + " The new TemporaryDirectory object is returned to avoid destruction of the\n", + " object, which results in deletion of the directory prematurely. This way\n", + " the cleanup() method can be used to delete the directory when required.\n", + " :param input_rel: file path relative to project root dir\n", + " :type input_rel: str\n", + " :return: temporary dir and absolute path to file in temp dir\n", + " :rtype: (TemporaryDirectory, pathlib.Path)\n", + " \"\"\"\n", + " # Create temporary dir to contain the run's outputs\n", + " temp_dir = TemporaryDirectory()\n", + " temp_dir_path = Path(temp_dir.name)\n", + "\n", + " # Define absolute path for input file\n", + " input_rel_path = Path(input_rel)\n", + " input_abs_path = PROJ_DIR / input_rel_path\n", + "\n", + " try:\n", + " assert input_abs_path.exists()\n", + " except AssertionError as err:\n", + " raise FileNotFoundError(\"Input file doesn't exist.\") from err\n", + "\n", + " # Copy input file to temp dir\n", + " copy(input_abs_path, temp_dir_path)\n", + " temp_input_path = temp_dir_path / input_abs_path.name\n", + "\n", + " return temp_dir, temp_input_path, temp_dir_path" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Basic run of Process\n", + "Run Process on an input file using the `SingleRun` class. This outputs an `MFILE.DAT` and an `OUT.DAT`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from process.main import SingleRun, setup_loggers\n", + "\n", + "# setup the loggers so that the output is not spammed with model errors/warnings\n", + "setup_loggers()\n", + "\n", + "# Define input file name relative to project dir, then copy to temp dir\n", + "script_dir = Path(\"__file__\").parent.resolve()\n", + "input_rel = script_dir / \"data/large_tokamak_IN.DAT\"\n", + "\n", + "temp_dir, temp_input_path, temp_dir_path = copy_to_temp_dir(input_rel)\n", + "\n", + "# Run process on an input file in a temporary directory\n", + "single_run = SingleRun(temp_input_path.as_posix())\n", + "single_run.run()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plot summary\n", + "Create a summary of the generated `MFILE.DAT` using `plot_proc`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from process.io import plot_proc\n", + "\n", + "# plot_proc uses command line arguments of the current process. Jupyter adds command line arguments under the hood causing plot_proc to fail. running plot proc in its own process isolates it from the jupyter command line arguments\n", + "# Use the --output-format argument \"none\" to not save outputs.\n", + "# Pdf and png output are also available\n", + "plot_proc.main(\n", + " args=[\"-f\", single_run.mfile_path.as_posix(), \"--output-format\", \"none\", \"--show\"]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## View key output variables\n", + "Using the `MFILE` we generated by running the large tokamak scenario above, we have set some values on the `CostModel` instance and can print them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import process.data_structure\n", + "\n", + "# Print some values on the CostModel instance\n", + "print(f\"Heat transport system: {process.data_structure.cost_variables.c226:.3e} M$\")\n", + "print(f\"Electrical plant equipment: {process.data_structure.cost_variables.c24:.3e} M$\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Convert to CSV format\n", + "This demonstrates how you would read from a PROCESS MFILE and write specified values into a csv using the `mfile_to_csv` function" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from process.io import mfile_to_csv\n", + "\n", + "data_dir = Path(\"data\")\n", + "\n", + "# mfile_to_csv requires two inputs:\n", + "# - path to the MFILE\n", + "# - .json containing the variable names to include in the csv file\n", + "\n", + "# This routine attempts to find every variable listed in the json file\n", + "# in the MFILE and writes the variable name, description and value\n", + "# to the output csv.\n", + "# Any listed variable that isn't in that MFILE will be skipped.\n", + "# The .csv file is saved to the directory of the input file\n", + "\n", + "mfile_to_csv.main(\n", + " args=[\n", + " \"-f\",\n", + " (data_dir / \"large_tokamak_1_MFILE.DAT\").as_posix(),\n", + " \"-v\",\n", + " (data_dir / \"mfile_to_csv_vars.json\").as_posix(),\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Clean up\n", + "temp_dir.cleanup()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## VaryRun\n", + "Vary iteration parameters until a feasible solution is found, using the `VaryRun` class." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from process.main import VaryRun\n", + "\n", + "input_rel = script_dir / \"data/run_process.conf\"\n", + "temp_dir, temp_input_path, _ = copy_to_temp_dir(input_rel)\n", + "\n", + "# .conf file relies on a separate input file too; copy this as well\n", + "# TODO This double input file requirement needs to be removed\n", + "input_rel_2 = script_dir / \"data/large_tokamak_IN.DAT\"\n", + "copy(PROJ_DIR / input_rel_2, temp_dir.name)\n", + "\n", + "# VaryRun uses process_config.py, which changes the current working directory\n", + "# via os.chdir() to the temporary dir. Apart from being bad practice, once the\n", + "# temp dir is removed, this causes Path.cwd() (as used in plot_scans.py) to\n", + "# throw an exception when trying to return the (now deleted) CWD. Hence it\n", + "# needs to be set back after VaryRun()\n", + "# TODO Remove the os.chdir() from VaryRun\n", + "cwd = Path.cwd()\n", + "\n", + "vary_run = VaryRun(temp_input_path.as_posix())\n", + "vary_run.run()\n", + "os.chdir(cwd)\n", + "\n", + "temp_dir.cleanup()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/tests/examples/test_examples.py b/tests/examples/test_examples.py index f3c76be4b8..d97015af65 100644 --- a/tests/examples/test_examples.py +++ b/tests/examples/test_examples.py @@ -44,40 +44,12 @@ def test_examples(examples_temp_data): """ example_notebook_location = examples_temp_data / "examples.ipynb" with testbook(example_notebook_location, execute=True, timeout=600): - pass - - -def test_scan(examples_temp_data): - """Run scan.ipynb notebook check no exceptions are raised and that an MFILE is created. - - scan.ipynb intentionally produces files when running the notebook, but remove - them when testing. - :param examples_temp_data: temporary dir containing examples files - :type examples_temp_data: Path - """ - scan_notebook_location = examples_temp_data / "scan.ipynb" - with testbook(scan_notebook_location, execute=True, timeout=1200): - # Run entire scan.ipynb notebook and assert an MFILE is created - assert os.path.exists(examples_temp_data / "data/scan_example_file_MFILE.DAT") - - -def test_csv(examples_temp_data): - """Run csv_output.ipynb, check no exceptions are raised, check a csv file exists and check the csv file contains data. - - csv_output.ipynb intentionally produces files when running the notebook, but remove - them when testing. - :param examples_temp_data: temporary dir containing examples files - :type examples_temp_data: Path - """ - csv_notebook_location = examples_temp_data / "csv_output.ipynb" - with testbook(csv_notebook_location, execute=True, timeout=600): # Check csv file is created assert os.path.exists(examples_temp_data / "data/large_tokamak_1_MFILE.csv") # Read in the csv file created by test and check it contains positive floats readcsv = pd.read_csv(examples_temp_data / "data/large_tokamak_1_MFILE.csv") - values = readcsv["Value"] - value_array = np.array(values) + value_array = np.array(readcsv["Value"]) check_float = False check_positive = False value_array_type = value_array.dtype @@ -91,6 +63,20 @@ def test_csv(examples_temp_data): assert check_positive +def test_scan(examples_temp_data): + """Run scan.ipynb notebook check no exceptions are raised and that an MFILE is created. + + scan.ipynb intentionally produces files when running the notebook, but remove + them when testing. + :param examples_temp_data: temporary dir containing examples files + :type examples_temp_data: Path + """ + scan_notebook_location = examples_temp_data / "scan.ipynb" + with testbook(scan_notebook_location, execute=True, timeout=1200): + # Run entire scan.ipynb notebook and assert an MFILE is created + assert os.path.exists(examples_temp_data / "data/scan_example_file_MFILE.DAT") + + def test_plot_solutions(examples_temp_data): """Run plot_solutions.ipynb and check no exceptions are raised. From cd8ef68cca6eb0f42ba6b9b95383e2b105fdef04 Mon Sep 17 00:00:00 2001 From: Clair Mould <86794332+clmould@users.noreply.github.com> Date: Thu, 18 Dec 2025 17:18:32 +0000 Subject: [PATCH 2/2] fix --- examples/examples.ipynb | 506 ++++++++++++++++++++-------------------- 1 file changed, 253 insertions(+), 253 deletions(-) diff --git a/examples/examples.ipynb b/examples/examples.ipynb index 986fc75e75..5c18a58e6d 100644 --- a/examples/examples.ipynb +++ b/examples/examples.ipynb @@ -1,254 +1,254 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Introduction to running PROCESS\n", - "\n", - "A Jupyter notebook to demonstrate usage of the `process` package.\n", - "\n", - "## Motivation\n", - "Process is moving away from being a runnable package with a single command-line entry-point to an importable package which can be scripted. This notebook is a good way of demonstrating the functionality of the package, and could provide a better way of working for modellers, who may wish to create their own scripts or notebooks for different tasks.\n", - "\n", - "## Setup\n", - "Currently the various classes and \"utilities\" scripts in Process have different interfaces and read and write files in differing manners and in locations that can't be easily controlled. To partially avoid the headaches associated with this, the code cell below defines a function to allow each example to be run in a temporary directory, much like a test. Input files are copied to this temporary directory and outputs contained there before the directory is removed.\n", - "\n", - "This temporary directory function is only required for running the examples below and removing any modifications afterwards, not in regular use of Process where the outputs will want to be preserved. Further development work will unify these disparate ways of running Process into a common Pythonic form." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2\n", - "\n", - "from pathlib import Path\n", - "from shutil import copy\n", - "from tempfile import TemporaryDirectory\n", - "\n", - "# Define project root dir; when running a notebook, the cwd is the dir the notebook is in\n", - "PROJ_DIR = Path.cwd().parent\n", - "\n", - "\n", - "def copy_to_temp_dir(input_rel):\n", - " \"\"\"Copy an input file to a new temp dir and return its new path.\n", - "\n", - " The new TemporaryDirectory object is returned to avoid destruction of the\n", - " object, which results in deletion of the directory prematurely. This way\n", - " the cleanup() method can be used to delete the directory when required.\n", - " :param input_rel: file path relative to project root dir\n", - " :type input_rel: str\n", - " :return: temporary dir and absolute path to file in temp dir\n", - " :rtype: (TemporaryDirectory, pathlib.Path)\n", - " \"\"\"\n", - " # Create temporary dir to contain the run's outputs\n", - " temp_dir = TemporaryDirectory()\n", - " temp_dir_path = Path(temp_dir.name)\n", - "\n", - " # Define absolute path for input file\n", - " input_rel_path = Path(input_rel)\n", - " input_abs_path = PROJ_DIR / input_rel_path\n", - "\n", - " try:\n", - " assert input_abs_path.exists()\n", - " except AssertionError as err:\n", - " raise FileNotFoundError(\"Input file doesn't exist.\") from err\n", - "\n", - " # Copy input file to temp dir\n", - " copy(input_abs_path, temp_dir_path)\n", - " temp_input_path = temp_dir_path / input_abs_path.name\n", - "\n", - " return temp_dir, temp_input_path, temp_dir_path" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "## Basic run of Process\n", - "Run Process on an input file using the `SingleRun` class. This outputs an `MFILE.DAT` and an `OUT.DAT`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from process.main import SingleRun, setup_loggers\n", - "\n", - "# setup the loggers so that the output is not spammed with model errors/warnings\n", - "setup_loggers()\n", - "\n", - "# Define input file name relative to project dir, then copy to temp dir\n", - "script_dir = Path(\"__file__\").parent.resolve()\n", - "input_rel = script_dir / \"data/large_tokamak_IN.DAT\"\n", - "\n", - "temp_dir, temp_input_path, temp_dir_path = copy_to_temp_dir(input_rel)\n", - "\n", - "# Run process on an input file in a temporary directory\n", - "single_run = SingleRun(temp_input_path.as_posix())\n", - "single_run.run()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Plot summary\n", - "Create a summary of the generated `MFILE.DAT` using `plot_proc`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from process.io import plot_proc\n", - "\n", - "# plot_proc uses command line arguments of the current process. Jupyter adds command line arguments under the hood causing plot_proc to fail. running plot proc in its own process isolates it from the jupyter command line arguments\n", - "# Use the --output-format argument \"none\" to not save outputs.\n", - "# Pdf and png output are also available\n", - "plot_proc.main(\n", - " args=[\"-f\", single_run.mfile_path.as_posix(), \"--output-format\", \"none\", \"--show\"]\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## View key output variables\n", - "Using the `MFILE` we generated by running the large tokamak scenario above, we have set some values on the `CostModel` instance and can print them." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import process.data_structure\n", - "\n", - "# Print some values on the CostModel instance\n", - "print(f\"Heat transport system: {process.data_structure.cost_variables.c226:.3e} M$\")\n", - "print(f\"Electrical plant equipment: {process.data_structure.cost_variables.c24:.3e} M$\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Convert to CSV format\n", - "This demonstrates how you would read from a PROCESS MFILE and write specified values into a csv using the `mfile_to_csv` function" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from process.io import mfile_to_csv\n", - "\n", - "data_dir = Path(\"data\")\n", - "\n", - "# mfile_to_csv requires two inputs:\n", - "# - path to the MFILE\n", - "# - .json containing the variable names to include in the csv file\n", - "\n", - "# This routine attempts to find every variable listed in the json file\n", - "# in the MFILE and writes the variable name, description and value\n", - "# to the output csv.\n", - "# Any listed variable that isn't in that MFILE will be skipped.\n", - "# The .csv file is saved to the directory of the input file\n", - "\n", - "mfile_to_csv.main(\n", - " args=[\n", - " \"-f\",\n", - " (data_dir / \"large_tokamak_1_MFILE.DAT\").as_posix(),\n", - " \"-v\",\n", - " (data_dir / \"mfile_to_csv_vars.json\").as_posix(),\n", - " ]\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Clean up\n", - "temp_dir.cleanup()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## VaryRun\n", - "Vary iteration parameters until a feasible solution is found, using the `VaryRun` class." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "from process.main import VaryRun\n", - "\n", - "input_rel = script_dir / \"data/run_process.conf\"\n", - "temp_dir, temp_input_path, _ = copy_to_temp_dir(input_rel)\n", - "\n", - "# .conf file relies on a separate input file too; copy this as well\n", - "# TODO This double input file requirement needs to be removed\n", - "input_rel_2 = script_dir / \"data/large_tokamak_IN.DAT\"\n", - "copy(PROJ_DIR / input_rel_2, temp_dir.name)\n", - "\n", - "# VaryRun uses process_config.py, which changes the current working directory\n", - "# via os.chdir() to the temporary dir. Apart from being bad practice, once the\n", - "# temp dir is removed, this causes Path.cwd() (as used in plot_scans.py) to\n", - "# throw an exception when trying to return the (now deleted) CWD. Hence it\n", - "# needs to be set back after VaryRun()\n", - "# TODO Remove the os.chdir() from VaryRun\n", - "cwd = Path.cwd()\n", - "\n", - "vary_run = VaryRun(temp_input_path.as_posix())\n", - "vary_run.run()\n", - "os.chdir(cwd)\n", - "\n", - "temp_dir.cleanup()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "env", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} \ No newline at end of file + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Introduction to running PROCESS\n", + "\n", + "A Jupyter notebook to demonstrate usage of the `process` package.\n", + "\n", + "## Motivation\n", + "Process is moving away from being a runnable package with a single command-line entry-point to an importable package which can be scripted. This notebook is a good way of demonstrating the functionality of the package, and could provide a better way of working for modellers, who may wish to create their own scripts or notebooks for different tasks.\n", + "\n", + "## Setup\n", + "Currently the various classes and \"utilities\" scripts in Process have different interfaces and read and write files in differing manners and in locations that can't be easily controlled. To partially avoid the headaches associated with this, the code cell below defines a function to allow each example to be run in a temporary directory, much like a test. Input files are copied to this temporary directory and outputs contained there before the directory is removed.\n", + "\n", + "This temporary directory function is only required for running the examples below and removing any modifications afterwards, not in regular use of Process where the outputs will want to be preserved. Further development work will unify these disparate ways of running Process into a common Pythonic form." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n", + "\n", + "from pathlib import Path\n", + "from shutil import copy\n", + "from tempfile import TemporaryDirectory\n", + "\n", + "# Define project root dir; when running a notebook, the cwd is the dir the notebook is in\n", + "PROJ_DIR = Path.cwd().parent\n", + "\n", + "\n", + "def copy_to_temp_dir(input_rel):\n", + " \"\"\"Copy an input file to a new temp dir and return its new path.\n", + "\n", + " The new TemporaryDirectory object is returned to avoid destruction of the\n", + " object, which results in deletion of the directory prematurely. This way\n", + " the cleanup() method can be used to delete the directory when required.\n", + " :param input_rel: file path relative to project root dir\n", + " :type input_rel: str\n", + " :return: temporary dir and absolute path to file in temp dir\n", + " :rtype: (TemporaryDirectory, pathlib.Path)\n", + " \"\"\"\n", + " # Create temporary dir to contain the run's outputs\n", + " temp_dir = TemporaryDirectory()\n", + " temp_dir_path = Path(temp_dir.name)\n", + "\n", + " # Define absolute path for input file\n", + " input_rel_path = Path(input_rel)\n", + " input_abs_path = PROJ_DIR / input_rel_path\n", + "\n", + " try:\n", + " assert input_abs_path.exists()\n", + " except AssertionError as err:\n", + " raise FileNotFoundError(\"Input file doesn't exist.\") from err\n", + "\n", + " # Copy input file to temp dir\n", + " copy(input_abs_path, temp_dir_path)\n", + " temp_input_path = temp_dir_path / input_abs_path.name\n", + "\n", + " return temp_dir, temp_input_path, temp_dir_path" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "## Basic run of Process\n", + "Run Process on an input file using the `SingleRun` class. This outputs an `MFILE.DAT` and an `OUT.DAT`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from process.main import SingleRun, setup_loggers\n", + "\n", + "# setup the loggers so that the output is not spammed with model errors/warnings\n", + "setup_loggers()\n", + "\n", + "# Define input file name relative to project dir, then copy to temp dir\n", + "script_dir = Path(\"__file__\").parent.resolve()\n", + "input_rel = script_dir / \"data/large_tokamak_IN.DAT\"\n", + "\n", + "temp_dir, temp_input_path, temp_dir_path = copy_to_temp_dir(input_rel)\n", + "\n", + "# Run process on an input file in a temporary directory\n", + "single_run = SingleRun(temp_input_path.as_posix())\n", + "single_run.run()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Plot summary\n", + "Create a summary of the generated `MFILE.DAT` using `plot_proc`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from process.io import plot_proc\n", + "\n", + "# plot_proc uses command line arguments of the current process. Jupyter adds command line arguments under the hood causing plot_proc to fail. running plot proc in its own process isolates it from the jupyter command line arguments\n", + "# Use the --output-format argument \"none\" to not save outputs.\n", + "# Pdf and png output are also available\n", + "plot_proc.main(\n", + " args=[\"-f\", single_run.mfile_path.as_posix(), \"--output-format\", \"none\", \"--show\"]\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## View key output variables\n", + "Using the `MFILE` we generated by running the large tokamak scenario above, we have set some values on the `CostModel` instance and can print them." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import process.data_structure\n", + "\n", + "# Print some values on the CostModel instance\n", + "print(f\"Heat transport system: {process.data_structure.cost_variables.c226:.3e} M$\")\n", + "print(f\"Electrical plant equipment: {process.data_structure.cost_variables.c24:.3e} M$\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Convert to CSV format\n", + "This demonstrates how you would read from a PROCESS MFILE and write specified values into a csv using the `mfile_to_csv` function" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from process.io import mfile_to_csv\n", + "\n", + "data_dir = Path(\"data\")\n", + "\n", + "# mfile_to_csv requires two inputs:\n", + "# - path to the MFILE\n", + "# - .json containing the variable names to include in the csv file\n", + "\n", + "# This routine attempts to find every variable listed in the json file\n", + "# in the MFILE and writes the variable name, description and value\n", + "# to the output csv.\n", + "# Any listed variable that isn't in that MFILE will be skipped.\n", + "# The .csv file is saved to the directory of the input file\n", + "\n", + "mfile_to_csv.main(\n", + " args=[\n", + " \"-f\",\n", + " (data_dir / \"large_tokamak_1_MFILE.DAT\").as_posix(),\n", + " \"-v\",\n", + " (data_dir / \"mfile_to_csv_vars.json\").as_posix(),\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Clean up\n", + "temp_dir.cleanup()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## VaryRun\n", + "Vary iteration parameters until a feasible solution is found, using the `VaryRun` class." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from process.main import VaryRun\n", + "\n", + "input_rel = script_dir / \"data/run_process.conf\"\n", + "temp_dir, temp_input_path, _ = copy_to_temp_dir(input_rel)\n", + "\n", + "# .conf file relies on a separate input file too; copy this as well\n", + "# TODO This double input file requirement needs to be removed\n", + "input_rel_2 = script_dir / \"data/large_tokamak_IN.DAT\"\n", + "copy(PROJ_DIR / input_rel_2, temp_dir.name)\n", + "\n", + "# VaryRun uses process_config.py, which changes the current working directory\n", + "# via os.chdir() to the temporary dir. Apart from being bad practice, once the\n", + "# temp dir is removed, this causes Path.cwd() (as used in plot_scans.py) to\n", + "# throw an exception when trying to return the (now deleted) CWD. Hence it\n", + "# needs to be set back after VaryRun()\n", + "# TODO Remove the os.chdir() from VaryRun\n", + "cwd = Path.cwd()\n", + "\n", + "vary_run = VaryRun(temp_input_path.as_posix())\n", + "vary_run.run()\n", + "os.chdir(cwd)\n", + "\n", + "temp_dir.cleanup()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}