From 04f44a931cb1437bf0341e9d8c1599eb1e533fe4 Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Wed, 11 Jan 2023 22:15:58 -0800 Subject: [PATCH 001/111] beuler solver: Add Jacobian scaling If scale_rhs = true then the approximate Jacobian calculated by PETSc coloring is used to scale the time derivatives (RHS) by row scaling the Jacobian. This results in pseudo-timestepping, since every equation is effectively using a different timestep. --- src/solver/impls/snes/snes.cxx | 124 +++++++++++++++++++++++++++++++-- src/solver/impls/snes/snes.hxx | 10 +++ 2 files changed, 130 insertions(+), 4 deletions(-) diff --git a/src/solver/impls/snes/snes.cxx b/src/solver/impls/snes/snes.cxx index 724d84e1c2..9536750a4d 100644 --- a/src/solver/impls/snes/snes.cxx +++ b/src/solver/impls/snes/snes.cxx @@ -57,6 +57,32 @@ static PetscErrorCode snesPCapply(PC pc, Vec x, Vec y) { PetscFunctionReturn(s->precon(x, y)); } +/// +/// Input Parameters: +/// snes - nonlinear solver object +/// x1 - location at which to evaluate Jacobian +/// ctx - MatFDColoring context or NULL +/// +/// Output Parameters: +/// J - Jacobian matrix (not altered in this routine) +/// B - newly computed Jacobian matrix to use with preconditioner (generally the same as J) +PetscErrorCode SNESComputeJacobianScaledColor(SNES snes, Vec x1, Mat J, Mat B, void *ctx) { + PetscErrorCode err = SNESComputeJacobianDefaultColor(snes, x1, J, B, ctx); + + if ((err != 0) or (ctx == nullptr)) { + return err; + } + + // Get the the SNESSolver pointer from the function call context + SNESSolver* fctx; + err = MatFDColoringGetFunction(static_cast(ctx), nullptr, + reinterpret_cast(&fctx)); + CHKERRQ(err); + + // Call the SNESSolver function + return fctx->scaleJacobian(B); +} + SNESSolver::SNESSolver(Options* opts) : Solver(opts), timestep( @@ -119,7 +145,10 @@ SNESSolver::SNESSolver(Options* opts) .withDefault(50)), use_coloring((*options)["use_coloring"] .doc("Use matrix coloring to calculate Jacobian?") - .withDefault(true)) {} + .withDefault(true)), + scale_rhs((*options)["scale_rhs"] + .doc("Scale time derivatives?") + .withDefault(false)) {} int SNESSolver::init() { @@ -158,12 +187,26 @@ int SNESSolver::init() { if (equation_form == BoutSnesEquationForm::rearranged_backward_euler) { // Need an intermediate vector for rearranged Backward Euler - VecDuplicate(snes_x, &delta_x); + ierr = VecDuplicate(snes_x, &delta_x); + CHKERRQ(ierr); } if (predictor) { // Storage for previous solution - VecDuplicate(snes_x, &x1); + ierr = VecDuplicate(snes_x, &x1); + CHKERRQ(ierr); + } + + if (scale_rhs) { + // Storage for rhs factors, one per evolving variable + ierr = VecDuplicate(snes_x, &rhs_scaling_factors); + CHKERRQ(ierr); + // Set all factors to 1 to start with + ierr = VecSet(rhs_scaling_factors, 1.0); + CHKERRQ(ierr); + // Array to store inverse Jacobian row norms + ierr = VecDuplicate(snes_x, &jac_row_inv_norms); + CHKERRQ(ierr); } // Nonlinear solver interface (SNES) @@ -556,7 +599,7 @@ int SNESSolver::init() { MatFDColoringSetUp(Jmf, iscoloring, fdcoloring); ISColoringDestroy(&iscoloring); - SNESSetJacobian(snes, Jmf, Jmf, SNESComputeJacobianDefaultColor, fdcoloring); + SNESSetJacobian(snes, Jmf, Jmf, SNESComputeJacobianScaledColor, fdcoloring); } else { // Brute force calculation @@ -921,6 +964,12 @@ PetscErrorCode SNESSolver::snes_function(Vec x, Vec f, bool linear) { } }; + if (scale_rhs) { + // f <- f * rhs_scaling_factors + ierr = VecPointwiseMult(f, f, rhs_scaling_factors); + CHKERRQ(ierr); + } + return 0; } @@ -967,4 +1016,71 @@ PetscErrorCode SNESSolver::precon(Vec x, Vec f) { return 0; } +PetscErrorCode SNESSolver::scaleJacobian(Mat B) { + if (!scale_rhs) { + return 0; // Not scaling the RHS values + } + + int ierr; + + // Get index of rows owned by this processor + int rstart, rend; + MatGetOwnershipRange(B, &rstart, &rend); + + // Check that the vector has the same ownership range + int istart, iend; + VecGetOwnershipRange(jac_row_inv_norms, &istart, &iend); + if ((rstart != istart) or (rend != iend)) { + throw BoutException("Ownership ranges different: [{}, {}) and [{}, {})\n", rstart, rend, istart, iend); + } + + // Calculate the norm of each row of the Jacobian + PetscScalar *row_inv_norm_data; + ierr = VecGetArray(jac_row_inv_norms, &row_inv_norm_data); + CHKERRQ(ierr); + + PetscInt ncols; + const PetscScalar *vals; + for (int row = rstart; row < rend; row++) { + MatGetRow(B, row, &ncols, nullptr, &vals); + + // Calculate a norm of this row of the Jacobian + PetscScalar norm = 0.0; + for (int col = 0; col < ncols; col++) { + PetscScalar absval = std::abs(vals[col]); + if (absval > norm) { + norm = absval; + } + // Can we identify small elements and remove them? + // so we don't need to calculate them next time + } + + // Store in the vector as 1 / norm + row_inv_norm_data[row - rstart] = 1. / norm; + + MatRestoreRow(B, row, &ncols, nullptr, &vals); + } + + ierr = VecRestoreArray(jac_row_inv_norms, &row_inv_norm_data); + CHKERRQ(ierr); + + // Modify the RHS scaling: factor = factor / norm + ierr = VecPointwiseMult(rhs_scaling_factors, rhs_scaling_factors, jac_row_inv_norms); + CHKERRQ(ierr); + + if (diagnose) { + // Print maximum and minimum scaling factors + PetscReal max_scale, min_scale; + VecMax(rhs_scaling_factors, nullptr, &max_scale); + VecMin(rhs_scaling_factors, nullptr, &min_scale); + output.write("RHS scaling: {} -> {}\n", min_scale, max_scale); + } + + // Scale the Jacobian rows by multiplying on the left by 1/norm + ierr = MatDiagonalScale(B, jac_row_inv_norms, nullptr); + CHKERRQ(ierr); + + return 0; +} + #endif // BOUT_HAS_PETSC diff --git a/src/solver/impls/snes/snes.hxx b/src/solver/impls/snes/snes.hxx index 599b66fe14..1bb28cbc1f 100644 --- a/src/solver/impls/snes/snes.hxx +++ b/src/solver/impls/snes/snes.hxx @@ -84,6 +84,12 @@ public: /// @param[out] f The result of the operation PetscErrorCode precon(Vec x, Vec f); + /// Scale an approximate Jacobian, + /// and update the internal RHS scaling factors + /// This is called by SNESComputeJacobianScaledColor with the + /// finite difference approximated Jacobian. + PetscErrorCode scaleJacobian(Mat B); + private: BoutReal timestep; ///< Internal timestep BoutReal dt; ///< Current timestep used in snes_function @@ -131,6 +137,10 @@ private: bool matrix_free; ///< Use matrix free Jacobian int lag_jacobian; ///< Re-use Jacobian bool use_coloring; ///< Use matrix coloring + + bool scale_rhs; ///< Scale time derivatives? + Vec rhs_scaling_factors; ///< Factors to multiply RHS function + Vec jac_row_inv_norms; ///< 1 / Norm of the rows of the Jacobian }; #else From 225f892ad554a1fbe57a98d45633ae39b474c532 Mon Sep 17 00:00:00 2001 From: bendudson Date: Thu, 12 Jan 2023 06:26:42 +0000 Subject: [PATCH 002/111] Apply clang-format changes --- src/solver/impls/snes/snes.cxx | 21 ++++++++++++--------- src/solver/impls/snes/snes.hxx | 6 +++--- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/solver/impls/snes/snes.cxx b/src/solver/impls/snes/snes.cxx index 9536750a4d..74e5cc8a07 100644 --- a/src/solver/impls/snes/snes.cxx +++ b/src/solver/impls/snes/snes.cxx @@ -65,8 +65,10 @@ static PetscErrorCode snesPCapply(PC pc, Vec x, Vec y) { /// /// Output Parameters: /// J - Jacobian matrix (not altered in this routine) -/// B - newly computed Jacobian matrix to use with preconditioner (generally the same as J) -PetscErrorCode SNESComputeJacobianScaledColor(SNES snes, Vec x1, Mat J, Mat B, void *ctx) { +/// B - newly computed Jacobian matrix to use with preconditioner (generally the same as +/// J) +PetscErrorCode SNESComputeJacobianScaledColor(SNES snes, Vec x1, Mat J, Mat B, + void* ctx) { PetscErrorCode err = SNESComputeJacobianDefaultColor(snes, x1, J, B, ctx); if ((err != 0) or (ctx == nullptr)) { @@ -145,10 +147,10 @@ SNESSolver::SNESSolver(Options* opts) .withDefault(50)), use_coloring((*options)["use_coloring"] .doc("Use matrix coloring to calculate Jacobian?") - .withDefault(true)), + .withDefault(true)), scale_rhs((*options)["scale_rhs"] - .doc("Scale time derivatives?") - .withDefault(false)) {} + .doc("Scale time derivatives?") + .withDefault(false)) {} int SNESSolver::init() { @@ -1022,7 +1024,7 @@ PetscErrorCode SNESSolver::scaleJacobian(Mat B) { } int ierr; - + // Get index of rows owned by this processor int rstart, rend; MatGetOwnershipRange(B, &rstart, &rend); @@ -1031,16 +1033,17 @@ PetscErrorCode SNESSolver::scaleJacobian(Mat B) { int istart, iend; VecGetOwnershipRange(jac_row_inv_norms, &istart, &iend); if ((rstart != istart) or (rend != iend)) { - throw BoutException("Ownership ranges different: [{}, {}) and [{}, {})\n", rstart, rend, istart, iend); + throw BoutException("Ownership ranges different: [{}, {}) and [{}, {})\n", rstart, + rend, istart, iend); } // Calculate the norm of each row of the Jacobian - PetscScalar *row_inv_norm_data; + PetscScalar* row_inv_norm_data; ierr = VecGetArray(jac_row_inv_norms, &row_inv_norm_data); CHKERRQ(ierr); PetscInt ncols; - const PetscScalar *vals; + const PetscScalar* vals; for (int row = rstart; row < rend; row++) { MatGetRow(B, row, &ncols, nullptr, &vals); diff --git a/src/solver/impls/snes/snes.hxx b/src/solver/impls/snes/snes.hxx index 1bb28cbc1f..4366f7169b 100644 --- a/src/solver/impls/snes/snes.hxx +++ b/src/solver/impls/snes/snes.hxx @@ -138,9 +138,9 @@ private: int lag_jacobian; ///< Re-use Jacobian bool use_coloring; ///< Use matrix coloring - bool scale_rhs; ///< Scale time derivatives? - Vec rhs_scaling_factors; ///< Factors to multiply RHS function - Vec jac_row_inv_norms; ///< 1 / Norm of the rows of the Jacobian + bool scale_rhs; ///< Scale time derivatives? + Vec rhs_scaling_factors; ///< Factors to multiply RHS function + Vec jac_row_inv_norms; ///< 1 / Norm of the rows of the Jacobian }; #else From ada60a787526ba260ac70917873008a1d5796032 Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Fri, 13 Jan 2023 14:29:32 -0800 Subject: [PATCH 003/111] Fix for PETSc >= 3.18 Removes function PCGAMGSetSymGraph, so symmetry should be set though matrix options. --- src/invert/laplace/impls/petsc3damg/petsc3damg.cxx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/invert/laplace/impls/petsc3damg/petsc3damg.cxx b/src/invert/laplace/impls/petsc3damg/petsc3damg.cxx index 210e41cbb3..ce9239d74b 100644 --- a/src/invert/laplace/impls/petsc3damg/petsc3damg.cxx +++ b/src/invert/laplace/impls/petsc3damg/petsc3damg.cxx @@ -508,7 +508,10 @@ void LaplacePetsc3dAmg::updateMatrix3D() { // Set the relative and absolute tolerances PCSetType(pc, pctype.c_str()); +#if !PETSC_VERSION_GE(3, 18, 0) + // This was removed in 3.18 PCGAMGSetSymGraph(pc, PETSC_TRUE); +#endif } lib.setOptionsFromInputFile(ksp); From 46fcf889e1d091eb3b4db2216f4628a8595edf27 Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Fri, 13 Jan 2023 14:33:15 -0800 Subject: [PATCH 004/111] beuler: Fix index calculation in boundaries If nproc > 1 then the local index offset should only be added to points inside the domain. --- src/solver/impls/snes/snes.cxx | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/solver/impls/snes/snes.cxx b/src/solver/impls/snes/snes.cxx index 74e5cc8a07..480ca2e63c 100644 --- a/src/solver/impls/snes/snes.cxx +++ b/src/solver/impls/snes/snes.cxx @@ -70,6 +70,7 @@ static PetscErrorCode snesPCapply(PC pc, Vec x, Vec y) { PetscErrorCode SNESComputeJacobianScaledColor(SNES snes, Vec x1, Mat J, Mat B, void* ctx) { PetscErrorCode err = SNESComputeJacobianDefaultColor(snes, x1, J, B, ctx); + CHKERRQ(err); if ((err != 0) or (ctx == nullptr)) { return err; @@ -266,24 +267,23 @@ int SNESSolver::init() { output_progress.write("Setting Jacobian matrix sizes\n"); - int localN = getLocalN(); // Number of rows on this processor int n2d = f2d.size(); int n3d = f3d.size(); - // Set size of Matrix on each processor to localN x localN + // Set size of Matrix on each processor to nlocal x nlocal MatCreate(BoutComm::get(), &Jmf); - MatSetSizes(Jmf, localN, localN, PETSC_DETERMINE, PETSC_DETERMINE); + MatSetSizes(Jmf, nlocal, nlocal, PETSC_DETERMINE, PETSC_DETERMINE); MatSetFromOptions(Jmf); - std::vector d_nnz(localN); - std::vector o_nnz(localN); + std::vector d_nnz(nlocal); + std::vector o_nnz(nlocal); // Set values for most points const auto star_pattern_2d = 5 * (n3d + n2d); const auto star_pattern_3d = 7 * n3d + 5 * n2d; const auto star_pattern = (mesh->LocalNz > 1) ? star_pattern_3d : star_pattern_2d; - for (int i = 0; i < localN; i++) { + for (int i = 0; i < nlocal; i++) { // Non-zero elements on this processor d_nnz[i] = star_pattern; // Non-zero elements on neighboring processor @@ -296,7 +296,7 @@ int SNESSolver::init() { for (int y = mesh->ystart; y <= mesh->yend; y++) { for (int z = 0; z < mesh->LocalNz; z++) { int localIndex = ROUND(index(mesh->xstart, y, z)); - ASSERT2((localIndex >= 0) && (localIndex < localN)); + ASSERT2((localIndex >= 0) && (localIndex < nlocal)); const int num_fields = (z == 0) ? n2d + n3d : n3d; for (int i = 0; i < num_fields; i++) { d_nnz[localIndex + i] -= (n3d + n2d); @@ -308,7 +308,7 @@ int SNESSolver::init() { for (int y = mesh->ystart; y <= mesh->yend; y++) { for (int z = 0; z < mesh->LocalNz; z++) { int localIndex = ROUND(index(mesh->xstart, y, z)); - ASSERT2((localIndex >= 0) && (localIndex < localN)); + ASSERT2((localIndex >= 0) && (localIndex < nlocal)); const int num_fields = (z == 0) ? n2d + n3d : n3d; for (int i = 0; i < num_fields; i++) { d_nnz[localIndex + i] -= (n3d + n2d); @@ -323,7 +323,7 @@ int SNESSolver::init() { for (int y = mesh->ystart; y <= mesh->yend; y++) { for (int z = 0; z < mesh->LocalNz; z++) { int localIndex = ROUND(index(mesh->xend, y, z)); - ASSERT2((localIndex >= 0) && (localIndex < localN)); + ASSERT2((localIndex >= 0) && (localIndex < nlocal)); const int num_fields = (z == 0) ? n2d + n3d : n3d; for (int i = 0; i < num_fields; i++) { d_nnz[localIndex + i] -= (n3d + n2d); @@ -335,7 +335,7 @@ int SNESSolver::init() { for (int y = mesh->ystart; y <= mesh->yend; y++) { for (int z = 0; z < mesh->LocalNz; z++) { int localIndex = ROUND(index(mesh->xend, y, z)); - ASSERT2((localIndex >= 0) && (localIndex < localN)); + ASSERT2((localIndex >= 0) && (localIndex < nlocal)); const int num_fields = (z == 0) ? n2d + n3d : n3d; for (int i = 0; i < num_fields; i++) { d_nnz[localIndex + i] -= (n3d + n2d); @@ -448,7 +448,10 @@ int SNESSolver::init() { MatGetOwnershipRange(Jmf, &Istart, &Iend); // Convert local into global indices - index += Istart; + // Note: Omit boundary cells since those are set to -1 to indicate boundaries + BOUT_FOR(i, index.getRegion("RGN_NOBNDRY")) { + index[i] += Istart; + } // Now communicate to fill guard cells mesh->communicate(index); From fc80bdc8fabb232cfa28ac2b0cc433cc8dfcc9ee Mon Sep 17 00:00:00 2001 From: bendudson Date: Fri, 13 Jan 2023 22:37:27 +0000 Subject: [PATCH 005/111] Apply clang-format changes --- src/solver/impls/snes/snes.cxx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/solver/impls/snes/snes.cxx b/src/solver/impls/snes/snes.cxx index 480ca2e63c..9e9fa0e6a7 100644 --- a/src/solver/impls/snes/snes.cxx +++ b/src/solver/impls/snes/snes.cxx @@ -449,9 +449,7 @@ int SNESSolver::init() { // Convert local into global indices // Note: Omit boundary cells since those are set to -1 to indicate boundaries - BOUT_FOR(i, index.getRegion("RGN_NOBNDRY")) { - index[i] += Istart; - } + BOUT_FOR(i, index.getRegion("RGN_NOBNDRY")) { index[i] += Istart; } // Now communicate to fill guard cells mesh->communicate(index); From 580e784040c1a6d721f97eaca974e89ec8adbf01 Mon Sep 17 00:00:00 2001 From: David Bold Date: Fri, 17 Feb 2023 13:54:07 +0100 Subject: [PATCH 006/111] make executable dir optional --- tests/integrated/test-squash/runtest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integrated/test-squash/runtest b/tests/integrated/test-squash/runtest index c00dfb2a42..574dd23fa8 100755 --- a/tests/integrated/test-squash/runtest +++ b/tests/integrated/test-squash/runtest @@ -77,7 +77,7 @@ def verify(f1, f2): parser = argparse.ArgumentParser(description="Test the bout-squashoutput wrapper") parser.add_argument( - "executable", help="Path to bout-squashoutput", default="../../../bin" + "executable", help="Path to bout-squashoutput", default="../../../bin", nargs="?" ) args = parser.parse_args() From 8b76a8f36bd1b7f7b0fee7c1f4b1fdf804834a45 Mon Sep 17 00:00:00 2001 From: David Bold Date: Fri, 17 Feb 2023 13:54:30 +0100 Subject: [PATCH 007/111] Fall back to using $PATH to find bout-squashoutput --- tests/integrated/test-squash/runtest | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integrated/test-squash/runtest b/tests/integrated/test-squash/runtest index 574dd23fa8..4fd1124679 100755 --- a/tests/integrated/test-squash/runtest +++ b/tests/integrated/test-squash/runtest @@ -7,6 +7,7 @@ import numpy as np from boututils.run_wrapper import launch_safe, shell_safe, build_and_log import argparse import re +import os.path # requires: all_tests @@ -85,6 +86,9 @@ build_and_log("Squash test") bout_squashoutput = args.executable + "/bout-squashoutput" +if not os.path.exists(bout_squashoutput): + bout_squashoutput = "bout-squashoutput" + print("Run once to get normal data") timed_shell_safe("./squash -q -q -q nout=2") timed_shell_safe("mv data/BOUT.dmp.0.nc f1.nc") From 8cdba94420d1d5a6b4d80d8a97286536444de966 Mon Sep 17 00:00:00 2001 From: David Bold Date: Fri, 17 Feb 2023 14:00:40 +0100 Subject: [PATCH 008/111] Error early with in-source-builds * Previously the second cmake invocation failed, e.g. after some source changes or after changing a configure time option * It also generated many files, some of which overwrite git-tracked files from autotools, and others where not git-ignored. --- CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index d4e8e73be4..46820025eb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,6 +15,10 @@ if(${CMAKE_VERSION} VERSION_GREATER_EQUAL 3.22) cmake_policy(SET CMP0127 NEW) endif() +if ("${CMAKE_CURRENT_BINARY_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") + message(FATAL_ERROR "BOUT++ does not currently supports in source buildds. Try building out of source, e.g. with `cmake -S . -B build`") +endif() + # CMake currently doesn't support proper semver # Set the version here, strip any extra tags to use in `project` # We try to use git to get a full description, inspired by setuptools_scm From 36cabdfe53644cdf8eeec7a61ea3cfb224ce6655 Mon Sep 17 00:00:00 2001 From: David Bold Date: Mon, 20 Feb 2023 09:07:44 +0100 Subject: [PATCH 009/111] Declare variable as unused to avoid warning --- include/bout/output.hxx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/bout/output.hxx b/include/bout/output.hxx index bb202cc8df..2f0d66811c 100644 --- a/include/bout/output.hxx +++ b/include/bout/output.hxx @@ -140,8 +140,8 @@ public: void write(const S&, const Args&...){}; template void print(const S&, const Args&...){}; - void write(const std::string& s) override{}; - void print(const std::string& s) override{}; + void write(const std::string& UNUSED(s)) override{}; + void print(const std::string& UNUSED(s)) override{}; void enable() override{}; void disable() override{}; void enable(MAYBE_UNUSED(bool enable)){}; From 2af404d3e99d21c76381c35e49c7b47755a58b15 Mon Sep 17 00:00:00 2001 From: David Bold Date: Mon, 20 Feb 2023 09:23:35 +0100 Subject: [PATCH 010/111] Avoid no-return warning in pvode --- externalpackages/PVODE/source/cvode.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/externalpackages/PVODE/source/cvode.cpp b/externalpackages/PVODE/source/cvode.cpp index e28a3b1ae6..065aa93595 100644 --- a/externalpackages/PVODE/source/cvode.cpp +++ b/externalpackages/PVODE/source/cvode.cpp @@ -1826,6 +1826,8 @@ static int CVnls(CVodeMem cv_mem, int nflag) case FUNCTIONAL : return(CVnlsFunctional(cv_mem)); case NEWTON : return(CVnlsNewton(cv_mem, nflag)); } + fprintf(errfp, "Should be unreachable ..."); + return(ERR_FAILURE); } /***************** CVnlsFunctional ******************************** @@ -2392,6 +2394,8 @@ static int CVHandleFailure(CVodeMem cv_mem, int kflag) case SOLVE_FAILED: fprintf(errfp, MSG_SOLVE_FAILED, tn); return(SOLVE_FAILURE); } + fprintf(errfp, "Should be unreachable ..."); + return(ERR_FAILURE); } /*******************************************************************/ From e593a9960265a8a59901ddc075bd5ebaffb3e7f3 Mon Sep 17 00:00:00 2001 From: David Bold Date: Mon, 20 Feb 2023 09:24:16 +0100 Subject: [PATCH 011/111] Avoid PVEC_REAL_MPI_TYPE redifinition warning --- externalpackages/PVODE/include/pvode/nvector.h | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/externalpackages/PVODE/include/pvode/nvector.h b/externalpackages/PVODE/include/pvode/nvector.h index 8232995847..c696b185d3 100644 --- a/externalpackages/PVODE/include/pvode/nvector.h +++ b/externalpackages/PVODE/include/pvode/nvector.h @@ -88,12 +88,23 @@ namespace pvode { /* Set types real and integer for MPI calls. */ - +#ifndef PVEC_REAL_MPI_TYPE #if (LLNL_DOUBLE == 1) #define PVEC_REAL_MPI_TYPE MPI_DOUBLE #else #define PVEC_REAL_MPI_TYPE MPI_FLOAT #endif +#else +#if (LLNL_DOUBLE == 1) +#if PVEC_REAL_MPI_TYPE != MPI_DOUBLE +#error PVEC_REAL_MPI_TYPE changed +#endif +#else +#if PVEC_REAL_MPI_TYPE != MPI_FLOAT +#error PVEC_REAL_MPI_TYPE changed +#endif +#endif +#endif // Ugh, potentionanlly bad but need to do this for now when compiling with both SUNDIALS and PVODE #ifndef PVEC_INTEGER_MPI_TYPE From 725bb74567adf5a39177ff153f8695e46ee74d40 Mon Sep 17 00:00:00 2001 From: David Bold Date: Mon, 20 Feb 2023 09:24:28 +0100 Subject: [PATCH 012/111] Avoid iter not used warning --- src/bout++.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bout++.cxx b/src/bout++.cxx index e5788230fb..52d8927edb 100644 --- a/src/bout++.cxx +++ b/src/bout++.cxx @@ -813,7 +813,7 @@ BoutMonitor::BoutMonitor(BoutReal timestep, Options& options) .doc(_("Name of file whose existence triggers a stop")) .withDefault("BOUT.stop"))) {} -int BoutMonitor::call(Solver* solver, BoutReal t, int iter, int NOUT) { +int BoutMonitor::call(Solver* solver, BoutReal t, MAYBE_UNUSED(int iter), int NOUT) { TRACE("BoutMonitor::call({:e}, {:d}, {:d})", t, iter, NOUT); // Increment Solver's iteration counter, and set the global `iteration` From a057a65b822c83121ecbd916ab7abe4272946502 Mon Sep 17 00:00:00 2001 From: dschwoerer Date: Mon, 20 Feb 2023 08:25:07 +0000 Subject: [PATCH 013/111] Apply clang-format changes --- externalpackages/PVODE/source/cvode.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/externalpackages/PVODE/source/cvode.cpp b/externalpackages/PVODE/source/cvode.cpp index 065aa93595..03d9f0b130 100644 --- a/externalpackages/PVODE/source/cvode.cpp +++ b/externalpackages/PVODE/source/cvode.cpp @@ -1827,7 +1827,7 @@ static int CVnls(CVodeMem cv_mem, int nflag) case NEWTON : return(CVnlsNewton(cv_mem, nflag)); } fprintf(errfp, "Should be unreachable ..."); - return(ERR_FAILURE); + return (ERR_FAILURE); } /***************** CVnlsFunctional ******************************** @@ -2395,7 +2395,7 @@ static int CVHandleFailure(CVodeMem cv_mem, int kflag) return(SOLVE_FAILURE); } fprintf(errfp, "Should be unreachable ..."); - return(ERR_FAILURE); + return (ERR_FAILURE); } /*******************************************************************/ From 668de4d371dbb7d1203542e71a3a1ebc25c0adbc Mon Sep 17 00:00:00 2001 From: David Bold Date: Mon, 20 Feb 2023 09:29:04 +0100 Subject: [PATCH 014/111] Allow in-source-builds, but recommend against --- CMakeLists.txt | 5 ++++- manual/sphinx/conf.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 46820025eb..b6548f60cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,10 @@ if(${CMAKE_VERSION} VERSION_GREATER_EQUAL 3.22) endif() if ("${CMAKE_CURRENT_BINARY_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") - message(FATAL_ERROR "BOUT++ does not currently supports in source buildds. Try building out of source, e.g. with `cmake -S . -B build`") + option(BOUT_ALLOW_INSOURCE_BUILD "Whether BOUT++ should really allow to build in source." OFF) + if (NOT ${BOUT_ALLOW_INSOURCE_BUILD}) + message(FATAL_ERROR "BOUT++ does not recommends in source builds. Try building out of source, e.g. with `cmake -S . -B build` or set -DBOUT_ALLOW_INSOURCE_BUILD=ON - but thinks may break!") + endif() endif() # CMake currently doesn't support proper semver diff --git a/manual/sphinx/conf.py b/manual/sphinx/conf.py index a643774553..40facef6e5 100755 --- a/manual/sphinx/conf.py +++ b/manual/sphinx/conf.py @@ -86,6 +86,7 @@ def __getattr__(cls, name): + " -DBOUT_ENABLE_PYTHON=ON" + " -DBOUT_UPDATE_GIT_SUBMODULE=OFF" + " -DBOUT_TESTS=OFF" + + " -DBOUT_ALLOW_INSOURCE_BUILD=ON" + f" -DPython_ROOT_DIR={pydir}" + f" -Dmpark_variant_DIR={pwd}/externalpackages/mpark.variant/" + f" -Dfmt_DIR={pwd}/externalpackages/fmt/" From 636eb567c33a54ec9635355f5995ba2e98dbead5 Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 22 Feb 2023 11:12:29 +0100 Subject: [PATCH 015/111] Fix: highlight code --- manual/sphinx/user_docs/installing.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/manual/sphinx/user_docs/installing.rst b/manual/sphinx/user_docs/installing.rst index 2141e67c0e..260909c8ba 100644 --- a/manual/sphinx/user_docs/installing.rst +++ b/manual/sphinx/user_docs/installing.rst @@ -443,27 +443,27 @@ configuration:: If not, see :ref:`sec-advancedinstall` for some things you can try to resolve common problems. -Working with an active `conda` environment +Working with an active ``conda`` environment ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -When `conda` is used, it installs separate versions of several libraries. These +When ``conda`` is used, it installs separate versions of several libraries. These can cause warnings or even failures when linking BOUT++ executables. There are several alternatives to deal with this problem: -* The simplest but least convenient option is to use `conda deactivate` before +* The simplest but least convenient option is to use ``conda deactivate`` before configuring, compiling, or running any BOUT++ program. * You might sometimes want to link to the conda-installed libraries. This is probably not ideal for production runs on an HPC system (as conda downloads binary packages that will not be optimized for specific hardware), but can be a simple way to get packages for testing or on a personal computer. In this - case just keep your `conda` environment active, and with luck the libraries + case just keep your ``conda`` environment active, and with luck the libraries should be picked up by the standard search mechanisms. * In case you do want a fully optimized and as-stable-as-possible build for production runs, it is probably best not to depend on any conda packages for - compiling or running BOUT++ executables (restrict `conda` to providing Python + compiling or running BOUT++ executables (restrict ``conda`` to providing Python packages for post-processing, and their dependencies). Passing - `-DBOUT_IGNORE_CONDA_ENV=ON` (default `OFF`) excludes anything in the conda + ``-DBOUT_IGNORE_CONDA_ENV=ON`` (default ``OFF``) excludes anything in the conda environment from CMake search paths. This should totally separate BOUT++ from - the `conda` environment. + the ``conda`` environment. .. _sec-config-nls: From ba77dd7e1db7f27740c179010c96c0f2d15b22ac Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 22 Feb 2023 11:48:02 +0100 Subject: [PATCH 016/111] Fix: all boutpp members are imported --- manual/sphinx/_apidoc/boutpp.rst | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/manual/sphinx/_apidoc/boutpp.rst b/manual/sphinx/_apidoc/boutpp.rst index d28d7c7d18..71a092ca17 100644 --- a/manual/sphinx/_apidoc/boutpp.rst +++ b/manual/sphinx/_apidoc/boutpp.rst @@ -13,16 +13,4 @@ Module contents .. automodule:: boutpp :members: - -Undocumented functions ----------------------- - -.. automodule:: boutpp - :undoc-members: - -Special and inherited functions -------------------------------- - -.. automodule:: boutpp - :special-members: - :inherited-members: + :imported-members: From fd3503044ee463b668e56a7a02b3cf54c826b3fa Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 22 Feb 2023 11:48:25 +0100 Subject: [PATCH 017/111] Fix more references that should be code --- manual/sphinx/developer_docs/file_io.rst | 2 +- tools/pylib/_boutpp_build/boutpp.pyx.jinja | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/manual/sphinx/developer_docs/file_io.rst b/manual/sphinx/developer_docs/file_io.rst index ad44c97277..aece8cfb1e 100644 --- a/manual/sphinx/developer_docs/file_io.rst +++ b/manual/sphinx/developer_docs/file_io.rst @@ -92,7 +92,7 @@ similar interface to ``Datafile``, and just like ``Datafile`` it also stores pointers to variables. This means that it still suffers from all of the downsides of ``Datafile``, and developers are encouraged to move to the ``outputVars`` approach. The `SAVE_ONCE`/`SAVE_REPEAT` -macros also work through `DataFileFacade`` -- this means that they +macros also work through ``DataFileFacade`` -- this means that they cannot be used outside of ``PhysicsModel`` methods! diff --git a/tools/pylib/_boutpp_build/boutpp.pyx.jinja b/tools/pylib/_boutpp_build/boutpp.pyx.jinja index 0d8cef173a..657e2f28c1 100644 --- a/tools/pylib/_boutpp_build/boutpp.pyx.jinja +++ b/tools/pylib/_boutpp_build/boutpp.pyx.jinja @@ -1500,7 +1500,7 @@ def interp_to(Field3D f3d,location): def setOption(name, value, source="PyInterface", force=False): """ Set an option in the global Options tree. Prefer - `Options.set` to avoid unexpected results if several Option + ``Options.set`` to avoid unexpected results if several Option roots are avalaible. Parameters From 87fa772de63d6949b0ee1e3dc70a386b2478d1f7 Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 22 Feb 2023 11:49:01 +0100 Subject: [PATCH 018/111] Fix: code needs to be separated by an empty line --- manual/sphinx/user_docs/advanced_install.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/manual/sphinx/user_docs/advanced_install.rst b/manual/sphinx/user_docs/advanced_install.rst index 2f816c4e0c..5e230b8ad8 100644 --- a/manual/sphinx/user_docs/advanced_install.rst +++ b/manual/sphinx/user_docs/advanced_install.rst @@ -142,32 +142,38 @@ Cori First set up the environment by loading the correct modules. For Bash shell use: .. code-block:: bash + source config/cori/setup-env-cgpu.sh and for C shell: .. code-block:: csh + source config/cori/setup-env-cgpu.sh Then configure BOUT++ by running a script which calls CMake. Under bash: .. code-block:: bash + ./config/cori/config-bout-cgpu.sh and C shell: .. code-block:: csh + ./config/cori/config-bout-cgpu.csh At the time of writing, Hypre linking is not working with CUDA. If you come across errors with the above configuration, try turning off Hypre support: .. code-block:: bash + ./config/cori/config-bout-cgpu-nohypre.sh or .. code-block:: csh + ./config/cori/config-bout-cgpu-nohypre.csh See section :ref:`sec-gpusupport` for details of compiling and running From 88956a0f64b3eea105fdb07210a39d8d6671dca5 Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 22 Feb 2023 11:49:21 +0100 Subject: [PATCH 019/111] RTD: Add more modules for mock --- manual/sphinx/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/manual/sphinx/conf.py b/manual/sphinx/conf.py index a643774553..6f943f0765 100755 --- a/manual/sphinx/conf.py +++ b/manual/sphinx/conf.py @@ -67,6 +67,8 @@ def __getattr__(cls, name): "scipy.ndimage.filters", "scipy.ndimage.morphology", "scipy.spatial", + "past", + "crosslines", ] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) print(os.environ) From e3ce78bcb7edcb91db9c2669b905ef5261b62eef Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Wed, 22 Feb 2023 11:26:05 +0000 Subject: [PATCH 020/111] Fix some warnings in output header --- include/bout/output.hxx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/bout/output.hxx b/include/bout/output.hxx index bb202cc8df..ef09aa7ee5 100644 --- a/include/bout/output.hxx +++ b/include/bout/output.hxx @@ -137,11 +137,11 @@ private: class DummyOutput : public Output { public: template - void write(const S&, const Args&...){}; + void write([[maybe_unused]] const S& format, [[maybe_unused]] const Args&... args){}; template - void print(const S&, const Args&...){}; - void write(const std::string& s) override{}; - void print(const std::string& s) override{}; + void print([[maybe_unused]] const S& format, [[maybe_unused]] const Args&... args){}; + void write([[maybe_unused]] const std::string& message) override{}; + void print([[maybe_unused]] const std::string& message) override{}; void enable() override{}; void disable() override{}; void enable(MAYBE_UNUSED(bool enable)){}; From d60a5a224982cb6d6bf5fc48f40dd6195b3f32f0 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Wed, 22 Feb 2023 11:27:42 +0000 Subject: [PATCH 021/111] Fix deprecated header includes --- src/invert/pardiv/impls/cyclic/pardiv_cyclic.cxx | 15 +++++++-------- src/invert/pardiv/impls/cyclic/pardiv_cyclic.hxx | 6 +++--- .../test-bout-override-default-option.cxx | 2 +- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/invert/pardiv/impls/cyclic/pardiv_cyclic.cxx b/src/invert/pardiv/impls/cyclic/pardiv_cyclic.cxx index 32cfea8306..c17e5af64d 100644 --- a/src/invert/pardiv/impls/cyclic/pardiv_cyclic.cxx +++ b/src/invert/pardiv/impls/cyclic/pardiv_cyclic.cxx @@ -40,16 +40,15 @@ #if not BOUT_USE_METRIC_3D +#include #include -#include -#include -#include -#include -#include -#include -#include - +#include +#include +#include +#include +#include #include +#include #include diff --git a/src/invert/pardiv/impls/cyclic/pardiv_cyclic.hxx b/src/invert/pardiv/impls/cyclic/pardiv_cyclic.hxx index 2ce4f1acc9..fdcdab055e 100644 --- a/src/invert/pardiv/impls/cyclic/pardiv_cyclic.hxx +++ b/src/invert/pardiv/impls/cyclic/pardiv_cyclic.hxx @@ -52,9 +52,9 @@ RegisterUnavailableInvertParDiv registerinvertpardivcyclic{ #else -#include "dcomplex.hxx" -#include "utils.hxx" -#include +#include +#include +#include class InvertParDivCR : public InvertParDiv { public: diff --git a/tests/integrated/test-bout-override-default-option/test-bout-override-default-option.cxx b/tests/integrated/test-bout-override-default-option/test-bout-override-default-option.cxx index 24cb6bc0b2..4c50579c47 100644 --- a/tests/integrated/test-bout-override-default-option/test-bout-override-default-option.cxx +++ b/tests/integrated/test-bout-override-default-option/test-bout-override-default-option.cxx @@ -1,4 +1,4 @@ -#include +#include // Use an integrated test for what is effectively a unit test of the // BOUT_OVERRIDE_DEFAULT_OPTION() macro because the functionality relies on the state of From 4c05600e4275c4e48bac02c8263b08ad81ab771c Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Wed, 22 Feb 2023 13:58:59 +0000 Subject: [PATCH 022/111] Some fixes for tokamak-2fluid example Not comprehensive, still fails to run successfully on basic input file (zero pivot in cyclic reduce) Fixes: - unused variables - wrong variable names (including case) - creating laplacians from wrong input sections - some issues with staggering --- examples/tokamak-2fluid/2fluid.cxx | 158 +++++++++++++++----------- examples/tokamak-2fluid/data/BOUT.inp | 10 -- 2 files changed, 89 insertions(+), 79 deletions(-) diff --git a/examples/tokamak-2fluid/2fluid.cxx b/examples/tokamak-2fluid/2fluid.cxx index 9108edb5bd..323a34df4c 100644 --- a/examples/tokamak-2fluid/2fluid.cxx +++ b/examples/tokamak-2fluid/2fluid.cxx @@ -4,6 +4,7 @@ * for cross-benchmarking etc. *******************************************************************************/ +#include "bout/bout_types.hxx" #include #include @@ -98,6 +99,9 @@ class TwoFluid : public PhysicsModel { std::unique_ptr aparSolver; Field2D acoef; // Coefficient in the Helmholtz equation + /// Location of possibly staggered fields + CELL_LOC stagger_loc = CELL_LOC::deflt; + int init(bool UNUSED(restarting)) override { TRACE("int init(bool) "); @@ -154,15 +158,15 @@ class TwoFluid : public PhysicsModel { // READ OPTIONS // Read some parameters - auto globalOptions = Options::root(); - auto options = globalOptions["2fluid"]; + auto& globalOptions = Options::root(); + auto& options = globalOptions["2fluid"]; AA = options["AA"].withDefault(2.0); ZZ = options["ZZ"].withDefault(1.0); estatic = options["estatic"].withDefault(false); ZeroElMass = options["ZeroElMass"].withDefault(false); - zeff = options["zeff"].withDefault(1.0); + zeff = options["Zeff"].withDefault(1.0); nu_perp = options["nu_perp"].withDefault(0.0); ShearFactor = options["ShearFactor"].withDefault(1.0); OhmPe = options["OhmPe"].withDefault(true); @@ -170,8 +174,7 @@ class TwoFluid : public PhysicsModel { curv_upwind = options["curv_upwind"].withDefault(false); // Choose method to use for Poisson bracket advection terms - int bracket_method; - bracket_method = options["bracket_method"].withDefault(0); + int bracket_method = options["bracket_method"].withDefault(0); switch (bracket_method) { case 0: { bm = BRACKET_STD; @@ -206,79 +209,92 @@ class TwoFluid : public PhysicsModel { laplace_extra_rho_term = options["laplace_extra_rho_term"].withDefault(false); vort_include_pi = options["vort_include_pi"].withDefault(false); - lowPass_z = options["lowPass_z"].withDefault(-1); + lowPass_z = options["low_pass_z"].withDefault(-1); + + //////////////////////////////////////////////////////// + // Equation terms + + auto& ni_options = globalOptions["Ni"]; + ni_options.setConditionallyUsed(); + evolve_ni = ni_options["evolve"].withDefault(true); + + auto& rho_options = globalOptions["rho"]; + rho_options.setConditionallyUsed(); + evolve_rho = rho_options["evolve"].withDefault(true); + + auto& vi_options = globalOptions["Vi"]; + vi_options.setConditionallyUsed(); + evolve_vi = vi_options["evolve"].withDefault(true); + + auto& te_options = globalOptions["Te"]; + te_options.setConditionallyUsed(); + evolve_ti = te_options["evolve"].withDefault(true); - evolve_ni = globalOptions["Ni"]["evolve"].withDefault(true); - evolve_rho = globalOptions["rho"]["evolve"].withDefault(true); - evolve_vi = globalOptions["vi"]["evolve"].withDefault(true); - evolve_ti = globalOptions["ti"]["evolve"].withDefault(true); - evolve_ajpar = globalOptions["Ajpar"]["evolve"].withDefault(true); + auto& ti_options = globalOptions["Ti"]; + ti_options.setConditionallyUsed(); + evolve_ti = ti_options["evolve"].withDefault(true); + + auto& ajpar_options = globalOptions["Ajpar"]; + ajpar_options.setConditionallyUsed(); + evolve_ajpar = ajpar_options["evolve"].withDefault(true); if (ZeroElMass) { - evolve_ajpar = 0; // Don't need ajpar - calculated from ohm's law + evolve_ajpar = false; // Don't need ajpar - calculated from ohm's law } - //////////////////////////////////////////////////////// - // Equation terms - if (evolve_ni) { - Options* options = &globalOptions["Ni"]; - options->get("ni1_phi0", ni_ni1_phi0, false); - options->get("ni0_phi1", ni_ni0_phi1, false); - options->get("ni1_phi1", ni_ni1_phi1, false); - options->get("nit_phit", ni_nit_phit, false); - options->get("vi1_ni0", ni_vi1_ni0, false); - options->get("vi0_ni1", ni_vi0_ni1, false); - options->get("vi1_ni1", ni_vi1_ni1, false); - options->get("vit_nit", ni_vit_nit, false); - options->get("jpar1", ni_jpar1, false); - options->get("pe1", ni_pe1, false); - options->get("ni0_curv_phi1", ni_ni0_curv_phi1, false); - options->get("ni1_curv_phi0", ni_ni1_curv_phi0, false); - options->get("ni1_curv_phi1", ni_ni1_curv_phi1, false); - options->get("nit_curv_phit", ni_nit_curv_phit, false); + ni_ni1_phi0 = ni_options["ni1_phi0"].withDefault(false); + ni_ni0_phi1 = ni_options["ni0_phi1"].withDefault(false); + ni_ni1_phi1 = ni_options["ni1_phi1"].withDefault(false); + ni_nit_phit = ni_options["nit_phit"].withDefault(false); + ni_vi1_ni0 = ni_options["vi1_ni0"].withDefault(false); + ni_vi0_ni1 = ni_options["vi0_ni1"].withDefault(false); + ni_vi1_ni1 = ni_options["vi1_ni1"].withDefault(false); + ni_vit_nit = ni_options["vit_nit"].withDefault(false); + ni_jpar1 = ni_options["jpar1"].withDefault(false); + ni_pe1 = ni_options["pe1"].withDefault(false); + ni_ni0_curv_phi1 = ni_options["ni0_curv_phi1"].withDefault(false); + ni_ni1_curv_phi0 = ni_options["ni1_curv_phi0"].withDefault(false); + ni_ni1_curv_phi1 = ni_options["ni1_curv_phi1"].withDefault(false); + ni_nit_curv_phit = ni_options["nit_curv_phit"].withDefault(false); } if (evolve_rho) { - Options* options = &globalOptions["rho"]; - options->get("rho0_phi1", rho_rho0_phi1, false); - options->get("rho1_phi0", rho_rho1_phi0, false); - options->get("rho1_phi1", rho_rho1_phi1, false); - options->get("vi1_rho0", rho_vi1_rho0, false); - options->get("vi0_rho1", rho_vi0_rho1, false); - options->get("vi1_rho1", rho_vi1_rho1, false); - options->get("pei1", rho_pei1, false); - options->get("jpar1", rho_jpar1, false); - options->get("rho1", rho_rho1, false); + rho_rho0_phi1 = rho_options["rho0_phi1"].withDefault(false); + rho_rho1_phi0 = rho_options["rho1_phi0"].withDefault(false); + rho_rho1_phi1 = rho_options["rho1_phi1"].withDefault(false); + rho_vi1_rho0 = rho_options["vi1_rho0"].withDefault(false); + rho_vi0_rho1 = rho_options["vi0_rho1"].withDefault(false); + rho_vi1_rho1 = rho_options["vi1_rho1"].withDefault(false); + rho_pei1 = rho_options["pei1"].withDefault(false); + rho_jpar1 = rho_options["jpar1"].withDefault(false); + rho_rho1 = rho_options["rho1"].withDefault(false); } if (evolve_vi) { - Options* options = &globalOptions["vi"]; - options->get("vi0_phi1", vi_vi0_phi1, false); - options->get("vi1_phi0", vi_vi1_phi0, false); - options->get("vi1_phi1", vi_vi1_phi1, false); - options->get("vit_phit", vi_vit_phit, false); - options->get("vi1_vi0", vi_vi1_vi0, false); - options->get("vi0_vi1", vi_vi0_vi1, false); - options->get("vi1_vi1", vi_vi1_vi1, false); - options->get("vit_vit", vi_vit_vit, false); - options->get("pei1", vi_pei1, false); - options->get("peit", vi_peit, false); - options->get("vi1", vi_vi1, false); + vi_vi0_phi1 = vi_options["vi0_phi1"].withDefault(false); + vi_vi1_phi0 = vi_options["vi1_phi0"].withDefault(false); + vi_vi1_phi1 = vi_options["vi1_phi1"].withDefault(false); + vi_vit_phit = vi_options["vit_phit"].withDefault(false); + vi_vi1_vi0 = vi_options["vi1_vi0"].withDefault(false); + vi_vi0_vi1 = vi_options["vi0_vi1"].withDefault(false); + vi_vi1_vi1 = vi_options["vi1_vi1"].withDefault(false); + vi_vit_vit = vi_options["vit_vit"].withDefault(false); + vi_pei1 = vi_options["pei1"].withDefault(false); + vi_peit = vi_options["peit"].withDefault(false); + vi_vi1 = vi_options["vi1"].withDefault(false); } if (evolve_te) { - Options* options = &globalOptions["te"]; - options->get("te1_phi0", te_te1_phi0, false); - options->get("te0_phi1", te_te0_phi1, false); - options->get("te1_phi1", te_te1_phi1, false); + te_te1_phi0 = te_options["te1_phi0"].withDefault(false); + te_te0_phi1 = te_options["te0_phi1"].withDefault(false); + te_te1_phi1 = te_options["te1_phi1"].withDefault(false); } if (evolve_ti) { - Options* options = &globalOptions["ti"]; - options->get("ti1_phi0", ti_ti1_phi0, false); - options->get("ti0_phi1", ti_ti0_phi1, false); - options->get("ti1_phi1", ti_ti1_phi1, false); + ti_ti1_phi0 = ti_options["ti1_phi0"].withDefault(false); + ti_ti0_phi1 = ti_options["ti0_phi1"].withDefault(false); + ti_ti1_phi1 = ti_options["ti1_phi1"].withDefault(false); } //////////////////////////////////////////////////////// @@ -337,13 +353,15 @@ class TwoFluid : public PhysicsModel { //////////////////////////////////////////////////////// // SHIFTED GRIDS LOCATION + stagger_loc = CELL_LOC::ylow; + // Velocities defined on cell boundaries - Vi.setLocation(CELL_YLOW); - Ajpar.setLocation(CELL_YLOW); + Vi.setLocation(stagger_loc); + Ajpar.setLocation(stagger_loc); // Apar and jpar too - Apar.setLocation(CELL_YLOW); - jpar.setLocation(CELL_YLOW); + Apar.setLocation(stagger_loc); + jpar.setLocation(stagger_loc); } //////////////////////////////////////////////////////// @@ -490,16 +508,17 @@ class TwoFluid : public PhysicsModel { dump.addOnce(wci, "wci"); // Create a solver for the Laplacian - phiSolver = Laplacian::create(&options["phiSolver"]); + phiSolver = Laplacian::create(&globalOptions["phiSolver"]); if (laplace_extra_rho_term) { // Include the first order term Grad_perp Ni dot Grad_perp phi phiSolver->setCoefC(Ni0); } + globalOptions["aparSolver"].setConditionallyUsed(); + if (!(estatic || ZeroElMass)) { // Create a solver for the electromagnetic potential - aparSolver = Laplacian::create(&options["aparSolver"], - mesh->StaggerGrids ? CELL_YLOW : CELL_CENTRE); + aparSolver = Laplacian::create(&globalOptions["aparSolver"], stagger_loc); if (mesh->StaggerGrids) { acoef = (-0.5 * beta_p / fmei) * interp_to(Ni0, CELL_YLOW); } else { @@ -599,10 +618,11 @@ class TwoFluid : public PhysicsModel { if (ZeroElMass) { // Set jpar,Ve,Ajpar neglecting the electron inertia term // Calculate Jpar, communicating across processors - jpar = -(Ni0 * Grad_par(phi, CELL_YLOW)) / (fmei * 0.51 * nu); + + jpar = -interp_to(Ni0, stagger_loc) * Grad_par(phi, stagger_loc) / (fmei * 0.51 * nu); if (OhmPe) { - jpar += (Te0 * Grad_par(Ni, CELL_YLOW)) / (fmei * 0.51 * nu); + jpar += interp_to(Te0, stagger_loc) * Grad_par(Ni, stagger_loc) / (fmei * 0.51 * nu); } // Need to communicate jpar diff --git a/examples/tokamak-2fluid/data/BOUT.inp b/examples/tokamak-2fluid/data/BOUT.inp index 563af66cd6..214bafc19a 100644 --- a/examples/tokamak-2fluid/data/BOUT.inp +++ b/examples/tokamak-2fluid/data/BOUT.inp @@ -45,14 +45,6 @@ first = C4 second = C4 upwind = U1 -################################################## -# Laplacian inversion settings - -[laplace] -all_terms = false -nonuniform = true -filter = 0.2 # Remove the top 20% of modes (BOUT-06 zwindow=0.4) - ################################################## # Solver settings @@ -81,8 +73,6 @@ estatic = true # if true, electrostatic (Apar = 0). (BOUT-06 = esop) ZeroElMass = true # Use Ohms law without electron inertia bout_jpar = true # Use BOUT-06 method to calculate ZeroElMass jpar -bout_exb = true # Use the BOUT-06 subset of ExB terms - curv_upwind = false # Use upwinding for b0xkappa_dot_Grad terms laplace_extra_rho_term = false # include Grad_perp(Ni) dot Grad_perp(phi) term From e004276220892ea74fb916053471a7e9eabc5bd2 Mon Sep 17 00:00:00 2001 From: ZedThree Date: Wed, 22 Feb 2023 16:32:17 +0000 Subject: [PATCH 023/111] Apply clang-format changes --- examples/tokamak-2fluid/2fluid.cxx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/tokamak-2fluid/2fluid.cxx b/examples/tokamak-2fluid/2fluid.cxx index 323a34df4c..03458d2d4e 100644 --- a/examples/tokamak-2fluid/2fluid.cxx +++ b/examples/tokamak-2fluid/2fluid.cxx @@ -619,10 +619,12 @@ class TwoFluid : public PhysicsModel { // Set jpar,Ve,Ajpar neglecting the electron inertia term // Calculate Jpar, communicating across processors - jpar = -interp_to(Ni0, stagger_loc) * Grad_par(phi, stagger_loc) / (fmei * 0.51 * nu); + jpar = + -interp_to(Ni0, stagger_loc) * Grad_par(phi, stagger_loc) / (fmei * 0.51 * nu); if (OhmPe) { - jpar += interp_to(Te0, stagger_loc) * Grad_par(Ni, stagger_loc) / (fmei * 0.51 * nu); + jpar += + interp_to(Te0, stagger_loc) * Grad_par(Ni, stagger_loc) / (fmei * 0.51 * nu); } // Need to communicate jpar From f1b2e049c82105fe7b530fce6d35b73a9ee9cf48 Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 22 Feb 2023 19:37:50 +0100 Subject: [PATCH 024/111] Do not check for PVEC_REAL_MPI_TYPE change MPI_DOUBLE is a macro, so this check can break, even if there is no mismatch. --- externalpackages/PVODE/include/pvode/nvector.h | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/externalpackages/PVODE/include/pvode/nvector.h b/externalpackages/PVODE/include/pvode/nvector.h index c696b185d3..1095bdb474 100644 --- a/externalpackages/PVODE/include/pvode/nvector.h +++ b/externalpackages/PVODE/include/pvode/nvector.h @@ -94,16 +94,6 @@ namespace pvode { #else #define PVEC_REAL_MPI_TYPE MPI_FLOAT #endif -#else -#if (LLNL_DOUBLE == 1) -#if PVEC_REAL_MPI_TYPE != MPI_DOUBLE -#error PVEC_REAL_MPI_TYPE changed -#endif -#else -#if PVEC_REAL_MPI_TYPE != MPI_FLOAT -#error PVEC_REAL_MPI_TYPE changed -#endif -#endif #endif // Ugh, potentionanlly bad but need to do this for now when compiling with both SUNDIALS and PVODE From bee8eeb67c8450fa817a4e4fc47f62dfeccad4d0 Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 23 Feb 2023 09:30:32 +0100 Subject: [PATCH 025/111] Fix versions after release --- CMakeLists.txt | 4 ++-- manual/RELEASE_HOWTO.md | 2 ++ tools/pylib/_boutpp_build/backend.py | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 80171598ac..41c04d1536 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,8 +18,8 @@ endif() # CMake currently doesn't support proper semver # Set the version here, strip any extra tags to use in `project` # We try to use git to get a full description, inspired by setuptools_scm -set(_bout_previous_version "v4.0.0") -set(_bout_next_version "5.0.0") +set(_bout_previous_version "v5.0.0") +set(_bout_next_version "5.0.1") execute_process( COMMAND "git" describe --tags --match=${_bout_previous_version} COMMAND sed -e s/${_bout_previous_version}-/${_bout_next_version}.dev/ -e s/-/+/ diff --git a/manual/RELEASE_HOWTO.md b/manual/RELEASE_HOWTO.md index 4b09fd5b46..67b0009ab2 100644 --- a/manual/RELEASE_HOWTO.md +++ b/manual/RELEASE_HOWTO.md @@ -50,6 +50,8 @@ Before merging PR: - [ ] [`manual/doxygen/Doxyfile_readthedocs`][Doxyfile_readthedocs]: `PROJECT_NUMBER` - [ ] [`manual/doxygen/Doxyfile`][Doxyfile]: `PROJECT_NUMBER` - [ ] [`CMakeLists.txt`][CMakeLists]: `_bout_previous_version`, `_bout_next_version` + - [ ] [`tools/pylib/_boutpp_build/backend.py`][backend.py]: `_bout_previous_version`, `_bout_next_version` + After PR is merged: diff --git a/tools/pylib/_boutpp_build/backend.py b/tools/pylib/_boutpp_build/backend.py index e299e51578..ab472b7558 100644 --- a/tools/pylib/_boutpp_build/backend.py +++ b/tools/pylib/_boutpp_build/backend.py @@ -27,8 +27,8 @@ def getversion(): """ global version if version is None: - _bout_previous_version = "v4.0.0" - _bout_next_version = "5.0.0" + _bout_previous_version = "v5.0.0" + _bout_next_version = "5.0.1" try: tmp = run2(f"git describe --tags --match={_bout_previous_version}").strip() From 9171e9eff314d446a8f30ef2c2d9be386de3bb1b Mon Sep 17 00:00:00 2001 From: David Bold Date: Fri, 24 Feb 2023 08:28:20 +0100 Subject: [PATCH 026/111] Fix typos Co-authored-by: Peter Hill --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b6548f60cc..7cf893bd18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ endif() if ("${CMAKE_CURRENT_BINARY_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") option(BOUT_ALLOW_INSOURCE_BUILD "Whether BOUT++ should really allow to build in source." OFF) if (NOT ${BOUT_ALLOW_INSOURCE_BUILD}) - message(FATAL_ERROR "BOUT++ does not recommends in source builds. Try building out of source, e.g. with `cmake -S . -B build` or set -DBOUT_ALLOW_INSOURCE_BUILD=ON - but thinks may break!") + message(FATAL_ERROR "BOUT++ does not recommend in source builds. Try building out of source, e.g. with `cmake -S . -B build` or set -DBOUT_ALLOW_INSOURCE_BUILD=ON - but things may break!") endif() endif() From 612188898c0e3af24c0257a470dca90e14006e56 Mon Sep 17 00:00:00 2001 From: David Bold Date: Tue, 14 Mar 2023 23:33:26 +0100 Subject: [PATCH 027/111] Explicitly cast to void* --- src/solver/impls/arkode/arkode.cxx | 2 +- src/solver/impls/cvode/cvode.cxx | 2 +- src/solver/impls/ida/ida.cxx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/solver/impls/arkode/arkode.cxx b/src/solver/impls/arkode/arkode.cxx index 09aa5b9ea2..c2edc533c1 100644 --- a/src/solver/impls/arkode/arkode.cxx +++ b/src/solver/impls/arkode/arkode.cxx @@ -226,7 +226,7 @@ ArkodeSolver::ArkodeSolver(Options* opts) .withDefault(false)), optimize( (*options)["optimize"].doc("Use ARKode optimal parameters").withDefault(false)), - suncontext(MPI_COMM_WORLD) { + suncontext(static_cast(MPI_COMM_WORLD)) { has_constraints = false; // This solver doesn't have constraints // Add diagnostics to output diff --git a/src/solver/impls/cvode/cvode.cxx b/src/solver/impls/cvode/cvode.cxx index 4701d02f69..a715e19d96 100644 --- a/src/solver/impls/cvode/cvode.cxx +++ b/src/solver/impls/cvode/cvode.cxx @@ -174,7 +174,7 @@ CvodeSolver::CvodeSolver(Options* opts) .doc("Use right preconditioner? Otherwise use left.") .withDefault(false)), use_jacobian((*options)["use_jacobian"].withDefault(false)), - suncontext(MPI_COMM_WORLD) { + suncontext(static_cast(MPI_COMM_WORLD)) { has_constraints = false; // This solver doesn't have constraints canReset = true; diff --git a/src/solver/impls/ida/ida.cxx b/src/solver/impls/ida/ida.cxx index cf057ac74c..02a3534d52 100644 --- a/src/solver/impls/ida/ida.cxx +++ b/src/solver/impls/ida/ida.cxx @@ -101,7 +101,7 @@ IdaSolver::IdaSolver(Options* opts) correct_start((*options)["correct_start"] .doc("Correct the initial values") .withDefault(true)), - suncontext(MPI_COMM_WORLD) { + suncontext(static_cast(MPI_COMM_WORLD)) { has_constraints = true; // This solver has constraints } From aa242f8e3e99735b225a20d237bb5cc5689fc805 Mon Sep 17 00:00:00 2001 From: David Bold Date: Tue, 14 Mar 2023 23:48:32 +0100 Subject: [PATCH 028/111] Update signature to match upstream --- include/bout/sundials_backports.hxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/bout/sundials_backports.hxx b/include/bout/sundials_backports.hxx index 9c96de1935..f34ce3fcc4 100644 --- a/include/bout/sundials_backports.hxx +++ b/include/bout/sundials_backports.hxx @@ -39,7 +39,7 @@ inline void SUNNonlinSolFree(MAYBE_UNUSED(SUNNonlinearSolver solver)) {} #if SUNDIALS_VERSION_MAJOR < 6 namespace sundials { struct Context { - Context(MPI_Comm comm MAYBE_UNUSED() = MPI_COMM_NULL) {} + Context(void* comm MAYBE_UNUSED() = MPI_COMM_NULL) {} }; } // namespace sundials From 759c0cbb31089a585ae82fe7c93b39aea141e7b2 Mon Sep 17 00:00:00 2001 From: dschwoerer Date: Tue, 14 Mar 2023 22:49:27 +0000 Subject: [PATCH 029/111] Apply clang-format changes --- externalpackages/PVODE/source/cvode.cpp | 4 ++-- src/solver/impls/arkode/arkode.cxx | 2 +- src/solver/impls/cvode/cvode.cxx | 2 +- src/solver/impls/ida/ida.cxx | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/externalpackages/PVODE/source/cvode.cpp b/externalpackages/PVODE/source/cvode.cpp index e28a3b1ae6..de97649d5e 100644 --- a/externalpackages/PVODE/source/cvode.cpp +++ b/externalpackages/PVODE/source/cvode.cpp @@ -1825,7 +1825,7 @@ static int CVnls(CVodeMem cv_mem, int nflag) switch(iter) { case FUNCTIONAL : return(CVnlsFunctional(cv_mem)); case NEWTON : return(CVnlsNewton(cv_mem, nflag)); - } + } } /***************** CVnlsFunctional ******************************** @@ -2391,7 +2391,7 @@ static int CVHandleFailure(CVodeMem cv_mem, int kflag) return(SETUP_FAILURE); case SOLVE_FAILED: fprintf(errfp, MSG_SOLVE_FAILED, tn); return(SOLVE_FAILURE); - } + } } /*******************************************************************/ diff --git a/src/solver/impls/arkode/arkode.cxx b/src/solver/impls/arkode/arkode.cxx index c2edc533c1..6df091288a 100644 --- a/src/solver/impls/arkode/arkode.cxx +++ b/src/solver/impls/arkode/arkode.cxx @@ -226,7 +226,7 @@ ArkodeSolver::ArkodeSolver(Options* opts) .withDefault(false)), optimize( (*options)["optimize"].doc("Use ARKode optimal parameters").withDefault(false)), - suncontext(static_cast(MPI_COMM_WORLD)) { + suncontext(static_cast(MPI_COMM_WORLD)) { has_constraints = false; // This solver doesn't have constraints // Add diagnostics to output diff --git a/src/solver/impls/cvode/cvode.cxx b/src/solver/impls/cvode/cvode.cxx index a715e19d96..88f2c8e8f9 100644 --- a/src/solver/impls/cvode/cvode.cxx +++ b/src/solver/impls/cvode/cvode.cxx @@ -174,7 +174,7 @@ CvodeSolver::CvodeSolver(Options* opts) .doc("Use right preconditioner? Otherwise use left.") .withDefault(false)), use_jacobian((*options)["use_jacobian"].withDefault(false)), - suncontext(static_cast(MPI_COMM_WORLD)) { + suncontext(static_cast(MPI_COMM_WORLD)) { has_constraints = false; // This solver doesn't have constraints canReset = true; diff --git a/src/solver/impls/ida/ida.cxx b/src/solver/impls/ida/ida.cxx index 02a3534d52..be13f4d7f7 100644 --- a/src/solver/impls/ida/ida.cxx +++ b/src/solver/impls/ida/ida.cxx @@ -101,7 +101,7 @@ IdaSolver::IdaSolver(Options* opts) correct_start((*options)["correct_start"] .doc("Correct the initial values") .withDefault(true)), - suncontext(static_cast(MPI_COMM_WORLD)) { + suncontext(static_cast(MPI_COMM_WORLD)) { has_constraints = true; // This solver has constraints } From 7bc12600f5967bb48415d4e0f2220f719e7c4c19 Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 15 Mar 2023 13:47:35 +0100 Subject: [PATCH 030/111] Ensure to pass a pointer `MPI_COMM_WORLD` can be int, thus pass the BOUT++ communicator Co-authored-by: Peter Hill --- src/solver/impls/arkode/arkode.cxx | 2 +- src/solver/impls/cvode/cvode.cxx | 2 +- src/solver/impls/ida/ida.cxx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/solver/impls/arkode/arkode.cxx b/src/solver/impls/arkode/arkode.cxx index 6df091288a..13e0dd817e 100644 --- a/src/solver/impls/arkode/arkode.cxx +++ b/src/solver/impls/arkode/arkode.cxx @@ -226,7 +226,7 @@ ArkodeSolver::ArkodeSolver(Options* opts) .withDefault(false)), optimize( (*options)["optimize"].doc("Use ARKode optimal parameters").withDefault(false)), - suncontext(static_cast(MPI_COMM_WORLD)) { + suncontext(static_cast(&BoutComm::get())) { has_constraints = false; // This solver doesn't have constraints // Add diagnostics to output diff --git a/src/solver/impls/cvode/cvode.cxx b/src/solver/impls/cvode/cvode.cxx index 88f2c8e8f9..23f76b4db2 100644 --- a/src/solver/impls/cvode/cvode.cxx +++ b/src/solver/impls/cvode/cvode.cxx @@ -174,7 +174,7 @@ CvodeSolver::CvodeSolver(Options* opts) .doc("Use right preconditioner? Otherwise use left.") .withDefault(false)), use_jacobian((*options)["use_jacobian"].withDefault(false)), - suncontext(static_cast(MPI_COMM_WORLD)) { + suncontext(static_cast(&BoutComm::get())) { has_constraints = false; // This solver doesn't have constraints canReset = true; diff --git a/src/solver/impls/ida/ida.cxx b/src/solver/impls/ida/ida.cxx index be13f4d7f7..b008ebf903 100644 --- a/src/solver/impls/ida/ida.cxx +++ b/src/solver/impls/ida/ida.cxx @@ -101,7 +101,7 @@ IdaSolver::IdaSolver(Options* opts) correct_start((*options)["correct_start"] .doc("Correct the initial values") .withDefault(true)), - suncontext(static_cast(MPI_COMM_WORLD)) { + suncontext(static_cast(&BoutComm::get())) { has_constraints = true; // This solver has constraints } From d48acbe3a0d4b790bf6f1d58e337aa3ada7bfc8e Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 16 Mar 2023 10:42:28 +0100 Subject: [PATCH 031/111] MPI_Comm is a pointer type Do not take the address of the pointer. Compiles with mpich and openmpi --- src/solver/impls/arkode/arkode.cxx | 2 +- src/solver/impls/cvode/cvode.cxx | 2 +- src/solver/impls/ida/ida.cxx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/solver/impls/arkode/arkode.cxx b/src/solver/impls/arkode/arkode.cxx index 13e0dd817e..ff6334ef90 100644 --- a/src/solver/impls/arkode/arkode.cxx +++ b/src/solver/impls/arkode/arkode.cxx @@ -226,7 +226,7 @@ ArkodeSolver::ArkodeSolver(Options* opts) .withDefault(false)), optimize( (*options)["optimize"].doc("Use ARKode optimal parameters").withDefault(false)), - suncontext(static_cast(&BoutComm::get())) { + suncontext(static_cast(BoutComm::get())) { has_constraints = false; // This solver doesn't have constraints // Add diagnostics to output diff --git a/src/solver/impls/cvode/cvode.cxx b/src/solver/impls/cvode/cvode.cxx index 23f76b4db2..320ead4732 100644 --- a/src/solver/impls/cvode/cvode.cxx +++ b/src/solver/impls/cvode/cvode.cxx @@ -174,7 +174,7 @@ CvodeSolver::CvodeSolver(Options* opts) .doc("Use right preconditioner? Otherwise use left.") .withDefault(false)), use_jacobian((*options)["use_jacobian"].withDefault(false)), - suncontext(static_cast(&BoutComm::get())) { + suncontext(static_cast(BoutComm::get())) { has_constraints = false; // This solver doesn't have constraints canReset = true; diff --git a/src/solver/impls/ida/ida.cxx b/src/solver/impls/ida/ida.cxx index b008ebf903..399f676892 100644 --- a/src/solver/impls/ida/ida.cxx +++ b/src/solver/impls/ida/ida.cxx @@ -101,7 +101,7 @@ IdaSolver::IdaSolver(Options* opts) correct_start((*options)["correct_start"] .doc("Correct the initial values") .withDefault(true)), - suncontext(static_cast(&BoutComm::get())) { + suncontext(static_cast(BoutComm::get())) { has_constraints = true; // This solver has constraints } From 47a70ed91c7406821edd8510e31e495b39507b08 Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 16 Mar 2023 12:12:11 +0100 Subject: [PATCH 032/111] Return reference This is needed so we can get a pointer to the communicator, as is needed by SUNContext --- include/bout/boutcomm.hxx | 4 ++-- src/sys/boutcomm.cxx | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/bout/boutcomm.hxx b/include/bout/boutcomm.hxx index 77038844a8..fea401af02 100644 --- a/include/bout/boutcomm.hxx +++ b/include/bout/boutcomm.hxx @@ -40,7 +40,7 @@ public: static BoutComm* getInstance(); /// Shortcut method - static MPI_Comm get(); + static MPI_Comm& get(); static void setArgs(int& c, char**& v); @@ -53,7 +53,7 @@ public: void setComm(MPI_Comm c); // Getters - MPI_Comm getComm(); + MPI_Comm& getComm(); bool isSet(); private: diff --git a/src/sys/boutcomm.cxx b/src/sys/boutcomm.cxx index ecaeb866bb..2d78a24076 100644 --- a/src/sys/boutcomm.cxx +++ b/src/sys/boutcomm.cxx @@ -25,7 +25,7 @@ void BoutComm::setComm(MPI_Comm c) { hasBeenSet = true; } -MPI_Comm BoutComm::getComm() { +MPI_Comm& BoutComm::getComm() { if (comm == MPI_COMM_NULL) { // No communicator set. Initialise MPI MPI_Init(pargc, pargv); @@ -39,7 +39,7 @@ MPI_Comm BoutComm::getComm() { bool BoutComm::isSet() { return hasBeenSet; } // Static functions below. Must use getInstance() -MPI_Comm BoutComm::get() { return getInstance()->getComm(); } +MPI_Comm& BoutComm::get() { return getInstance()->getComm(); } void BoutComm::setArgs(int& c, char**& v) { getInstance()->pargc = &c; From c31e948978980b4458ab9ae0e3c7b826dad70a01 Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 16 Mar 2023 12:13:35 +0100 Subject: [PATCH 033/111] Pass pointer to to SUNContext pass MPI_Comm* to SUNContext_Create as indicated in the docs https://sundials.readthedocs.io/en/latest/sundials/SUNContext_link.html --- src/solver/impls/arkode/arkode.cxx | 2 +- src/solver/impls/cvode/cvode.cxx | 2 +- src/solver/impls/ida/ida.cxx | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/solver/impls/arkode/arkode.cxx b/src/solver/impls/arkode/arkode.cxx index ff6334ef90..13e0dd817e 100644 --- a/src/solver/impls/arkode/arkode.cxx +++ b/src/solver/impls/arkode/arkode.cxx @@ -226,7 +226,7 @@ ArkodeSolver::ArkodeSolver(Options* opts) .withDefault(false)), optimize( (*options)["optimize"].doc("Use ARKode optimal parameters").withDefault(false)), - suncontext(static_cast(BoutComm::get())) { + suncontext(static_cast(&BoutComm::get())) { has_constraints = false; // This solver doesn't have constraints // Add diagnostics to output diff --git a/src/solver/impls/cvode/cvode.cxx b/src/solver/impls/cvode/cvode.cxx index 320ead4732..23f76b4db2 100644 --- a/src/solver/impls/cvode/cvode.cxx +++ b/src/solver/impls/cvode/cvode.cxx @@ -174,7 +174,7 @@ CvodeSolver::CvodeSolver(Options* opts) .doc("Use right preconditioner? Otherwise use left.") .withDefault(false)), use_jacobian((*options)["use_jacobian"].withDefault(false)), - suncontext(static_cast(BoutComm::get())) { + suncontext(static_cast(&BoutComm::get())) { has_constraints = false; // This solver doesn't have constraints canReset = true; diff --git a/src/solver/impls/ida/ida.cxx b/src/solver/impls/ida/ida.cxx index 399f676892..b008ebf903 100644 --- a/src/solver/impls/ida/ida.cxx +++ b/src/solver/impls/ida/ida.cxx @@ -101,7 +101,7 @@ IdaSolver::IdaSolver(Options* opts) correct_start((*options)["correct_start"] .doc("Correct the initial values") .withDefault(true)), - suncontext(static_cast(BoutComm::get())) { + suncontext(static_cast(&BoutComm::get())) { has_constraints = true; // This solver has constraints } From 4a288925e744bb0988ba869191fe68ede485f8c9 Mon Sep 17 00:00:00 2001 From: David Bold Date: Fri, 17 Feb 2023 10:32:08 +0100 Subject: [PATCH 034/111] Update bundled boututils and boutdata --- externalpackages/boutdata | 2 +- externalpackages/boututils | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/externalpackages/boutdata b/externalpackages/boutdata index 2d4c5bb4b2..68797c212e 160000 --- a/externalpackages/boutdata +++ b/externalpackages/boutdata @@ -1 +1 @@ -Subproject commit 2d4c5bb4b2bdc0b74d3d267559a5624aa599b95f +Subproject commit 68797c212ef07a7a4edab7ae39eee3267c7210d3 diff --git a/externalpackages/boututils b/externalpackages/boututils index a79a00a54f..70c0645c5d 160000 --- a/externalpackages/boututils +++ b/externalpackages/boututils @@ -1 +1 @@ -Subproject commit a79a00a54f69663117a93dd42ffa6004783432c8 +Subproject commit 70c0645c5d7461dec9735d1352b83ac1abb6b81d From f54a2acb5f00d112055b920dc9c30af4df63f3d2 Mon Sep 17 00:00:00 2001 From: David Bold Date: Fri, 17 Feb 2023 10:35:33 +0100 Subject: [PATCH 035/111] Add checking bundled libs to release schedule --- manual/RELEASE_HOWTO.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/manual/RELEASE_HOWTO.md b/manual/RELEASE_HOWTO.md index 67b0009ab2..1077aaf375 100644 --- a/manual/RELEASE_HOWTO.md +++ b/manual/RELEASE_HOWTO.md @@ -19,7 +19,8 @@ releases - [ ] Run `make check-all` - Raise issues for any tests that fail - Possibly run `clang-tidy`, `clang-check`, `coverity`, etc. -- [ ] Review pinned pip package versions for Travis +- [ ] Review pinned pip package versions for CI +- [ ] Review bundled libraries Before merging PR: From b511ccbeefba128bbf97e6df0d1185a9990360e1 Mon Sep 17 00:00:00 2001 From: David Bold Date: Fri, 17 Feb 2023 12:56:28 +0100 Subject: [PATCH 036/111] install natsort for CI --- .ci_fedora.sh | 2 +- .github/workflows/tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci_fedora.sh b/.ci_fedora.sh index bf43f76fc6..e6d9f61e8f 100755 --- a/.ci_fedora.sh +++ b/.ci_fedora.sh @@ -49,7 +49,7 @@ then sudo -u test ${0/\/tmp/\/home\/test} $mpi ## If we are called as normal user, run test else - pip install --user zoidberg + pip install --user zoidberg natsort . /etc/profile.d/modules.sh module load mpi/${1}-x86_64 export OMPI_MCA_rmaps_base_oversubscribe=yes diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d51835c82c..7fd98b01dd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -137,7 +137,7 @@ jobs: - name: Install pip packages run: | - ./.pip_install_for_ci.sh 'cython~=0.29' 'netcdf4~=1.5' 'sympy~=1.5' 'gcovr' 'cmake' 'h5py' zoidberg + ./.pip_install_for_ci.sh 'cython~=0.29' 'netcdf4~=1.5' 'sympy~=1.5' 'gcovr' 'cmake' 'h5py' 'zoidberg' 'natsort' # Add the pip install location to the runner's PATH echo ~/.local/bin >> $GITHUB_PATH From 73f5d4d0bb56eb699921e65c1c5d7512d55f7af0 Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 15 Mar 2023 10:47:30 +0100 Subject: [PATCH 037/111] Workaround for test-restart-io for recent boututils boututils changed the default for datafile to netcdf4, which breaks reading the y position of the FieldPerp slices. --- tests/integrated/test-restart-io/runtest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integrated/test-restart-io/runtest b/tests/integrated/test-restart-io/runtest index 00ee907958..31649cc519 100755 --- a/tests/integrated/test-restart-io/runtest +++ b/tests/integrated/test-restart-io/runtest @@ -51,7 +51,7 @@ except FileExistsError: pass with DataFile( - os.path.join(restartdir, "BOUT.restart.0.nc"), create=True + os.path.join(restartdir, "BOUT.restart.0.nc"), create=True, format="NETCDF3_64BIT" ) as base_restart: base_restart.write("MXSUB", nx) base_restart.write("MYSUB", ny) From fefd31c27fb895c720cc83274235fe6ae4818610 Mon Sep 17 00:00:00 2001 From: David Bold Date: Fri, 17 Mar 2023 13:17:35 +0100 Subject: [PATCH 038/111] update boutdata to include fix for yindex_global --- externalpackages/boutdata | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externalpackages/boutdata b/externalpackages/boutdata index 68797c212e..ab59ef9138 160000 --- a/externalpackages/boutdata +++ b/externalpackages/boutdata @@ -1 +1 @@ -Subproject commit 68797c212ef07a7a4edab7ae39eee3267c7210d3 +Subproject commit ab59ef913884918a5bc5ee84101e6d6b833dcd6c From 7f5e7c78fe027685723d1b35fe0466bff6dc176f Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 22 Mar 2023 12:46:18 +0100 Subject: [PATCH 039/111] CI: switch to openmpi for fedora GHA --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d51835c82c..d1b8031fc9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -187,7 +187,7 @@ jobs: with: submodules: true - name: Build Fedora rawhide - run: ./.ci_fedora.sh setup mpich rawhide + run: ./.ci_fedora.sh setup openmpi rawhide shell: bash env: TRAVIS_BUILD_DIR: ${{ github.workspace }} From 8cc5ee69749071fcde53ed61682c97e2684abb66 Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 22 Mar 2023 13:22:21 +0100 Subject: [PATCH 040/111] If we are on a tag, we should use that as version --- tools/pylib/_boutpp_build/backend.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/tools/pylib/_boutpp_build/backend.py b/tools/pylib/_boutpp_build/backend.py index e299e51578..7551ba4d54 100644 --- a/tools/pylib/_boutpp_build/backend.py +++ b/tools/pylib/_boutpp_build/backend.py @@ -31,13 +31,20 @@ def getversion(): _bout_next_version = "5.0.0" try: - tmp = run2(f"git describe --tags --match={_bout_previous_version}").strip() - tmp = re.sub(f"{_bout_previous_version}-", f"{_bout_next_version}.dev", tmp) - if useLocalVersion: - tmp = re.sub("-", "+", tmp) - else: - tmp = re.sub("-.*", "", tmp) - version = tmp + try: + version = run2("git describe --exact-match --tags HEAD").strip() + except subprocess.CalledProcessError: + tmp = run2( + f"git describe --tags --match={_bout_previous_version}" + ).strip() + tmp = re.sub( + f"{_bout_previous_version}-", f"{_bout_next_version}.dev", tmp + ) + if useLocalVersion: + tmp = re.sub("-", "+", tmp) + else: + tmp = re.sub("-.*", "", tmp) + version = tmp with open("_version.txt", "w") as f: f.write(version + "\n") except subprocess.CalledProcessError: From 9949e41735c3d7d6e649fa90203a774930a0416c Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 23 Mar 2023 08:38:45 +0100 Subject: [PATCH 041/111] Do not assign in if statement --- externalpackages/PVODE/precon/band.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/externalpackages/PVODE/precon/band.cpp b/externalpackages/PVODE/precon/band.cpp index c1d0f6f1e2..43151320d0 100644 --- a/externalpackages/PVODE/precon/band.cpp +++ b/externalpackages/PVODE/precon/band.cpp @@ -185,8 +185,8 @@ integer gbfa(real **a, integer n, integer mu, integer ml, integer smu, if (col_k[storage_l] == ZERO) return(k+1); /* swap a(l,k) and a(k,k) if necessary */ - - if (swap = (l != k)) { + swap = (l != k); + if (swap) { temp = col_k[storage_l]; col_k[storage_l] = *diag_k; *diag_k = temp; From 1d6b6b144e525c8f61c453c79ac5cf13d765bf36 Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 23 Mar 2023 08:39:50 +0100 Subject: [PATCH 042/111] Fix type in printf format string --- externalpackages/PVODE/source/cvode.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externalpackages/PVODE/source/cvode.cpp b/externalpackages/PVODE/source/cvode.cpp index 03d9f0b130..48ac3209f2 100644 --- a/externalpackages/PVODE/source/cvode.cpp +++ b/externalpackages/PVODE/source/cvode.cpp @@ -179,7 +179,7 @@ namespace pvode { #define MSG_Y0_NULL CVM "y0=NULL illegal.\n\n" -#define MSG_BAD_N CVM "N=%ld < 1 illegal.\n\n" +#define MSG_BAD_N CVM "N=%d < 1 illegal.\n\n" #define MSG_BAD_LMM_1 CVM "lmm=%d illegal.\n" #define MSG_BAD_LMM_2 "The legal values are ADAMS=%d and BDF=%d.\n\n" From e21b0df6acdc145ef576085c2ff5b1f27045f81a Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 23 Mar 2023 08:45:11 +0100 Subject: [PATCH 043/111] Revert "Workaround for test-restart-io for recent boututils" This reverts commit 73f5d4d0bb56eb699921e65c1c5d7512d55f7af0. --- tests/integrated/test-restart-io/runtest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integrated/test-restart-io/runtest b/tests/integrated/test-restart-io/runtest index 31649cc519..00ee907958 100755 --- a/tests/integrated/test-restart-io/runtest +++ b/tests/integrated/test-restart-io/runtest @@ -51,7 +51,7 @@ except FileExistsError: pass with DataFile( - os.path.join(restartdir, "BOUT.restart.0.nc"), create=True, format="NETCDF3_64BIT" + os.path.join(restartdir, "BOUT.restart.0.nc"), create=True ) as base_restart: base_restart.write("MXSUB", nx) base_restart.write("MYSUB", ny) From 5b83ddb1f076e20f4500999810e9889bf393f1ae Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 23 Mar 2023 09:03:56 +0100 Subject: [PATCH 044/111] Fall back to pkgconfig for finding netcdf-cxx4 Backport of https://cgit.freebsd.org/ports/tree/science/bout++/files/patch-CMakeLists.txt --- cmake/SetupBOUTThirdParty.cmake | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cmake/SetupBOUTThirdParty.cmake b/cmake/SetupBOUTThirdParty.cmake index c04cc98a69..7f716fc424 100644 --- a/cmake/SetupBOUTThirdParty.cmake +++ b/cmake/SetupBOUTThirdParty.cmake @@ -167,11 +167,19 @@ if (BOUT_USE_NETCDF) # Use our own FindnetCDF module which uses nc-config find_package(netCDF REQUIRED) FetchContent_MakeAvailable(netcdf-cxx4) + target_link_libraries(bout++ PUBLIC netCDF::netcdf-cxx4) else() - find_package(netCDFCxx REQUIRED) - set(CONFIG_LDFLAGS "${CONFIG_LDFLAGS} ${netCDF_CXX_LIBRARY} ${netCDF_LIBRARIES}") + find_package(netCDFCxx) + if (netCDFCxx_FOUND) { + set(CONFIG_LDFLAGS "${CONFIG_LDFLAGS} ${netCDF_CXX_LIBRARY} ${netCDF_LIBRARIES}") + target_link_libraries(bout++ PUBLIC netCDF::netcdf-cxx4) + else() + find_package(PkgConfig REQUIRED) + pkg_check_modules(NETCDF REQUIRED IMPORTED_TARGET netcdf-cxx4) + set(CONFIG_LDFLAGS "${CONFIG_LDFLAGS} ${netCDF_CXX_LIBRARY} ${NETCDF_LDFLAGS}") + target_link_libraries(bout++ PUBLIC ${NETCDF_LDFLAGS}) + endif() endif() - target_link_libraries(bout++ PUBLIC netCDF::netcdf-cxx4) endif() message(STATUS "NetCDF support: ${BOUT_USE_NETCDF}") set(BOUT_HAS_NETCDF ${BOUT_USE_NETCDF}) From c04a11a36f55c407c2eb37bb5615282f9b985c88 Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 23 Mar 2023 09:20:16 +0100 Subject: [PATCH 045/111] cmake does not need {} for blocks --- cmake/SetupBOUTThirdParty.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/SetupBOUTThirdParty.cmake b/cmake/SetupBOUTThirdParty.cmake index 7f716fc424..fd41fe5bfe 100644 --- a/cmake/SetupBOUTThirdParty.cmake +++ b/cmake/SetupBOUTThirdParty.cmake @@ -170,7 +170,7 @@ if (BOUT_USE_NETCDF) target_link_libraries(bout++ PUBLIC netCDF::netcdf-cxx4) else() find_package(netCDFCxx) - if (netCDFCxx_FOUND) { + if (netCDFCxx_FOUND) set(CONFIG_LDFLAGS "${CONFIG_LDFLAGS} ${netCDF_CXX_LIBRARY} ${netCDF_LIBRARIES}") target_link_libraries(bout++ PUBLIC netCDF::netcdf-cxx4) else() From 4e01fc360996130bb11a03a888bedef2eefbc92b Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 23 Mar 2023 09:53:48 +0100 Subject: [PATCH 046/111] Switch to updated boututils --- externalpackages/boututils | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externalpackages/boututils b/externalpackages/boututils index 70c0645c5d..c2e97a226a 160000 --- a/externalpackages/boututils +++ b/externalpackages/boututils @@ -1 +1 @@ -Subproject commit 70c0645c5d7461dec9735d1352b83ac1abb6b81d +Subproject commit c2e97a226a8a7728b8f5085792f7744484943512 From 7cbbf54115e59a35e81bfc109052f1fd26a9e12d Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 23 Mar 2023 10:06:03 +0100 Subject: [PATCH 047/111] Recommend `requirements.txt` for dependencies --- manual/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/manual/README.md b/manual/README.md index 1003f0ed19..fb696f8ab0 100644 --- a/manual/README.md +++ b/manual/README.md @@ -12,8 +12,7 @@ To build the manual locally, you need at least "sphinx" and "recommonmark", which you can install using pip (or pip3): ```bash -$ pip install --user sphinx -$ pip install --user recommonmark +$ pip install --user -r sphinx/requirements.txt ``` These documents can be built into a PDF using "sphinx-build": From e50112e428e981f820f7d5bd0e5c95b1618eba0e Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 23 Mar 2023 10:15:25 +0100 Subject: [PATCH 048/111] Build html docs by default --- manual/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manual/Makefile b/manual/Makefile index 1e1d674a3f..2f032cd7b1 100644 --- a/manual/Makefile +++ b/manual/Makefile @@ -14,7 +14,7 @@ manual: all pdf: sphinx-pdf html: sphinx-html man: sphinx-man -sphinx: sphinx-pdf +sphinx: sphinx-html sphinx-pdf: doxygen PYTHONPATH=$(BOUT_TOP)/tools/pylib:$$PYTHONPATH $(sphinx-build) -b latex sphinx/ build/ From 4e640f7dd16889b55d3c57c5a8505674903b0a4e Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 23 Mar 2023 10:20:54 +0100 Subject: [PATCH 049/111] Do not ignore doxygen errors if explicitly requested --- manual/Makefile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/manual/Makefile b/manual/Makefile index 2f032cd7b1..6da41e4a71 100644 --- a/manual/Makefile +++ b/manual/Makefile @@ -1,5 +1,5 @@ # Makefile for the reference and user manuals -.PHONY:all clean sphinx html pdf doxygen breathe-autogen +.PHONY:all clean sphinx html pdf doxygen breathe-autogen maybe-doxygen BOUT_TOP?=.. @@ -16,24 +16,28 @@ html: sphinx-html man: sphinx-man sphinx: sphinx-html -sphinx-pdf: doxygen +sphinx-pdf: maybe-doxygen PYTHONPATH=$(BOUT_TOP)/tools/pylib:$$PYTHONPATH $(sphinx-build) -b latex sphinx/ build/ cd build && latexmk -pdf BOUT -interaction=batchmode test -e BOUT.pdf || ln -s build/BOUT.pdf . @echo "Documentation is available in $$(pwd)/BOUT.pdf" -sphinx-html: doxygen +sphinx-html: maybe-doxygen PYTHONPATH=$(BOUT_TOP)/tools/pylib:$$PYTHONPATH $(sphinx-build) -b html sphinx/ html/ @echo "Documentation available in file://$$(pwd)/html/index.html" -sphinx-man: doxygen +sphinx-man: maybe-doxygen PYTHONPATH=$(BOUT_TOP)/tools/pylib:$$PYTHONPATH $(sphinx-build) -b man sphinx/ man/ @echo "Documentation available in $$(pwd)/man/bout.1" # Run doxygen, ignore if it fails (leading '-') -doxygen: +maybe-doxygen: -cd doxygen && doxygen Doxyfile + +doxygen: + cd doxygen && doxygen Doxyfile + # Run breathe-apidoc, ignore if it fails (leading '-') breathe-autogen: doxygen -breathe-apidoc -f -o sphinx/_breathe_autogen doxygen/bout/xml From 2b85c8bd354516916d027ff3df0c874367d046d0 Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 23 Mar 2023 10:22:24 +0100 Subject: [PATCH 050/111] Do not recommend `--user` for `pip` It does not work in venv, and is not needed otherwise anymore. --- manual/README.md | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/manual/README.md b/manual/README.md index fb696f8ab0..966bbe8767 100644 --- a/manual/README.md +++ b/manual/README.md @@ -12,7 +12,7 @@ To build the manual locally, you need at least "sphinx" and "recommonmark", which you can install using pip (or pip3): ```bash -$ pip install --user -r sphinx/requirements.txt +$ pip install -r sphinx/requirements.txt ``` These documents can be built into a PDF using "sphinx-build": @@ -52,14 +52,7 @@ It is possible to build the API documentation into the main manual using "breathe". Install breathe: ```bash -$ pip install --user breathe -``` - -and comment out the following line in `sphinx/conf.py`: - -```python -# Disable breathe -has_breathe = False +$ pip install breathe ``` You can then build the sphinx documentation as normal. From 9b1fe87d73c5c269dab90f9c59f23a9738bcd2d0 Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 23 Mar 2023 10:22:56 +0100 Subject: [PATCH 051/111] Update docs to updated defaults --- manual/README.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/manual/README.md b/manual/README.md index 966bbe8767..9f69c00548 100644 --- a/manual/README.md +++ b/manual/README.md @@ -15,26 +15,25 @@ To build the manual locally, you need at least "sphinx" and $ pip install -r sphinx/requirements.txt ``` -These documents can be built into a PDF using "sphinx-build": +To get a local html version, run ```bash -$ make +$ make html ``` +This should create a file "index.html" in the "manual/html" directory. + To use e.g. "sphinx-build-3" instead of "sphinx-build", run ```bash $ make sphinx-build=sphinx-build-3 ``` -This should create a file "BOUT.pdf" in the "manual" directory. - -To get a local html version, run +These documents can be built into a PDF using "sphinx-build": ```bash -$ make html +$ make ``` - -This should create a file "index.html" in the "manual/html" directory. +This should create a file "BOUT.pdf" in the "manual" directory. ### API documentation From e23b828415963d1d8ea755d22dfb1497c0556cf1 Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 23 Mar 2023 10:32:25 +0100 Subject: [PATCH 052/111] Remove default arguments --- include/bout/sundials_backports.hxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/bout/sundials_backports.hxx b/include/bout/sundials_backports.hxx index f34ce3fcc4..c5e0f3ab15 100644 --- a/include/bout/sundials_backports.hxx +++ b/include/bout/sundials_backports.hxx @@ -39,7 +39,7 @@ inline void SUNNonlinSolFree(MAYBE_UNUSED(SUNNonlinearSolver solver)) {} #if SUNDIALS_VERSION_MAJOR < 6 namespace sundials { struct Context { - Context(void* comm MAYBE_UNUSED() = MPI_COMM_NULL) {} + Context(void* comm MAYBE_UNUSED()) {} }; } // namespace sundials From 9180d280ff0b4cfff6cde1544a2cce8082ac56cc Mon Sep 17 00:00:00 2001 From: dschwoerer Date: Thu, 23 Mar 2023 10:10:04 +0000 Subject: [PATCH 053/111] Apply clang-format changes --- externalpackages/PVODE/precon/band.cpp | 2 +- externalpackages/PVODE/source/cvode.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/externalpackages/PVODE/precon/band.cpp b/externalpackages/PVODE/precon/band.cpp index c1d0f6f1e2..8b444de191 100644 --- a/externalpackages/PVODE/precon/band.cpp +++ b/externalpackages/PVODE/precon/band.cpp @@ -185,7 +185,7 @@ integer gbfa(real **a, integer n, integer mu, integer ml, integer smu, if (col_k[storage_l] == ZERO) return(k+1); /* swap a(l,k) and a(k,k) if necessary */ - + if (swap = (l != k)) { temp = col_k[storage_l]; col_k[storage_l] = *diag_k; diff --git a/externalpackages/PVODE/source/cvode.cpp b/externalpackages/PVODE/source/cvode.cpp index 03d9f0b130..1bad94b376 100644 --- a/externalpackages/PVODE/source/cvode.cpp +++ b/externalpackages/PVODE/source/cvode.cpp @@ -179,7 +179,7 @@ namespace pvode { #define MSG_Y0_NULL CVM "y0=NULL illegal.\n\n" -#define MSG_BAD_N CVM "N=%ld < 1 illegal.\n\n" +#define MSG_BAD_N CVM "N=%ld < 1 illegal.\n\n" #define MSG_BAD_LMM_1 CVM "lmm=%d illegal.\n" #define MSG_BAD_LMM_2 "The legal values are ADAMS=%d and BDF=%d.\n\n" From a4aa8669acdab17a214171d9e33f2b2325fd0011 Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 29 Mar 2023 10:57:06 +0200 Subject: [PATCH 054/111] Do not fail if numbers are only almost equal --- include/bout/field.hxx | 3 ++- include/bout/utils.hxx | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/include/bout/field.hxx b/include/bout/field.hxx index 97b98fee13..ecd547db5e 100644 --- a/include/bout/field.hxx +++ b/include/bout/field.hxx @@ -345,7 +345,8 @@ inline bool isUniform(const T& f, bool allpe = false, auto element = f[*f.getRegion(region).begin()]; // TODO: maybe parallise this loop, as the early return is unlikely BOUT_FOR_SERIAL(i, f.getRegion(region)) { - if (f[i] != element) { + // by default we only check for exact equality, as that should cover most cases + if (f[i] != element and (not almost_equal(f[i], element))) { result = false; break; } diff --git a/include/bout/utils.hxx b/include/bout/utils.hxx index d221abcbad..5d815c7a32 100644 --- a/include/bout/utils.hxx +++ b/include/bout/utils.hxx @@ -661,6 +661,19 @@ std::string trimComments(const std::string& s, const std::string& c = "#;"); /// https://en.wikipedia.org/wiki/Damerau%E2%80%93Levenshtein_distance#Optimal_string_alignment_distance std::string::size_type editDistance(const std::string& str1, const std::string& str2); +// from https://en.cppreference.com/w/cpp/types/numeric_limits/epsilon +template +typename std::enable_if::is_integer, bool>::type + almost_equal(T x, T y, int ulp = 2) +{ + // the machine epsilon has to be scaled to the magnitude of the values used + // and multiplied by the desired precision in ULPs (units in the last place) + return std::fabs(x - y) <= std::numeric_limits::epsilon() * std::fabs(x + y) * ulp + // unless the result is subnormal + || std::fabs(x - y) < std::numeric_limits::min(); +} + + /// the bout_vsnprintf macro: /// The first argument is an char * buffer of length len. /// It needs to have been allocated with new[], as it may be From b0adf3f6545773d8f3a0a9f6fc1525b0bf0da221 Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 29 Mar 2023 10:58:37 +0200 Subject: [PATCH 055/111] Remove unused vsnprintf macro We have switch to fmt a while ago --- include/bout/utils.hxx | 26 -------------------------- src/sys/output.cxx | 2 -- 2 files changed, 28 deletions(-) diff --git a/include/bout/utils.hxx b/include/bout/utils.hxx index 5d815c7a32..e57768c643 100644 --- a/include/bout/utils.hxx +++ b/include/bout/utils.hxx @@ -673,32 +673,6 @@ typename std::enable_if::is_integer, bool>::type || std::fabs(x - y) < std::numeric_limits::min(); } - -/// the bout_vsnprintf macro: -/// The first argument is an char * buffer of length len. -/// It needs to have been allocated with new[], as it may be -/// reallocated. -/// len: the length of said buffer. May be changed, mussn't be const. -/// fmt: the const char * descriping the format. -/// note that fmt should be the first argument of the function of type -/// const char * and has to be directly followed by the variable arguments. -#define bout_vsnprintf(buf, len, fmt) \ - { \ - va_list va; \ - va_start(va, fmt); \ - int _vsnprintflen = vsnprintf(buf, len, fmt, va); \ - va_end(va); \ - if (_vsnprintflen + 1 > int(len)) { \ - _vsnprintflen += 1; \ - delete[] buf; \ - buf = new char[_vsnprintflen]; \ - len = _vsnprintflen; \ - va_start(va, fmt); \ - vsnprintf(buf, len, fmt, va); \ - va_end(va); \ - } \ - } - /// Convert pointer or reference to pointer /// This allows consistent handling of both in macros, templates template diff --git a/src/sys/output.cxx b/src/sys/output.cxx index 0ab1d2e0c8..c07cd53474 100644 --- a/src/sys/output.cxx +++ b/src/sys/output.cxx @@ -109,5 +109,3 @@ ConditionalOutput output_progress(Output::getInstance()); ConditionalOutput output_error(Output::getInstance()); ConditionalOutput output_verbose(Output::getInstance(), false); ConditionalOutput output(Output::getInstance()); - -#undef bout_vsnprint_pre From 2f30d2bb9d89b7ed0affee1acac282f447845895 Mon Sep 17 00:00:00 2001 From: dschwoerer Date: Wed, 29 Mar 2023 09:08:28 +0000 Subject: [PATCH 056/111] Apply clang-format changes --- include/bout/utils.hxx | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/include/bout/utils.hxx b/include/bout/utils.hxx index e57768c643..450d3eaf87 100644 --- a/include/bout/utils.hxx +++ b/include/bout/utils.hxx @@ -662,15 +662,14 @@ std::string trimComments(const std::string& s, const std::string& c = "#;"); std::string::size_type editDistance(const std::string& str1, const std::string& str2); // from https://en.cppreference.com/w/cpp/types/numeric_limits/epsilon -template +template typename std::enable_if::is_integer, bool>::type - almost_equal(T x, T y, int ulp = 2) -{ - // the machine epsilon has to be scaled to the magnitude of the values used - // and multiplied by the desired precision in ULPs (units in the last place) - return std::fabs(x - y) <= std::numeric_limits::epsilon() * std::fabs(x + y) * ulp - // unless the result is subnormal - || std::fabs(x - y) < std::numeric_limits::min(); +almost_equal(T x, T y, int ulp = 2) { + // the machine epsilon has to be scaled to the magnitude of the values used + // and multiplied by the desired precision in ULPs (units in the last place) + return std::fabs(x - y) <= std::numeric_limits::epsilon() * std::fabs(x + y) * ulp + // unless the result is subnormal + || std::fabs(x - y) < std::numeric_limits::min(); } /// Convert pointer or reference to pointer From 0eea867bbf5314c7e87fe847687ea7e6eb68feb0 Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 30 Mar 2023 00:18:49 +0200 Subject: [PATCH 057/111] Increase limit for isUniform --- include/bout/field.hxx | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/include/bout/field.hxx b/include/bout/field.hxx index ecd547db5e..300a09a62c 100644 --- a/include/bout/field.hxx +++ b/include/bout/field.hxx @@ -346,7 +346,7 @@ inline bool isUniform(const T& f, bool allpe = false, // TODO: maybe parallise this loop, as the early return is unlikely BOUT_FOR_SERIAL(i, f.getRegion(region)) { // by default we only check for exact equality, as that should cover most cases - if (f[i] != element and (not almost_equal(f[i], element))) { + if (f[i] != element and (not almost_equal(f[i], element, 10))) { result = false; break; } @@ -371,8 +371,11 @@ inline BoutReal getUniform(const T& f, bool allpe = false, const std::string& region = "RGN_ALL") { #if CHECK > 1 if (not isUniform(f, allpe, region)) { - throw BoutException("Requested getUniform({}, {}, {}) but Field is not const", f.name, - allpe, region); + const BoutReal f1 = min(f); + const BoutReal f2 = max(f); + throw BoutException("Requested getUniform({}, {}, {}) but Field is not const " + "([{:.12f}...{:.12f}] Δ={:e})", + f.name, allpe, region, f1, f2, f2 - f1); } #endif return f[*f.getRegion(region).begin()]; From 0afa74ec4be002d291239ec45bf284af167ceb60 Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 30 Mar 2023 00:19:52 +0200 Subject: [PATCH 058/111] Improve error message in case field is not uniform --- include/bout/field.hxx | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/include/bout/field.hxx b/include/bout/field.hxx index 300a09a62c..80f3dc37c9 100644 --- a/include/bout/field.hxx +++ b/include/bout/field.hxx @@ -371,11 +371,12 @@ inline BoutReal getUniform(const T& f, bool allpe = false, const std::string& region = "RGN_ALL") { #if CHECK > 1 if (not isUniform(f, allpe, region)) { - const BoutReal f1 = min(f); - const BoutReal f2 = max(f); + const BoutReal f1 = min(f, allpe, region); + const BoutReal f2 = max(f, allpe, region); throw BoutException("Requested getUniform({}, {}, {}) but Field is not const " - "([{:.12f}...{:.12f}] Δ={:e})", - f.name, allpe, region, f1, f2, f2 - f1); + "([{:.15f}...{:.15f}] Δ={:e} {:e}ε)", + f.name, allpe, region, f1, f2, f2 - f1, + (f2 - f1) / (f1 + f2) / std::numeric_limits::epsilon()); } #endif return f[*f.getRegion(region).begin()]; From f01db8390aa6e459dc1accb14c76daaff024ac30 Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 19 Apr 2023 09:49:35 +0200 Subject: [PATCH 059/111] CI: do not run on removed old ubuntu --- .github/workflows/tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7ac4f0977f..1c581a4c2d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,13 +31,13 @@ jobs: is_master_or_next: - ${{ github.ref == 'refs/heads/master' || github.ref == 'refs/heads/next' || github.base_ref == 'master' || github.base_ref == 'next' }} config: - - name: "Default options, Ubuntu 18.04" - os: ubuntu-18.04 + - name: "Default options, Ubuntu 20.04" + os: ubuntu-20.04 configure_options: "" script_flags: "-uim" - name: "Optimised, shared, Python" - os: ubuntu-18.04 + os: ubuntu-20.04 configure_options: "--enable-shared --enable-checks=no --enable-optimize=3 @@ -51,7 +51,7 @@ jobs: omp_num_threads: 1 - name: "Debug, shared" - os: ubuntu-18.04 + os: ubuntu-20.04 configure_options: "--enable-shared --enable-sigfpe --enable-debug From 33315d8166d471b0305ffc718e350d5bc5d05fee Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 19 Apr 2023 09:52:14 +0200 Subject: [PATCH 060/111] CI: move some runs to ubunutu-latest --- .github/workflows/tests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1c581a4c2d..dc19609951 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -62,8 +62,8 @@ jobs: script_flags: "-uim" omp_num_threads: 1 - - name: "CMake, shared, release, Ubuntu 20.04" - os: ubuntu-20.04 + - name: "CMake, shared, release" + os: ubuntu-latest cmake_options: "-DBUILD_SHARED_LIBS=ON -DBOUT_ENABLE_OPENMP=ON -DCMAKE_BUILD_TYPE=Release @@ -74,8 +74,8 @@ jobs: -DSUNDIALS_ROOT=/home/runner/local" omp_num_threads: 2 - - name: "CMake, shared, OpenMP, Ubuntu 20.04, 3D metrics" - os: ubuntu-20.04 + - name: "CMake, shared, OpenMP, 3D metrics" + os: ubuntu-latest cmake_options: "-DBUILD_SHARED_LIBS=ON -DBOUT_ENABLE_METRIC_3D=ON -DBOUT_ENABLE_OPENMP=ON @@ -87,7 +87,7 @@ jobs: omp_num_threads: 2 - name: "Coverage" - os: ubuntu-18.04 + os: ubuntu-latest configure_options: "--enable-shared --enable-code-coverage --enable-debug @@ -181,7 +181,7 @@ jobs: # This is its own job as it doesn't use most of the steps of the # standard_tests timeout-minutes: 60 - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 with: From b507ce7fe238d12c27318f51b34955956400c5df Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 19 Apr 2023 10:50:27 +0200 Subject: [PATCH 061/111] CI: Update paths for ubuntu 20.04 --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dc19609951..7cfe35383e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,9 +16,9 @@ jobs: runs-on: ${{ matrix.config.os }} env: BOUT_TEST_TIMEOUT: "6m" - PETSC_DIR: /usr/lib/petscdir/3.7.7/x86_64-linux-gnu-real + PETSC_DIR: /usr/lib/petscdir/3.12.4/x86_64-linux-gnu-real PETSC_ARCH: "" - SLEPC_DIR: /usr/lib/slepcdir/3.7.4/x86_64-linux-gnu-real + SLEPC_DIR: /usr/lib/slepcdir/3.12.2/x86_64-linux-gnu-real SLEPC_ARCH: "" OMP_NUM_THREADS: ${{ matrix.config.omp_num_threads }} PYTHONPATH: ${{ github.workspace }}/tools/pylib From 85ea85702e4a595c008fda486225e5dd4b949e93 Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 19 Apr 2023 12:51:07 +0200 Subject: [PATCH 062/111] CI: fix slepc path Co-authored-by: Peter Hill --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7cfe35383e..be847188f4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -18,7 +18,7 @@ jobs: BOUT_TEST_TIMEOUT: "6m" PETSC_DIR: /usr/lib/petscdir/3.12.4/x86_64-linux-gnu-real PETSC_ARCH: "" - SLEPC_DIR: /usr/lib/slepcdir/3.12.2/x86_64-linux-gnu-real + SLEPC_DIR: /usr/lib/slepcdir/slepc3.12/x86_64-linux-gnu-real/ SLEPC_ARCH: "" OMP_NUM_THREADS: ${{ matrix.config.omp_num_threads }} PYTHONPATH: ${{ github.workspace }}/tools/pylib From bcb302fa51f825a551b6147fdc0f0d51fba7e661 Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 19 Apr 2023 12:56:33 +0200 Subject: [PATCH 063/111] CI: fix petsc path --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index be847188f4..d97d27678d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,7 +16,7 @@ jobs: runs-on: ${{ matrix.config.os }} env: BOUT_TEST_TIMEOUT: "6m" - PETSC_DIR: /usr/lib/petscdir/3.12.4/x86_64-linux-gnu-real + PETSC_DIR: /usr/lib/petscdir/petsc3.12/x86_64-linux-gnu-real PETSC_ARCH: "" SLEPC_DIR: /usr/lib/slepcdir/slepc3.12/x86_64-linux-gnu-real/ SLEPC_ARCH: "" From c635f25379e99a6fe506f917f784273dbeec4b12 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Wed, 19 Apr 2023 13:54:29 +0100 Subject: [PATCH 064/111] CI: Drop coverage build OS down to ubuntu-20.04 Incompatible PETSc/SLEPc version This seems to only be a problem for `configure` and so should disappear when we switch completely to CMake anyway --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d97d27678d..6883c7077c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -87,7 +87,7 @@ jobs: omp_num_threads: 2 - name: "Coverage" - os: ubuntu-latest + os: ubuntu-20.04 configure_options: "--enable-shared --enable-code-coverage --enable-debug From 6c31487b05721525ab3cb46dbbfe7a9a450d83b0 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Wed, 19 Apr 2023 17:53:09 +0100 Subject: [PATCH 065/111] Link against PkgConfig's imported target for netCDF --- cmake/SetupBOUTThirdParty.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/SetupBOUTThirdParty.cmake b/cmake/SetupBOUTThirdParty.cmake index fd41fe5bfe..82d36198ce 100644 --- a/cmake/SetupBOUTThirdParty.cmake +++ b/cmake/SetupBOUTThirdParty.cmake @@ -176,8 +176,8 @@ if (BOUT_USE_NETCDF) else() find_package(PkgConfig REQUIRED) pkg_check_modules(NETCDF REQUIRED IMPORTED_TARGET netcdf-cxx4) - set(CONFIG_LDFLAGS "${CONFIG_LDFLAGS} ${netCDF_CXX_LIBRARY} ${NETCDF_LDFLAGS}") - target_link_libraries(bout++ PUBLIC ${NETCDF_LDFLAGS}) + set(CONFIG_LDFLAGS "${CONFIG_LDFLAGS} ${NETCDF_LDFLAGS}") + target_link_libraries(bout++ PUBLIC PkgConfig::NETCDF) endif() endif() endif() From 419a7e79b7b892af421168f8619b63c8d95ff08d Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Thu, 20 Apr 2023 09:28:53 +0100 Subject: [PATCH 066/111] CMake: Fix setting `CONFIG_LDFLAGS` for netCDF from PkgConfig --- cmake/SetupBOUTThirdParty.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/SetupBOUTThirdParty.cmake b/cmake/SetupBOUTThirdParty.cmake index 82d36198ce..25ce8f5581 100644 --- a/cmake/SetupBOUTThirdParty.cmake +++ b/cmake/SetupBOUTThirdParty.cmake @@ -176,8 +176,9 @@ if (BOUT_USE_NETCDF) else() find_package(PkgConfig REQUIRED) pkg_check_modules(NETCDF REQUIRED IMPORTED_TARGET netcdf-cxx4) - set(CONFIG_LDFLAGS "${CONFIG_LDFLAGS} ${NETCDF_LDFLAGS}") target_link_libraries(bout++ PUBLIC PkgConfig::NETCDF) + list(JOIN NETCDF_LDFLAGS " " NETCDF_LDFLAGS_STRING) + set(CONFIG_LDFLAGS "${CONFIG_LDFLAGS} ${NETCDF_LDFLAGS_STRING}") endif() endif() endif() From a54b51dbf315a7f2d1241e29b64a89823ca1d014 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 21 Apr 2023 09:11:19 +0100 Subject: [PATCH 067/111] Docs: Fix default branch for "suggest edit" Fixes #2688 [skip ci] --- manual/sphinx/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/manual/sphinx/conf.py b/manual/sphinx/conf.py index 05c33240ea..cd8e858a72 100755 --- a/manual/sphinx/conf.py +++ b/manual/sphinx/conf.py @@ -234,6 +234,7 @@ def __getattr__(cls, name): # html_theme_options = dict( repository_url="https://github.com/boutproject/BOUT-dev", + repository_branch="master", path_to_docs="manual/sphinx", use_edit_page_button=True, use_repository_button=True, From 6a1ec40d7cbff2e7b76d9caa0cddbf14db13316f Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 21 Apr 2023 09:13:04 +0100 Subject: [PATCH 068/111] Docs: Update copyright year [skip ci] --- manual/sphinx/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manual/sphinx/conf.py b/manual/sphinx/conf.py index cd8e858a72..c5a9f4716d 100755 --- a/manual/sphinx/conf.py +++ b/manual/sphinx/conf.py @@ -177,8 +177,8 @@ def __getattr__(cls, name): # General information about the project. project = "BOUT++" -copyright = "2017, B. Dudson" -author = "The BOUT++ team" +copyright = "2017-2023" +author = "B. Dudson and The BOUT++ team" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the From 7617cc3f79fc42415b7d0b7223830e1733ca6a04 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Wed, 26 Apr 2023 14:56:14 +0100 Subject: [PATCH 069/111] Remove last uses of `bout::globals::dump` Fixes #2667 --- .../simple-hypre/test-laplacexy-hypre.cxx | 10 ++-- .../test-laplace-hypre3d/test-laplace3d.cxx | 9 ++- .../test-laplacexy/test-laplacexy.cxx | 59 +++++++++---------- .../test-laplacexy2-hypre/test-laplacexy.cxx | 29 ++++----- 4 files changed, 53 insertions(+), 54 deletions(-) diff --git a/examples/laplacexy/simple-hypre/test-laplacexy-hypre.cxx b/examples/laplacexy/simple-hypre/test-laplacexy-hypre.cxx index 68caa2222b..e38a285cfc 100644 --- a/examples/laplacexy/simple-hypre/test-laplacexy-hypre.cxx +++ b/examples/laplacexy/simple-hypre/test-laplacexy-hypre.cxx @@ -13,12 +13,14 @@ int main(int argc, char** argv) { bout::globals::mesh); /// Solution - Field2D x = 0.0; + Field2D solution = 0.0; - x = laplacexy.solve(rhs, x); + solution = laplacexy.solve(rhs, solution); - SAVE_ONCE2(rhs, x); - bout::globals::dump.write(); // Save output file + Options dump; + dump["rhs"] = rhs; + dump["x"] = solution; + bout::writeDefaultOutputFile(dump); } BoutFinalise(); #if BOUT_USE_CUDA diff --git a/tests/integrated/test-laplace-hypre3d/test-laplace3d.cxx b/tests/integrated/test-laplace-hypre3d/test-laplace3d.cxx index 69524d44da..64b5dec1d4 100644 --- a/tests/integrated/test-laplace-hypre3d/test-laplace3d.cxx +++ b/tests/integrated/test-laplace-hypre3d/test-laplace3d.cxx @@ -125,8 +125,13 @@ int main(int argc, char** argv) { Field3D error = rhs_check - rhs; BoutReal error_max = max(abs(error), true); - SAVE_ONCE(f, rhs, rhs_check, error, error_max); - bout::globals::dump.write(); + Options dump; + dump["f"] = f; + dump["rhs"] = rhs; + dump["rhs_check"] = rhs_check; + dump["error"] = error; + dump["error_max"] = error_max; + bout::writeDefaultOutputFile(dump); laplace_solver.reset(nullptr); BoutFinalise(); diff --git a/tests/integrated/test-laplacexy/test-laplacexy.cxx b/tests/integrated/test-laplacexy/test-laplacexy.cxx index 62449a76ee..97f2cc14d3 100644 --- a/tests/integrated/test-laplacexy/test-laplacexy.cxx +++ b/tests/integrated/test-laplacexy/test-laplacexy.cxx @@ -30,27 +30,20 @@ #include #include -using bout::globals::dump; using bout::globals::mesh; int main(int argc, char** argv) { BoutInitialise(argc, argv); - auto coords = mesh->getCoordinates(); - - auto& opt = Options::root(); - - LaplaceXY laplacexy; - - bool include_y_derivs = opt["laplacexy"]["include_y_derivs"]; + auto* coords = mesh->getCoordinates(); // Solving equations of the form // Div(A Grad_perp(f)) + B*f = rhs // A*Laplace_perp(f) + Grad_perp(A).Grad_perp(f) + B*f = rhs - Field2D f, a, b, sol; - Field2D error, absolute_error; //Absolute value of relative error: abs((f - sol)/f) - BoutReal max_error; //Output of test + Field2D f; + Field2D a; + Field2D b; initial_profile("f", f); initial_profile("a", a); @@ -63,42 +56,44 @@ int main(int argc, char** argv) { //////////////////////////////////////////////////////////////////////////////////////// - Field2D rhs, rhs_check; + Field2D rhs; + const bool include_y_derivs = Options::root()["laplacexy"]["include_y_derivs"]; if (include_y_derivs) { rhs = a * Laplace_perp(f) + Grad_perp(a) * Grad_perp(f) + b * f; } else { rhs = a * Delp2(f, CELL_DEFAULT, false) + coords->g11 * DDX(a) * DDX(f) + b * f; } + LaplaceXY laplacexy; laplacexy.setCoefs(a, b); - sol = laplacexy.solve(rhs, 0.); - error = (f - sol) / f; - absolute_error = f - sol; - max_error = max(abs(absolute_error), true); + Field2D solution = laplacexy.solve(rhs, 0.); + Field2D relative_error = (f - solution) / f; + Field2D absolute_error = f - solution; + BoutReal max_error = max(abs(absolute_error), true); - output << "Magnitude of maximum absolute error is " << max_error << endl; + output.write("Magnitude of maximum absolute error is {}\n", max_error); - mesh->communicate(sol); + mesh->communicate(solution); + Field2D rhs_check; if (include_y_derivs) { - rhs_check = a * Laplace_perp(sol) + Grad_perp(a) * Grad_perp(sol) + b * sol; + rhs_check = a * Laplace_perp(solution) + Grad_perp(a) * Grad_perp(solution) + b * solution; } else { rhs_check = - a * Delp2(sol, CELL_DEFAULT, false) + coords->g11 * DDX(a) * DDX(sol) + b * sol; + a * Delp2(solution, CELL_DEFAULT, false) + coords->g11 * DDX(a) * DDX(solution) + b * solution; } - dump.add(a, "a"); - dump.add(b, "b"); - dump.add(f, "f"); - dump.add(sol, "sol"); - dump.add(error, "error"); - dump.add(absolute_error, "absolute_error"); - dump.add(max_error, "max_error"); - dump.add(rhs, "rhs"); - dump.add(rhs_check, "rhs_check"); - - dump.write(); - dump.close(); + Options dump; + dump["a"] = a; + dump["b"] = b; + dump["f"] = f; + dump["sol"] = solution; + dump["relative_error"] = relative_error; + dump["absolute_error"] = absolute_error; + dump["max_error"] = max_error; + dump["rhs"] = rhs; + dump["rhs_check"] = rhs_check; + bout::writeDefaultOutputFile(dump); MPI_Barrier(BoutComm::get()); // Wait for all processors to write data diff --git a/tests/integrated/test-laplacexy2-hypre/test-laplacexy.cxx b/tests/integrated/test-laplacexy2-hypre/test-laplacexy.cxx index 77fbdb197c..0ae2cbf312 100644 --- a/tests/integrated/test-laplacexy2-hypre/test-laplacexy.cxx +++ b/tests/integrated/test-laplacexy2-hypre/test-laplacexy.cxx @@ -59,8 +59,8 @@ int main(int argc, char** argv) { Field2D guess = 0.0; Field2D sol = laplacexy.solve(rhs, guess); Field2D error = (f - sol) / f; - Field2D absolute_error = - abs(f - sol); // Absolute value of relative error: abs((f - sol)/f) + // Absolute value of relative error: abs((f - sol)/f) + Field2D absolute_error = abs(f - sol); BoutReal max_error = max(absolute_error, true); output << "Magnitude of maximum absolute error is " << max_error << endl; @@ -68,20 +68,17 @@ int main(int argc, char** argv) { sol.getMesh()->communicate(sol); Field2D rhs_check = Laplace_perpXY(a, sol); - using bout::globals::dump; - - dump.add(a, "a"); - dump.add(b, "b"); - dump.add(f, "f"); - dump.add(sol, "sol"); - dump.add(error, "error"); - dump.add(absolute_error, "absolute_error"); - dump.add(max_error, "max_error"); - dump.add(rhs, "rhs"); - dump.add(rhs_check, "rhs_check"); - - dump.write(); - dump.close(); + Options dump; + dump["a"] = a; + dump["b"] = b; + dump["f"] = f; + dump["sol"] = sol; + dump["error"] = error; + dump["absolute_error"] = absolute_error; + dump["max_error"] = max_error; + dump["rhs"] = rhs; + dump["rhs_check"] = rhs_check; + bout::writeDefaultOutputFile(dump); MPI_Barrier(BoutComm::get()); // Wait for all processors to write data From 3051629d660f7ca7e3f7dc1095dea75c90b73e36 Mon Sep 17 00:00:00 2001 From: ZedThree Date: Wed, 26 Apr 2023 13:58:17 +0000 Subject: [PATCH 070/111] Apply clang-format changes --- tests/integrated/test-laplacexy/test-laplacexy.cxx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/integrated/test-laplacexy/test-laplacexy.cxx b/tests/integrated/test-laplacexy/test-laplacexy.cxx index 97f2cc14d3..3142d359b1 100644 --- a/tests/integrated/test-laplacexy/test-laplacexy.cxx +++ b/tests/integrated/test-laplacexy/test-laplacexy.cxx @@ -77,10 +77,11 @@ int main(int argc, char** argv) { mesh->communicate(solution); Field2D rhs_check; if (include_y_derivs) { - rhs_check = a * Laplace_perp(solution) + Grad_perp(a) * Grad_perp(solution) + b * solution; - } else { rhs_check = - a * Delp2(solution, CELL_DEFAULT, false) + coords->g11 * DDX(a) * DDX(solution) + b * solution; + a * Laplace_perp(solution) + Grad_perp(a) * Grad_perp(solution) + b * solution; + } else { + rhs_check = a * Delp2(solution, CELL_DEFAULT, false) + + coords->g11 * DDX(a) * DDX(solution) + b * solution; } Options dump; From 333dfc49034eca28dadfccedfa86f83bfbb566f1 Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 27 Apr 2023 11:27:49 +0200 Subject: [PATCH 071/111] paralleltransform is a section --- manual/sphinx/user_docs/parallel-transforms.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/manual/sphinx/user_docs/parallel-transforms.rst b/manual/sphinx/user_docs/parallel-transforms.rst index ad7295bbee..3ee3eccfb8 100644 --- a/manual/sphinx/user_docs/parallel-transforms.rst +++ b/manual/sphinx/user_docs/parallel-transforms.rst @@ -70,8 +70,8 @@ setting is .. code-block:: cfg - [mesh] - paralleltransform = identity + [mesh:paralleltransform] + type = identity This then uses the `ParallelTransformIdentity` class to calculate the @@ -100,8 +100,8 @@ The shifted metric method is selected using: .. code-block:: cfg - [mesh] - paralleltransform = shifted + [mesh:paralleltransform] + type = shifted so that mesh uses the `ShiftedMetric` class to calculate parallel transforms. During initialisation, this class reads a quantity zShift @@ -141,8 +141,8 @@ calculation of parallel slices. Select it by using: .. code-block:: cfg - [mesh] - paralleltransform = shifted + [mesh:paralleltransform] + type = shifted calcParallelSlices_on_communicate = false With these settings, inputs to parallel derivative or interpolation @@ -177,8 +177,8 @@ To use the FCI method for parallel transforms, set .. code-block:: cfg - [mesh] - paralleltransform = fci + [mesh:paralleltransform] + type = fci which causes the `FCITransform` class to be used for parallel transforms. This reads four variables (3D fields) from the input From f73b1edc2c9da11ff569777eafa1bdb9d30b0d47 Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 17 Nov 2022 13:37:10 +0100 Subject: [PATCH 072/111] CI: Add testing with new PETSc --- .build_petsc_for_ci.sh | 41 +++++++++++++++++++++++++++++++++++++ .github/workflows/tests.yml | 19 ++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100755 .build_petsc_for_ci.sh diff --git a/.build_petsc_for_ci.sh b/.build_petsc_for_ci.sh new file mode 100755 index 0000000000..4d23774315 --- /dev/null +++ b/.build_petsc_for_ci.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -e + +if test $BUILD_PETSC ; then + if [[ ! -d $HOME/local/include/petsc ]]; then + echo "****************************************" + echo "Building PETSc" + echo "****************************************" + + git clone -b release https://gitlab.com/petsc/petsc.git petsc --depth=1 + + unset PETSC_DIR + unset PETSC_ARCH + + pushd petsc + ./configure \ + --with-mpi=yes \ + --with-precision=double \ + --with-scalar-type=real \ + --with-shared-libraries=1 \ + --with-debugging=0 \ + {C,CXX,F}OPTFLAGS="-O3 -march=native" \ + --prefix=$HOME/local/petsc + + make && make install + popd + + echo "****************************************" + echo " Finished building PETSc" + echo "****************************************" + else + echo "****************************************" + echo " PETSc already installed" + echo "****************************************" + fi +else + echo "****************************************" + echo " PETSc not requested" + echo "****************************************" +fi diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6883c7077c..0fbe2e91ca 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -86,6 +86,20 @@ jobs: -DSUNDIALS_ROOT=/home/runner/local" omp_num_threads: 2 + - name: "new PETSc" + os: ubuntu-20.04 + cmake_options: "-DBUILD_SHARED_LIBS=ON + -DBOUT_ENABLE_METRIC_3D=ON + -DBOUT_ENABLE_OPENMP=ON + -DBOUT_USE_PETSC=ON + -DBOUT_USE_SLEPC=ON + -DBOUT_USE_SUNDIALS=ON + -DBOUT_ENABLE_PYTHON=ON + -DSUNDIALS_ROOT=/home/runner/local + -DPETSC_DIR=/home/runner/local/petsc" + + build_petsc: -petsc + - name: "Coverage" os: ubuntu-20.04 configure_options: "--enable-shared @@ -145,11 +159,14 @@ jobs: uses: actions/cache@v2 with: path: /home/runner/local - key: bout-sundials-${{ matrix.config.os }} + key: bout-sundials-${{ matrix.config.os }}${{ matrix.config.build_petsc }} - name: Build SUNDIALS run: ./.build_sundials_for_ci.sh + - name: Build PETSc + run: BUILD_PETSC=${{ matrix.config.build_petsc }} ./.build_petsc_for_ci.sh + - name: Build (configure) if: ${{ ! contains(matrix.config.name, 'CMake') }} run: ./.ci_script.sh ${{ matrix.config.script_flags }} From 6c0d6b487f19a7b50fa69646a2cf6b565c0bc5ad Mon Sep 17 00:00:00 2001 From: David Bold Date: Thu, 17 Nov 2022 13:37:33 +0100 Subject: [PATCH 073/111] Call PCGAMGSetSymGraph only with old PETSc For newer PETSc set the option on the matrix --- src/invert/laplace/impls/petsc3damg/petsc3damg.cxx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/invert/laplace/impls/petsc3damg/petsc3damg.cxx b/src/invert/laplace/impls/petsc3damg/petsc3damg.cxx index 460bec865b..88f53f34fd 100644 --- a/src/invert/laplace/impls/petsc3damg/petsc3damg.cxx +++ b/src/invert/laplace/impls/petsc3damg/petsc3damg.cxx @@ -460,6 +460,10 @@ void LaplacePetsc3dAmg::updateMatrix3D() { operator3D.assemble(); MatSetBlockSize(*operator3D.get(), 1); +#if PETSC_VERSION_GE(3, 18, 0) + MatSetOption(*operator3D.get(), MAT_SYMMETRIC, PETSC_TRUE); +#endif + // Declare KSP Context (abstract PETSc object that manages all Krylov methods) if (kspInitialised) { KSPDestroy(&ksp); @@ -511,7 +515,9 @@ void LaplacePetsc3dAmg::updateMatrix3D() { // Set the relative and absolute tolerances PCSetType(pc, pctype.c_str()); +#if PETSC_VERSION_LT(3, 18, 0) PCGAMGSetSymGraph(pc, PETSC_TRUE); +#endif } lib.setOptionsFromInputFile(ksp); From 4fd461c94b692d866bd4c98ce86a8705e651e790 Mon Sep 17 00:00:00 2001 From: David Bold Date: Mon, 21 Nov 2022 12:10:24 +0100 Subject: [PATCH 074/111] Do not specify that the matrix is symmetric --- src/invert/laplace/impls/petsc3damg/petsc3damg.cxx | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/invert/laplace/impls/petsc3damg/petsc3damg.cxx b/src/invert/laplace/impls/petsc3damg/petsc3damg.cxx index 88f53f34fd..40cbf621b6 100644 --- a/src/invert/laplace/impls/petsc3damg/petsc3damg.cxx +++ b/src/invert/laplace/impls/petsc3damg/petsc3damg.cxx @@ -460,10 +460,6 @@ void LaplacePetsc3dAmg::updateMatrix3D() { operator3D.assemble(); MatSetBlockSize(*operator3D.get(), 1); -#if PETSC_VERSION_GE(3, 18, 0) - MatSetOption(*operator3D.get(), MAT_SYMMETRIC, PETSC_TRUE); -#endif - // Declare KSP Context (abstract PETSc object that manages all Krylov methods) if (kspInitialised) { KSPDestroy(&ksp); From 43868704cc932f6aa4da077ebe1728a24cab1d43 Mon Sep 17 00:00:00 2001 From: David Bold Date: Mon, 21 Nov 2022 13:30:48 +0100 Subject: [PATCH 075/111] CI: Specify that it uses cmake --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0fbe2e91ca..944b730f66 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -86,7 +86,7 @@ jobs: -DSUNDIALS_ROOT=/home/runner/local" omp_num_threads: 2 - - name: "new PETSc" + - name: "CMake, new PETSc" os: ubuntu-20.04 cmake_options: "-DBUILD_SHARED_LIBS=ON -DBOUT_ENABLE_METRIC_3D=ON From ec1678bb0b5d616331c121f2e6f9c341323b70b1 Mon Sep 17 00:00:00 2001 From: David Bold Date: Mon, 21 Nov 2022 13:32:05 +0100 Subject: [PATCH 076/111] CI: Fix path for checking installation --- .build_petsc_for_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.build_petsc_for_ci.sh b/.build_petsc_for_ci.sh index 4d23774315..03afb8abc7 100755 --- a/.build_petsc_for_ci.sh +++ b/.build_petsc_for_ci.sh @@ -3,7 +3,7 @@ set -e if test $BUILD_PETSC ; then - if [[ ! -d $HOME/local/include/petsc ]]; then + if [[ ! -d $HOME/local/petsc/include/petsc ]]; then echo "****************************************" echo "Building PETSc" echo "****************************************" From b04c939c8eb725bdf755ed9b96c378d23235c09e Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 26 Apr 2023 10:26:34 +0200 Subject: [PATCH 077/111] PETSC_DEFAULT is enum, and cannot be used as Options type --- src/solver/impls/slepc/slepc.cxx | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/solver/impls/slepc/slepc.cxx b/src/solver/impls/slepc/slepc.cxx index 3f6b7a91bf..22b0e6c93e 100644 --- a/src/solver/impls/slepc/slepc.cxx +++ b/src/solver/impls/slepc/slepc.cxx @@ -162,11 +162,17 @@ SlepcSolver::SlepcSolver(Options* options) { .withDefault(0); tol = options_ref["tol"].doc("SLEPc tolerance").withDefault(1.0e-6); - maxIt = options_ref["maxIt"].doc("Maximum iterations").withDefault(PETSC_DEFAULT); + maxIt = options_ref["maxIt"].doc("Maximum iterations").withDefault(-1); + if (maxIt == -1){ + maxIt = PETSC_DEFAULT; + } mpd = options_ref["mpd"] .doc("Maximum dimension allowed for the projected problem") - .withDefault(PETSC_DEFAULT); + .withDefault(-1); + if (mpd == -1) { + mpd = PETSC_DEFAULT; + } ddtMode = options_ref["ddtMode"].withDefault(true); From 6e37df37f512af93d3c44f90aae9780ba6c51682 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 28 Apr 2023 13:34:53 +0100 Subject: [PATCH 078/111] Make sure `PETSC_DEFAULT` is `int` when using it in `Options` --- src/solver/impls/slepc/slepc.cxx | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/solver/impls/slepc/slepc.cxx b/src/solver/impls/slepc/slepc.cxx index 22b0e6c93e..f90afe717e 100644 --- a/src/solver/impls/slepc/slepc.cxx +++ b/src/solver/impls/slepc/slepc.cxx @@ -162,17 +162,13 @@ SlepcSolver::SlepcSolver(Options* options) { .withDefault(0); tol = options_ref["tol"].doc("SLEPc tolerance").withDefault(1.0e-6); - maxIt = options_ref["maxIt"].doc("Maximum iterations").withDefault(-1); - if (maxIt == -1){ - maxIt = PETSC_DEFAULT; - } + maxIt = options_ref["maxIt"] + .doc("Maximum iterations") + .withDefault(static_cast(PETSC_DEFAULT)); mpd = options_ref["mpd"] .doc("Maximum dimension allowed for the projected problem") - .withDefault(-1); - if (mpd == -1) { - mpd = PETSC_DEFAULT; - } + .withDefault(static_cast(PETSC_DEFAULT)); ddtMode = options_ref["ddtMode"].withDefault(true); From b75942d21446ab64b051e8e18078bb2f93dc3682 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 28 Apr 2023 14:05:04 +0100 Subject: [PATCH 079/111] Replace deprecated `PETSC_NULL` with `nullptr` --- cmake/FindSLEPc.cmake | 2 +- .../laplace/impls/petsc/petsc_laplace.cxx | 2 - src/solver/impls/imex-bdf2/imex-bdf2.cxx | 4 +- src/solver/impls/petsc/petsc.cxx | 44 +++++++++---------- src/solver/impls/snes/snes.cxx | 4 +- src/sys/petsclib.cxx | 2 +- src/sys/slepclib.cxx | 2 +- 7 files changed, 29 insertions(+), 31 deletions(-) diff --git a/cmake/FindSLEPc.cmake b/cmake/FindSLEPc.cmake index 2db1a25a1f..ad873822d5 100644 --- a/cmake/FindSLEPc.cmake +++ b/cmake/FindSLEPc.cmake @@ -204,7 +204,7 @@ int main() PetscErrorCode ierr; int argc = 0; char** argv = NULL; - ierr = SlepcInitialize(&argc, &argv, PETSC_NULL, PETSC_NULL); + ierr = SlepcInitialize(&argc, &argv, nullptr, nullptr); EPS eps; ierr = EPSCreate(PETSC_COMM_SELF, &eps); CHKERRQ(ierr); //ierr = EPSSetFromOptions(eps); CHKERRQ(ierr); diff --git a/src/invert/laplace/impls/petsc/petsc_laplace.cxx b/src/invert/laplace/impls/petsc/petsc_laplace.cxx index 107f3912d9..d96a9ff129 100644 --- a/src/invert/laplace/impls/petsc/petsc_laplace.cxx +++ b/src/invert/laplace/impls/petsc/petsc_laplace.cxx @@ -148,8 +148,6 @@ LaplacePetsc::LaplacePetsc(Options* opt, const CELL_LOC loc, Mesh* mesh_in, MatCreate(comm, &MatA); MatSetSizes(MatA, localN, localN, size, size); MatSetFromOptions(MatA); - // if (fourth_order) MatMPIAIJSetPreallocation( MatA, 25, PETSC_NULL, 10, PETSC_NULL ); - // else MatMPIAIJSetPreallocation( MatA, 9, PETSC_NULL, 3, PETSC_NULL ); /* Pre allocate memory * nnz denotes an array containing the number of non-zeros in the various rows diff --git a/src/solver/impls/imex-bdf2/imex-bdf2.cxx b/src/solver/impls/imex-bdf2/imex-bdf2.cxx index aae92ad27d..425c3e0073 100644 --- a/src/solver/impls/imex-bdf2/imex-bdf2.cxx +++ b/src/solver/impls/imex-bdf2/imex-bdf2.cxx @@ -676,9 +676,9 @@ void IMEXBDF2::constructSNES(SNES* snesIn) { BoutComm::get(), nlocal, nlocal, // Local sizes PETSC_DETERMINE, PETSC_DETERMINE, // Global sizes 3, // Number of nonzero entries in diagonal portion of local submatrix - PETSC_NULL, + nullptr, 0, // Number of nonzeros per row in off-diagonal portion of local submatrix - PETSC_NULL, &Jmf); + nullptr, &Jmf); #if PETSC_VERSION_GE(3, 4, 0) SNESSetJacobian(*snesIn, Jmf, Jmf, SNESComputeJacobianDefault, this); diff --git a/src/solver/impls/petsc/petsc.cxx b/src/solver/impls/petsc/petsc.cxx index ad8e99dd38..1e4450ae43 100644 --- a/src/solver/impls/petsc/petsc.cxx +++ b/src/solver/impls/petsc/petsc.cxx @@ -261,11 +261,11 @@ int PetscSolver::init() { CHKERRQ(ierr); #if PETSC_VERSION_GE(3, 7, 0) - ierr = PetscOptionsGetBool(PETSC_NULL, PETSC_NULL, "-interpolate", &interpolate, - PETSC_NULL); + ierr = PetscOptionsGetBool(nullptr, nullptr, "-interpolate", &interpolate, + nullptr); CHKERRQ(ierr); #else - ierr = PetscOptionsGetBool(PETSC_NULL, "-interpolate", &interpolate, PETSC_NULL); + ierr = PetscOptionsGetBool(nullptr, "-interpolate", &interpolate, nullptr); CHKERRQ(ierr); #endif @@ -273,21 +273,21 @@ int PetscSolver::init() { // run, if they didn't then use the standard monitor function. TODO: // use PetscFList #if PETSC_VERSION_GE(3, 7, 0) - ierr = PetscOptionsGetString(PETSC_NULL, PETSC_NULL, "-output_name", this->output_name, + ierr = PetscOptionsGetString(nullptr, nullptr, "-output_name", this->output_name, sizeof this->output_name, &output_flag); CHKERRQ(ierr); #else - ierr = PetscOptionsGetString(PETSC_NULL, "-output_name", this->output_name, + ierr = PetscOptionsGetString(nullptr, "-output_name", this->output_name, sizeof this->output_name, &output_flag); CHKERRQ(ierr); #endif // If the output_name is not specified then use the standard monitor function if (output_flag) { - ierr = SNESMonitorSet(snes, PetscSNESMonitor, this, PETSC_NULL); + ierr = SNESMonitorSet(snes, PetscSNESMonitor, this, nullptr); CHKERRQ(ierr); } else { - ierr = TSMonitorSet(ts, PetscMonitor, this, PETSC_NULL); + ierr = TSMonitorSet(ts, PetscMonitor, this, nullptr); CHKERRQ(ierr); } @@ -399,11 +399,11 @@ int PetscSolver::init() { // Create Jacobian matrix to be used by preconditioner output_info << " Get Jacobian matrix at simtime " << simtime << "\n"; #if PETSC_VERSION_GE(3, 7, 0) - ierr = PetscOptionsGetString(PETSC_NULL, PETSC_NULL, "-J_load", load_file, + ierr = PetscOptionsGetString(nullptr, nullptr, "-J_load", load_file, PETSC_MAX_PATH_LEN - 1, &J_load); CHKERRQ(ierr); #else - ierr = PetscOptionsGetString(PETSC_NULL, "-J_load", load_file, PETSC_MAX_PATH_LEN - 1, + ierr = PetscOptionsGetString(nullptr, "-J_load", load_file, PETSC_MAX_PATH_LEN - 1, &J_load); CHKERRQ(ierr); #endif @@ -451,26 +451,26 @@ int PetscSolver::init() { // Get nonzero pattern of J - color_none !!! prealloc = cols * dof * dof; - ierr = MatSeqAIJSetPreallocation(J, prealloc, PETSC_NULL); + ierr = MatSeqAIJSetPreallocation(J, prealloc, nullptr); CHKERRQ(ierr); - ierr = MatMPIAIJSetPreallocation(J, prealloc, PETSC_NULL, prealloc, PETSC_NULL); + ierr = MatMPIAIJSetPreallocation(J, prealloc, nullptr, prealloc, nullptr); CHKERRQ(ierr); prealloc = cols; // why nonzeros=295900, allocated nonzeros=2816000/12800000 (*dof*dof), number of mallocs used during MatSetValues calls =256? - ierr = MatSeqBAIJSetPreallocation(J, dof, prealloc, PETSC_NULL); + ierr = MatSeqBAIJSetPreallocation(J, dof, prealloc, nullptr); CHKERRQ(ierr); - ierr = MatMPIBAIJSetPreallocation(J, dof, prealloc, PETSC_NULL, prealloc, PETSC_NULL); + ierr = MatMPIBAIJSetPreallocation(J, dof, prealloc, nullptr, prealloc, nullptr); CHKERRQ(ierr); #if PETSC_VERSION_GE(3, 7, 0) - ierr = PetscOptionsHasName(PETSC_NULL, PETSC_NULL, "-J_slowfd", &J_slowfd); + ierr = PetscOptionsHasName(nullptr, nullptr, "-J_slowfd", &J_slowfd); CHKERRQ(ierr); #else - ierr = PetscOptionsHasName(PETSC_NULL, "-J_slowfd", &J_slowfd); + ierr = PetscOptionsHasName(nullptr, "-J_slowfd", &J_slowfd); CHKERRQ(ierr); #endif if (J_slowfd) { // create Jacobian matrix by slow fd - ierr = SNESSetJacobian(snes, J, J, SNESComputeJacobianDefault, PETSC_NULL); + ierr = SNESSetJacobian(snes, J, J, SNESComputeJacobianDefault, nullptr); CHKERRQ(ierr); output_info << "SNESComputeJacobian J by slow fd...\n"; @@ -524,10 +524,10 @@ int PetscSolver::init() { // Write J in binary for study - see ~petsc/src/mat/examples/tests/ex124.c #if PETSC_VERSION_GE(3, 7, 0) - ierr = PetscOptionsHasName(PETSC_NULL, PETSC_NULL, "-J_write", &J_write); + ierr = PetscOptionsHasName(nullptr, nullptr, "-J_write", &J_write); CHKERRQ(ierr); #else - ierr = PetscOptionsHasName(PETSC_NULL, "-J_write", &J_write); + ierr = PetscOptionsHasName(nullptr, "-J_write", &J_write); CHKERRQ(ierr); #endif if (J_write) { @@ -876,7 +876,7 @@ PetscErrorCode PhysicsSNESApply(SNES snes, Vec x) { Mat A, B; PetscFunctionBegin; - ierr = SNESGetJacobian(snes, &A, &B, PETSC_NULL, PETSC_NULL); + ierr = SNESGetJacobian(snes, &A, &B, nullptr, nullptr); CHKERRQ(ierr); #if PETSC_VERSION_GE(3, 5, 0) ierr = SNESComputeJacobian(snes, x, A, B); @@ -890,7 +890,7 @@ PetscErrorCode PhysicsSNESApply(SNES snes, Vec x) { CHKERRQ(ierr); ierr = KSPGetPC(ksp, &pc); CHKERRQ(ierr); - ierr = SNESGetFunction(snes, &F, PETSC_NULL, PETSC_NULL); + ierr = SNESGetFunction(snes, &F, nullptr, nullptr); CHKERRQ(ierr); ierr = SNESComputeFunction(snes, x, F); CHKERRQ(ierr); @@ -913,7 +913,7 @@ PetscErrorCode PhysicsSNESApply(SNES snes, Vec x) { << ", F \\cdot Fout " << dot << " "; #if PETSC_VERSION_GE(3, 5, 0) Vec func; - ierr = SNESGetFunction(snes, &func, PETSC_NULL, PETSC_NULL); + ierr = SNESGetFunction(snes, &func, nullptr, nullptr); CHKERRQ(ierr); ierr = VecNorm(func, NORM_2, &fnorm); CHKERRQ(ierr); @@ -962,7 +962,7 @@ PetscErrorCode PetscMonitor(TS ts, PetscInt UNUSED(step), PetscReal t, Vec X, vo ierr = TSGetMaxTime(ts, &tfinal); CHKERRQ(ierr); #else - ierr = TSGetDuration(ts, PETSC_NULL, &tfinal); + ierr = TSGetDuration(ts, nullptr, &tfinal); CHKERRQ(ierr); #endif diff --git a/src/solver/impls/snes/snes.cxx b/src/solver/impls/snes/snes.cxx index 69280bd6c2..65ada40d21 100644 --- a/src/solver/impls/snes/snes.cxx +++ b/src/solver/impls/snes/snes.cxx @@ -564,9 +564,9 @@ int SNESSolver::init() { BoutComm::get(), nlocal, nlocal, // Local sizes PETSC_DETERMINE, PETSC_DETERMINE, // Global sizes 3, // Number of nonzero entries in diagonal portion of local submatrix - PETSC_NULL, + nullptr, 0, // Number of nonzeros per row in off-diagonal portion of local submatrix - PETSC_NULL, &Jmf); + nullptr, &Jmf); #if PETSC_VERSION_GE(3, 4, 0) SNESSetJacobian(snes, Jmf, Jmf, SNESComputeJacobianDefault, this); #else diff --git a/src/sys/petsclib.cxx b/src/sys/petsclib.cxx index 91db0c44ea..85108b522f 100644 --- a/src/sys/petsclib.cxx +++ b/src/sys/petsclib.cxx @@ -25,7 +25,7 @@ PetscLib::PetscLib(Options* opt) { output << "Initialising PETSc\n"; PETSC_COMM_WORLD = BoutComm::getInstance()->getComm(); - PetscInitialize(pargc, pargv, PETSC_NULL, help); + PetscInitialize(pargc, pargv, nullptr, help); PetscPopSignalHandler(); PetscLogEventRegister("Total BOUT++", 0, &USER_EVENT); diff --git a/src/sys/slepclib.cxx b/src/sys/slepclib.cxx index 1dc89bc047..79fdfa9bbc 100644 --- a/src/sys/slepclib.cxx +++ b/src/sys/slepclib.cxx @@ -18,7 +18,7 @@ SlepcLib::SlepcLib() { // Initialise SLEPc output << "Initialising SLEPc\n"; - SlepcInitialize(pargc, pargv, PETSC_NULL, help); + SlepcInitialize(pargc, pargv, nullptr, help); PetscLogEventRegister("Total BOUT++", 0, &USER_EVENT); PetscLogEventBegin(USER_EVENT, 0, 0, 0, 0); } From 28e87602a2ecb217714e1cbb04f5bf5c38bf4450 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 28 Apr 2023 14:46:29 +0100 Subject: [PATCH 080/111] Replace calls to PETSc printf functions with `Output` --- src/solver/impls/petsc/petsc.cxx | 38 +++++++++----------------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/src/solver/impls/petsc/petsc.cxx b/src/solver/impls/petsc/petsc.cxx index 1e4450ae43..ed827ee05b 100644 --- a/src/solver/impls/petsc/petsc.cxx +++ b/src/solver/impls/petsc/petsc.cxx @@ -532,7 +532,7 @@ int PetscSolver::init() { #endif if (J_write) { PetscViewer viewer; - output_info << "\n[" << rank << "] Test TSComputeRHSJacobian() ...\n"; + output_info.write("\n[{:d}] Test TSComputeRHSJacobian() ...\n", rank); #if PETSC_VERSION_GE(3, 5, 0) ierr = TSComputeRHSJacobian(ts, simtime, u, J, J); CHKERRQ(ierr); @@ -542,22 +542,14 @@ int PetscSolver::init() { CHKERRQ(ierr); #endif - ierr = PetscSynchronizedPrintf(comm, "[{:d}] TSComputeRHSJacobian is done\n", rank); - CHKERRQ(ierr); + output.write("[{:d}] TSComputeRHSJacobian is done\n", rank); -#if PETSC_VERSION_GE(3, 5, 0) - ierr = PetscSynchronizedFlush(comm, PETSC_STDOUT); - CHKERRQ(ierr); -#else - ierr = PetscSynchronizedFlush(comm); - CHKERRQ(ierr); -#endif if (J_slowfd) { - output_info << "[" << rank << "] writing J in binary to data/Jrhs_dense.dat...\n"; + output_info.write("[{:d}] writing J in binary to data/Jrhs_dense.dat...\n", rank); ierr = PetscViewerBinaryOpen(comm, "data/Jrhs_dense.dat", FILE_MODE_WRITE, &viewer); CHKERRQ(ierr); } else { - output_info << "[" << rank << "] writing J in binary to data/Jrhs_sparse.dat...\n"; + output_info.write("[{:d}] writing J in binary to data/Jrhs_sparse.dat...\n", rank); ierr = PetscViewerBinaryOpen(comm, "data/Jrhs_sparse.dat", FILE_MODE_WRITE, &viewer); CHKERRQ(ierr); @@ -579,8 +571,6 @@ int PetscSolver::init() { **************************************************************************/ PetscErrorCode PetscSolver::run() { - PetscErrorCode ierr; - FILE* fp = nullptr; // Set when the next call to monitor is desired next_output = simtime + getOutputTimestep(); @@ -592,23 +582,17 @@ PetscErrorCode PetscSolver::run() { bout_snes_time = bout::globals::mpi->MPI_Wtime(); } - ierr = TSSolve(ts, u); - CHKERRQ(ierr); + CHKERRQ(TSSolve(ts, u)); // Gawd, everything is a hack - if (this->output_flag) { - ierr = PetscFOpen(PETSC_COMM_WORLD, this->output_name, "w", &fp); - CHKERRQ(ierr); - ierr = PetscFPrintf(PETSC_COMM_WORLD, fp, - "SNES Iteration, KSP Iterations, Wall Time, Norm\n"); - CHKERRQ(ierr); + if (this->output_flag and BoutComm::rank() == 0) { + Output petsc_info(output_name); + // Don't write to stdout + petsc_info.disable(); + petsc_info.write("SNES Iteration, KSP Iterations, Wall Time, Norm\n"); for (const auto& info : snes_list) { - ierr = PetscFPrintf(PETSC_COMM_WORLD, fp, "{:d}, {:d}, {:e}, {:e}\n", info.it, - info.linear_its, info.time, info.norm); - CHKERRQ(ierr); + petsc_info.write("{:d}, {:d}, {:e}, {:e}\n", info.it, info.linear_its, info.time, info.norm); } - ierr = PetscFClose(PETSC_COMM_WORLD, fp); - CHKERRQ(ierr); } PetscFunctionReturn(0); From 64735e597ae87b58a53acdea0d7b8473809f30b7 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 28 Apr 2023 15:10:13 +0100 Subject: [PATCH 081/111] Use default inner preconditioner for laplace-petsc3d test PETSc changed the default in 3.17 to PCJACOBI. There doesn't appear to be a programmatic way of setting this --- .../integrated/test-laplace-petsc3d/data_circular_core/BOUT.inp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integrated/test-laplace-petsc3d/data_circular_core/BOUT.inp b/tests/integrated/test-laplace-petsc3d/data_circular_core/BOUT.inp index ae7cfaa5b4..6474b2604b 100644 --- a/tests/integrated/test-laplace-petsc3d/data_circular_core/BOUT.inp +++ b/tests/integrated/test-laplace-petsc3d/data_circular_core/BOUT.inp @@ -76,6 +76,7 @@ atol = 1e-13 [laplace:petsc] mg_levels_ksp_max_it = 3 +mg_levels_pc_type = sor [input] transform_from_field_aligned = false From 0f96be9c5e3721005a792baab66dc72dab8b8562 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 28 Apr 2023 15:14:37 +0100 Subject: [PATCH 082/111] Include failure reason for PETSc solvers --- src/invert/laplace/impls/petsc/petsc_laplace.cxx | 8 +++++--- src/invert/laplace/impls/petsc3damg/petsc3damg.cxx | 5 +++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/invert/laplace/impls/petsc/petsc_laplace.cxx b/src/invert/laplace/impls/petsc/petsc_laplace.cxx index d96a9ff129..1df560f288 100644 --- a/src/invert/laplace/impls/petsc/petsc_laplace.cxx +++ b/src/invert/laplace/impls/petsc/petsc_laplace.cxx @@ -852,9 +852,11 @@ FieldPerp LaplacePetsc::solve(const FieldPerp& b, const FieldPerp& x0) { KSPGetConvergedReason(ksp, &reason); if (reason == -3) { // Too many iterations, might be fixed by taking smaller timestep throw BoutIterationFail("petsc_laplace: too many iterations"); - } else if (reason <= 0) { - output << "KSPConvergedReason is " << reason << endl; - throw BoutException("petsc_laplace: inversion failed to converge."); + } + if (reason <= 0) { + throw BoutException( + "petsc_laplace: inversion failed to converge. KSPConvergedReason: {} ({})", + KSPConvergedReasons[reason], reason); } // Add data to FieldPerp Object diff --git a/src/invert/laplace/impls/petsc3damg/petsc3damg.cxx b/src/invert/laplace/impls/petsc3damg/petsc3damg.cxx index 40cbf621b6..24d61af783 100644 --- a/src/invert/laplace/impls/petsc3damg/petsc3damg.cxx +++ b/src/invert/laplace/impls/petsc3damg/petsc3damg.cxx @@ -265,8 +265,9 @@ Field3D LaplacePetsc3dAmg::solve(const Field3D& b_in, const Field3D& x0) { throw BoutIterationFail("Petsc3dAmg: too many iterations"); } if (reason <= 0) { - output << "KSPConvergedReason is " << reason << "\n"; - throw BoutException("Petsc3dAmg: inversion failed to converge."); + throw BoutException( + "Petsc3dAmg: inversion failed to converge. KSPConvergedReason: {} ({})", + KSPConvergedReasons[reason], reason); } // Create field from result From 5fbf1e08bb0ef0cdfed1aa033a98c8291096ec32 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 28 Apr 2023 15:34:11 +0100 Subject: [PATCH 083/111] Use references instead of pointers in setting `Options` --- .../laplace/test_laplace_petsc3damg.cxx | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/unit/invert/laplace/test_laplace_petsc3damg.cxx b/tests/unit/invert/laplace/test_laplace_petsc3damg.cxx index 832ccc8e6b..aac3b8a8fe 100644 --- a/tests/unit/invert/laplace/test_laplace_petsc3damg.cxx +++ b/tests/unit/invert/laplace/test_laplace_petsc3damg.cxx @@ -99,7 +99,7 @@ class Petsc3dAmgTest public: WithQuietOutput info{output_info}, warn{output_warn}, progress{output_progress}, all{output}; - Petsc3dAmgTest() : FakeMeshFixture(), solver(getOptions(GetParam())) { + Petsc3dAmgTest() : solver(&getOptions(GetParam())) { PetscErrorPrintf = PetscErrorPrintfNone; int nx = mesh->GlobalNx, ny = mesh->GlobalNy, nz = mesh->GlobalNz; static_cast(bout::globals::mesh) @@ -139,20 +139,20 @@ class Petsc3dAmgTest ForwardOperator forward; private: - static Options* getOptions(std::tuple param) { - Options* options = Options::getRoot()->getSection("laplace"); - (*options)["type"] = "petsc3damg"; - (*options)["inner_boundary_flags"] = + static Options& getOptions(std::tuple param) { + auto& options = Options::root()["laplace"]; + options["type"] = "petsc3damg"; + options["inner_boundary_flags"] = (std::get<0>(param) ? INVERT_AC_GRAD : 0) + INVERT_RHS; - (*options)["outer_boundary_flags"] = + options["outer_boundary_flags"] = (std::get<1>(param) ? INVERT_AC_GRAD : 0) + INVERT_RHS; - (*options)["lower_boundary_flags"] = + options["lower_boundary_flags"] = (std::get<2>(param) ? INVERT_AC_GRAD : 0) + INVERT_RHS; - (*options)["upper_boundary_flags"] = + options["upper_boundary_flags"] = (std::get<3>(param) ? INVERT_AC_GRAD : 0) + INVERT_RHS; - (*options)["fourth_order"] = false; - (*options)["atol"] = tol / 30; // Need to specify smaller than desired tolerance to - (*options)["rtol"] = tol / 30; // ensure it is satisfied for every element. + options["fourth_order"] = false; + options["atol"] = tol / 30; // Need to specify smaller than desired tolerance to + options["rtol"] = tol / 30; // ensure it is satisfied for every element. return options; } }; From 3f60cd6e1523eca2d0e1b0036be5bfd35f35af37 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 28 Apr 2023 15:34:40 +0100 Subject: [PATCH 084/111] Fix unit test for recent changes in PETSc Need to set old smoother and increase KSP max iterations to regain old tolerance --- tests/unit/invert/laplace/test_laplace_petsc3damg.cxx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/invert/laplace/test_laplace_petsc3damg.cxx b/tests/unit/invert/laplace/test_laplace_petsc3damg.cxx index aac3b8a8fe..b2831808a4 100644 --- a/tests/unit/invert/laplace/test_laplace_petsc3damg.cxx +++ b/tests/unit/invert/laplace/test_laplace_petsc3damg.cxx @@ -153,6 +153,9 @@ class Petsc3dAmgTest options["fourth_order"] = false; options["atol"] = tol / 30; // Need to specify smaller than desired tolerance to options["rtol"] = tol / 30; // ensure it is satisfied for every element. + auto& petsc_options = options["petsc"]; + petsc_options["mg_levels_ksp_max_it"] = 4; + petsc_options["mg_levels_pc_type"] = "sor"; return options; } }; From de1f258134f48954630d4600090b07f8d064ffec Mon Sep 17 00:00:00 2001 From: ZedThree Date: Fri, 28 Apr 2023 14:45:41 +0000 Subject: [PATCH 085/111] Apply clang-format changes --- src/solver/impls/petsc/petsc.cxx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/solver/impls/petsc/petsc.cxx b/src/solver/impls/petsc/petsc.cxx index ed827ee05b..45400fc132 100644 --- a/src/solver/impls/petsc/petsc.cxx +++ b/src/solver/impls/petsc/petsc.cxx @@ -261,8 +261,7 @@ int PetscSolver::init() { CHKERRQ(ierr); #if PETSC_VERSION_GE(3, 7, 0) - ierr = PetscOptionsGetBool(nullptr, nullptr, "-interpolate", &interpolate, - nullptr); + ierr = PetscOptionsGetBool(nullptr, nullptr, "-interpolate", &interpolate, nullptr); CHKERRQ(ierr); #else ierr = PetscOptionsGetBool(nullptr, "-interpolate", &interpolate, nullptr); @@ -591,7 +590,8 @@ PetscErrorCode PetscSolver::run() { petsc_info.disable(); petsc_info.write("SNES Iteration, KSP Iterations, Wall Time, Norm\n"); for (const auto& info : snes_list) { - petsc_info.write("{:d}, {:d}, {:e}, {:e}\n", info.it, info.linear_its, info.time, info.norm); + petsc_info.write("{:d}, {:d}, {:e}, {:e}\n", info.it, info.linear_its, info.time, + info.norm); } } From 7e42ddcaee99f1db7b34cfdb862c5369ebfce733 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 28 Apr 2023 16:32:42 +0100 Subject: [PATCH 086/111] CI: Build latest SLEPc with latest PETSc --- .build_petsc_for_ci.sh | 16 ++++++++++++++++ .github/workflows/tests.yml | 3 ++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/.build_petsc_for_ci.sh b/.build_petsc_for_ci.sh index 03afb8abc7..c29902c850 100755 --- a/.build_petsc_for_ci.sh +++ b/.build_petsc_for_ci.sh @@ -29,6 +29,22 @@ if test $BUILD_PETSC ; then echo "****************************************" echo " Finished building PETSc" echo "****************************************" + + echo "****************************************" + echo "Building SLEPc" + echo "****************************************" + + git clone -b release https://gitlab.com/slepc/slepc.git slepc --depth=1 + + pushd slepc + PETSC_DIR=$HOME/local/petsc ./configure --prefix=$HOME/local/slepc + + make && make install + popd + + echo "****************************************" + echo " Finished building SLEPc" + echo "****************************************" else echo "****************************************" echo " PETSc already installed" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 944b730f66..ec0c7f2759 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -96,7 +96,8 @@ jobs: -DBOUT_USE_SUNDIALS=ON -DBOUT_ENABLE_PYTHON=ON -DSUNDIALS_ROOT=/home/runner/local - -DPETSC_DIR=/home/runner/local/petsc" + -DPETSC_DIR=/home/runner/local/petsc + -DSLEPC_DIR=/home/runner/local/slepc" build_petsc: -petsc From 128b38e8072bbd457ed07de498394379cdb56ffa Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 28 Apr 2023 16:37:01 +0100 Subject: [PATCH 087/111] Clang-tidy fixes --- src/solver/impls/petsc/petsc.cxx | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/solver/impls/petsc/petsc.cxx b/src/solver/impls/petsc/petsc.cxx index 45400fc132..1b81ca36b6 100644 --- a/src/solver/impls/petsc/petsc.cxx +++ b/src/solver/impls/petsc/petsc.cxx @@ -282,7 +282,7 @@ int PetscSolver::init() { #endif // If the output_name is not specified then use the standard monitor function - if (output_flag) { + if (output_flag != 0U) { ierr = SNESMonitorSet(snes, PetscSNESMonitor, this, nullptr); CHKERRQ(ierr); } else { @@ -455,8 +455,8 @@ int PetscSolver::init() { ierr = MatMPIAIJSetPreallocation(J, prealloc, nullptr, prealloc, nullptr); CHKERRQ(ierr); - prealloc = - cols; // why nonzeros=295900, allocated nonzeros=2816000/12800000 (*dof*dof), number of mallocs used during MatSetValues calls =256? + // why nonzeros=295900, allocated nonzeros=2816000/12800000 (*dof*dof), number of mallocs used during MatSetValues calls =256? + prealloc = cols; ierr = MatSeqBAIJSetPreallocation(J, dof, prealloc, nullptr); CHKERRQ(ierr); ierr = MatMPIBAIJSetPreallocation(J, dof, prealloc, nullptr, prealloc, nullptr); @@ -468,7 +468,7 @@ int PetscSolver::init() { ierr = PetscOptionsHasName(nullptr, "-J_slowfd", &J_slowfd); CHKERRQ(ierr); #endif - if (J_slowfd) { // create Jacobian matrix by slow fd + if (J_slowfd != 0U) { // create Jacobian matrix by slow fd ierr = SNESSetJacobian(snes, J, J, SNESComputeJacobianDefault, nullptr); CHKERRQ(ierr); output_info << "SNESComputeJacobian J by slow fd...\n"; @@ -529,8 +529,8 @@ int PetscSolver::init() { ierr = PetscOptionsHasName(nullptr, "-J_write", &J_write); CHKERRQ(ierr); #endif - if (J_write) { - PetscViewer viewer; + if (J_write != 0U) { + PetscViewer viewer = nullptr; output_info.write("\n[{:d}] Test TSComputeRHSJacobian() ...\n", rank); #if PETSC_VERSION_GE(3, 5, 0) ierr = TSComputeRHSJacobian(ts, simtime, u, J, J); @@ -543,7 +543,7 @@ int PetscSolver::init() { output.write("[{:d}] TSComputeRHSJacobian is done\n", rank); - if (J_slowfd) { + if (J_slowfd != 0U) { output_info.write("[{:d}] writing J in binary to data/Jrhs_dense.dat...\n", rank); ierr = PetscViewerBinaryOpen(comm, "data/Jrhs_dense.dat", FILE_MODE_WRITE, &viewer); CHKERRQ(ierr); @@ -584,7 +584,7 @@ PetscErrorCode PetscSolver::run() { CHKERRQ(TSSolve(ts, u)); // Gawd, everything is a hack - if (this->output_flag and BoutComm::rank() == 0) { + if ((this->output_flag != 0U) and (BoutComm::rank() == 0)) { Output petsc_info(output_name); // Don't write to stdout petsc_info.disable(); From 4f9843f7d2c39f1e352f3f57c1dd438b02e1c159 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 28 Apr 2023 17:29:48 +0100 Subject: [PATCH 088/111] CI: Unset SLEPC variables before building slepc --- .build_petsc_for_ci.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.build_petsc_for_ci.sh b/.build_petsc_for_ci.sh index c29902c850..c9bd2e45bb 100755 --- a/.build_petsc_for_ci.sh +++ b/.build_petsc_for_ci.sh @@ -37,6 +37,8 @@ if test $BUILD_PETSC ; then git clone -b release https://gitlab.com/slepc/slepc.git slepc --depth=1 pushd slepc + unset SLEPC_DIR + unset SLEPC_ARCH PETSC_DIR=$HOME/local/petsc ./configure --prefix=$HOME/local/slepc make && make install From 78dbc0fad0c1d43349d68617cb631c2c70cda468 Mon Sep 17 00:00:00 2001 From: Peter Hill Date: Fri, 28 Apr 2023 18:00:02 +0100 Subject: [PATCH 089/111] CI: Set SLEPC vars when building slepc --- .build_petsc_for_ci.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.build_petsc_for_ci.sh b/.build_petsc_for_ci.sh index c9bd2e45bb..f9516b882a 100755 --- a/.build_petsc_for_ci.sh +++ b/.build_petsc_for_ci.sh @@ -41,7 +41,8 @@ if test $BUILD_PETSC ; then unset SLEPC_ARCH PETSC_DIR=$HOME/local/petsc ./configure --prefix=$HOME/local/slepc - make && make install + make SLEPC_DIR=$(pwd) PETSC_DIR=$HOME/local/petsc + make SLEPC_DIR=$(pwd) PETSC_DIR=$HOME/local/petsc install popd echo "****************************************" From 58143ebf73ffb0ce306fb67c89e122a20dc12a25 Mon Sep 17 00:00:00 2001 From: David Bold Date: Tue, 2 May 2023 09:47:17 +0200 Subject: [PATCH 090/111] Update `make dist` invocation --- manual/RELEASE_HOWTO.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manual/RELEASE_HOWTO.md b/manual/RELEASE_HOWTO.md index 1077aaf375..3c12c4fdc4 100644 --- a/manual/RELEASE_HOWTO.md +++ b/manual/RELEASE_HOWTO.md @@ -56,7 +56,7 @@ Before merging PR: After PR is merged: -- [ ] Make tarball: `./configure && make dist` +- [ ] Make tarball: `make dist` from build directory. Ensure you are on a tag and correct version is used for archive and folder within. - [ ] Try to summarise the changes! - [ ] Make [GitHub Release][gh_release], include change summary **NB:** tag should have leading `v` From 8fbaad7c0b21e991e1eb83d92ebcf82174d235f1 Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 3 May 2023 09:35:50 +0200 Subject: [PATCH 091/111] Do not run PETSc and SLEPc in cmake step PETSc result is anyway ignored, so that only speed ups the cmake step. SLEPc can probably fail for similar reasons, so lets skip --- cmake/FindPETSc.cmake | 9 ++---- cmake/FindSLEPc.cmake | 66 ++----------------------------------------- 2 files changed, 5 insertions(+), 70 deletions(-) diff --git a/cmake/FindPETSc.cmake b/cmake/FindPETSc.cmake index 0ba9c909f8..efa74dcc60 100644 --- a/cmake/FindPETSc.cmake +++ b/cmake/FindPETSc.cmake @@ -281,11 +281,8 @@ int main(int argc,char *argv[]) { return 0; } ") - multipass_source_runs ("${includes}" "${libraries}" "${_PETSC_TEST_SOURCE}" ${runs} "${PETSC_LANGUAGE_BINDINGS}") - if (${${runs}}) - set (PETSC_EXECUTABLE_RUNS "YES" CACHE BOOL - "Can the system successfully run a PETSc executable? This variable can be manually set to \"YES\" to force CMake to accept a given PETSc configuration, but this will almost always result in a broken build. If you change PETSC_DIR, PETSC_ARCH, or PETSC_CURRENT you would have to reset this variable." FORCE) - endif (${${runs}}) + set (PETSC_EXECUTABLE_RUNS "YES" BOOL + "Can the system successfully run a PETSc executable? This variable can be manually set to \"YES\" to force CMake to accept a given PETSc configuration, but this will almost always result in a broken build. If you change PETSC_DIR, PETSC_ARCH, or PETSC_CURRENT you would have to reset this variable." FORCE) endmacro (PETSC_TEST_RUNS) @@ -366,7 +363,7 @@ if (NOT PETSC_INCLUDES AND NOT TARGET PETSc::PETSc) pkg_search_module(PkgPETSC PETSc>3.4.0 petsc>3.4.0) set (PETSC_LIBRARIES ${PkgPETSC_LINK_LIBRARIES} CACHE STRING "PETSc libraries" FORCE) set (PETSC_INCLUDES ${PkgPETSC_INCLUDE_DIRS} CACHE STRING "PETSc include path" FORCE) - set (PETSC_EXECUTABLE_RUNS "YES" CACHE BOOL + set (PETSC_EXECUTABLE_RUNS "YES" BOOL "Can the system successfully run a PETSc executable? This variable can be manually set to \"YES\" to force CMake to accept a given PETSc configuration, but this will almost always result in a broken build. If you change PETSC_DIR, PETSC_ARCH, or PETSC_CURRENT you would have to reset this variable." FORCE) endif() endif() diff --git a/cmake/FindSLEPc.cmake b/cmake/FindSLEPc.cmake index ad873822d5..caf2f828e3 100644 --- a/cmake/FindSLEPc.cmake +++ b/cmake/FindSLEPc.cmake @@ -193,71 +193,9 @@ int main() { endif() mark_as_advanced(SLEPC_VERSION_OK) - # Run SLEPc test program - set(SLEPC_TEST_LIB_CPP - "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/slepc_test_lib.cpp") - file(WRITE ${SLEPC_TEST_LIB_CPP} " -#include \"petsc.h\" -#include \"slepceps.h\" -int main() -{ - PetscErrorCode ierr; - int argc = 0; - char** argv = NULL; - ierr = SlepcInitialize(&argc, &argv, nullptr, nullptr); - EPS eps; - ierr = EPSCreate(PETSC_COMM_SELF, &eps); CHKERRQ(ierr); - //ierr = EPSSetFromOptions(eps); CHKERRQ(ierr); -#if PETSC_VERSION_MAJOR == 3 && PETSC_VERSION_MINOR <= 1 - ierr = EPSDestroy(eps); CHKERRQ(ierr); -#else - ierr = EPSDestroy(&eps); CHKERRQ(ierr); -#endif - ierr = SlepcFinalize(); CHKERRQ(ierr); - return 0; -} -") - - try_run( - SLEPC_TEST_LIB_EXITCODE - SLEPC_TEST_LIB_COMPILED - ${CMAKE_CURRENT_BINARY_DIR} - ${SLEPC_TEST_LIB_CPP} - CMAKE_FLAGS "-DINCLUDE_DIRECTORIES:STRING=${CMAKE_REQUIRED_INCLUDES}" - LINK_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} - COMPILE_OUTPUT_VARIABLE SLEPC_TEST_LIB_COMPILE_OUTPUT - RUN_OUTPUT_VARIABLE SLEPC_TEST_LIB_OUTPUT - ) + # Do not run SLEPc test program - if (SLEPC_TEST_LIB_COMPILED AND SLEPC_TEST_LIB_EXITCODE EQUAL 0) - message(STATUS "Performing test SLEPC_TEST_RUNS - Success") - set(SLEPC_TEST_RUNS TRUE CACHE BOOL "SLEPc test program can run") - else() - message(STATUS "Performing test SLEPC_TEST_RUNS - Failed") - - # Test program does not run - try adding SLEPc 3rd party libs and test again - list(APPEND CMAKE_REQUIRED_LIBRARIES ${SLEPC_EXTERNAL_LIBRARIES}) - - try_run( - SLEPC_TEST_3RD_PARTY_LIBS_EXITCODE - SLEPC_TEST_3RD_PARTY_LIBS_COMPILED - ${CMAKE_CURRENT_BINARY_DIR} - ${SLEPC_TEST_LIB_CPP} - CMAKE_FLAGS "-DINCLUDE_DIRECTORIES:STRING=${CMAKE_REQUIRED_INCLUDES}" - LINK_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} - COMPILE_OUTPUT_VARIABLE SLEPC_TEST_3RD_PARTY_LIBS_COMPILE_OUTPUT - RUN_OUTPUT_VARIABLE SLEPC_TEST_3RD_PARTY_LIBS_OUTPUT - ) - - if (SLEPC_TEST_3RD_PARTY_LIBS_COMPILED AND SLEPC_TEST_3RD_PARTY_LIBS_EXITCODE EQUAL 0) - message(STATUS "Performing test SLEPC_TEST_3RD_PARTY_LIBS_RUNS - Success") - set(SLEPC_LIBRARIES ${SLEPC_LIBRARIES} ${SLEPC_EXTERNAL_LIBRARIES} - CACHE STRING "SLEPc libraries." FORCE) - set(SLEPC_TEST_RUNS TRUE CACHE BOOL "SLEPc test program can run") - else() - message(STATUS "Performing test SLEPC_TEST_3RD_PARTY_LIBS_RUNS - Failed") - endif() - endif() + set(SLEPC_TEST_RUNS TRUE CACHE BOOL "SLEPc test program can run") endif() # Standard package handling From a22ffa26d800f5798598dff90a0d5557a3e27d79 Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 3 May 2023 09:43:53 +0200 Subject: [PATCH 092/111] remove more unused code --- cmake/FindPETSc.cmake | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/cmake/FindPETSc.cmake b/cmake/FindPETSc.cmake index efa74dcc60..eb7f85f688 100644 --- a/cmake/FindPETSc.cmake +++ b/cmake/FindPETSc.cmake @@ -259,29 +259,7 @@ show : include(Check${PETSC_LANGUAGE_BINDINGS}SourceRuns) macro (PETSC_TEST_RUNS includes libraries runs) - message(STATUS "PETSc test with : ${includes} ${libraries}" ) - if (PETSC_VERSION VERSION_GREATER 3.1) - set (_PETSC_TSDestroy "TSDestroy(&ts)") - else () - set (_PETSC_TSDestroy "TSDestroy(ts)") - endif () - - set(_PETSC_TEST_SOURCE " -static const char help[] = \"PETSc test program.\"; -#include -int main(int argc,char *argv[]) { - PetscErrorCode ierr; - TS ts; - - ierr = PetscInitialize(&argc,&argv,0,help);CHKERRQ(ierr); - ierr = TSCreate(PETSC_COMM_WORLD,&ts);CHKERRQ(ierr); - ierr = TSSetFromOptions(ts);CHKERRQ(ierr); - ierr = ${_PETSC_TSDestroy};CHKERRQ(ierr); - ierr = PetscFinalize();CHKERRQ(ierr); - return 0; -} -") - set (PETSC_EXECUTABLE_RUNS "YES" BOOL + set (PETSC_EXECUTABLE_RUNS "YES" CACHE BOOL "Can the system successfully run a PETSc executable? This variable can be manually set to \"YES\" to force CMake to accept a given PETSc configuration, but this will almost always result in a broken build. If you change PETSC_DIR, PETSC_ARCH, or PETSC_CURRENT you would have to reset this variable." FORCE) endmacro (PETSC_TEST_RUNS) From e232d34923c89c47fbe494d8ddbcab15117b8116 Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 3 May 2023 09:49:51 +0200 Subject: [PATCH 093/111] Fix findPetsc.cmake --- cmake/FindPETSc.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/FindPETSc.cmake b/cmake/FindPETSc.cmake index eb7f85f688..cc76489cbb 100644 --- a/cmake/FindPETSc.cmake +++ b/cmake/FindPETSc.cmake @@ -341,7 +341,7 @@ if (NOT PETSC_INCLUDES AND NOT TARGET PETSc::PETSc) pkg_search_module(PkgPETSC PETSc>3.4.0 petsc>3.4.0) set (PETSC_LIBRARIES ${PkgPETSC_LINK_LIBRARIES} CACHE STRING "PETSc libraries" FORCE) set (PETSC_INCLUDES ${PkgPETSC_INCLUDE_DIRS} CACHE STRING "PETSc include path" FORCE) - set (PETSC_EXECUTABLE_RUNS "YES" BOOL + set (PETSC_EXECUTABLE_RUNS "YES" CACHE BOOL "Can the system successfully run a PETSc executable? This variable can be manually set to \"YES\" to force CMake to accept a given PETSc configuration, but this will almost always result in a broken build. If you change PETSC_DIR, PETSC_ARCH, or PETSC_CURRENT you would have to reset this variable." FORCE) endif() endif() From 8365ee274d8b3d99e7d9c80183d197dc39f404c5 Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 3 May 2023 15:23:11 +0200 Subject: [PATCH 094/111] The additional check is not needed The parallel transform already checks whether it is needed. For the identity transform this is always needed, so the second check is harmful in that case, and otherwise not needed. --- src/mesh/impls/bout/boutmesh.cxx | 66 +++++++++++++++----------------- 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/src/mesh/impls/bout/boutmesh.cxx b/src/mesh/impls/bout/boutmesh.cxx index 173fec50da..4a778115fe 100644 --- a/src/mesh/impls/bout/boutmesh.cxx +++ b/src/mesh/impls/bout/boutmesh.cxx @@ -1415,48 +1415,42 @@ int BoutMesh::wait(comm_handle handle) { // Loop over 3D fields for (const auto& var : ch->var_list.field3d()) { if (var->requiresTwistShift(TwistShift)) { - // Twist-shift only needed for field-aligned fields int jx = 0; int jy = 0; // Perform Twist-shift using shifting method - if (var->getDirectionY() == YDirectionType::Aligned) { - // Only variables in field-aligned coordinates need the twist-shift boundary - // condition to be applied - // Lower boundary - if (TS_down_in && (DDATA_INDEST != -1)) { - for (jx = 0; jx < DDATA_XSPLIT; jx++) { - for (jy = 0; jy != MYG; jy++) { - shiftZ(*var, jx, jy, ShiftAngle[jx]); - } - } - } - if (TS_down_out && (DDATA_OUTDEST != -1)) { - for (jx = DDATA_XSPLIT; jx < LocalNx; jx++) { - for (jy = 0; jy != MYG; jy++) { - shiftZ(*var, jx, jy, ShiftAngle[jx]); - } - } - } - - // Upper boundary - if (TS_up_in && (UDATA_INDEST != -1)) { - for (jx = 0; jx < UDATA_XSPLIT; jx++) { - for (jy = LocalNy - MYG; jy != LocalNy; jy++) { - shiftZ(*var, jx, jy, -ShiftAngle[jx]); - } - } - } - if (TS_up_out && (UDATA_OUTDEST != -1)) { - for (jx = UDATA_XSPLIT; jx < LocalNx; jx++) { - for (jy = LocalNy - MYG; jy != LocalNy; jy++) { - shiftZ(*var, jx, jy, -ShiftAngle[jx]); - } - } - } - } + if (TS_down_in && (DDATA_INDEST != -1)) { + for (jx = 0; jx < DDATA_XSPLIT; jx++) { + for (jy = 0; jy != MYG; jy++) { + shiftZ(*var, jx, jy, ShiftAngle[jx]); + } + } + } + if (TS_down_out && (DDATA_OUTDEST != -1)) { + for (jx = DDATA_XSPLIT; jx < LocalNx; jx++) { + for (jy = 0; jy != MYG; jy++) { + shiftZ(*var, jx, jy, ShiftAngle[jx]); + } + } + } + + // Upper boundary + if (TS_up_in && (UDATA_INDEST != -1)) { + for (jx = 0; jx < UDATA_XSPLIT; jx++) { + for (jy = LocalNy - MYG; jy != LocalNy; jy++) { + shiftZ(*var, jx, jy, -ShiftAngle[jx]); + } + } + } + if (TS_up_out && (UDATA_OUTDEST != -1)) { + for (jx = UDATA_XSPLIT; jx < LocalNx; jx++) { + for (jy = LocalNy - MYG; jy != LocalNy; jy++) { + shiftZ(*var, jx, jy, -ShiftAngle[jx]); + } + } + } } } } From ff9c7b961214a09f7fdde670f29917660180b991 Mon Sep 17 00:00:00 2001 From: dschwoerer Date: Wed, 3 May 2023 13:40:53 +0000 Subject: [PATCH 095/111] Apply clang-format changes --- src/mesh/impls/bout/boutmesh.cxx | 62 ++++++++++++++++---------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/src/mesh/impls/bout/boutmesh.cxx b/src/mesh/impls/bout/boutmesh.cxx index 4a778115fe..a802d3f5b3 100644 --- a/src/mesh/impls/bout/boutmesh.cxx +++ b/src/mesh/impls/bout/boutmesh.cxx @@ -1420,37 +1420,37 @@ int BoutMesh::wait(comm_handle handle) { int jy = 0; // Perform Twist-shift using shifting method - // Lower boundary - if (TS_down_in && (DDATA_INDEST != -1)) { - for (jx = 0; jx < DDATA_XSPLIT; jx++) { - for (jy = 0; jy != MYG; jy++) { - shiftZ(*var, jx, jy, ShiftAngle[jx]); - } - } - } - if (TS_down_out && (DDATA_OUTDEST != -1)) { - for (jx = DDATA_XSPLIT; jx < LocalNx; jx++) { - for (jy = 0; jy != MYG; jy++) { - shiftZ(*var, jx, jy, ShiftAngle[jx]); - } - } - } - - // Upper boundary - if (TS_up_in && (UDATA_INDEST != -1)) { - for (jx = 0; jx < UDATA_XSPLIT; jx++) { - for (jy = LocalNy - MYG; jy != LocalNy; jy++) { - shiftZ(*var, jx, jy, -ShiftAngle[jx]); - } - } - } - if (TS_up_out && (UDATA_OUTDEST != -1)) { - for (jx = UDATA_XSPLIT; jx < LocalNx; jx++) { - for (jy = LocalNy - MYG; jy != LocalNy; jy++) { - shiftZ(*var, jx, jy, -ShiftAngle[jx]); - } - } - } + // Lower boundary + if (TS_down_in && (DDATA_INDEST != -1)) { + for (jx = 0; jx < DDATA_XSPLIT; jx++) { + for (jy = 0; jy != MYG; jy++) { + shiftZ(*var, jx, jy, ShiftAngle[jx]); + } + } + } + if (TS_down_out && (DDATA_OUTDEST != -1)) { + for (jx = DDATA_XSPLIT; jx < LocalNx; jx++) { + for (jy = 0; jy != MYG; jy++) { + shiftZ(*var, jx, jy, ShiftAngle[jx]); + } + } + } + + // Upper boundary + if (TS_up_in && (UDATA_INDEST != -1)) { + for (jx = 0; jx < UDATA_XSPLIT; jx++) { + for (jy = LocalNy - MYG; jy != LocalNy; jy++) { + shiftZ(*var, jx, jy, -ShiftAngle[jx]); + } + } + } + if (TS_up_out && (UDATA_OUTDEST != -1)) { + for (jx = UDATA_XSPLIT; jx < LocalNx; jx++) { + for (jy = LocalNy - MYG; jy != LocalNy; jy++) { + shiftZ(*var, jx, jy, -ShiftAngle[jx]); + } + } + } } } } From fa83d8eda4e8bd890ca8b12e7989de23855842bb Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Fri, 19 May 2023 20:55:38 -0700 Subject: [PATCH 096/111] Get mesh outputs after physics init Physics models often modify metric tensor components during initialisation (e.g. normalisation), so get their values for saving to output only after init has been called. --- src/physics/physicsmodel.cxx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/physics/physicsmodel.cxx b/src/physics/physicsmodel.cxx index f0fbffb0fb..cac4bda5cc 100644 --- a/src/physics/physicsmodel.cxx +++ b/src/physics/physicsmodel.cxx @@ -99,7 +99,6 @@ void PhysicsModel::initialise(Solver* s) { solver = s; bout::experimental::addBuildFlagsToOptions(output_options); - mesh->outputVars(output_options); // Restart option const bool restarting = Options::root()["restart"].withDefault(false); @@ -113,6 +112,8 @@ void PhysicsModel::initialise(Solver* s) { throw BoutException("Couldn't initialise physics model"); } + mesh->outputVars(output_options); + // Post-initialise, which reads restart files // This function can be overridden by the user if (postInit(restarting) != 0) { From 07f7b8fe989e6218a85955e8e8af0c5e3dc40768 Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Wed, 24 May 2023 21:18:08 -0700 Subject: [PATCH 097/111] beuler: Fix stencils, pre-allocation and hypre defaults No mallocs now needed when setting matrix elements in serial or parallel for 1D-recycling example. Hypre preconditioner type can now be set with pc_hypre_type option. Defaults to pilut, as this seems to give the best performance in parallel. --- src/solver/impls/snes/snes.cxx | 304 ++++++++++++++++++--------------- src/solver/impls/snes/snes.hxx | 1 + 2 files changed, 172 insertions(+), 133 deletions(-) diff --git a/src/solver/impls/snes/snes.cxx b/src/solver/impls/snes/snes.cxx index 65ada40d21..2c15e61b3a 100644 --- a/src/solver/impls/snes/snes.cxx +++ b/src/solver/impls/snes/snes.cxx @@ -108,6 +108,9 @@ SNESSolver::SNESSolver(Options* opts) (*options)["pc_type"] .doc("Preconditioner type. By default lets PETSc decide (ilu or bjacobi)") .withDefault("default")), + pc_hypre_type((*options)["pc_hypre_type"] + .doc("hypre preconditioner type: euclid, pilut, parasails, boomeramg, ams, ads") + .withDefault("pilut")), line_search_type((*options)["line_search_type"] .doc("Line search type: basic, bt, l2, cp, nleqerr") .withDefault("default")), @@ -146,6 +149,7 @@ int SNESSolver::init() { int ierr; // Vectors + output.write("Creating vector\n"); ierr = VecCreate(BoutComm::get(), &snes_x); CHKERRQ(ierr); ierr = VecSetSizes(snes_x, nlocal, PETSC_DECIDE); @@ -167,11 +171,12 @@ int SNESSolver::init() { } // Nonlinear solver interface (SNES) + output.write("Create SNES\n"); SNESCreate(BoutComm::get(), &snes); - + // Set the callback function SNESSetFunction(snes, snes_f, FormFunction, this); - + SNESSetType(snes, snes_type.c_str()); // Line search @@ -234,10 +239,26 @@ int SNESSolver::init() { std::vector o_nnz(localN); // Set values for most points + const int ncells_x = (mesh->LocalNx > 1) ? 2 : 0; + const int ncells_y = (mesh->LocalNy > 1) ? 2 : 0; + const int ncells_z = (mesh->LocalNz > 1) ? 2 : 0; + + const auto star_pattern = (1 + ncells_x + ncells_y) * (n3d + n2d) + ncells_z * n3d; + + // Offsets. Start with the central cell + std::vector> xyoffsets {{0, 0}}; + if (ncells_x != 0) { + // Stencil includes points in X + xyoffsets.push_back({-1, 0}); + xyoffsets.push_back({1, 0}); + } + if (ncells_y != 0) { + // Stencil includes points in Y + xyoffsets.push_back({0, -1}); + xyoffsets.push_back({0, 1}); + } - const auto star_pattern_2d = 5 * (n3d + n2d); - const auto star_pattern_3d = 7 * n3d + 5 * n2d; - const auto star_pattern = (mesh->LocalNz > 1) ? star_pattern_3d : star_pattern_2d; + output.write("Star pattern: {} non-zero entries\n", star_pattern); for (int i = 0; i < localN; i++) { // Non-zero elements on this processor d_nnz[i] = star_pattern; @@ -246,148 +267,154 @@ int SNESSolver::init() { } // X boundaries - if (mesh->firstX()) { - // Lower X boundary - for (int y = mesh->ystart; y <= mesh->yend; y++) { - for (int z = 0; z < mesh->LocalNz; z++) { - int localIndex = ROUND(index(mesh->xstart, y, z)); - ASSERT2((localIndex >= 0) && (localIndex < localN)); - const int num_fields = (z == 0) ? n2d + n3d : n3d; - for (int i = 0; i < num_fields; i++) { - d_nnz[localIndex + i] -= (n3d + n2d); + if (ncells_x != 0) { + if (mesh->firstX()) { + // Lower X boundary + for (int y = mesh->ystart; y <= mesh->yend; y++) { + for (int z = 0; z < mesh->LocalNz; z++) { + int localIndex = ROUND(index(mesh->xstart, y, z)); + ASSERT2((localIndex >= 0) && (localIndex < localN)); + const int num_fields = (z == 0) ? n2d + n3d : n3d; + for (int i = 0; i < num_fields; i++) { + d_nnz[localIndex + i] -= (n3d + n2d); + } } } - } - } else { - // On another processor - for (int y = mesh->ystart; y <= mesh->yend; y++) { - for (int z = 0; z < mesh->LocalNz; z++) { - int localIndex = ROUND(index(mesh->xstart, y, z)); - ASSERT2((localIndex >= 0) && (localIndex < localN)); - const int num_fields = (z == 0) ? n2d + n3d : n3d; - for (int i = 0; i < num_fields; i++) { - d_nnz[localIndex + i] -= (n3d + n2d); - o_nnz[localIndex + i] += (n3d + n2d); + } else { + // On another processor + for (int y = mesh->ystart; y <= mesh->yend; y++) { + for (int z = 0; z < mesh->LocalNz; z++) { + int localIndex = ROUND(index(mesh->xstart, y, z)); + ASSERT2((localIndex >= 0) && (localIndex < localN)); + const int num_fields = (z == 0) ? n2d + n3d : n3d; + for (int i = 0; i < num_fields; i++) { + d_nnz[localIndex + i] -= (n3d + n2d); + o_nnz[localIndex + i] += (n3d + n2d); + } } } } - } - - if (mesh->lastX()) { - // Upper X boundary - for (int y = mesh->ystart; y <= mesh->yend; y++) { - for (int z = 0; z < mesh->LocalNz; z++) { - int localIndex = ROUND(index(mesh->xend, y, z)); - ASSERT2((localIndex >= 0) && (localIndex < localN)); - const int num_fields = (z == 0) ? n2d + n3d : n3d; - for (int i = 0; i < num_fields; i++) { - d_nnz[localIndex + i] -= (n3d + n2d); + if (mesh->lastX()) { + // Upper X boundary + for (int y = mesh->ystart; y <= mesh->yend; y++) { + for (int z = 0; z < mesh->LocalNz; z++) { + int localIndex = ROUND(index(mesh->xend, y, z)); + ASSERT2((localIndex >= 0) && (localIndex < localN)); + const int num_fields = (z == 0) ? n2d + n3d : n3d; + for (int i = 0; i < num_fields; i++) { + d_nnz[localIndex + i] -= (n3d + n2d); + } } } - } - } else { - // On another processor - for (int y = mesh->ystart; y <= mesh->yend; y++) { - for (int z = 0; z < mesh->LocalNz; z++) { - int localIndex = ROUND(index(mesh->xend, y, z)); - ASSERT2((localIndex >= 0) && (localIndex < localN)); - const int num_fields = (z == 0) ? n2d + n3d : n3d; - for (int i = 0; i < num_fields; i++) { - d_nnz[localIndex + i] -= (n3d + n2d); - o_nnz[localIndex + i] += (n3d + n2d); + } else { + // On another processor + for (int y = mesh->ystart; y <= mesh->yend; y++) { + for (int z = 0; z < mesh->LocalNz; z++) { + int localIndex = ROUND(index(mesh->xend, y, z)); + ASSERT2((localIndex >= 0) && (localIndex < localN)); + const int num_fields = (z == 0) ? n2d + n3d : n3d; + for (int i = 0; i < num_fields; i++) { + d_nnz[localIndex + i] -= (n3d + n2d); + o_nnz[localIndex + i] += (n3d + n2d); + } } } } } // Y boundaries - - for (int x = mesh->xstart; x <= mesh->xend; x++) { - // Default to no boundary - // NOTE: This assumes that communications in Y are to other - // processors. If Y is communicated with this processor (e.g. NYPE=1) - // then this will result in PETSc warnings about out of range allocations - - // z = 0 case - int localIndex = ROUND(index(x, mesh->ystart, 0)); - ASSERT2(localIndex >= 0); - - // All 2D and 3D fields - for (int i = 0; i < n2d + n3d; i++) { - o_nnz[localIndex + i] += (n3d + n2d); - } - - for (int z = 1; z < mesh->LocalNz; z++) { - localIndex = ROUND(index(x, mesh->ystart, z)); - - // Only 3D fields - for (int i = 0; i < n3d; i++) { + if (ncells_y != 0) { + for (int x = mesh->xstart; x <= mesh->xend; x++) { + // Default to no boundary + // NOTE: This assumes that communications in Y are to other + // processors. If Y is communicated with this processor (e.g. NYPE=1) + // then this will result in PETSc warnings about out of range allocations + + // z = 0 case + int localIndex = ROUND(index(x, mesh->ystart, 0)); + ASSERT2(localIndex >= 0); + + // All 2D and 3D fields + for (int i = 0; i < n2d + n3d; i++) { o_nnz[localIndex + i] += (n3d + n2d); + d_nnz[localIndex + i] -= (n3d + n2d); } - } - // z = 0 case - localIndex = ROUND(index(x, mesh->yend, 0)); - // All 2D and 3D fields - for (int i = 0; i < n2d + n3d; i++) { - o_nnz[localIndex + i] += (n3d + n2d); - } + for (int z = 1; z < mesh->LocalNz; z++) { + localIndex = ROUND(index(x, mesh->ystart, z)); - for (int z = 1; z < mesh->LocalNz; z++) { - localIndex = ROUND(index(x, mesh->yend, z)); + // Only 3D fields + for (int i = 0; i < n3d; i++) { + o_nnz[localIndex + i] += (n3d + n2d); + d_nnz[localIndex + i] -= (n3d + n2d); + } + } - // Only 3D fields - for (int i = 0; i < n3d; i++) { + // z = 0 case + localIndex = ROUND(index(x, mesh->yend, 0)); + // All 2D and 3D fields + for (int i = 0; i < n2d + n3d; i++) { o_nnz[localIndex + i] += (n3d + n2d); + d_nnz[localIndex + i] -= (n3d + n2d); } - } - } - for (RangeIterator it = mesh->iterateBndryLowerY(); !it.isDone(); it++) { - // A boundary, so no communication + for (int z = 1; z < mesh->LocalNz; z++) { + localIndex = ROUND(index(x, mesh->yend, z)); - // z = 0 case - int localIndex = ROUND(index(it.ind, mesh->ystart, 0)); - if (localIndex < 0) { - // This can occur because it.ind includes values in x boundary e.g. x=0 - continue; - } - // All 2D and 3D fields - for (int i = 0; i < n2d + n3d; i++) { - o_nnz[localIndex + i] -= (n3d + n2d); + // Only 3D fields + for (int i = 0; i < n3d; i++) { + o_nnz[localIndex + i] += (n3d + n2d); + d_nnz[localIndex + i] -= (n3d + n2d); + } + } } - for (int z = 1; z < mesh->LocalNz; z++) { - int localIndex = ROUND(index(it.ind, mesh->ystart, z)); + for (RangeIterator it = mesh->iterateBndryLowerY(); !it.isDone(); it++) { + // A boundary, so no communication - // Only 3D fields - for (int i = 0; i < n3d; i++) { + // z = 0 case + int localIndex = ROUND(index(it.ind, mesh->ystart, 0)); + if (localIndex < 0) { + // This can occur because it.ind includes values in x boundary e.g. x=0 + continue; + } + // All 2D and 3D fields + for (int i = 0; i < n2d + n3d; i++) { o_nnz[localIndex + i] -= (n3d + n2d); } - } - } - for (RangeIterator it = mesh->iterateBndryUpperY(); !it.isDone(); it++) { - // A boundary, so no communication + for (int z = 1; z < mesh->LocalNz; z++) { + int localIndex = ROUND(index(it.ind, mesh->ystart, z)); - // z = 0 case - int localIndex = ROUND(index(it.ind, mesh->yend, 0)); - if (localIndex < 0) { - continue; // Out of domain + // Only 3D fields + for (int i = 0; i < n3d; i++) { + o_nnz[localIndex + i] -= (n3d + n2d); + } + } } - // All 2D and 3D fields - for (int i = 0; i < n2d + n3d; i++) { - o_nnz[localIndex + i] -= (n3d + n2d); - } + for (RangeIterator it = mesh->iterateBndryUpperY(); !it.isDone(); it++) { + // A boundary, so no communication - for (int z = 1; z < mesh->LocalNz; z++) { - int localIndex = ROUND(index(it.ind, mesh->yend, z)); + // z = 0 case + int localIndex = ROUND(index(it.ind, mesh->yend, 0)); + if (localIndex < 0) { + continue; // Out of domain + } - // Only 3D fields - for (int i = 0; i < n3d; i++) { + // All 2D and 3D fields + for (int i = 0; i < n2d + n3d; i++) { o_nnz[localIndex + i] -= (n3d + n2d); } + + for (int z = 1; z < mesh->LocalNz; z++) { + int localIndex = ROUND(index(it.ind, mesh->yend, z)); + + // Only 3D fields + for (int i = 0; i < n3d; i++) { + o_nnz[localIndex + i] -= (n3d + n2d); + } + } } } @@ -395,15 +422,19 @@ int SNESSolver::init() { // Pre-allocate MatMPIAIJSetPreallocation(Jmf, 0, d_nnz.data(), 0, o_nnz.data()); + MatSeqAIJSetPreallocation(Jmf, 0, d_nnz.data()); MatSetUp(Jmf); - MatSetOption(Jmf, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_FALSE); + MatSetOption(Jmf, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE); // Determine which row/columns of the matrix are locally owned int Istart, Iend; MatGetOwnershipRange(Jmf, &Istart, &Iend); // Convert local into global indices - index += Istart; + // Note: Not in the boundary cells, to keep -1 values + for (const auto& i : mesh->getRegion3D("RGN_NOBNDRY")) { + index[i] += Istart; + } // Now communicate to fill guard cells mesh->communicate(index); @@ -413,11 +444,6 @@ int SNESSolver::init() { output_progress.write("Marking non-zero Jacobian entries\n"); - // Offsets for a 5-point pattern - constexpr std::size_t stencil_size = 5; - const std::array xoffset = {0, -1, 1, 0, 0}; - const std::array yoffset = {0, 0, 0, -1, 1}; - PetscScalar val = 1.0; for (int x = mesh->xstart; x <= mesh->xend; x++) { @@ -430,9 +456,9 @@ int SNESSolver::init() { PetscInt row = ind0 + i; // Loop through each point in the 5-point stencil - for (std::size_t c = 0; c < stencil_size; c++) { - int xi = x + xoffset[c]; - int yi = y + yoffset[c]; + for (const auto& xyoffset : xyoffsets) { + int xi = x + xyoffset.first; + int yi = y + xyoffset.second; if ((xi < 0) || (yi < 0) || (xi >= mesh->LocalNx) || (yi >= mesh->LocalNy)) { @@ -448,8 +474,8 @@ int SNESSolver::init() { // Depends on all variables on this cell for (int j = 0; j < n2d; j++) { PetscInt col = ind2 + j; - - MatSetValues(Jmf, 1, &row, 1, &col, &val, INSERT_VALUES); + ierr = MatSetValues(Jmf, 1, &row, 1, &col, &val, INSERT_VALUES); + CHKERRQ(ierr); } } } @@ -468,13 +494,14 @@ int SNESSolver::init() { // Depends on 2D fields for (int j = 0; j < n2d; j++) { PetscInt col = ind0 + j; - MatSetValues(Jmf, 1, &row, 1, &col, &val, INSERT_VALUES); + ierr = MatSetValues(Jmf, 1, &row, 1, &col, &val, INSERT_VALUES); + CHKERRQ(ierr); } - // 5 point star pattern - for (std::size_t c = 0; c < stencil_size; c++) { - int xi = x + xoffset[c]; - int yi = y + yoffset[c]; + // Star pattern + for (const auto& xyoffset : xyoffsets) { + int xi = x + xyoffset.first; + int yi = y + xyoffset.second; if ((xi < 0) || (yi < 0) || (xi >= mesh->LocalNx) || (yi >= mesh->LocalNy)) { @@ -489,11 +516,15 @@ int SNESSolver::init() { if (z == 0) { ind2 += n2d; } - + // 3D fields on this cell for (int j = 0; j < n3d; j++) { PetscInt col = ind2 + j; - MatSetValues(Jmf, 1, &row, 1, &col, &val, INSERT_VALUES); + ierr = MatSetValues(Jmf, 1, &row, 1, &col, &val, INSERT_VALUES); + if (ierr != 0) { + output.write("ERROR: {} : ({}, {}) -> ({}, {}) : {} -> {}\n", row, x, y, xi, yi, ind2, ind2 + n3d - 1); + } + CHKERRQ(ierr); } } @@ -509,7 +540,8 @@ int SNESSolver::init() { } for (int j = 0; j < n3d; j++) { PetscInt col = ind2 + j; - MatSetValues(Jmf, 1, &row, 1, &col, &val, INSERT_VALUES); + ierr = MatSetValues(Jmf, 1, &row, 1, &col, &val, INSERT_VALUES); + CHKERRQ(ierr); } int zm = (z - 1 + nz) % nz; @@ -519,7 +551,8 @@ int SNESSolver::init() { } for (int j = 0; j < n3d; j++) { PetscInt col = ind2 + j; - MatSetValues(Jmf, 1, &row, 1, &col, &val, INSERT_VALUES); + ierr = MatSetValues(Jmf, 1, &row, 1, &col, &val, INSERT_VALUES); + CHKERRQ(ierr); } } } @@ -633,6 +666,11 @@ int SNESSolver::init() { // Set PC type from input if (pc_type != "default") { PCSetType(pc, pc_type.c_str()); + + if (pc_type == "hypre") { + // Set the type of hypre preconditioner + PCHYPRESetType(pc, pc_hypre_type.c_str()); + } } } diff --git a/src/solver/impls/snes/snes.hxx b/src/solver/impls/snes/snes.hxx index c00f4b2581..2021402cd7 100644 --- a/src/solver/impls/snes/snes.hxx +++ b/src/solver/impls/snes/snes.hxx @@ -125,6 +125,7 @@ private: bool kspsetinitialguessnonzero; ///< Set initial guess to non-zero int maxl; ///< Maximum linear iterations std::string pc_type; ///< Preconditioner type + std::string pc_hypre_type; ///< Hypre preconditioner type std::string line_search_type; ///< Line search type bool matrix_free; ///< Use matrix free Jacobian int lag_jacobian; ///< Re-use Jacobian From 6e3ed0a4af05d1c7864505e0d73097be0f42c554 Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Wed, 24 May 2023 21:38:22 -0700 Subject: [PATCH 098/111] cvode solver: Add linear and nonlinear convergence options cvode_nonlinear_convergence_coef Sets parameter through call to CVodeSetNonlinConvCoef https://sundials.readthedocs.io/en/latest/cvodes/Usage/SIM.html#c.CVodeSetNonlinConvCoef cvode_linear_convergence_coef Sets parameter through call to CVodeSetEpsLin https://sundials.readthedocs.io/en/latest/cvodes/Usage/SIM.html#c.CVodeSetEpsLin --- src/solver/impls/cvode/cvode.cxx | 10 ++++++++++ src/solver/impls/cvode/cvode.hxx | 2 ++ 2 files changed, 12 insertions(+) diff --git a/src/solver/impls/cvode/cvode.cxx b/src/solver/impls/cvode/cvode.cxx index 23f76b4db2..13d7703fcc 100644 --- a/src/solver/impls/cvode/cvode.cxx +++ b/src/solver/impls/cvode/cvode.cxx @@ -174,6 +174,12 @@ CvodeSolver::CvodeSolver(Options* opts) .doc("Use right preconditioner? Otherwise use left.") .withDefault(false)), use_jacobian((*options)["use_jacobian"].withDefault(false)), + cvode_nonlinear_convergence_coef((*options)["cvode_nonlinear_convergence_coef"] + .doc("Safety factor used in the nonlinear convergence test") + .withDefault(0.1)), + cvode_linear_convergence_coef((*options)["cvode_linear_convergence_coef"] + .doc("Factor by which the Krylov linear solver’s convergence test constant is reduced from the nonlinear solver test constant.") + .withDefault(0.05)), suncontext(static_cast(&BoutComm::get())) { has_constraints = false; // This solver doesn't have constraints canReset = true; @@ -458,6 +464,10 @@ int CvodeSolver::init() { #endif } + // Set internal tolerance factors + CVodeSetNonlinConvCoef(cvode_mem, cvode_nonlinear_convergence_coef); + CVodeSetEpsLin(cvode_mem, cvode_linear_convergence_coef); + cvode_initialised = true; return 0; diff --git a/src/solver/impls/cvode/cvode.hxx b/src/solver/impls/cvode/cvode.hxx index b2ed83568b..fa8b972bca 100644 --- a/src/solver/impls/cvode/cvode.hxx +++ b/src/solver/impls/cvode/cvode.hxx @@ -123,6 +123,8 @@ private: /// Use right preconditioner? Otherwise use left. bool rightprec; bool use_jacobian; + BoutReal cvode_nonlinear_convergence_coef; + BoutReal cvode_linear_convergence_coef; // Diagnostics from CVODE int nsteps{0}; From 5c7365be4681b6b97b1eba128270eb7c56c79474 Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Wed, 24 May 2023 21:58:15 -0700 Subject: [PATCH 099/111] Solver documentation Add some documentation of new cvode and beuler solver options. --- manual/sphinx/user_docs/time_integration.rst | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/manual/sphinx/user_docs/time_integration.rst b/manual/sphinx/user_docs/time_integration.rst index 7118039633..06aa1cf535 100644 --- a/manual/sphinx/user_docs/time_integration.rst +++ b/manual/sphinx/user_docs/time_integration.rst @@ -151,6 +151,20 @@ for a certain variable (e.g. ``[n]``), setting the option ``positivity_constraint`` to one of ``positive``, ``non_negative``, ``negative``, or ``non_positive``. +Additional options can be used to modify the behaviour of the linear and +nonlinear solvers: + +- ``cvode_nonlinear_convergence_coef`` specifies the safety factor + used in the nonlinear convergence test. Passed as a parameter to + `CVodeSetNonlinConvCoef + `_. + +- ``cvode_linear_convergence_coef`` specifies the factor by which the + Krylov linear solver’s convergence test constant is reduced from the + nonlinear solver test constant. Passed as a parameter to + `CVodeSetEpsLin + `_. + IMEX-BDF2 --------- @@ -358,7 +372,10 @@ iterations within a given range. +---------------------------+---------------+----------------------------------------------------+ | ksp_type | gmres | PETSc KSP linear solver | +---------------------------+---------------+----------------------------------------------------+ -| pc_type | ilu / bjacobi | PETSc PC preconditioner | +| pc_type | ilu / bjacobi | PETSc PC preconditioner (try hypre in parallel) | ++---------------------------+---------------+----------------------------------------------------+ +| pc_hypre_type | pilut | If ``pc_type = hypre``. | +| | | Hypre preconditioner type: euclid, boomeramg | +---------------------------+---------------+----------------------------------------------------+ | max_nonlinear_iterations | 20 | If exceeded, solve restarts with timestep / 2 | +---------------------------+---------------+----------------------------------------------------+ From 6875ea105da443fe08a64c0d367aef2e898336ce Mon Sep 17 00:00:00 2001 From: bendudson Date: Thu, 25 May 2023 05:01:28 +0000 Subject: [PATCH 100/111] Apply clang-format changes --- src/solver/impls/cvode/cvode.cxx | 15 +++++++++------ src/solver/impls/snes/snes.cxx | 16 +++++++++------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/solver/impls/cvode/cvode.cxx b/src/solver/impls/cvode/cvode.cxx index 13d7703fcc..70eb3b8841 100644 --- a/src/solver/impls/cvode/cvode.cxx +++ b/src/solver/impls/cvode/cvode.cxx @@ -174,12 +174,15 @@ CvodeSolver::CvodeSolver(Options* opts) .doc("Use right preconditioner? Otherwise use left.") .withDefault(false)), use_jacobian((*options)["use_jacobian"].withDefault(false)), - cvode_nonlinear_convergence_coef((*options)["cvode_nonlinear_convergence_coef"] - .doc("Safety factor used in the nonlinear convergence test") - .withDefault(0.1)), - cvode_linear_convergence_coef((*options)["cvode_linear_convergence_coef"] - .doc("Factor by which the Krylov linear solver’s convergence test constant is reduced from the nonlinear solver test constant.") - .withDefault(0.05)), + cvode_nonlinear_convergence_coef( + (*options)["cvode_nonlinear_convergence_coef"] + .doc("Safety factor used in the nonlinear convergence test") + .withDefault(0.1)), + cvode_linear_convergence_coef( + (*options)["cvode_linear_convergence_coef"] + .doc("Factor by which the Krylov linear solver’s convergence test constant " + "is reduced from the nonlinear solver test constant.") + .withDefault(0.05)), suncontext(static_cast(&BoutComm::get())) { has_constraints = false; // This solver doesn't have constraints canReset = true; diff --git a/src/solver/impls/snes/snes.cxx b/src/solver/impls/snes/snes.cxx index 2c15e61b3a..0c64aa5c22 100644 --- a/src/solver/impls/snes/snes.cxx +++ b/src/solver/impls/snes/snes.cxx @@ -109,8 +109,9 @@ SNESSolver::SNESSolver(Options* opts) .doc("Preconditioner type. By default lets PETSc decide (ilu or bjacobi)") .withDefault("default")), pc_hypre_type((*options)["pc_hypre_type"] - .doc("hypre preconditioner type: euclid, pilut, parasails, boomeramg, ams, ads") - .withDefault("pilut")), + .doc("hypre preconditioner type: euclid, pilut, parasails, " + "boomeramg, ams, ads") + .withDefault("pilut")), line_search_type((*options)["line_search_type"] .doc("Line search type: basic, bt, l2, cp, nleqerr") .withDefault("default")), @@ -173,10 +174,10 @@ int SNESSolver::init() { // Nonlinear solver interface (SNES) output.write("Create SNES\n"); SNESCreate(BoutComm::get(), &snes); - + // Set the callback function SNESSetFunction(snes, snes_f, FormFunction, this); - + SNESSetType(snes, snes_type.c_str()); // Line search @@ -246,7 +247,7 @@ int SNESSolver::init() { const auto star_pattern = (1 + ncells_x + ncells_y) * (n3d + n2d) + ncells_z * n3d; // Offsets. Start with the central cell - std::vector> xyoffsets {{0, 0}}; + std::vector> xyoffsets{{0, 0}}; if (ncells_x != 0) { // Stencil includes points in X xyoffsets.push_back({-1, 0}); @@ -516,13 +517,14 @@ int SNESSolver::init() { if (z == 0) { ind2 += n2d; } - + // 3D fields on this cell for (int j = 0; j < n3d; j++) { PetscInt col = ind2 + j; ierr = MatSetValues(Jmf, 1, &row, 1, &col, &val, INSERT_VALUES); if (ierr != 0) { - output.write("ERROR: {} : ({}, {}) -> ({}, {}) : {} -> {}\n", row, x, y, xi, yi, ind2, ind2 + n3d - 1); + output.write("ERROR: {} : ({}, {}) -> ({}, {}) : {} -> {}\n", row, x, + y, xi, yi, ind2, ind2 + n3d - 1); } CHKERRQ(ierr); } From ece5b3fbfd0008c4a401a37a67eae0c8cc75211a Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Thu, 25 May 2023 10:01:14 -0700 Subject: [PATCH 101/111] snes solver: Check if PETSc has Hypre If PETSc is not configured with Hypre then PCHYPRESetType is not defined. --- src/solver/impls/snes/snes.cxx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/solver/impls/snes/snes.cxx b/src/solver/impls/snes/snes.cxx index 0c64aa5c22..303ea952f0 100644 --- a/src/solver/impls/snes/snes.cxx +++ b/src/solver/impls/snes/snes.cxx @@ -670,8 +670,12 @@ int SNESSolver::init() { PCSetType(pc, pc_type.c_str()); if (pc_type == "hypre") { +#if PETSC_HAVE_HYPRE // Set the type of hypre preconditioner PCHYPRESetType(pc, pc_hypre_type.c_str()); +#else + throw BoutException("PETSc was not configured with Hypre."); +#endif } } } From bfe8f7c759506f83c54bfdd76596ee95123df963 Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Thu, 25 May 2023 13:21:31 -0700 Subject: [PATCH 102/111] beuler: Recalculate Jacobian on solver failure If a step fails, re-calculate the Jacobian and preconditioner. This can be an expensive operation, so doing this on failure allows lag_jacobian to be increased. --- src/solver/impls/snes/snes.cxx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/solver/impls/snes/snes.cxx b/src/solver/impls/snes/snes.cxx index 303ea952f0..9ab9ea34a7 100644 --- a/src/solver/impls/snes/snes.cxx +++ b/src/solver/impls/snes/snes.cxx @@ -792,6 +792,12 @@ int SNESSolver::run() { // Restore state VecCopy(x0, snes_x); + // Recalculate the Jacobian + if (saved_jacobian_lag == 0) { + SNESGetLagJacobian(snes, &saved_jacobian_lag); + SNESSetLagJacobian(snes, 1); + } + // Check lock state PetscInt lock_state; VecLockGet(snes_x, &lock_state); From 7152948fbde505f6708d5ca4a9c21e5828d1e0a1 Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Thu, 25 May 2023 15:30:38 -0700 Subject: [PATCH 103/111] Set PETSc options before initialize Enables `petsc:log_view=true` option to set the PETSc `-log_view` switch correctly. --- src/sys/petsclib.cxx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/sys/petsclib.cxx b/src/sys/petsclib.cxx index 85108b522f..f1ced63a21 100644 --- a/src/sys/petsclib.cxx +++ b/src/sys/petsclib.cxx @@ -23,6 +23,11 @@ PetscLib::PetscLib(Options* opt) { if (count == 0) { // Initialise PETSc + // Load global PETSc options from the [petsc] section of the input + // Note: This should be before PetscInitialize so that some options + // can modify initialization e.g. -log_view. + setPetscOptions(Options::root()["petsc"], ""); + output << "Initialising PETSc\n"; PETSC_COMM_WORLD = BoutComm::getInstance()->getComm(); PetscInitialize(pargc, pargv, nullptr, help); @@ -30,9 +35,6 @@ PetscLib::PetscLib(Options* opt) { PetscLogEventRegister("Total BOUT++", 0, &USER_EVENT); PetscLogEventBegin(USER_EVENT, 0, 0, 0, 0); - - // Load global PETSc options from the [petsc] section of the input - setPetscOptions(Options::root()["petsc"], ""); } if (opt != nullptr and opt->isSection()) { From 7a24114ab8a574ac849f56b0fbdb46f3d0100b04 Mon Sep 17 00:00:00 2001 From: David Bold Date: Tue, 30 May 2023 13:34:22 +0200 Subject: [PATCH 104/111] Remove _TEST_ variables They are not used anyway --- cmake/FindPETSc.cmake | 6 ------ cmake/FindSLEPc.cmake | 10 ++-------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/cmake/FindPETSc.cmake b/cmake/FindPETSc.cmake index cc76489cbb..7a491054e5 100644 --- a/cmake/FindPETSc.cmake +++ b/cmake/FindPETSc.cmake @@ -258,12 +258,6 @@ show : include(Check${PETSC_LANGUAGE_BINDINGS}SourceRuns) - macro (PETSC_TEST_RUNS includes libraries runs) - set (PETSC_EXECUTABLE_RUNS "YES" CACHE BOOL - "Can the system successfully run a PETSc executable? This variable can be manually set to \"YES\" to force CMake to accept a given PETSc configuration, but this will almost always result in a broken build. If you change PETSC_DIR, PETSC_ARCH, or PETSC_CURRENT you would have to reset this variable." FORCE) - endmacro (PETSC_TEST_RUNS) - - find_path (PETSC_INCLUDE_DIR petscts.h HINTS "${PETSC_DIR}" PATH_SUFFIXES include diff --git a/cmake/FindSLEPc.cmake b/cmake/FindSLEPc.cmake index caf2f828e3..3add8eba8b 100644 --- a/cmake/FindSLEPc.cmake +++ b/cmake/FindSLEPc.cmake @@ -137,10 +137,9 @@ show : endif() if (SLEPC_SKIP_BUILD_TESTS) - set(SLEPC_TEST_RUNS TRUE) set(SLEPC_VERSION "UNKNOWN") set(SLEPC_VERSION_OK TRUE) -elseif (SLEPC_LIBRARIES AND SLEPC_INCLUDE_DIRS AND NOT SLEPC_TEST_RUNS) +elseif (SLEPC_LIBRARIES AND SLEPC_INCLUDE_DIRS) # Set flags for building test program set(CMAKE_REQUIRED_INCLUDES ${SLEPC_INCLUDE_DIRS}) @@ -192,10 +191,6 @@ int main() { set(SLEPC_VERSION_OK TRUE CACHE BOOL "") endif() mark_as_advanced(SLEPC_VERSION_OK) - - # Do not run SLEPc test program - - set(SLEPC_TEST_RUNS TRUE CACHE BOOL "SLEPc test program can run") endif() # Standard package handling @@ -204,8 +199,7 @@ find_package_handle_standard_args(SLEPc FOUND_VAR SLEPC_FOUND FAIL_MESSAGE "SLEPc could not be found. Be sure to set SLEPC_DIR, PETSC_DIR, and PETSC_ARCH." VERSION_VAR SLEPC_VERSION - REQUIRED_VARS SLEPC_LIBRARIES SLEPC_DIR SLEPC_INCLUDE_DIRS SLEPC_TEST_RUNS - SLEPC_VERSION_OK) + REQUIRED_VARS SLEPC_LIBRARIES SLEPC_DIR SLEPC_INCLUDE_DIRS SLEPC_VERSION_OK) if (SLEPC_FOUND) if (NOT TARGET SLEPc::SLEPc) From ebca99bae3ebdbdcb06cf4be4614f8886bd69386 Mon Sep 17 00:00:00 2001 From: David Bold Date: Tue, 30 May 2023 13:40:21 +0200 Subject: [PATCH 105/111] Remove unchecked petsc_test_runs --- cmake/FindPETSc.cmake | 39 --------------------------------------- 1 file changed, 39 deletions(-) diff --git a/cmake/FindPETSc.cmake b/cmake/FindPETSc.cmake index 7a491054e5..81c2f171c3 100644 --- a/cmake/FindPETSc.cmake +++ b/cmake/FindPETSc.cmake @@ -277,45 +277,6 @@ show : set (petsc_mpi_include_dirs "${MPI_${PETSC_LANGUAGE_BINDINGS}_INCLUDE_DIRS}") #set (petsc_additional_libraries "MPI::MPI_${PETSC_LANGUAGE_BINDINGS}${petsc_openmp_library}") - petsc_test_runs ("${petsc_includes_minimal};${petsc_mpi_include_dirs}" - "${PETSC_LIBRARIES_TS};${petsc_additional_libraries}" - petsc_works_minimal) - if (petsc_works_minimal) - message (STATUS "Minimal PETSc includes and libraries work. This probably means we are building with shared libs.") - set (petsc_includes_needed "${petsc_includes_minimal}") - else (petsc_works_minimal) # Minimal includes fail, see if just adding full includes fixes it - petsc_test_runs ("${petsc_includes_all};${petsc_mpi_include_dirs}" - "${PETSC_LIBRARIES_TS};${petsc_additional_libraries}" - petsc_works_allincludes) - if (petsc_works_allincludes) # It does, we just need all the includes ( - message (STATUS "PETSc requires extra include paths, but links correctly with only interface libraries. This is an unexpected configuration (but it seems to work fine).") - set (petsc_includes_needed ${petsc_includes_all}) - else (petsc_works_allincludes) # We are going to need to link the external libs explicitly - resolve_libraries (petsc_libraries_external "${petsc_libs_external}") - foreach (pkg SYS VEC MAT DM KSP SNES TS ALL) - list (APPEND PETSC_LIBRARIES_${pkg} ${petsc_libraries_external}) - endforeach (pkg) - petsc_test_runs ("${petsc_includes_minimal};${petsc_mpi_include_dirs}" - "${PETSC_LIBRARIES_TS};${petsc_additional_libraries}" - petsc_works_alllibraries) - if (petsc_works_alllibraries) - message (STATUS "PETSc only need minimal includes, but requires explicit linking to all dependencies. This is expected when PETSc is built with static libraries.") - set (petsc_includes_needed ${petsc_includes_minimal}) - else (petsc_works_alllibraries) - # It looks like we really need everything, should have listened to Matt - set (petsc_includes_needed ${petsc_includes_all}) - petsc_test_runs ("${petsc_includes_all};${petsc_mpi_include_dirs}" - "${PETSC_LIBRARIES_TS};${petsc_additional_libraries}" - petsc_works_all) - if (petsc_works_all) # We fail anyways - message (STATUS "PETSc requires extra include paths and explicit linking to all dependencies. This probably means you have static libraries and something unexpected in PETSc headers.") - else (petsc_works_all) # We fail anyways - message (STATUS "PETSc could not be used, maybe the install is broken.") - endif (petsc_works_all) - endif (petsc_works_alllibraries) - endif (petsc_works_allincludes) - endif (petsc_works_minimal) - # We do an out-of-source build so __FILE__ will be an absolute path, hence __INSDIR__ is superfluous if (${PETSC_VERSION} VERSION_LESS 3.1) set (PETSC_DEFINITIONS "-D__SDIR__=\"\"" CACHE STRING "PETSc definitions" FORCE) From af75e7f4735987ff9d329eac6e35499dcb0726b9 Mon Sep 17 00:00:00 2001 From: David Bold Date: Mon, 5 Jun 2023 11:18:51 +0200 Subject: [PATCH 106/111] Ensure we can compile PETSc Reverts some of the deling commits and changes _runs to _compiles --- cmake/FindPETSc.cmake | 74 +++++++++++++++++++++++++++++++- cmake/FindPackageMultipass.cmake | 26 +++++++++++ 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/cmake/FindPETSc.cmake b/cmake/FindPETSc.cmake index 81c2f171c3..6f2e895b1e 100644 --- a/cmake/FindPETSc.cmake +++ b/cmake/FindPETSc.cmake @@ -258,6 +258,37 @@ show : include(Check${PETSC_LANGUAGE_BINDINGS}SourceRuns) + macro (petsc_test_compiles includes libraries runs) + message(STATUS "PETSc test with : ${includes} ${libraries}" ) + if (PETSC_VERSION VERSION_GREATER 3.1) + set (_PETSC_TSDestroy "TSDestroy(&ts)") + else () + set (_PETSC_TSDestroy "TSDestroy(ts)") + endif () + + set(_PETSC_TEST_SOURCE " +static const char help[] = \"PETSc test program.\"; +#include +int main(int argc,char *argv[]) { + PetscErrorCode ierr; + TS ts; + + ierr = PetscInitialize(&argc,&argv,0,help);CHKERRQ(ierr); + ierr = TSCreate(PETSC_COMM_WORLD,&ts);CHKERRQ(ierr); + ierr = TSSetFromOptions(ts);CHKERRQ(ierr); + ierr = ${_PETSC_TSDestroy};CHKERRQ(ierr); + ierr = PetscFinalize();CHKERRQ(ierr); + return 0; +} +") + multipass_source_compiles ("${includes}" "${libraries}" "${_PETSC_TEST_SOURCE}" ${runs} "${PETSC_LANGUAGE_BINDINGS}") + if (${${runs}}) + set (PETSC_EXECUTABLE_COMPILES "YES" CACHE BOOL + "Can the system successfully run a PETSc executable? This variable can be manually set to \"YES\" to force CMake to accept a given PETSc configuration, but this will almost always result in a broken build. If you change PETSC_DIR, PETSC_ARCH, or PETSC_CURRENT you would have to reset this variable." FORCE) + endif (${${runs}}) + endmacro () + + find_path (PETSC_INCLUDE_DIR petscts.h HINTS "${PETSC_DIR}" PATH_SUFFIXES include @@ -277,6 +308,45 @@ show : set (petsc_mpi_include_dirs "${MPI_${PETSC_LANGUAGE_BINDINGS}_INCLUDE_DIRS}") #set (petsc_additional_libraries "MPI::MPI_${PETSC_LANGUAGE_BINDINGS}${petsc_openmp_library}") + petsc_test_compiles ("${petsc_includes_minimal};${petsc_mpi_include_dirs}" + "${PETSC_LIBRARIES_TS};${petsc_additional_libraries}" + petsc_works_minimal) + if (petsc_works_minimal) + message (STATUS "Minimal PETSc includes and libraries work. This probably means we are building with shared libs.") + set (petsc_includes_needed "${petsc_includes_minimal}") + else (petsc_works_minimal) # Minimal includes fail, see if just adding full includes fixes it + petsc_test_compiles ("${petsc_includes_all};${petsc_mpi_include_dirs}" + "${PETSC_LIBRARIES_TS};${petsc_additional_libraries}" + petsc_works_allincludes) + if (petsc_works_allincludes) # It does, we just need all the includes ( + message (STATUS "PETSc requires extra include paths, but links correctly with only interface libraries. This is an unexpected configuration (but it seems to work fine).") + set (petsc_includes_needed ${petsc_includes_all}) + else (petsc_works_allincludes) # We are going to need to link the external libs explicitly + resolve_libraries (petsc_libraries_external "${petsc_libs_external}") + foreach (pkg SYS VEC MAT DM KSP SNES TS ALL) + list (APPEND PETSC_LIBRARIES_${pkg} ${petsc_libraries_external}) + endforeach (pkg) + petsc_test_compiles ("${petsc_includes_minimal};${petsc_mpi_include_dirs}" + "${PETSC_LIBRARIES_TS};${petsc_additional_libraries}" + petsc_works_alllibraries) + if (petsc_works_alllibraries) + message (STATUS "PETSc only need minimal includes, but requires explicit linking to all dependencies. This is expected when PETSc is built with static libraries.") + set (petsc_includes_needed ${petsc_includes_minimal}) + else (petsc_works_alllibraries) + # It looks like we really need everything, should have listened to Matt + set (petsc_includes_needed ${petsc_includes_all}) + petsc_test_compiles ("${petsc_includes_all};${petsc_mpi_include_dirs}" + "${PETSC_LIBRARIES_TS};${petsc_additional_libraries}" + petsc_works_all) + if (petsc_works_all) # We fail anyways + message (STATUS "PETSc requires extra include paths and explicit linking to all dependencies. This probably means you have static libraries and something unexpected in PETSc headers.") + else (petsc_works_all) # We fail anyways + message (STATUS "PETSc could not be used, maybe the install is broken.") + endif (petsc_works_all) + endif (petsc_works_alllibraries) + endif (petsc_works_allincludes) + endif (petsc_works_minimal) + # We do an out-of-source build so __FILE__ will be an absolute path, hence __INSDIR__ is superfluous if (${PETSC_VERSION} VERSION_LESS 3.1) set (PETSC_DEFINITIONS "-D__SDIR__=\"\"" CACHE STRING "PETSc definitions" FORCE) @@ -296,7 +366,7 @@ if (NOT PETSC_INCLUDES AND NOT TARGET PETSc::PETSc) pkg_search_module(PkgPETSC PETSc>3.4.0 petsc>3.4.0) set (PETSC_LIBRARIES ${PkgPETSC_LINK_LIBRARIES} CACHE STRING "PETSc libraries" FORCE) set (PETSC_INCLUDES ${PkgPETSC_INCLUDE_DIRS} CACHE STRING "PETSc include path" FORCE) - set (PETSC_EXECUTABLE_RUNS "YES" CACHE BOOL + set (PETSC_EXECUTABLE_COMPILES "YES" CACHE BOOL "Can the system successfully run a PETSc executable? This variable can be manually set to \"YES\" to force CMake to accept a given PETSc configuration, but this will almost always result in a broken build. If you change PETSC_DIR, PETSC_ARCH, or PETSC_CURRENT you would have to reset this variable." FORCE) endif() endif() @@ -304,7 +374,7 @@ endif() # Note that we have forced values for all these choices. If you # change these, you are telling the system to trust you that they # work. It is likely that you will end up with a broken build. -mark_as_advanced (PETSC_INCLUDES PETSC_LIBRARIES PETSC_COMPILER PETSC_DEFINITIONS PETSC_MPIEXEC PETSC_EXECUTABLE_RUNS) +mark_as_advanced (PETSC_INCLUDES PETSC_LIBRARIES PETSC_COMPILER PETSC_DEFINITIONS PETSC_MPIEXEC PETSC_EXECUTABLE_COMPILES) include (FindPackageHandleStandardArgs) find_package_handle_standard_args (PETSc diff --git a/cmake/FindPackageMultipass.cmake b/cmake/FindPackageMultipass.cmake index fbf06a7f0f..2452096b56 100644 --- a/cmake/FindPackageMultipass.cmake +++ b/cmake/FindPackageMultipass.cmake @@ -32,6 +32,8 @@ # Always runs the given test, use this when you need to re-run tests # because parent variables have made old cache entries stale. +include(CheckCXXSourceCompiles) + macro (FIND_PACKAGE_MULTIPASS _name _current) string (TOUPPER ${_name} _NAME) set (_args ${ARGV}) @@ -104,3 +106,27 @@ endmacro (MULTIPASS_SOURCE_RUNS) macro (MULTIPASS_C_SOURCE_RUNS includes libraries source runs) multipass_source_runs("${includes}" "${libraries}" "${source}" ${runs} "C") endmacro (MULTIPASS_C_SOURCE_RUNS) + +macro (MULTIPASS_SOURCE_COMPILES includes libraries source runs language) + include (Check${language}SourceRuns) + # This is a ridiculous hack. CHECK_${language}_SOURCE_* thinks that if the + # *name* of the return variable doesn't change, then the test does + # not need to be re-run. We keep an internal count which we + # increment to guarantee that every test name is unique. If we've + # gotten here, then the configuration has changed enough that the + # test *needs* to be rerun. + if (NOT MULTIPASS_TEST_COUNT) + set (MULTIPASS_TEST_COUNT 00) + endif (NOT MULTIPASS_TEST_COUNT) + math (EXPR _tmp "${MULTIPASS_TEST_COUNT} + 1") # Why can't I add to a cache variable? + set (MULTIPASS_TEST_COUNT ${_tmp} CACHE INTERNAL "Unique test ID") + set (testname MULTIPASS_TEST_${MULTIPASS_TEST_COUNT}_${runs}) + set (CMAKE_REQUIRED_INCLUDES ${includes}) + set (CMAKE_REQUIRED_LIBRARIES ${libraries}) + if(${language} STREQUAL "C") + check_c_source_compiles ("${source}" ${testname}) + elseif(${language} STREQUAL "CXX") + check_cxx_source_compiles ("${source}" ${testname}) + endif() + set (${runs} "${${testname}}") +endmacro () From e0b24ebf7a36d4a62212affb917e88dc8531d189 Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Tue, 13 Jun 2023 10:42:53 -0700 Subject: [PATCH 107/111] Update src/solver/impls/snes/snes.cxx Co-authored-by: David Bold --- src/solver/impls/snes/snes.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/solver/impls/snes/snes.cxx b/src/solver/impls/snes/snes.cxx index 9ab9ea34a7..9c5d2578dd 100644 --- a/src/solver/impls/snes/snes.cxx +++ b/src/solver/impls/snes/snes.cxx @@ -150,7 +150,7 @@ int SNESSolver::init() { int ierr; // Vectors - output.write("Creating vector\n"); + output_info.write("Creating vector\n"); ierr = VecCreate(BoutComm::get(), &snes_x); CHKERRQ(ierr); ierr = VecSetSizes(snes_x, nlocal, PETSC_DECIDE); From 54f5a891362e045c99a6b05dda7a1bb8569b313f Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Tue, 13 Jun 2023 10:43:02 -0700 Subject: [PATCH 108/111] Update src/solver/impls/snes/snes.cxx Co-authored-by: David Bold --- src/solver/impls/snes/snes.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/solver/impls/snes/snes.cxx b/src/solver/impls/snes/snes.cxx index 9c5d2578dd..3a06c94191 100644 --- a/src/solver/impls/snes/snes.cxx +++ b/src/solver/impls/snes/snes.cxx @@ -172,7 +172,7 @@ int SNESSolver::init() { } // Nonlinear solver interface (SNES) - output.write("Create SNES\n"); + output_info.write("Create SNES\n"); SNESCreate(BoutComm::get(), &snes); // Set the callback function From ce059a72f11b54e1b2451f2bd8489d70df61efc2 Mon Sep 17 00:00:00 2001 From: Ben Dudson Date: Tue, 13 Jun 2023 10:43:18 -0700 Subject: [PATCH 109/111] Update src/solver/impls/snes/snes.cxx Co-authored-by: David Bold --- src/solver/impls/snes/snes.cxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/solver/impls/snes/snes.cxx b/src/solver/impls/snes/snes.cxx index 3a06c94191..db878cdb75 100644 --- a/src/solver/impls/snes/snes.cxx +++ b/src/solver/impls/snes/snes.cxx @@ -259,7 +259,7 @@ int SNESSolver::init() { xyoffsets.push_back({0, 1}); } - output.write("Star pattern: {} non-zero entries\n", star_pattern); + output_info.write("Star pattern: {} non-zero entries\n", star_pattern); for (int i = 0; i < localN; i++) { // Non-zero elements on this processor d_nnz[i] = star_pattern; From 912de4d427642c68377a230314f003eef460cf40 Mon Sep 17 00:00:00 2001 From: David Bold Date: Wed, 28 Jun 2023 15:46:07 +0200 Subject: [PATCH 110/111] Update to dnf5 --- .ci_fedora.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci_fedora.sh b/.ci_fedora.sh index e6d9f61e8f..badb1daa29 100755 --- a/.ci_fedora.sh +++ b/.ci_fedora.sh @@ -37,7 +37,7 @@ then cat /etc/os-release # Ignore weak depencies echo "install_weak_deps=False" >> /etc/dnf/dnf.conf - time dnf -y install dnf-plugins-core python3-pip cmake + time dnf -y install dnf5-plugins python3-pip cmake # Allow to override packages - see #2073 time dnf copr enable -y davidsch/fixes4bout || : time dnf -y upgrade From 743ae552112f95005276666bbd990ed2a79400a9 Mon Sep 17 00:00:00 2001 From: bendudson Date: Sat, 5 Aug 2023 19:16:38 +0000 Subject: [PATCH 111/111] Apply clang-format changes --- externalpackages/PVODE/source/cvode.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/externalpackages/PVODE/source/cvode.cpp b/externalpackages/PVODE/source/cvode.cpp index 48ac3209f2..135662a40f 100644 --- a/externalpackages/PVODE/source/cvode.cpp +++ b/externalpackages/PVODE/source/cvode.cpp @@ -179,7 +179,7 @@ namespace pvode { #define MSG_Y0_NULL CVM "y0=NULL illegal.\n\n" -#define MSG_BAD_N CVM "N=%d < 1 illegal.\n\n" +#define MSG_BAD_N CVM "N=%d < 1 illegal.\n\n" #define MSG_BAD_LMM_1 CVM "lmm=%d illegal.\n" #define MSG_BAD_LMM_2 "The legal values are ADAMS=%d and BDF=%d.\n\n"