diff --git a/.github/workflows/run_pr_tests.yml b/.github/workflows/run_pr_tests.yml index 45305499..fbd04d42 100644 --- a/.github/workflows/run_pr_tests.yml +++ b/.github/workflows/run_pr_tests.yml @@ -1,14 +1,9 @@ name: Run PR tests, graded and discrim on: - pull_request: - # only run when PRs target the main branch + push: branches: - - main - types: - - opened - - synchronize - - reopened + - test/maccGradedCase jobs: run-tests: @@ -57,11 +52,8 @@ jobs: sudo apt-get install git-all -y # Getting current branch - REPO="${{ github.repository }}" - CURRENT_BRANCH="${{ github.head_ref }}" - echo "Repository*^*: $REPO" - echo "Branch name*^*: $CURRENT_BRANCH" - git clone -b "${CURRENT_BRANCH}" "https://github.com/${REPO}.git" + CURRENT_BRANCH="${{ github.ref_name }}" + git clone -b "$CURRENT_BRANCH" https://github.com/stacs-cp/AutoIG.git # Install Necessary Dependencies into AutoIG Bin bash bin/install-mininzinc.sh @@ -78,20 +70,18 @@ jobs: cd scripts/testScripts # Run the two test scripts associated with PRs - echo "About to run pr_discrim tests" bash check_pr_discrim.sh - echo "About to run pr_graded tests" - bash check_pr_graded.sh + bash check_pr.sh # if script fails reject PR - name: Fail if: ${{ failure() }} run: | - echo "PR tests failed, rejecting." + echo "This tests failed, rejecting PR." exit 1 # if script passes approve PR - name: Pass if: ${{ success() }} run: | - echo "PR tests passed! allowing PR." + echo "This tests passed! allowing PR." exit 0 diff --git a/.github/workflows/run_push_tests.yml b/.github/workflows/run_push_tests.yml index 4fbeec27..782dad42 100644 --- a/.github/workflows/run_push_tests.yml +++ b/.github/workflows/run_push_tests.yml @@ -4,7 +4,7 @@ name: Run push tests, graded and discrim on: push: branches: - - "**" + - "*" jobs: run-tests: @@ -52,13 +52,9 @@ jobs: sudo apt-get install r-base -y sudo apt-get install git-all -y - # Getting current branch - REPO="${{ github.repository }}" - + # Getting current branch CURRENT_BRANCH="${{ github.ref_name }}" - echo "Repository*^*: $REPO" - - git clone -b "$CURRENT_BRANCH" https://github.com/${REPO}.git + git clone -b "$CURRENT_BRANCH" https://github.com/stacs-cp/AutoIG.git # Install Necessary Dependencies into AutoIG Bin bash bin/install-mininzinc.sh diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 8f1dcdb7..00000000 --- a/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Ignore __pycache__ directories -__pycache__/ diff --git a/DEV_README.md b/DEV_README.md index 0d9d2433..d5059a8b 100644 --- a/DEV_README.md +++ b/DEV_README.md @@ -1,4 +1,4 @@ - # Docker Build And Run Commands in a Container +# Docker Build And Run Commands in a Container ### Builds an image of using Docker @@ -33,35 +33,12 @@ At this point, AutoIG is fully configured and ready for use as normal. ## Example sequence of commands for setting up an experiment: -### To set up for MiniZinc - -### Graded Example `mkdir -p experiments/macc-graded/` `cd experiments/macc-graded/` `python $AUTOIG/scripts/setup.py --generatorModel $AUTOIG/data/models/macc/generator-small.essence --problemModel $AUTOIG/data/models/macc/problem.mzn --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver chuffed --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 5` -### Discriminating Example -`mkdir -p experiments/macc-discriminating/` - -`cd experiments/macc-discriminating/` - -`python $AUTOIG/scripts/setup.py --generatorModel $AUTOIG/data/models/macc/generator-small.essence --problemModel $AUTOIG/data/models/macc/problem.mzn --instanceSetting discriminating --minSolverTime 1 --maxSolverTime 3 --baseSolver chuffed --solverFlags="-f" --favouredSolver or-tools --favouredSolverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 5` - - -### To set up for essence - -#### Graded Example -For "vessel_loading" essence problem -`python $AUTOIG/scripts/setup.py --generatorModel $AUTOIG/data/models/vessel-loading/generator.essence --problemModel $AUTOIG/data/models/vessel-loading/problem.essence --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver chuffed --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 5` - -For "car-sequencing" essence problem -`python $AUTOIG/scripts/setup.py --generatorModel $AUTOIG/data/models/car-sequencing/generator.essence --problemModel $AUTOIG/data/models/car-sequencing/problem.essence --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver chuffed --solverFlags="-f" --maxEvaluations 300 --genSolverTimeLimit 5` - -#### Discriminating Example - -`python $AUTOIG/scripts/setup.py --generatorModel $AUTOIG/data/models/vessel-loading/generator.essence --problemModel $AUTOIG/data/models/vessel-loading/problem.essence --instanceSetting discriminating --minSolverTime 1 --maxSolverTime 3 --baseSolver chuffed --solverFlags="-f" --favouredSolver ortools --favouredSolverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 5` ### To Run The Generated Bash Script bash run.sh @@ -69,3 +46,4 @@ bash run.sh # Considerations for Use of Dockerfile The build Docker image allows for the program to be run in a container. It is worth noting though that the container could take up more storage than running AutoIG through Linux directly, as it will download dependencies within the container such as Python and R. If a users system already has these, it could be more efficient to run it directly on the system without a VM. In addition, data does not persist within the container, so it is important to save the results of AutoIG runs, perhaps with a Docker Volume. Instructions for setting up the Docker Volume are listed above. + diff --git a/Dockerfile b/Dockerfile index c2e5d3c7..c87932d5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,7 @@ RUN echo "$CACHEBUST" # Clone into AutoIG directory on Vincent fork # Not incorrect, but will need to be changed later to clone from stacs-cp/AutoIG instead -RUN git clone --depth 1 -b main https://github.com/vincepick/AutoIG.git +RUN git clone -b build/update-docker https://github.com/vincepick/AutoIG.git WORKDIR /AutoIG @@ -59,8 +59,6 @@ RUN bash bin/update-or-path.sh RUN bash bin/update-conjure-paths.sh # For use during development - -RUN apt-get update RUN apt-get install -y \ vim \ file diff --git a/bin/R-packages.R b/bin/R-packages.R index b3d78974..5759fa46 100644 --- a/bin/R-packages.R +++ b/bin/R-packages.R @@ -5,4 +5,4 @@ for (p in c("R6","data.table")){ install.packages(paste(binDir,paths[[p]],sep='/'), lib=binDir) library(p,character.only = TRUE, lib.loc=binDir) } -} \ No newline at end of file +} diff --git a/bin/data.table_1.14.2.tar.gz b/bin/data.table_1.14.2.tar.gz new file mode 100644 index 00000000..590bbd27 Binary files /dev/null and b/bin/data.table_1.14.2.tar.gz differ diff --git a/bin/install-irace.sh b/bin/install-irace.sh index c239ec95..78d8e2e7 100755 --- a/bin/install-irace.sh +++ b/bin/install-irace.sh @@ -5,7 +5,7 @@ echo "" echo "============= INSTALLING $name ===================" echo "$name version: $version" -BIN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +BIN_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" # check if R is installed if ! [ -x "$(command -v R)" ]; then @@ -17,7 +17,7 @@ url="https://cran.r-project.org/src/contrib/irace_3.4.1.tar.gz" pushd $BIN_DIR -mkdir -p $name +mkdir -p $name SOURCE_DIR="$name-source" mkdir -p $SOURCE_DIR diff --git a/scripts/.DS_Store b/scripts/.DS_Store index d869121e..d41c00c8 100644 Binary files a/scripts/.DS_Store and b/scripts/.DS_Store differ diff --git a/scripts/collect_results.py b/scripts/collect_results.py index 01b467a3..42877a1d 100644 --- a/scripts/collect_results.py +++ b/scripts/collect_results.py @@ -6,13 +6,11 @@ import sys import os -import re scriptDir = os.path.dirname(os.path.realpath(__file__)) sys.path.append(scriptDir) from minizinc_utils import calculate_minizinc_borda_scores, get_minizinc_problem_type -from essence_pipeline_utils import get_essence_problem_type, calculate_essence_borda_scores pd.options.mode.chained_assignment = None @@ -80,8 +78,8 @@ def rename_status_dis(status, score): else: return status tRs.loc[:,"status"] = [rename_status_dis(s[0],s[1]) for s in zip(tRs.status,tRs.score)] - - + #display(tRs[tRs.status.str.contains("Wins")]) + # rename some columns and re-order the columns tRs.rename(columns={"hashValue":"instanceHashValue","score":"iraceScore"}, inplace=True) tRs = tRs[["genInstance","instance","genResults","instanceResults","status","iraceScore","totalTime","instanceHashValue"]] @@ -103,7 +101,6 @@ def print_stats(config, tRs, tRsNoDup): # number of instances generated nInstances = len(tRsNoDup.instance.unique()) - # number of runs for each run status runStats = tRs.groupby('status').genResults.count().to_dict() runStatsWithoutDuplicates = tRsNoDup.groupby('status').genResults.count().to_dict() @@ -154,100 +151,44 @@ def extract_graded_and_discriminating_instances(runDir): """ extract information about graded/discriminating instances and save to a .csv file """ - - outFile = None - if re.search(r'\.mzn$', config["problemModel"]): - if config["instanceSetting"] == "graded": - # filter out non-graded instances - tInfo = tRsNoDup.loc[tRsNoDup.status=="graded",:] - # extract instance type - tInfo.loc[:,"instanceType"] = [x["results"]["main"]["runs"][0]["extra"]["instanceType"] for x in tInfo.instanceResults] - # calculate average solving time for each instance - tInfo.loc[:,"avgSolvingTime"] = [np.mean([rs["time"] for rs in x["results"]["main"]["runs"]]) for x in tInfo.instanceResults] - # re-order columns - tInfo = tInfo[["instance","instanceType","avgSolvingTime","instanceResults","genInstance","genResults","status","iraceScore","totalTime","instanceHashValue"]] - # save to a .csv file - outFile = f"{runDir}/graded-instances-info.csv" - print(f"\nInfo of graded instances is saved to {os.path.abspath(outFile)}") - tInfo.to_csv(outFile, index=False) - else: - # filter out non-discriminating instances or instances where the favoured solver lost - tInfo = tRsNoDup.loc[tRsNoDup.status.isin(["favouredSolverWins"]),:] - # extract instance type - tInfo.loc[:,"instanceType"] = [x["results"]["favoured"]["runs"][0]["extra"]["instanceType"] for x in tInfo.instanceResults] - # extract MiniZinc Borda score of the favoured and the base solvers - print("about to try to get problem type", config["problemModel"]) - - problemType = get_minizinc_problem_type(config["problemModel"]) - - - - def extract_minizinc_score(r): - results = calculate_minizinc_borda_scores(r['base']['runs'][0]['status'], r['favoured']['runs'][0]['status'], - r['base']['runs'][0]['time'], r['favoured']['runs'][0]['time'], - problemType, - r['base']['runs'][0]['extra']['objs'], r['favoured']['runs'][0]['extra']['objs'], - True) - scores = results["complete"] # first element: base solver's score, second element: favoured solver's score - return scores[1] - tInfo.loc[:,"favouredSolverMiniZincScore"] = [extract_minizinc_score(x["results"]) for x in tInfo.instanceResults] - tInfo.loc[:,"baseSolverMiniZincScore"] = [1 - x for x in tInfo.favouredSolverMiniZincScore] - tInfo.loc[:,"discriminatingPower"] = tInfo["favouredSolverMiniZincScore"] / tInfo["baseSolverMiniZincScore"] - # re-order columns - tInfo = tInfo[["instance","discriminatingPower","favouredSolverMiniZincScore","baseSolverMiniZincScore","instanceType","instanceResults","genInstance","genResults","status","iraceScore","totalTime","instanceHashValue"]] - # save to a .csv file - outFile = f"{runDir}/discriminating-instances-info.csv" - print(f"\nInfo of discriminating instances is saved to {os.path.abspath(outFile)}") - tInfo.to_csv(outFile, index=False) - - elif re.search(r'\.essence$', config["problemModel"]): - if config["instanceSetting"] == "graded": - # filter out non-graded instances - tInfo = tRsNoDup.loc[tRsNoDup.status=="graded",:] - - # extract instance type - tInfo.loc[:,"instanceType"] = [x["results"]["main"]["runs"][0]["status"] for x in tInfo.instanceResults] - # calculate average solving time for each instance - tInfo.loc[:,"avgSolvingTime"] = [np.mean([rs["time"] for rs in x["results"]["main"]["runs"]]) for x in tInfo.instanceResults] - # re-order columns - tInfo = tInfo[["instance","instanceType","avgSolvingTime","instanceResults","genInstance","genResults","status","iraceScore","totalTime","instanceHashValue"]] - # save to a .csv file - outFile = f"{runDir}/graded-instances-info.csv" - print(f"\nInfo of graded instances is saved to {os.path.abspath(outFile)}") - tInfo.to_csv(outFile, index=False) - else: - # filter out non-discriminating instances or instances where the favoured solver lost - tInfo = tRsNoDup.loc[tRsNoDup.status.isin(["favouredSolverWins"]),:] - # The instance type - tInfo.loc[:,"instanceType"] = [x["results"]["favoured"]["runs"][0]["status"] for x in tInfo.instanceResults] - # extract Esesnce Borda score of the favoured and the base solvers - - problemType = get_essence_problem_type(config["problemModel"]) - - - def extract_essence_score(r): - # Calculated using only solver time rather than total time of SR + Solver Time - results = calculate_essence_borda_scores(r['base']['runs'][0]['status'], r['favoured']['runs'][0]['status'], - r['base']['runs'][0]['solverTime'], r['favoured']['runs'][0]['solverTime'], - problemType, - True) - # scores = results # first element: base solver's score, second element: favoured solver's score - # Different than the essence pipeline, instaed the calculate_essence_borda_scores calculates the score directly - return results[1] - tInfo.loc[:,"favouredSolverEssenceScore"] = [extract_essence_score(x["results"]) for x in tInfo.instanceResults] - tInfo.loc[:,"baseSolverEssenceScore"] = [1 - x for x in tInfo.favouredSolverEssenceScore] - tInfo.loc[:,"discriminatingPower"] = tInfo["favouredSolverEssenceScore"] / tInfo["baseSolverEssenceScore"] - # re-order columns - tInfo = tInfo[["instance","discriminatingPower","favouredSolverEssenceScore","baseSolverEssenceScore","instanceType","instanceResults","genInstance","genResults","status","iraceScore","totalTime","instanceHashValue"]] - # save to a .csv file - outFile = f"{runDir}/discriminating-instances-info.csv" - print(f"\nInfo of discriminating instances is saved to {os.path.abspath(outFile)}") - tInfo.to_csv(outFile, index=False) + if config["instanceSetting"] == "graded": + # filter out non-graded instances + tInfo = tRsNoDup.loc[tRsNoDup.status=="graded",:] + # extract instance type + tInfo.loc[:,"instanceType"] = [x["results"]["main"]["runs"][0]["extra"]["instanceType"] for x in tInfo.instanceResults] + # calculate average solving time for each instance + tInfo.loc[:,"avgSolvingTime"] = [np.mean([rs["time"] for rs in x["results"]["main"]["runs"]]) for x in tInfo.instanceResults] + # re-order columns + tInfo = tInfo[["instance","instanceType","avgSolvingTime","instanceResults","genInstance","genResults","status","iraceScore","totalTime","instanceHashValue"]] + # save to a .csv file + outFile = f"{runDir}/graded-instances-info.csv" + print(f"\nInfo of graded instances is saved to {os.path.abspath(outFile)}") + tInfo.to_csv(outFile, index=False) else: - # there are no other supported model types for now - print("Unsupported model type, please try agian with Essence or Mzn problem model") - + # filter out non-discriminating instances or instances where the favoured solver lost + tInfo = tRsNoDup.loc[tRsNoDup.status.isin(["favouredSolverWins"]),:] + # extract instance type + tInfo.loc[:,"instanceType"] = [x["results"]["favoured"]["runs"][0]["extra"]["instanceType"] for x in tInfo.instanceResults] + # extract MiniZinc Borda score of the favoured and the base solvers + problemType = get_minizinc_problem_type(config["problemModel"]) + def extract_minizinc_score(r): + results = calculate_minizinc_borda_scores(r['base']['runs'][0]['status'], r['favoured']['runs'][0]['status'], + r['base']['runs'][0]['time'], r['favoured']['runs'][0]['time'], + problemType, + r['base']['runs'][0]['extra']['objs'], r['favoured']['runs'][0]['extra']['objs'], + True) + scores = results["complete"] # first element: base solver's score, second element: favoured solver's score + return scores[1] + tInfo.loc[:,"favouredSolverMiniZincScore"] = [extract_minizinc_score(x["results"]) for x in tInfo.instanceResults] + tInfo.loc[:,"baseSolverMiniZincScore"] = [1 - x for x in tInfo.favouredSolverMiniZincScore] + tInfo.loc[:,"discriminatingPower"] = tInfo["favouredSolverMiniZincScore"] / tInfo["baseSolverMiniZincScore"] + # re-order columns + tInfo = tInfo[["instance","discriminatingPower","favouredSolverMiniZincScore","baseSolverMiniZincScore","instanceType","instanceResults","genInstance","genResults","status","iraceScore","totalTime","instanceHashValue"]] + # save to a .csv file + outFile = f"{runDir}/discriminating-instances-info.csv" + print(f"\nInfo of discriminating instances is saved to {os.path.abspath(outFile)}") + tInfo.to_csv(outFile, index=False) return tInfo diff --git a/scripts/conf.py b/scripts/conf.py index c6c9320b..c9b1e4a3 100644 --- a/scripts/conf.py +++ b/scripts/conf.py @@ -1,26 +1 @@ problemType = None - -# Define constants for scoring - -# General -SCORE_UNWANTED_TYPE = 0 -SCORE_TOO_EASY = 0 -SCORE_INCORRECT_ANSWER = 0 -SCORE_TOO_DIFFICULT = 0 - -# Graded -SCORE_GRADED = -1 - -# Discriminating -SCORE_BASE_TOO_EASY = 0 -SCORE_FAVOURED_TOO_DIFFICULT = 0 -# Best when one can do it but the other can't -SCORE_BEST = -9999 - - - -# Define constants for outputs -detailedOutputDir = "./detailed-output" - -# for minizinc experiments only: solvers where -r doesn't work when being called via minizinc -deterministicSolvers = ["ortools"] \ No newline at end of file diff --git a/scripts/essence_pipeline_utils.py b/scripts/essence_pipeline_utils.py index 75781456..9b0f2248 100644 --- a/scripts/essence_pipeline_utils.py +++ b/scripts/essence_pipeline_utils.py @@ -2,7 +2,6 @@ import sys import time import glob -import json scriptDir = os.path.dirname(os.path.realpath(__file__)) sys.path.append(scriptDir) @@ -65,47 +64,6 @@ "timelimitPrefix": "--time=", "randomSeedPrefix": "--seed=", } -solverInfo["or-tools"] = { - "timelimitUnit": "s", - "timelimitPrefix": "-t ", - "randomSeedPrefix": "--fz_seed=", -} - - -def get_essence_problem_type(modelFile: str): - """ - Read an Essence model and return its type (MIN/MAX/SAT) - Uses the conjure pretty print feature, then interprets the generated JSON - """ - cmd = f"conjure pretty {modelFile} --output-format=astjson" - results_dict = run_cmd(cmd) - try: - parsed_json = json.loads(results_dict[0]) - except json.JSONDecodeError as e: - print("Failed to parse JSON:", e) - exit() - - # Now check for the Maximising objective recursively - def check_type(data): - if isinstance(data, dict): - for key, value in data.items(): - if key == "Objective" and isinstance(value, list): - if value[0] == "Maximising": - return "MAX" - elif value[0] == "Minimising": - return "MIN" - result = check_type(value) - if result in ("MAX", "MIN"): - return result - elif isinstance(data, list): - for item in data: - result = check_type(item) - if result in ("MAX", "MIN"): - return result - return "SAT" - - return check_type(parsed_json) - def conjure_translate_parameter(eprimeModelFile, paramFile, eprimeParamFile): @@ -143,7 +101,6 @@ def savilerow_translate( + " " + flags ) - log(cmd) start = time.time() @@ -342,7 +299,7 @@ def make_conjure_solve_command( SRFlags="", solverTimeLimit=0, solverFlags="", - seed=None, # no seed as default + seed=None, ): # temporary files that will be removed lsTempFiles = [] @@ -412,25 +369,16 @@ def make_conjure_solve_command( + " --solver=" + solver ) - print(conjureCmd) return conjureCmd, lsTempFiles -# Changed to take in the paramters directly, rather than through a provided settings dictionary -def call_conjure_solve( - essenceModelFile: str, - eprimeModelFile: str, - instFile: str, - solver: str, - SRTimeLimit, - SRFlags, - solverTimeLimit, - solverFlags, - seed): - - lsTempFiles = [] - print() +def call_conjure_solve(essenceModelFile, eprimeModelFile, instFile, setting, seed): + if "name" in setting: + solver = setting["name"] + elif "solver" in setting: + solver = setting["solver"] + lsTempFiles = [] # make conjure solve command line conjureCmd, tempFiles = make_conjure_solve_command( @@ -438,10 +386,10 @@ def call_conjure_solve( eprimeModelFile, instFile, solver, - SRTimeLimit, - SRFlags, - solverTimeLimit, - solverFlags, + setting["SRTimeLimit"], + setting["SRFlags"], + setting["solverTimeLimit"], + setting["solverFlags"], seed, ) lsTempFiles.extend(tempFiles) @@ -470,7 +418,7 @@ def call_conjure_solve( ): status = "solverMemOut" elif ("Sub-process exited with error code:139" in cmdOutput) and ( - setting["abortIfSolverCrash"] is False # TODO This is deprecated, but not sure what to replace with + setting["abortIfSolverCrash"] is False ): status = "solverCrash" elif returnCode != 0: @@ -533,7 +481,7 @@ def call_conjure_solve( # parse SR info file infoStatus, SRTime, solverTime = parse_SR_info_file( - infoFile, timelimit=solverTimeLimit + infoFile, timelimit=setting["solverTimeLimit"] ) if status != "solverCrash": status = infoStatus @@ -591,55 +539,3 @@ def get_val(field): else: status = "unsat" return status, SRTime, solverTime - -def calculate_essence_borda_scores( - status1: str, - status2: str, - time1: float, - time2: float, - problemType: str, - # no obj funciton for - zeroScoreWhenBothFail: bool = False, -): - """ - Compute a replica of the MiniZinc competition's Borda scores between runs of two solvers. - Different than the one for the MiniZinc pipeline because there is no optimization score - """ - - - possible_status = { - "sat", - "nsat", - "SRTimeOut", - "SRMemOut", - "solverTimeOut", - "solverMemOut", - "solverCrash", - "solverNodeOut" - } - # Initial Assertions - assert status1 in possible_status and status2 in possible_status - assert problemType in ["MIN", "MAX", "SAT"] - - def solved(status): - return status in ["sat", "nsat"] - - - def calculateMnzScore(time1, time2): - return ( time2 / (time1 + time2),time1 / (time1 + time2)) - - # General Logic used across both optimization and sat problems - if solved(status1) and not solved(status2): - return (1,0) - elif solved(status2) and not solved(status1): - return (0,1) - - # Special Cases - if not solved(status1) and not solved(status2): - if zeroScoreWhenBothFail: - return (0,0) - else: - return (0, 1) - return calculateMnzScore(time1, time2) - - diff --git a/scripts/evaluators/__init__.py b/scripts/evaluators/__init__.py deleted file mode 100644 index 0246fd53..00000000 --- a/scripts/evaluators/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .essence_graded import evaluate_essence_instance_graded -from .essence_discriminating import evaluate_essence_instance_discriminating -from .mzn_graded import evaluate_mzn_instance_graded -from .mzn_discriminating import evaluate_mzn_instance_discriminating diff --git a/scripts/evaluators/essence_discriminating.py b/scripts/evaluators/essence_discriminating.py deleted file mode 100644 index 5c8ff935..00000000 --- a/scripts/evaluators/essence_discriminating.py +++ /dev/null @@ -1,258 +0,0 @@ -import os -from utils import log -from essence_pipeline_utils import call_conjure_solve, get_essence_problem_type, calculate_essence_borda_scores -import conf - - - -def evaluate_essence_instance_discriminating( - modelFile: str, # Path to the Essence model file - instFile: str, # Path to the instance parameter file - scoringMethod: str = "complete", # Scoring method, complete by default - unwantedTypes: list = [], # A list of unwanted types, empty by default - nEvaluations: int = 1, # The number of iterations for each generator model, 1 by default - baseSolver: str = "ortools", # Default baseSolver if none is provided - baseSolverFlags: str = "-f", # Flags for the base solver, fed into Conjure - baseMinTime: int = 0, # The minimum time threshold for the base solver - favouredSolver: str = "yuck", # The default favoured solver, if none is provided - favouredSolverFlags: str = "-f", # Flags for the favoured solver, fed into Conjure - totalTimeLimit: int = 1200, # The default time limit for each solver run - initSeed: int = None, # The initial seed - totalMemLimit=8192, # Total memory limit for solver runs (currently unused) - SRTimeLimit: int = 0, # The timelimit for SR - SRFlags: str = "", # Flags for SR -): - - """evaluate a generated instance based on discriminating power with two solvers ### - " NOTE: - " - this function can be improved using parallelisation, as there are various cases in the scoring where runs can be safely terminated before they finished. Things to consider - " + gnu-parallel for implementation - " + runsolver for safely terminating a solver run - " - " scoring scheme for discriminating solvers: - " - gen unsat/SR memout/SR timeout: Inf - " - gen solver timeout: 2 - " - inst unwanted type or SR timeout (either solver): 1 (ISSUE: with this new implementation, we can't recognise SR timeout, so we treat it as both solver timeout, i.e., score=0) - " - favoured solver timeout (any run) or base solver too easy (any run): 0 - " - otherwise: max{-minRatio, -baseSolver/favouredSolver} - " - note: if minRatio>0, ideally we should set timelimit_baseSolver = minRatio * timelimit_favouredSolver - """ - - - # Set up file paths for Essence and EPrime files - essenceModelFile = "./problem.essence" - eprimeModelFile = conf.detailedOutputDir + "/problem.eprime" - instance = os.path.basename(instFile).replace(".param", "") - - - # Assertions to validate unwanted type and scoring method inputs - assert scoringMethod in ["complete", "incomplete"], ( - "ERROR: scoringMethod " + scoringMethod + " is not supported" - ) - - if len(unwantedTypes) > 0: - for s in unwantedTypes: - assert s in [ - "sat", - "unsat", - ], "ERROR: elements of unwantedTypes must be in {'sat','unsat'}" - assert nEvaluations > 0 - - # Initialize info and results dictionaries - info = { - "base": {"name": baseSolver, "flags": baseSolverFlags}, - "favoured": {"name": favouredSolver, "flags": favouredSolverFlags}, - } - - # Initialize results dictionary for each solver - results = {"base": {}, "favoured": {}} - for solverType in ["base", "favoured"]: - results[solverType]["runs"] = [] - score = status = None - instanceType = None - - - # Create a function to return the results of the run - def get_results(): - assert (score is not None) and ( - status is not None - ), "ERROR: score/status is missing" - rs = { - "instance": instFile, - "status": status, - "score": score, - "results": results, - } - return rs - - - print("\n") - log("Solving " + instFile + "...") - - correctedType = None - - # solve the instance using each solver - stop = False # when to stop the evaluation early - lsSolvingTime = {} # solving time of each solver per random seed - lsSolvingTime["favoured"] = [] - lsSolvingTime["base"] = [] - - for solverType in ["favoured", "base"]: - solved = False - - if solverType == "favouredSolver": - solverSetting = favouredSolverFlags - current_solver = favouredSolver - - else: - solverSetting = baseSolverFlags - current_solver = baseSolver - - # solverSetting = str(solver) + "Flags" - print("Solversetting: ", solverSetting) - print("About to enter loop for nEvaluations") - - for i in range(nEvaluations): - rndSeed = initSeed + i - - # Making the call to Conjure Solve - runStatus, SRTime, solverTime = call_conjure_solve( - essenceModelFile, eprimeModelFile, instFile, current_solver, SRTimeLimit, SRFlags, totalTimeLimit, solverSetting, rndSeed - ) - localVars = locals() - - # Checking the produced run status - if runStatus in ["sat", "nsat"]: - if instanceType is None: - instanceType = runStatus - assert instanceType in ["sat", "nsat"] - solved = True - - - # Condition if it hasn't already been run in a previous nEvaluation - else: - if instanceType is None: - # If a different result appears, verify with a third solver (chuffed) - if correctedType is None: - # use a third solver, chuffed, to solve the instance - c_runStatus, c_SRTime, c_solverTime = call_conjure_solve( - essenceModelFile, - eprimeModelFile, - instFile, - "chuffed", - SRTimeLimit, - SRFlags, - totalTimeLimit, - None, - rndSeed - ) - assert c_runStatus in [ - "sat", - "nsat" - ], "Error: Third solver (chuffed) also fails to prove sat or unsat" - correctedType = c_runStatus - if instanceType == correctedType: - solver = info[solverType]["name"] - print( - f"WARNING: incorrect results by {solver} on {instFile} with seed {rndSeed}. Chuffed returns {correctedType}" - ) - runStatus = "ERR" - if status == correctedType: - for st in results.keys(): - for r in results[st]["runs"]: - if r["status"] == instanceType: - print( - f"WARNING: incorrect results by {info[st]['name']} on {instFile} with seed {r['seed']}. Results returned: {r['extra']['instanceType']}, while chuffed returns {correctedType}" - ) - r["status"] = "ERR" - instanceType = correctedType - - # Append run results - results[solverType]["runs"].append( - { - "seed":rndSeed, - "status":runStatus, - "SRTime":SRTime, - "solverTime":solverTime, - } - ) - - # Early exit if instance type is unwanted - if ( - len(unwantedTypes) > 0 - and instanceType - and (instanceType in unwantedTypes) - ): - print("Unwanted instance type. Quitting...") - score = conf.SCORE_UNWANTED_TYPE - status = "unwantedType" - return score, get_results() - - - # For the case that the instance cannot be solved by the favoured solver - if (solverType == "favoured") and (solved is False): - print("\nCannot be solved by favoured solver. Quitting...") - score = conf.SCORE_FAVOURED_TOO_DIFFICULT - status = "favouredTooDifficult" - return score, get_results() - - # Check if the instance is too easy for the base solver - baseAvgTime = sum([r["solverTime"] for r in results["base"]["runs"]]) / nEvaluations - solvedByAllBaseRuns = True - for r in results["base"]["runs"]: - if r["status"] != "C": - solvedByAllBaseRuns = False - break - print(solvedByAllBaseRuns) - if solvedByAllBaseRuns and (baseAvgTime < baseMinTime): - print("\nInstance is too easy for the base solver. Quitting...") - score = conf.SCORE_BASE_TOO_EASY # Setting the type appropriately - status = "baseTooEasy" - return score, get_results() - - # Getting the problem model type, using the AST produced by Conjure - problemType = get_essence_problem_type(modelFile) - - - # In minizinc pipeline here there were checks for objective value, is a TODO here for a later implementation - baseScores = [] - favouredScores = [] - for i in range(nEvaluations): - baseResults = results["base"]["runs"][i] - favouredResults = results["favoured"]["runs"][i] - - # Calculating the borda scores based off only the solver time, no objective function implemented yet - bordaScore = calculate_essence_borda_scores( - baseResults["status"], - favouredResults["status"], - baseResults["solverTime"], - favouredResults["solverTime"], - problemType, - True, - ) - baseScores.append(bordaScore[0]) - favouredScores.append(bordaScore[1]) - - # Adding up for the total scores - baseSum = sum(baseScores) - favouredSum = sum(favouredScores) - - - # Aiming to maximise favouredSum / baseSum for these discriminating instances - if favouredSum == 0: - score = 0 - elif baseSum == 0: - assert ( - favouredSum == nEvaluations - # the best kind of instance possible, where base fails andfavoured succeeds - - ) - score = conf.SCORE_BEST - else: - score = ( - -favouredSum / baseSum - ) - - - status = "ok" - return score, get_results() diff --git a/scripts/evaluators/essence_graded.py b/scripts/evaluators/essence_graded.py deleted file mode 100644 index f536c31b..00000000 --- a/scripts/evaluators/essence_graded.py +++ /dev/null @@ -1,157 +0,0 @@ -import os -from utils import log -from essence_pipeline_utils import call_conjure_solve, get_essence_problem_type - -import conf # External file for holding static configurations, no need to redeclare here - - -def evaluate_essence_instance_graded( - modelFile: str, # Path to the Essence model file - instFile: str, # Path to the instance parameter file - unwantedTypes: list = [], # List of unwanted types - nEvaluations: int = 1, # Number of repeated solver runs - solver: str = "ortools", #Name of the solver to use, default is ortools - solverFlags: str = "-f", # Flags for the solver, fed into Conjure - solverType: str = "complete", # The type of the provided solver, default is complete - minTime: int = 10, # The minimum time threshold for the solver - timeLimit: int = 1200, # The default time limit for each solver run - initSeed: int = None, # The initial seed - SRTimeLimit: int = 0, # The timelimit for SR - SRFlags: str = "", # Flags for SR - - # Currently no oracle implemented, left here in case it is done later on - oracleSolver: str = None, - oracleSolverFlags: str = "-f", - oracleSolverTimeLimit: int = 3600, - memLimit=8192, # Total memory limit for solver runs (currently unused) -): - - - """ - evaluate an Essence instance with a single solver (goal: find graded instance for the given solver) - """ - # same implementation used as in the existing Essence pipeline - # using the new parameters passed in: - essenceModelFile = "./" + modelFile # Model file has sting "problem.essence" passed in - eprimeModelFile = conf.detailedOutputDir + "/problem.eprime" - instance = os.path.basename(instFile).replace(".param", "") - - score = None - results = {} - status = "ok" - - # check validity of input: similar to how it was done for MZN implementation - if len(unwantedTypes) > 0: - for s in unwantedTypes: - assert s in [ - "sat", - "unsat", - ], "ERROR: elements of unwantedTypes must be in {'sat','unsat'}" - assert nEvaluations > 0 - assert solverType in [ - "complete", - "incomplete", - ], "ERROR: solver type must be either complete or incomplete" - # Again, leaving so can be used in the future if the oracle is implemented - if solverType == "incomplete": - assert ( - oracleSolver != None - ), "ERROR: for incomplete solver, an oracle solver must be used" - assert ( - minTime < timeLimit - ), "ERROR: min solving time must be less than total time limit" - - # Get the essence problem type using Conjure AST functionality - problemType = get_essence_problem_type(modelFile) - conf.problemType = problemType - - # initialise results dictionaries - results = {"main": {}, "oracle": {}} - for st in ["main", "oracle"]: - results[st]["runs"] = [] - score = status = None - - # Function to get results - def get_results(): - assert (score is not None) and ( - status is not None - ), "ERROR: score/status is missing" - rs = { - "instance": instFile, - "status": status, - "score": score, - "results": results, - } - # print("\n",rs) - return rs - - - - # Starting the actual Essence instance generation object - print("\n") - log("Solving " + instFile + "...") - # run the main solver - # These variables aren't currently used, left for future implementation - instanceType = None - optimalObj = None - lsSolverTime = [] - - # Looping for each iteration of the nEvaluations parameter - for i in range(nEvaluations): - # Change seed each time - if initSeed: - seed = initSeed + i - else: - seed = None - - print( - "\n\n----------- With seed " + str(i) + "th (" + str(seed) + ")" - ) - - # call conjure solve - runStatus, SRTime, solverTime = call_conjure_solve( - essenceModelFile, eprimeModelFile, instFile, solver, SRTimeLimit, SRFlags, timeLimit, solverFlags, seed - ) - - # Append each iteration to the runs directory - results["main"]["runs"].append( - {"seed": seed, "status": runStatus, "solverTime": solverTime,"SRTime": SRTime } # there is no extra attribute to print - ) - - # Minizinc had a check here for if the instance type was the same as the previous command - # checked for inconsistencies but isn't possible here, no "EXTRA" field returned - # there was a check here based on the optimal objective, but not possible - - # Checking if the result is an unwanted type - if len(unwantedTypes) > 0 and runStatus and (runStatus in unwantedTypes): - print("Unwanted instance type. Quitting...") - score = conf.SCORE_UNWANTED_TYPE - status = "unwantedType" - return score, get_results() - - - # Calculate median runtime - results["main"]["runs"] = sorted( - results["main"]["runs"], key=lambda run: run["solverTime"] - ) - nRuns = len(results["main"]["runs"]) - medianRun = results["main"]["runs"][int(nRuns / 2)] - - - # if the instance is too easy by the main solver, can just return now, no need to run the oracle - if (medianRun["status"] == "sat") and (medianRun["solverTime"] < minTime): - print("Instance too easy. Quitting...") - score = conf.SCORE_TOO_EASY - status = "tooEasy" - return score, get_results() - - # if the instance is unsolvable by the main solver, there's no need to run the oracle - if medianRun["status"] not in ["sat"]: - print("Instance not satisfiable or timeout occurred. Quitting...") - score = conf.SCORE_TOO_DIFFICULT - status = "tooDifficult" - return score, get_results() - - status = "ok" - score = conf.SCORE_GRADED - return score, get_results() diff --git a/scripts/evaluators/mzn_discriminating.py b/scripts/evaluators/mzn_discriminating.py deleted file mode 100644 index 12c7aea1..00000000 --- a/scripts/evaluators/mzn_discriminating.py +++ /dev/null @@ -1,282 +0,0 @@ -from functools import cmp_to_key -from minizinc_utils import minizinc_solve, calculate_minizinc_borda_scores, get_minizinc_problem_type, has_better_objective - -import copy -import conf - - -def evaluate_mzn_instance_discriminating( - modelFile: str, - instFile: str, - scoringMethod: str = "complete", - unwantedTypes: list = [], - nEvaluations: int = 1, - baseSolver: str = "ortools", - baseSolverFlags: str = "-f", - baseMinTime: int = 0, - favouredSolver: str = "yuck", - favouredSolverFlags: str = "-f", - totalTimeLimit: int = 1200, - initSeed: int = None, - totalMemLimit=8192, -): - """ - Evaluate a mzn instance under the solver-discriminating criteria - """ - # Scores moved to be global variables so can be used elsewhere - - # check validity of input - assert scoringMethod in ["complete", "incomplete"], ( - "ERROR: scoringMethod " + scoringMethod + " is not supported" - ) - if len(unwantedTypes) > 0: - for s in unwantedTypes: - assert s in [ - "sat", - "unsat", - ], "ERROR: elements of unwantedTypes must be in {'sat','unsat'}" - assert nEvaluations > 0 - - # initialise info and results - info = { - "base": {"name": baseSolver, "flags": baseSolverFlags}, - "favoured": {"name": favouredSolver, "flags": favouredSolverFlags}, - } - results = {"base": {}, "favoured": {}} - for solverType in ["base", "favoured"]: - results[solverType]["runs"] = [] - score = status = None - instanceType = None - - def get_results(): - assert (score is not None) and ( - status is not None - ), "ERROR: score/status is missing" - rs = { - "instance": instFile, - "status": status, - "score": score, - "results": results, - } - # print("\n",rs) - return rs - - # run each solver on the instance and record results - correctedType = None - for solverType in ["favoured", "base"]: - solved = False # check if the instance is solved by this solver - - for i in range(nEvaluations): - if initSeed: - seed = initSeed + i - else: - seed = None - - # there are two cases where we only need to run a solver once and copy results to the remaining runs - # - case 1: the solver is deterministic - # - case 2: minizinc's flattening process fails - if i > 0: - assert len(results[solverType]["runs"]) > 0 - flattenStatus = results[solverType]["runs"][0]["extra"]["flattenStatus"] - if (info[solverType]["name"] in conf.deterministicSolvers) or ( - flattenStatus != "ok" - ): - r = copy.deepcopy(results[solverType]["runs"][0]) - r["seed"] = seed - results[solverType]["runs"].append(r) - continue - - print("\n") - runStatus, runTotalTime, extra = minizinc_solve( - modelFile, - instFile, - info[solverType]["name"], - info[solverType]["flags"], - seed, - totalTimeLimit, - totalMemLimit, - ) - - # for testing only - # if solverType=='favoured': - # if extra['instanceType'] == 'sat': - # extra['instanceType']='unsat' - - # for testing only - # if (solverType=='base') and (extra['instanceType']=='sat'): - # v = extra['objs'][-1] - # extra['objs'][-1] = (v[0], v[1]+1) - - # if the instance is solved by this run, update instanceType - if runStatus in ["S", "C"]: - assert extra["instanceType"] in ["sat", "unsat"] - - # if this is the first run where the instance is solved - if instanceType is None: - instanceType = extra["instanceType"] - assert instanceType in ["sat", "unsat"] - solved = True - if len(extra["objs"]) > 0 and extra["objs"][-1]: - bestObj = extra["objs"][-1] - - # otherwise, check if two solvers or two runs of the same solvers return different answers - else: - # if instance types (sat/unsat) are inconsistent - if instanceType != extra["instanceType"]: - if correctedType is None: - # use a third solver (chuffed) to solve the instance - c_runStatus, c_runTotalTime, c_extra = minizinc_solve( - modelFile, - instFile, - "chuffed", - "-f", - None, - totalTimeLimit, - totalMemLimit, - ) - # TODO: what if chuffed fails to solve the instance? - assert c_extra["instanceType"] in [ - "sat", - "unsat", - ], "ERROR: inconsistent results between solvers or between runs of the same solvers, and the third solver (chuffed) fails to determine which one is correct" - correctedType = c_extra["instanceType"] - # if the current run is the incorrected one, mark its status as ERR - if instanceType == correctedType: - solver = info[solverType]["name"] - print( - f"WARNING: incorrect results by {solver} on {instFile} with seed {seed}. Results returned: {extra['instanceType']}, while chuffed returns {correctedType}" - ) - runStatus = "ERR" - # if the previous runs were the incorrected ones, mark their statuses as ERR - if extra["instanceType"] == correctedType: - for st in results.keys(): - for r in results[st]["runs"]: - if r["extra"]["instanceType"] == instanceType: - print( - f"WARNING: incorrect results by {info[st]['name']} on {instFile} with seed {r['seed']}. Results returned: {r['extra']['instanceType']}, while chuffed returns {correctedType}" - ) - r["status"] = "ERR" - # assign the correct type - instanceType = correctedType - - results[solverType]["runs"].append( - { - "seed": seed, - "status": runStatus, - "time": runTotalTime, - "extra": extra, - } - ) - - # if the instance is of an unwanted type, we stop immediately - if ( - len(unwantedTypes) > 0 - and instanceType - and (instanceType in unwantedTypes) - ): - print("Unwanted instance type. Quitting...") - score = conf.SCORE_UNWANTED_TYPE - status = "unwantedType" - return score, get_results() - - # if the favoured solver cannot solve the instance on all runs, there's no need to run the base solver. We can just stop - if (solverType == "favoured") and (solved is False): - print("\nCannot be solved by favoured solver. Quitting...") - score = conf.SCORE_FAVOURED_TOO_DIFFICULT - status = "favouredTooDifficult" - return score, get_results() - - # check if the instance is too easy for the base solver - baseAvgTime = sum([r["time"] for r in results["base"]["runs"]]) / nEvaluations - solvedByAllBaseRuns = True - for r in results["base"]["runs"]: - if r["status"] != "C": - solvedByAllBaseRuns = False - break - print(solvedByAllBaseRuns) - if solvedByAllBaseRuns and (baseAvgTime < baseMinTime): - print("\nInstance is too easy for the base solver. Quitting...") - score = conf.SCORE_BASE_TOO_EASY - status = "baseTooEasy" - return score, get_results() - - problemType = get_minizinc_problem_type(modelFile) - - # if one of the two solvers is ortools and the instance is solved to optimality by ortools, use it to check correctness of the other solver in term of objective values before calculating borda score - if instanceType == "sat": - correctResults = toCheckResults = None - toCheckSolver = None - if info["base"]["name"] == "ortools": - correctResults = results["base"]["runs"] - toCheckResults = results["favoured"]["runs"] - toCheckSolver = info["favoured"]["name"] - elif info["favoured"]["name"] == "ortools": - correctResults = results["favoured"]["runs"] - toCheckResults = results["base"]["runs"] - toCheckSolver = info["base"]["name"] - if correctResults: - optimal = None - for r in correctResults: - if r["status"] == "C": - optimal = r["extra"]["objs"][-1][1] - break - if optimal: - for r in toCheckResults: - if r["status"] in ["S", "C"]: - assert len(r["extra"]["objs"]) > 0 - if has_better_objective( - r["extra"]["objs"][-1][1], optimal, problemType - ): - print( - f"WARNING: incorrect objective value by {toCheckSolver} on {instFile}. Best objective returned: {r['extra']['objs'][-1][1]}, while ortools returns {optimal}" - ) - r["status"] = "ERR" - - # calculate minizinc score of each solver per run - baseScores = [] - favouredScores = [] - print("\nBorda score for base and favoured solvers: ") - for i in range(nEvaluations): - baseResults = results["base"]["runs"][i] - favouredResults = results["favoured"]["runs"][i] - - bordaScores = calculate_minizinc_borda_scores( - baseResults["status"], - favouredResults["status"], - baseResults["time"], - favouredResults["time"], - problemType, - baseResults["extra"]["objs"], - favouredResults["extra"]["objs"], - True, - ) - print("bordascores****") - print(bordaScores) - sc = bordaScores[scoringMethod] - print("sc: ", sc) - baseScores.append(sc[0]) - favouredScores.append(sc[1]) - print("sc 0 is: ", sc[0]) - print("sc 1 is: ", sc[1]) - - - # summarise over all runs - baseSum = sum(baseScores) - favouredSum = sum(favouredScores) - - # we want to maximise favouredSum / baseSum - if favouredSum == 0: - score = 0 - elif baseSum == 0: - assert ( - favouredSum == nEvaluations - ) # the best type of instance we can achieve, where the base solver fails and the favoured solver succeeds - score = conf.SCORE_BEST - else: - score = ( - -favouredSum / baseSum - ) # when both solvers succeeds at solving the instance, we maximise the ratio of their scores - - status = "ok" - return score, get_results() - diff --git a/scripts/evaluators/mzn_graded.py b/scripts/evaluators/mzn_graded.py deleted file mode 100644 index 156be1a9..00000000 --- a/scripts/evaluators/mzn_graded.py +++ /dev/null @@ -1,251 +0,0 @@ -from functools import cmp_to_key - -# Import minizinc pipeline functions -from minizinc_utils import minizinc_solve, run_comparator, get_minizinc_problem_type, has_better_objective - -# Import configurations file for using constants -import conf - -def evaluate_mzn_instance_graded( - modelFile: str, - instFile: str, - unwantedTypes: list = [], - nEvaluations: int = 1, - solver: str = "ortools", - solverFlags: str = "-f", - solverType: str = "complete", - minTime: int = 10, - timeLimit: int = 1200, - initSeed: int = None, - oracleSolver: str = None, - oracleSolverFlags: str = "-f", - oracleSolverTimeLimit: int = 3600, - memLimit=8192, -): - """ - Evaluate a mzn instance under the gradedness criteria - """ - - - # check validity of input - if len(unwantedTypes) > 0: - for s in unwantedTypes: - assert s in [ - "sat", - "unsat", - ], "ERROR: elements of unwantedTypes must be in {'sat','unsat'}" - assert nEvaluations > 0 - assert solverType in [ - "complete", - "incomplete", - ], "ERROR: solver type must be either complete or incomplete" - if solverType == "incomplete": - assert ( - oracleSolver != None - ), "ERROR: for incomplete solver, an oracle solver must be used" - assert ( - minTime < timeLimit - ), "ERROR: min solving time must be less than total time limit" - - # this is used by minizinc_utils.run_comparator - problemType = get_minizinc_problem_type(modelFile) - conf.problemType = problemType - - # initialise results - results = {"main": {}, "oracle": {}} - for st in ["main", "oracle"]: - results[st]["runs"] = [] - score = status = None - - def get_results(): - assert (score is not None) and ( - status is not None - ), "ERROR: score/status is missing" - rs = { - "instance": instFile, - "status": status, - "score": score, - "results": results, - } - # print("\n",rs) - return rs - - # if it's a deterministic solver, we only need to run it once - # if (nEvaluations>1) and (solver in deterministicSolvers): - # nEvaluations = 1 - # print(f"{solver} is a deterministic solver via minizinc, so we only need to run it once.") - - # TODO: if main solver is incomplete and we don't want unsat instances, it's better to run the oracle with a small amount of time to check for satisfiability first - if (solverType == "incomplete") and ("unsat" in unwantedTypes): - smallTimeLimit = 120 - oracleRunStatus, oracleRunTotalTime, oracleExtra = minizinc_solve( - modelFile, - instFile, - oracleSolver, - oracleSolverFlags, - seed, - smallTimeLimit, - memLimit, - ) - if oracleExtra["instanceType"] == "unsat": - print("Unwanted instance type (checked by oracle). Quitting...") - score = conf.SCORE_UNWANTED_TYPE - status = "unwantedType" - # TODO: in this context, we don't really need to run the oracle to check correctness of instance type, since return scores for unwanted type and incorrect results are the same. But if we decide to have the two scores being different, we may need to use the oracle here - return score, get_results() - - # run the main solver - instanceType = None - optimalObj = None - for i in range(nEvaluations): - if initSeed: - seed = initSeed + i - else: - seed = None - - print("\n") - runStatus, runTotalTime, extra = minizinc_solve( - modelFile, instFile, solver, solverFlags, seed, timeLimit, memLimit - ) - results["main"]["runs"].append( - {"seed": seed, "status": runStatus, "time": runTotalTime, "extra": extra} - ) - - # just for testing - # extra['instanceType']='unsat' - - # update instance type & check for inconsistency - if runStatus in ["S", "C"]: - if instanceType is None: - instanceType = extra["instanceType"] - elif ( - instanceType != extra["instanceType"] - ): # inconsistent results between runs, return immediately - print("Inconsistent instance type between runs. Quitting...") - score = conf.SCORE_INCORRECT_ANSWER - status = "inconsistentInstanceTypes" - return score, get_results() - - # update optimal objective & check for inconsistency - if (runStatus == "C") and (instanceType == "sat"): - if optimalObj is None: - assert len(extra["objs"]) > 0 - optimalObj = extra["objs"][-1][1] - elif optimalObj != extra["objs"][-1][1]: - print("Inconsistent optimal objective value between runs. Quitting...") - score = conf.SCORE_INCORRECT_ANSWER - status = "inconsistentOptimalValues" - return score, get_results() - - # if the instance is of an unwanted type, we stop immediately - if len(unwantedTypes) > 0 and instanceType and (instanceType in unwantedTypes): - print("Unwanted instance type. Quitting...") - score = conf.SCORE_UNWANTED_TYPE - status = "unwantedType" - # TODO: in this context, we don't really need to run the oracle to check correctness of instance type, since return scores for unwanted type and incorrect results are the same. But if we decide to have the two scores being different, we may need to use the oracle here - return score, get_results() - - # get the median run - results["main"]["runs"] = sorted( - results["main"]["runs"], key=cmp_to_key(run_comparator) - ) - nRuns = len(results["main"]["runs"]) - medianRun = results["main"]["runs"][int(nRuns / 2)] - # pprint.pprint(results['main']['runs']) - - # if the instance is too easy by the main solver, there's no need to run the oracle - if (medianRun["status"] == "C") and (medianRun["time"] < minTime): - print("Instance too easy. Quitting...") - score = conf.SCORE_TOO_EASY - status = "tooEasy" - return score, get_results() - - # if the instance is unsolvable by the main solver, there's no need to run the oracle - if medianRun["status"] not in ["S", "C"]: - print("Instance too difficult. Quitting...") - score = conf.SCORE_TOO_DIFFICULT - status = "tooDifficult" - return score, get_results() - - if oracleSolver: - # run the oracle - # TODO: depending on results of the main solver, we do not necessarily run the oracle until the timelimit, e.g., if the main solver returns unsat, the oracle can stop as soon as it can find a (correct) solution. That might help to save lots of computation time. - print("\nRunning the oracle") - oracleRunStatus, oracleRunTotalTime, oracleExtra = minizinc_solve( - modelFile, - instFile, - oracleSolver, - oracleSolverFlags, - seed, - oracleSolverTimeLimit, - memLimit, - ) - results["oracle"]["runs"].append( - { - "status": oracleRunStatus, - "time": oracleRunTotalTime, - "extra": oracleExtra, - } - ) - - # for testing only - # v = oracleExtra['objs'][-1] - # oracleExtra['objs'][-1] = (v[0], v[1]-1) - - if oracleRunStatus != "C": - print("Instance cannot be solved by the oracle. Quitting...") - score = conf.SCORE_TOO_DIFFICULT - status = "tooDifficultOracle" - return score, get_results() - - # check correctness using the oracle - for r in results["main"]["runs"]: - # instance type - # print(r) - if (r["status"] in ["S", "C"]) and ( - r["extra"]["instanceType"] != oracleExtra["instanceType"] - ): - print("Incorrect results (checked by oracle). Quitting...") - score = conf.SCORE_INCORRECT_ANSWER - status = "incorrectInstanceType" - return score, get_results() - # objective value - if (r["status"] in ["S", "C"]) and (oracleExtra["instanceType"] == "sat"): - assert len(r["extra"]["objs"]) > 0 - optimal = oracleExtra["objs"][-1][1] - for o in r["extra"]["objs"]: - if has_better_objective(o[1], optimal, problemType): - print("Incorrect results (checked by oracle). Quitting...") - score = conf.SCORE_INCORRECT_ANSWER - status = "incorrectObjectiveValue" - return score, get_results() - if (r["status"] == "C") and (r["extra"]["objs"][-1][1] != optimal): - print("Incorrect results (checked by oracle). Quitting...") - score = conf.SCORE_INCORRECT_ANSWER - status = "incorrectOptimalValue" - return score, get_results() - - # for incomplete solver, use oracle to determine status - if (solverType == "incomplete") and (oracleExtra["instanceType"] == "sat"): - assert medianRun["status"] in ["S", "C"] - optimal = oracleExtra["objs"][-1][1] - o = medianRun["extra"]["objs"][-1][1] - assert has_better_objective(o, optimal, problemType) is False - if o != optimal: - print("Instance too difficult. Quitting...") - score = conf.SCORE_TOO_DIFFICULT - status = "tooDifficult" - return score, get_results() - else: - lastTime = medianRun["extra"]["objs"][-1][0] - # if the main solver is incomplete and the optimal value is reached within in less than minTime seconds, consider the instance as too easy - if lastTime < minTime: - print("Instance too easy. Quitting...") - score = conf.SCORE_TOO_EASY - status = "tooEasy" - return score, get_results() - - status = "ok" - score = conf.SCORE_GRADED - return score, get_results() - diff --git a/scripts/extract-md5sum.sh b/scripts/extract-md5sum.sh index d2e42832..4029e010 100755 --- a/scripts/extract-md5sum.sh +++ b/scripts/extract-md5sum.sh @@ -1,37 +1,16 @@ -#!/usr/bin/env bash -# set -euo pipefail # safer bash - outFile="instance-md5sum.csv" -echo "instance,hashValue" >"$outFile" - -shopt -s nullglob # let globs expand to an empty array instead of a literal - -# Look for .dzn -files=(inst-*.dzn) +echo "instance,hashValue">$outFile -# If no .dzn, go to default -if [ ${#files[@]} -eq 0 ]; then - files=(inst-*.param) # Default patter -fi - -# Exit if still none found -if [ ${#files[@]} -eq 0 ]; then - echo "No instance files found (neither .dzn nor .param)." - exit 1 -fi - -# Hash each file -for fn in "${files[@]}"; do - if [[ "$(uname)" == "Darwin" ]]; then - val=$(md5 "$fn" | cut -d"=" -f2 | xargs) - elif [[ "$(expr substr $(uname -s) 1 5)" == "Linux" ]]; then - val=$(md5sum "$fn" | cut -d' ' -f1) +for fn in $(ls inst-*.dzn) +do + if [ "$(uname)" == "Darwin" ]; then + val=$(md5 $fn| cut -d"=" -f2 |xargs) + elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then + val=$(md5sum $fn | cut -d' ' -f1) else - echo "Sorry, we only support Linux and macOS at the moment." + echo "Sorry, we only support Linux and MacOS at the moment" exit 1 fi - bfn=$(basename "$fn") - echo "$bfn,$val" >>"$outFile" + bfn=$(basename $fn) + echo "$bfn,$val" >>$outFile done - -shopt -u nullglob # restore default globbing diff --git a/scripts/scenario.R b/scripts/scenario.R index c04f6b31..e73ad6a2 100755 --- a/scripts/scenario.R +++ b/scripts/scenario.R @@ -3,11 +3,12 @@ repairConfiguration <- function(id, allConfigurations, parameters, digits, nConf outputDir <- './detailed-output/' configuration <- allConfigurations[id-nConfsPreviousRaces,] + require(data.table) + # if there is no repairing model, just return the current configuration repairModel <- paste(outputDir,'/repair.eprime',sep='') - if (!file.exists(repairModel)){ + if (!file.exists(repairModel)) return (configuration) - } # just for debug start_time <- Sys.time() @@ -19,7 +20,7 @@ repairConfiguration <- function(id, allConfigurations, parameters, digits, nConf # check if repairing results are already available outFile <- paste(outputDir,'/repairout-',baseFileName,sep='') if (file.exists(outFile)){ - configuration <- read.csv(outFile) + configuration <- fread(outFile) return (configuration) } diff --git a/scripts/setup.py b/scripts/setup.py index 830526a1..b9cc779a 100755 --- a/scripts/setup.py +++ b/scripts/setup.py @@ -35,15 +35,12 @@ def read_config(args): "genSolver", "genSolverTimeLimit", "genSolverFlags", - "repairModel", - ] instSettings = [ "instanceSetting", "instanceValidTypes", "minSolverTime", "maxSolverTime", - "SRTimeLimit", "nRunsPerInstance", ] @@ -178,19 +175,7 @@ def setup(config): assert os.path.isfile(iraceParamFile), "ERROR: params.irace is missing" tmpPath = os.path.join(config["runDir"], "repair.essence") - # Check if there is an already provided repair model if os.path.isfile(tmpPath): - print("Repair model provided at: ", tmpPath) - repairModelFile = tmpPath - # Check if a path to a repair model has been provided - elif config.get("repairModel") != None: - repair_dir = config["repairModel"] - if not os.path.isfile(repair_dir): - raise FileNotFoundError(f"Repair model not found: {repair_dir}") - print("Repair model provided at: ", config["repairModel"]) - - # Copy the repair model to the current directory - copy(repair_dir, tmpPath) repairModelFile = tmpPath # generate problem's eprime model @@ -357,23 +342,6 @@ def main(): type=int, help="time limit when solving an instance (in seconds)", ) - - - parser.add_argument( - "--SRTimeLimit", - default=0, - type=int, - help="Time limit of SR for instance solving (in seconds)", - ) - - - parser.add_argument( - "--repairModel", - default=None, - help="Optional parameter to specify location of a repair model", - ) - - parser.add_argument( "--nRunsPerInstance", default=1, type=int, help="number of runs per instance" ) diff --git a/scripts/target-runner b/scripts/target-runner index 62d15a95..b169673c 100755 --- a/scripts/target-runner +++ b/scripts/target-runner @@ -35,7 +35,7 @@ fi if [ ! -f $outfn ] || [ "${reRun}" = "1" ] ; then scriptDir="$( cd "$( dirname "${BASH_SOURCE[0]}"; )" >/dev/null 2>&1 && pwd )" cmd="python3 -u $scriptDir/wrapper.py $@ > ${outfn} 2>&1" - # echo $cmd + #echo $cmd eval $cmd fi diff --git a/scripts/testScripts/.DS_Store b/scripts/testScripts/.DS_Store index b11d88b1..951604e2 100644 Binary files a/scripts/testScripts/.DS_Store and b/scripts/testScripts/.DS_Store differ diff --git a/scripts/testScripts/check_pr_graded.sh b/scripts/testScripts/check_pr.sh similarity index 100% rename from scripts/testScripts/check_pr_graded.sh rename to scripts/testScripts/check_pr.sh diff --git a/scripts/testScripts/check_push_discrim.sh b/scripts/testScripts/check_push_discrim.sh index cda0c68c..29415c0e 100644 --- a/scripts/testScripts/check_push_discrim.sh +++ b/scripts/testScripts/check_push_discrim.sh @@ -14,7 +14,9 @@ COMMENT # Lines being checked for lines=( - "too difficult instances for the favoured solver" + "instances where the base solver wins" + "too easy instances for the base solver" + "Info of discriminating instances is saved to" ) testsPassed=0 diff --git a/scripts/testScripts/essence_testing_data/.DS_Store b/scripts/testScripts/essence_testing_data/.DS_Store deleted file mode 100644 index 2f246370..00000000 Binary files a/scripts/testScripts/essence_testing_data/.DS_Store and /dev/null differ diff --git a/scripts/testScripts/essence_testing_data/conjure-output/.conjure-checksum b/scripts/testScripts/essence_testing_data/conjure-output/.conjure-checksum deleted file mode 100644 index 3717555e..00000000 --- a/scripts/testScripts/essence_testing_data/conjure-output/.conjure-checksum +++ /dev/null @@ -1 +0,0 @@ --2292257769284198277 \ No newline at end of file diff --git a/scripts/testScripts/essence_testing_data/conjure-output/model000001-test-solution000001.eprime-solution b/scripts/testScripts/essence_testing_data/conjure-output/model000001-test-solution000001.eprime-solution deleted file mode 100644 index a17c156a..00000000 --- a/scripts/testScripts/essence_testing_data/conjure-output/model000001-test-solution000001.eprime-solution +++ /dev/null @@ -1,3 +0,0 @@ -language ESSENCE' 1.0 - -letting y be 6 diff --git a/scripts/testScripts/essence_testing_data/conjure-output/model000001-test-solution000001.solution b/scripts/testScripts/essence_testing_data/conjure-output/model000001-test-solution000001.solution deleted file mode 100644 index ce150573..00000000 --- a/scripts/testScripts/essence_testing_data/conjure-output/model000001-test-solution000001.solution +++ /dev/null @@ -1,3 +0,0 @@ -language Essence 1.3 - -letting y be 6 diff --git a/scripts/testScripts/essence_testing_data/conjure-output/model000001-test.eprime-info b/scripts/testScripts/essence_testing_data/conjure-output/model000001-test.eprime-info deleted file mode 100644 index 1c228b6f..00000000 --- a/scripts/testScripts/essence_testing_data/conjure-output/model000001-test.eprime-info +++ /dev/null @@ -1,7 +0,0 @@ -SolverTotalTime:0.153 -SavileRowClauseOut:0 -SavileRowTotalTime:0.123 -SolverFailures:0 -SolverSatisfiable:1 -SavileRowTimeOut:0 -SolverTimeOut:0 diff --git a/scripts/testScripts/essence_testing_data/conjure-output/model000001-test.eprime-infor b/scripts/testScripts/essence_testing_data/conjure-output/model000001-test.eprime-infor deleted file mode 100644 index bc22cc56..00000000 --- a/scripts/testScripts/essence_testing_data/conjure-output/model000001-test.eprime-infor +++ /dev/null @@ -1 +0,0 @@ -"conjure-output/model000001.eprime" "conjure-output/model000001-test.eprime-param" 1 0 1 NA 0.153 NA NA 0 1 0.123 NA 0 0 NA NA diff --git a/scripts/testScripts/essence_testing_data/conjure-output/model000001-test.eprime-minion b/scripts/testScripts/essence_testing_data/conjure-output/model000001-test.eprime-minion deleted file mode 100644 index c08f875b..00000000 --- a/scripts/testScripts/essence_testing_data/conjure-output/model000001-test.eprime-minion +++ /dev/null @@ -1,14 +0,0 @@ -MINION 3 -# CSETopLevel_number = 0 -# CSETopLevel_eliminated_expressions = 0 -# CSETopLevel_total_size = 0 -# CSE_active_number = 0 -# CSE_active_eliminated_expressions = 0 -# CSE_active_total_size = 0 -**VARIABLES** -DISCRETE y # -{6..10} -**SEARCH** -PRINT[[y]] -**CONSTRAINTS** -true()**EOF** diff --git a/scripts/testScripts/essence_testing_data/conjure-output/model000001-test.eprime-param b/scripts/testScripts/essence_testing_data/conjure-output/model000001-test.eprime-param deleted file mode 100644 index 7c43adac..00000000 --- a/scripts/testScripts/essence_testing_data/conjure-output/model000001-test.eprime-param +++ /dev/null @@ -1,3 +0,0 @@ -language ESSENCE' 1.0 - -letting x be 5 diff --git a/scripts/testScripts/essence_testing_data/conjure-output/model000001-test.eprime-param.fzn b/scripts/testScripts/essence_testing_data/conjure-output/model000001-test.eprime-param.fzn deleted file mode 100644 index 49c98f43..00000000 --- a/scripts/testScripts/essence_testing_data/conjure-output/model000001-test.eprime-param.fzn +++ /dev/null @@ -1,3 +0,0 @@ -var 6..10: y:: output_var ; -constraint bool_eq(true,true); -solve :: int_search([y], input_order, indomain_min, complete) satisfy; diff --git a/scripts/testScripts/essence_testing_data/conjure-output/model000001-test.stats.json b/scripts/testScripts/essence_testing_data/conjure-output/model000001-test.stats.json deleted file mode 100644 index 2bcb6b8d..00000000 --- a/scripts/testScripts/essence_testing_data/conjure-output/model000001-test.stats.json +++ /dev/null @@ -1,16 +0,0 @@ -{"computer": "eb6af18f1c49", - "conjureVersion": "Conjure v2.5.1 (Repository version b988be2 (2025-03-24 22:07:42 +0000))", - "essence": "model.essence", "essenceParams": ["test.param"], "runsolverInfo": {}, - "savilerowInfo": - {"SavileRowClauseOut": "0", "SavileRowTimeOut": "0", "SavileRowTotalTime": "0.123", "SolverFailures": "0", - "SolverSatisfiable": "1", "SolverTimeOut": "0", "SolverTotalTime": "0.153"}, - "savilerowLogs": - {"exitCode": 0, - "stdout": - ["Created output file for domain filtering conjure-output/model000001-test.eprime-minion", - "Created output file conjure-output/model000001-test.eprime-param.fzn", - "Created information file conjure-output/model000001-test.eprime-info"]}, - "savilerowOptions": [], - "savilerowVersion": "Savile Row 1.10.0 (Repository Version: 1aded299f (2024-03-10 10:56:22 +0000))", - "solver": "or-tools", "solverOptions": [], "status": "OK", "timestamp": "2025-04-09T09:35:40.444945173Z", - "totalTime": 0.28, "useExistingModels": []} \ No newline at end of file diff --git a/scripts/testScripts/essence_testing_data/conjure-output/model000001.eprime b/scripts/testScripts/essence_testing_data/conjure-output/model000001.eprime deleted file mode 100644 index f5266bc5..00000000 --- a/scripts/testScripts/essence_testing_data/conjure-output/model000001.eprime +++ /dev/null @@ -1,42 +0,0 @@ -language ESSENCE' 1.0 - -given x: int(1..10) -find y: int(1..10) -branching on [y] -such that y > x - -$ Conjure's -$ {"finds": [{"Name": "y"}], "givens": [{"Name": "x"}], "enumGivens": [], "enumLettings": [], "lettings": [], -$ "unnameds": [], "strategyQ": {"PickFirst": []}, "strategyA": {"Compact": []}, -$ "trailCompact": [[1, 1, 1], [1, 1, 1], [1, 1, 1]], "nameGenState": [], "nbExtraGivens": 0, -$ "representations": -$ [[{"Name": "y"}, -$ {"DomainInt": -$ [{"TagInt": []}, -$ [{"RangeBounded": -$ [{"Constant": {"ConstantInt": [{"TagInt": []}, 1]}}, -$ {"Constant": {"ConstantInt": [{"TagInt": []}, 10]}}]}]]}], -$ [{"Name": "x"}, -$ {"DomainInt": -$ [{"TagInt": []}, -$ [{"RangeBounded": -$ [{"Constant": {"ConstantInt": [{"TagInt": []}, 1]}}, -$ {"Constant": {"ConstantInt": [{"TagInt": []}, 10]}}]}]]}]], -$ "representationsTree": [[{"Name": "x"}, [{"subForest": []}]], [{"Name": "y"}, [{"subForest": []}]]], -$ "originalDomains": -$ [[{"Name": "x"}, -$ {"DomainInt": -$ [{"TagInt": []}, -$ [{"RangeBounded": -$ [{"Constant": {"ConstantInt": [{"TagInt": []}, 1]}}, -$ {"Constant": {"ConstantInt": [{"TagInt": []}, 10]}}]}]]}], -$ [{"Name": "y"}, -$ {"DomainInt": -$ [{"TagInt": []}, -$ [{"RangeBounded": -$ [{"Constant": {"ConstantInt": [{"TagInt": []}, 1]}}, -$ {"Constant": {"ConstantInt": [{"TagInt": []}, 10]}}]}]]}]], -$ "trailGeneralised": -$ [[4041027393977006316, -9050029345243486990], [8770190182173682593, 2013375366846517838], -$ [3811834608841468655, 4778594758532549542]], -$ "trailVerbose": [], "trailRewrites": []} diff --git a/scripts/testScripts/essence_testing_data/model-test.solution b/scripts/testScripts/essence_testing_data/model-test.solution index ce150573..4b01251b 100644 --- a/scripts/testScripts/essence_testing_data/model-test.solution +++ b/scripts/testScripts/essence_testing_data/model-test.solution @@ -1,3 +1,3 @@ language Essence 1.3 -letting y be 6 +letting y be 6 \ No newline at end of file diff --git a/scripts/testScripts/essence_testing_data/model.essence b/scripts/testScripts/essence_testing_data/model.essence index 945d55b5..9bdd5792 100644 --- a/scripts/testScripts/essence_testing_data/model.essence +++ b/scripts/testScripts/essence_testing_data/model.essence @@ -1,4 +1,3 @@ -$ satisfaction problem given x: int(1..10) find y: int(1..10) such that diff --git a/scripts/testScripts/essence_testing_data/optimisation/conjure-output/.conjure-checksum b/scripts/testScripts/essence_testing_data/optimisation/conjure-output/.conjure-checksum deleted file mode 100644 index dc775401..00000000 --- a/scripts/testScripts/essence_testing_data/optimisation/conjure-output/.conjure-checksum +++ /dev/null @@ -1 +0,0 @@ --8939236543211074461 \ No newline at end of file diff --git a/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test-solution000001.eprime-solution b/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test-solution000001.eprime-solution deleted file mode 100644 index c3e1d3b7..00000000 --- a/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test-solution000001.eprime-solution +++ /dev/null @@ -1,3 +0,0 @@ -language ESSENCE' 1.0 - -letting y be 10 diff --git a/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test-solution000001.solution b/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test-solution000001.solution deleted file mode 100644 index 530b0f7d..00000000 --- a/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test-solution000001.solution +++ /dev/null @@ -1,3 +0,0 @@ -language Essence 1.3 - -letting y be 10 diff --git a/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test.eprime-info b/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test.eprime-info deleted file mode 100644 index 7fda9e4e..00000000 --- a/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test.eprime-info +++ /dev/null @@ -1,8 +0,0 @@ -SolverTotalTime:0.013 -SavileRowClauseOut:0 -SavileRowTotalTime:0.103 -SolverFailures:0 -SolverSatisfiable:1 -SavileRowTimeOut:0 -SolverTimeOut:0 -SolverNodes:9 diff --git a/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test.eprime-infor b/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test.eprime-infor deleted file mode 100644 index 9935e169..00000000 --- a/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test.eprime-infor +++ /dev/null @@ -1 +0,0 @@ -"conjure-output/model000001.eprime" "conjure-output/model000001-test.eprime-param" 1 0 1 NA 0.013 NA 9 0 1 0.103 NA 0 0 NA NA diff --git a/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test.eprime-minion b/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test.eprime-minion deleted file mode 100644 index f5234f50..00000000 --- a/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test.eprime-minion +++ /dev/null @@ -1,22 +0,0 @@ -MINION 3 -# CSETopLevel_number = 0 -# CSETopLevel_eliminated_expressions = 0 -# CSETopLevel_total_size = 0 -# CSE_active_number = 0 -# CSE_active_eliminated_expressions = 0 -# CSE_active_total_size = 0 -**VARIABLES** -DISCRETE y # -{6..10} -DISCRETE aux0 #(5*y) -{30..50} -**CONSTRAINTS** -w-inintervalset(aux0, [30,30,35,35,40,40,45,45,50,50]) -**VARIABLES** -**SEARCH** -PRINT[[y],[aux0]] -MAXIMISING aux0 -**CONSTRAINTS** -weightedsumleq([5],[y],aux0) -weightedsumgeq([5],[y],aux0) -**EOF** diff --git a/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test.eprime-param b/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test.eprime-param deleted file mode 100644 index 7c43adac..00000000 --- a/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test.eprime-param +++ /dev/null @@ -1,3 +0,0 @@ -language ESSENCE' 1.0 - -letting x be 5 diff --git a/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test.eprime-param.fzn b/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test.eprime-param.fzn deleted file mode 100644 index 28028444..00000000 --- a/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test.eprime-param.fzn +++ /dev/null @@ -1,4 +0,0 @@ -var 6..10: y:: output_var ; -var {30,35,40,45,50}: aux0:: var_is_introduced ; -constraint int_lin_eq([5,-1],[y,aux0],0); -solve :: int_search([y], input_order, indomain_min, complete) maximize aux0; diff --git a/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test.stats.json b/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test.stats.json deleted file mode 100644 index ce9b5197..00000000 --- a/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001-test.stats.json +++ /dev/null @@ -1,16 +0,0 @@ -{"computer": "cf473687a707", - "conjureVersion": "Conjure v2.5.1 (Repository version e1bba43 (2025-03-06 15:40:14 +0000))", - "essence": "model.essence", "essenceParams": ["test.param"], "runsolverInfo": {}, - "savilerowInfo": - {"SavileRowClauseOut": "0", "SavileRowTimeOut": "0", "SavileRowTotalTime": "0.103", "SolverFailures": "0", - "SolverNodes": "9", "SolverSatisfiable": "1", "SolverTimeOut": "0", "SolverTotalTime": "0.013"}, - "savilerowLogs": - {"exitCode": 0, - "stdout": - ["Created output file for domain filtering conjure-output/model000001-test.eprime-minion", - "Created output file conjure-output/model000001-test.eprime-param.fzn", - "Created information file conjure-output/model000001-test.eprime-info"]}, - "savilerowOptions": [], - "savilerowVersion": "Savile Row 1.10.0 (Repository Version: 1aded299f (2024-03-10 10:56:22 +0000))", - "solver": "chuffed", "solverOptions": [], "status": "OK", "timestamp": "2025-03-25T23:11:27.765280134Z", - "totalTime": 0.12, "useExistingModels": []} \ No newline at end of file diff --git a/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001.eprime b/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001.eprime deleted file mode 100644 index 684b3115..00000000 --- a/scripts/testScripts/essence_testing_data/optimisation/conjure-output/model000001.eprime +++ /dev/null @@ -1,44 +0,0 @@ -language ESSENCE' 1.0 - -given x: int(1..10) -find y: int(1..10) -branching on [y] -maximising y * x -such that y > x - -$ Conjure's -$ {"finds": [{"Name": "y"}], "givens": [{"Name": "x"}], "enumGivens": [], "enumLettings": [], "lettings": [], -$ "unnameds": [], "strategyQ": {"PickFirst": []}, "strategyA": {"Compact": []}, -$ "trailCompact": [[1, 1, 1], [1, 1, 1], [1, 1, 1]], "nameGenState": [], "nbExtraGivens": 0, -$ "representations": -$ [[{"Name": "y"}, -$ {"DomainInt": -$ [{"TagInt": []}, -$ [{"RangeBounded": -$ [{"Constant": {"ConstantInt": [{"TagInt": []}, 1]}}, -$ {"Constant": {"ConstantInt": [{"TagInt": []}, 10]}}]}]]}], -$ [{"Name": "x"}, -$ {"DomainInt": -$ [{"TagInt": []}, -$ [{"RangeBounded": -$ [{"Constant": {"ConstantInt": [{"TagInt": []}, 1]}}, -$ {"Constant": {"ConstantInt": [{"TagInt": []}, 10]}}]}]]}]], -$ "representationsTree": -$ [[{"Name": "x"}, [{"rootLabel": null, "subForest": []}]], [{"Name": "y"}, [{"rootLabel": null, "subForest": []}]]], -$ "originalDomains": -$ [[{"Name": "x"}, -$ {"DomainInt": -$ [{"TagInt": []}, -$ [{"RangeBounded": -$ [{"Constant": {"ConstantInt": [{"TagInt": []}, 1]}}, -$ {"Constant": {"ConstantInt": [{"TagInt": []}, 10]}}]}]]}], -$ [{"Name": "y"}, -$ {"DomainInt": -$ [{"TagInt": []}, -$ [{"RangeBounded": -$ [{"Constant": {"ConstantInt": [{"TagInt": []}, 1]}}, -$ {"Constant": {"ConstantInt": [{"TagInt": []}, 10]}}]}]]}]], -$ "trailGeneralised": -$ [[1640593561417914027, -4526452141366588034], [1723353391399597555, 226908959003800808], -$ [4221928815309945609, -410178887729722368]], -$ "trailVerbose": [], "trailRewrites": []} diff --git a/scripts/testScripts/essence_testing_data/optimisation/model-test.solution b/scripts/testScripts/essence_testing_data/optimisation/model-test.solution deleted file mode 100644 index 530b0f7d..00000000 --- a/scripts/testScripts/essence_testing_data/optimisation/model-test.solution +++ /dev/null @@ -1,3 +0,0 @@ -language Essence 1.3 - -letting y be 10 diff --git a/scripts/testScripts/essence_testing_data/optimisation/model.essence b/scripts/testScripts/essence_testing_data/optimisation/model.essence deleted file mode 100644 index 6aa89b40..00000000 --- a/scripts/testScripts/essence_testing_data/optimisation/model.essence +++ /dev/null @@ -1,7 +0,0 @@ -$ optimisation -given x: int(1..10) -find y: int(1..10) -such that - y > x - -maximizing y * x \ No newline at end of file diff --git a/scripts/testScripts/essence_testing_data/optimisation/pretty.txt b/scripts/testScripts/essence_testing_data/optimisation/pretty.txt deleted file mode 100644 index 922567a4..00000000 --- a/scripts/testScripts/essence_testing_data/optimisation/pretty.txt +++ /dev/null @@ -1,36 +0,0 @@ -{"mInfo": - {"finds": [], "givens": [], "enumGivens": [], "enumLettings": [], "lettings": [], "unnameds": [], - "strategyQ": {"Auto": {"Interactive": []}}, "strategyA": {"Auto": {"Interactive": []}}, "trailCompact": [], - "nameGenState": [], "nbExtraGivens": 0, "representations": [], "representationsTree": [], "originalDomains": [], - "trailGeneralised": [], "trailVerbose": [], "trailRewrites": []}, - "mLanguage": {"language": {"Name": "Essence"}, "version": [1, 3]}, - "mStatements": - [{"Declaration": - {"FindOrGiven": - ["Given", {"Name": "x"}, - {"DomainInt": - [{"TagInt": []}, - [{"RangeBounded": - [{"Constant": {"ConstantInt": [{"TagInt": []}, 1]}}, - {"Constant": {"ConstantInt": [{"TagInt": []}, 10]}}]}]]}]}}, - {"Declaration": - {"FindOrGiven": - ["Find", {"Name": "y"}, - {"DomainInt": - [{"TagInt": []}, - [{"RangeBounded": - [{"Constant": {"ConstantInt": [{"TagInt": []}, 1]}}, - {"Constant": {"ConstantInt": [{"TagInt": []}, 10]}}]}]]}]}}, - {"SuchThat": [{"Op": {"MkOpGt": [{"Reference": [{"Name": "y"}, null]}, {"Reference": [{"Name": "x"}, null]}]}}]}, - {"Objective": - ["Maximising", - {"Op": - {"MkOpProduct": - {"AbstractLiteral": - {"AbsLitMatrix": - [{"DomainInt": - [{"TagInt": []}, - [{"RangeBounded": - [{"Constant": {"ConstantInt": [{"TagInt": []}, 1]}}, - {"Constant": {"ConstantInt": [{"TagInt": []}, 2]}}]}]]}, - [{"Reference": [{"Name": "y"}, null]}, {"Reference": [{"Name": "x"}, null]}]]}}}}]}]} diff --git a/scripts/testScripts/essence_testing_data/optimisation/temp.txt b/scripts/testScripts/essence_testing_data/optimisation/temp.txt deleted file mode 100644 index 922567a4..00000000 --- a/scripts/testScripts/essence_testing_data/optimisation/temp.txt +++ /dev/null @@ -1,36 +0,0 @@ -{"mInfo": - {"finds": [], "givens": [], "enumGivens": [], "enumLettings": [], "lettings": [], "unnameds": [], - "strategyQ": {"Auto": {"Interactive": []}}, "strategyA": {"Auto": {"Interactive": []}}, "trailCompact": [], - "nameGenState": [], "nbExtraGivens": 0, "representations": [], "representationsTree": [], "originalDomains": [], - "trailGeneralised": [], "trailVerbose": [], "trailRewrites": []}, - "mLanguage": {"language": {"Name": "Essence"}, "version": [1, 3]}, - "mStatements": - [{"Declaration": - {"FindOrGiven": - ["Given", {"Name": "x"}, - {"DomainInt": - [{"TagInt": []}, - [{"RangeBounded": - [{"Constant": {"ConstantInt": [{"TagInt": []}, 1]}}, - {"Constant": {"ConstantInt": [{"TagInt": []}, 10]}}]}]]}]}}, - {"Declaration": - {"FindOrGiven": - ["Find", {"Name": "y"}, - {"DomainInt": - [{"TagInt": []}, - [{"RangeBounded": - [{"Constant": {"ConstantInt": [{"TagInt": []}, 1]}}, - {"Constant": {"ConstantInt": [{"TagInt": []}, 10]}}]}]]}]}}, - {"SuchThat": [{"Op": {"MkOpGt": [{"Reference": [{"Name": "y"}, null]}, {"Reference": [{"Name": "x"}, null]}]}}]}, - {"Objective": - ["Maximising", - {"Op": - {"MkOpProduct": - {"AbstractLiteral": - {"AbsLitMatrix": - [{"DomainInt": - [{"TagInt": []}, - [{"RangeBounded": - [{"Constant": {"ConstantInt": [{"TagInt": []}, 1]}}, - {"Constant": {"ConstantInt": [{"TagInt": []}, 2]}}]}]]}, - [{"Reference": [{"Name": "y"}, null]}, {"Reference": [{"Name": "x"}, null]}]]}}}}]}]} diff --git a/scripts/testScripts/essence_testing_data/optimisation/test.param b/scripts/testScripts/essence_testing_data/optimisation/test.param deleted file mode 100644 index a063dcc8..00000000 --- a/scripts/testScripts/essence_testing_data/optimisation/test.param +++ /dev/null @@ -1 +0,0 @@ -letting x be 5 \ No newline at end of file diff --git a/scripts/testScripts/pr_discrim_tests/chuffed-maccDiscriminatingGen.sh b/scripts/testScripts/pr_discrim_tests/chuffed-maccDiscriminatingGen.sh index f9b575bf..da8af1e5 100644 --- a/scripts/testScripts/pr_discrim_tests/chuffed-maccDiscriminatingGen.sh +++ b/scripts/testScripts/pr_discrim_tests/chuffed-maccDiscriminatingGen.sh @@ -1,7 +1,7 @@ #!/bin/bash -# Tests discriminating instance generation with Vessel_loading problem, and basesolver: Chuffed and favouredSolver: ortools. +# Tests discriminating instance generation with Macc problem, using small generator, and basesolver: Chuffed and favouredSolver: ortools. echo $AUTOIG mkdir -p "$AUTOIG/experiments/macc-discrim-gen" cd "$AUTOIG/experiments/macc-discrim-gen" -python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/macc/generator.essence" --problemModel "$AUTOIG/data/models/macc/problem.mzn" --instanceSetting discriminating --minSolverTime 1 --maxSolverTime 5 --baseSolver chuffed --baseSolverFlags="-f" --favouredSolver picat --favouredSolverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 3 +python $AUTOIG/scripts/setup.py --generatorModel $AUTOIG/data/models/macc/generator.essence --problemModel $AUTOIG/data/models/macc/problem.mzn --instanceSetting discriminating --minSolverTime 1 --maxSolverTime 3 --baseSolver chuffed --solverFlags="-f" --favouredSolver ortools --favouredSolverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 5 bash "$AUTOIG/experiments/macc-discrim-gen/run.sh" diff --git a/scripts/testScripts/pr_discrim_tests/chuffed-vesselDiscriminatingGen.sh b/scripts/testScripts/pr_discrim_tests/chuffed-vesselDiscriminatingGen.sh deleted file mode 100644 index 7550409c..00000000 --- a/scripts/testScripts/pr_discrim_tests/chuffed-vesselDiscriminatingGen.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -# Testing graded instance generation for the Vessel Loading problem using its small generator with the chuffed solver" -mkdir -p "$AUTOIG/experiments/vessel_loading_discrim" -cd "$AUTOIG/experiments/vessel_loading_discrim" -python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/vessel-loading/generator.essence" --problemModel "$AUTOIG/data/models/vessel-loading/problem.essence" --instanceSetting discriminating --minSolverTime 1 --maxSolverTime 20 --baseSolver chuffed --baseSolverFlags="-f" --favouredSolver or-tools --favouredSolverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 200 -bash "$AUTOIG/experiments/vessel_loading_discrim/run.sh" diff --git a/scripts/testScripts/pr_graded_tests/chuffed-vesselGraded.sh b/scripts/testScripts/pr_graded_tests/chuffed-vesselGraded.sh deleted file mode 100644 index a53a6d5c..00000000 --- a/scripts/testScripts/pr_graded_tests/chuffed-vesselGraded.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# Testing graded instance generation for the Vessel problem using the chuffed solver" -mkdir -p "$AUTOIG/experiments/vessel-graded" -cd "$AUTOIG/experiments/vessel-graded" -python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/vessel-loading/generator.essence" --problemModel "$AUTOIG/data/models/vessel-loading/problem.essence" --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver chuffed --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 3 - -bash "$AUTOIG/experiments/vessel-graded/run.sh" diff --git a/scripts/testScripts/push_discrim_tests/chuffed-maccDiscriminatingGen.sh b/scripts/testScripts/push_discrim_tests/chuffed-maccDiscriminatingGen.sh index eec5c903..b2e3e659 100644 --- a/scripts/testScripts/push_discrim_tests/chuffed-maccDiscriminatingGen.sh +++ b/scripts/testScripts/push_discrim_tests/chuffed-maccDiscriminatingGen.sh @@ -3,5 +3,5 @@ echo $AUTOIG mkdir -p "$AUTOIG/experiments/macc-discrim-small-gen" cd "$AUTOIG/experiments/macc-discrim-small-gen" -python $AUTOIG/scripts/setup.py --generatorModel $AUTOIG/data/models/macc/generator-small.essence --problemModel $AUTOIG/data/models/macc/problem.mzn --instanceSetting discriminating --minSolverTime 1 --maxSolverTime 3 --baseSolver chuffed --solverFlags="-f" --favouredSolver picat --favouredSolverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 5 +python $AUTOIG/scripts/setup.py --generatorModel $AUTOIG/data/models/macc/generator-small.essence --problemModel $AUTOIG/data/models/macc/problem.mzn --instanceSetting discriminating --minSolverTime 1 --maxSolverTime 3 --baseSolver chuffed --solverFlags="-f" --favouredSolver ortools --favouredSolverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 5 bash "$AUTOIG/experiments/macc-discrim-small-gen/run.sh" diff --git a/scripts/testScripts/push_essence_graded.sh b/scripts/testScripts/push_essence_graded.sh deleted file mode 100644 index 1fefa1cc..00000000 --- a/scripts/testScripts/push_essence_graded.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/bash - -: <<'COMMENT' - Tests for Graded Instance Generation - - Runs all scripts put in ./push_graded_essence_tests and makes sure that the run contains provided lines. - - Each script involves a full run of the vessel_loading problem, and each one uses a different solver. - - Solvers tested with include chuffed, cpsat, gecode, picat, and yuck. - - This script runs less intensive tests (vessel_loading runs with the small generator), and is intended for pushes to any branch. -COMMENT - -# Lines being checked for -lines=( - "# Best configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks):" - "# Best configurations as commandlines (first number is the configuration ID; same order as above):" -) - -testsPassed=0 -testsRun=0 - -start=$(date +%s) - -# Loop through each script in the tests directory -for file in push_graded_essence_tests/*; do - ((testsRun++)) - # Check if file - if [[ -f "$file" ]]; then - - # Run contents of file - output=$(bash "$file") - all_lines_found=true - - # Check for each line in the array - for line in "${lines[@]}"; do - if [[ "$output" != *"$line"* ]]; then - all_lines_found=false - echo "Test $testsRun: $file failed, line not found: $line" - fi - done - - # If all lines are found, count as passed - if $all_lines_found; then - echo "Test $testsRun: $file passed, all lines found in output" - ((testsPassed++)) - fi - fi - # Record end time and calculate elapsed time - end=$(date +%s) - elapsedTime=$((end - start)) - - # Display time elapsed - echo "Time elapsed: $elapsedTime seconds" -done - -# Final results -if [[ "$testsRun" -eq "$testsPassed" ]]; then - printf "\e[32mAll tests passed: %d/%d! :D\e[0m\n" "$testsPassed" "$testsRun" - exit 0 -else - printf "\e[31mSome cases failing, only %d/%d passed.\e[0m\n" "$testsPassed" "$testsRun" - exit 1 -fi diff --git a/scripts/testScripts/push_graded_essence_tests/chuffed-maccGradedSmallGen.sh b/scripts/testScripts/push_graded_essence_tests/chuffed-maccGradedSmallGen.sh deleted file mode 100644 index 47794458..00000000 --- a/scripts/testScripts/push_graded_essence_tests/chuffed-maccGradedSmallGen.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# Testing graded instance generation for the Macc problem using its small generator with the chuffed solver" -mkdir -p "$AUTOIG/experiments/vessel_loading" -cd "$AUTOIG/experiments/vessel_loading" -python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/vessel-loading/generator.essence" --problemModel "$AUTOIG/data/models/vessel-loading/problem.essence" --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver chuffed --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 3 - -bash "$AUTOIG/experiments/vessel_loading/run.sh" diff --git a/scripts/testScripts/push_graded_essence_tests/cpsat-maccGradedSmallGen.sh b/scripts/testScripts/push_graded_essence_tests/cpsat-maccGradedSmallGen.sh deleted file mode 100644 index 3575fbbd..00000000 --- a/scripts/testScripts/push_graded_essence_tests/cpsat-maccGradedSmallGen.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -# Testing graded instance generation for the Macc problem using its small generator with the cpsat solver" -mkdir -p "$AUTOIG/experiments/vessel_loading" -cd "$AUTOIG/experiments/vessel_loading" -python3 "$AUTOIG/scripts/setup.py" --generatorModel "$AUTOIG/data/models/vessel-loading/generator.essence" --problemModel "$AUTOIG/data/models/problem.essence" --instanceSetting graded --minSolverTime 0 --maxSolverTime 5 --solver cpsat --solverFlags="-f" --maxEvaluations 180 --genSolverTimeLimit 3 - -bash "$AUTOIG/experiments/vessel_loading/run.sh" diff --git a/scripts/testing.essence b/scripts/testing.essence deleted file mode 100644 index 6aa89b40..00000000 --- a/scripts/testing.essence +++ /dev/null @@ -1,7 +0,0 @@ -$ optimisation -given x: int(1..10) -find y: int(1..10) -such that - y > x - -maximizing y * x \ No newline at end of file diff --git a/scripts/wrapper.py b/scripts/wrapper.py index 1f7b4f25..5c0b8308 100755 --- a/scripts/wrapper.py +++ b/scripts/wrapper.py @@ -19,28 +19,923 @@ import pprint import math import conf -import sys - -from wrapper_helpers import read_setting, read_args -from conf import ( - detailedOutputDir, -) +import sys -from evaluators.essence_graded import evaluate_essence_instance_graded -from evaluators.essence_discriminating import evaluate_essence_instance_discriminating -from evaluators.mzn_graded import evaluate_mzn_instance_graded -from evaluators.mzn_discriminating import evaluate_mzn_instance_discriminating -# Imports -import conf scriptDir = os.path.dirname(os.path.realpath(__file__)) sys.path.append(scriptDir) from utils import log, read_file, search_string, run_cmd, delete_file -from essence_pipeline_utils import encode_negative_table +from essence_pipeline_utils import call_conjure_solve, encode_negative_table +from minizinc_utils import ( + minizinc_solve, + calculate_minizinc_borda_scores, + get_minizinc_problem_type, + has_better_objective, + run_comparator, +) from generator import solve_generator from convert import convert_essence_instance_to_mzn +detailedOutputDir = "./detailed-output" + +# for minizinc experiments only: solvers where -r doesn't work when being called via minizinc +deterministicSolvers = ["ortools"] + + +def evaluate_essence_instance_graded(instFile, seed, setting): + # TODO: we need a similar function for minizinc instance (or we can modify this function to make it support minizinc instances) + # TODO: we need to return a dictionary of results, as in evaluate_mzn_instance_discriminating + # TODO: make all inputs of the function explicit, as in evaluate_mzn_instance_discriminating + """ + evaluate an Essence instance with a single solver (goal: find graded instance for the given solver) + """ + essenceModelFile = "./problem.essence" + eprimeModelFile = detailedOutputDir + "/problem.eprime" + instance = os.path.basename(instFile).replace(".param", "") + solver = setting["solver"] + + score = None + results = {} + status = "ok" + + def get_results(): + results["score"] = score + results["status"] = status + + def get_results(): + assert (score is not None) and (status is not None) + rs = { + "insttance": instFile, + "status": status, + "score": score, + "results": results, + } + return rs + + # TODO: add values for variable "results" (see evaluate_mzn_instance_discriminating for example) + + print("\n") + log("Solving " + instFile + "...") + + lsSolverTime = [] + for i in range(setting["nEvaluations"]): + rndSeed = seed + i + print( + "\n\n----------- With random seed " + str(i) + "th (" + str(rndSeed) + ")" + ) + runStatus, SRTime, solverTime = call_conjure_solve( + essenceModelFile, eprimeModelFile, instFile, setting, rndSeed + ) + + # print out results + localVars = locals() + log( + "\nRun results: solverType=" + + solver + + ", solver=" + + solver + + ", instance=" + + instance + + ", runId=" + + str(i) + + ", " + + ", ".join( + [ + s + "=" + str(localVars[s]) + for s in ["runStatus", "SRTime", "solverTime"] + ] + ) + ) + + # make score + # inst unwanted type: score=1 + if ( + (setting["gradedTypes"] != "both") + and (runStatus in ["sat", "unsat"]) + and (runStatus != setting["gradedTypes"]) + ): + print("\nunwanted instance type. Quitting!...") + score = 1 + stop = True + status = "unwantedType" + break + # SR timeout or SR memout: score=1 + if runStatus in ["SRTimeOut", "SRMemOut"]: + print("\nSR timeout/memout while translating the instance. Quitting!...") + score = 1 + status = runStatus + break + # solverTimeout or solverMemOut: score=0 + if runStatus in ["solverTimeOut", "solverMemOut"]: + print("\nsolver timeout or out of memory. Quitting!...") + score = 0 + status = runStatus + break + if runStatus == "solverCrash": + print("\nsolver crashes. Quitting!...") + score = 1 + status = runStatus + break + lsSolverTime.append(solverTime) + + # summary of results + meanSolverTime = 0 + if status == "ok": + meanSolverTime = sum(lsSolverTime) / len(lsSolverTime) + if meanSolverTime < setting["solverMinTime"]: + status = "tooEasy" + else: + status = "graded" + s = ( + "\nInstance summary: instance=" + + instance + + ", status=" + + status + + ", meanSolverTime=" + + str(meanSolverTime) + ) + print(s) + + # make final score + if score != None: + return score + # - otherwise, for each evaluation: if the run is too easy: score=-solverTime, if the run is graded: score=nEvaluations*-solverMinTime + score = 0 + for i in range(len(lsSolverTime)): + if lsSolverTime[i] < setting["solverMinTime"]: + score -= lsSolverTime[i] + else: + score -= setting["nEvaluations"] * setting["solverMinTime"] + return score, get_results() + + +def evaluate_essence_instance_discriminating(instFile, seed, setting): + # TODO: we need to return a dictionary of results, as in evaluate_mzn_instance_discriminating + # TODO: make all inputs of the function explicit, as in evaluate_mzn_instance_discriminating + """evaluate a generated instance based on discriminating power with two solvers ### + " NOTE: + " - this function can be improved using parallelisation, as there are various cases in the scoring where runs can be safely terminated before they finished. Things to consider + " + gnu-parallel for implementation + " + runsolver for safely terminating a solver run + " + " scoring scheme for discriminating solvers: + " - gen unsat/SR memout/SR timeout: Inf + " - gen solver timeout: 2 + " - inst unwanted type or SR timeout (either solver): 1 (ISSUE: with this new implementation, we can't recognise SR timeout, so we treat it as both solver timeout, i.e., score=0) + " - favoured solver timeout (any run) or base solver too easy (any run): 0 + " - otherwise: max{-minRatio, -baseSolver/favouredSolver} + " - note: if minRatio>0, ideally we should set timelimit_baseSolver = minRatio * timelimit_favouredSolver + """ + + essenceModelFile = "./problem.essence" + eprimeModelFile = detailedOutputDir + "/problem.eprime" + instance = os.path.basename(instFile).replace(".param", "") + + score = None + results = {} + + def get_results(): + results["score"] = score + results["status"] = status + + print("\n") + log("Solving " + instFile + "...") + + # solve the instance using each solver + stop = False # when to stop the evaluation early + lsSolvingTime = {} # solving time of each solver per random seed + lsSolvingTime["favouredSolver"] = [] + lsSolvingTime["baseSolver"] = [] + for i in range(setting["nEvaluations"]): + rndSeed = seed + i + + status = "ok" + for solver in ["favouredSolver", "baseSolver"]: + solverSetting = setting[solver] + print( + "\n\n---- With random seed " + + str(i) + + "th (" + + str(rndSeed) + + ") and solver " + + solverSetting["name"] + + " (" + + solver + + ")" + ) + + runStatus, SRTime, solverTime = call_conjure_solve( + essenceModelFile, eprimeModelFile, instFile, solverSetting, rndSeed + ) + localVars = locals() + log( + "\nRun results: solverType=" + + solver + + ", solver=" + + solverSetting["name"] + + ", instance=" + + instance + + ", runId=" + + str(i) + + ", " + + ", ".join( + [ + s + "=" + str(localVars[s]) + for s in ["runStatus", "SRTime", "solverTime"] + ] + ) + ) + + lsSolvingTime[solver].append(solverTime) + + # ------------ update score + # inst unwanted type: score=1 + if ( + (setting["gradedTypes"] != "both") + and (runStatus in ["sat", "unsat"]) + and (runStatus != setting["gradedTypes"]) + ): + print("\nunwanted instance type. Quitting!...") + score = 1 + stop = True + status = "unwantedType" + break + # SR timeout or SR memout: score=1 + if runStatus in ["SRTimeOut", "SRMemOut"]: + print( + "\nSR timeout/memout while translating the instance. Quitting!..." + ) + score = 1 + stop = True + status = runStatus + break + # solver crashes + if runStatus == "solverCrash": + print("\nsolver crashes. Quitting!...") + score = 1 + stop = True + status = runStatus + break + # favoured solver timeout (any run) or base solver too easy (any run): score=0 + if (solver == "favouredSolver") and (runStatus == "solverTimeOut"): + print("\nfavoured solver timeout. Quitting!...") + score = 0 + stop = True + status = "favouredTimeOut" + break + if (solver == "baseSolver") and ( + solverTime < solverSetting["solverMinTime"] + ): + print("\ntoo easy run for base solver. Quitting!...") + score = 0 + stop = True + status = "baseTooEasy" + break + + # evaluation is stopped as there's no need to test the rest + if stop: + break + + # if nothing is stop prematurely, calculate mean solving time & ratio, and update score + ratio = 0 + if stop is False: + meanSolverTime_favouredSolver = np.mean(lsSolvingTime["favouredSolver"]) + meanSolverTime_baseSolver = np.mean(lsSolvingTime["baseSolver"]) + ratio = meanSolverTime_baseSolver / meanSolverTime_favouredSolver + # if minRatio is provided, use it + if setting["minRatio"] != 0: + score = max(-setting["minValue"], -ratio) + else: # otherwise, simply use the current ratio + score = -ratio + + print("\n\nMean solving time: ") + print( + "\t- Favoured solver: " + + str(np.round(meanSolverTime_favouredSolver, 2)) + + "s" + ) + print("\t- Base solver: " + str(np.round(meanSolverTime_baseSolver, 2)) + "s") + print("\t- Ratio: " + str(np.round(ratio, 2))) + + # print summary for later analysis + favouredSolverTotalTime = baseSolverTotalTime = 0 + if len(lsSolvingTime["favouredSolver"]) > 0: + favouredSolverTotalTime = sum(lsSolvingTime["favouredSolver"]) + if len(lsSolvingTime["baseSolver"]) > 0: + baseSolverTotalTime = sum(lsSolvingTime["baseSolver"]) + s = ( + "\nInstance summary: instance=" + + instance + + ", status=" + + status + + ", favouredSolverTotalTime=" + + str(favouredSolverTotalTime) + + ", baseSolverTotalTime=" + + str(baseSolverTotalTime) + + ", ratio=" + + str(ratio) + ) + print(s) + + return score, get_results() + + +def evaluate_mzn_instance_discriminating( + modelFile: str, + instFile: str, + scoringMethod: str = "complete", + unwantedTypes: list = [], + nEvaluations: int = 1, + baseSolver: str = "ortools", + baseSolverFlags: str = "-f", + baseMinTime: int = 0, + favouredSolver: str = "yuck", + favouredSolverFlags: str = "-f", + totalTimeLimit: int = 1200, + initSeed: int = None, + totalMemLimit=8192, +): + """ + Evaluate a mzn instance under the solver-discriminating criteria + """ + # define constants for scores + SCORE_UNWANTED_TYPE = 0 + SCORE_FAVOURED_TOO_DIFFICULT = 0 + SCORE_BASE_TOO_EASY = 0 + + # check validity of input + assert scoringMethod in ["complete", "incomplete"], ( + "ERROR: scoringMethod " + scoringMethod + " is not supported" + ) + if len(unwantedTypes) > 0: + for s in unwantedTypes: + assert s in [ + "sat", + "unsat", + ], "ERROR: elements of unwantedTypes must be in {'sat','unsat'}" + assert nEvaluations > 0 + + # initialise info and results + info = { + "base": {"name": baseSolver, "flags": baseSolverFlags}, + "favoured": {"name": favouredSolver, "flags": favouredSolverFlags}, + } + results = {"base": {}, "favoured": {}} + for solverType in ["base", "favoured"]: + results[solverType]["runs"] = [] + score = status = None + instanceType = None + + def get_results(): + assert (score is not None) and ( + status is not None + ), "ERROR: score/status is missing" + rs = { + "instance": instFile, + "status": status, + "score": score, + "results": results, + } + # print("\n",rs) + return rs + + # run each solver on the instance and record results + correctedType = None + for solverType in ["favoured", "base"]: + solved = False # check if the instance is solved by this solver + + for i in range(nEvaluations): + if initSeed: + seed = initSeed + i + else: + seed = None + + # there are two cases where we only need to run a solver once and copy results to the remaining runs + # - case 1: the solver is deterministic + # - case 2: minizinc's flattening process fails + if i > 0: + assert len(results[solverType]["runs"]) > 0 + flattenStatus = results[solverType]["runs"][0]["extra"]["flattenStatus"] + if (info[solverType]["name"] in deterministicSolvers) or ( + flattenStatus != "ok" + ): + r = copy.deepcopy(results[solverType]["runs"][0]) + r["seed"] = seed + results[solverType]["runs"].append(r) + continue + + print("\n") + runStatus, runTotalTime, extra = minizinc_solve( + modelFile, + instFile, + info[solverType]["name"], + info[solverType]["flags"], + seed, + totalTimeLimit, + totalMemLimit, + ) + + # for testing only + # if solverType=='favoured': + # if extra['instanceType'] == 'sat': + # extra['instanceType']='unsat' + + # for testing only + # if (solverType=='base') and (extra['instanceType']=='sat'): + # v = extra['objs'][-1] + # extra['objs'][-1] = (v[0], v[1]+1) + + # if the instance is solved by this run, update instanceType + if runStatus in ["S", "C"]: + assert extra["instanceType"] in ["sat", "unsat"] + + # if this is the first run where the instance is solved + if instanceType is None: + instanceType = extra["instanceType"] + assert instanceType in ["sat", "unsat"] + solved = True + if len(extra["objs"]) > 0 and extra["objs"][-1]: + bestObj = extra["objs"][-1] + + # otherwise, check if two solvers or two runs of the same solvers return different answers + else: + # if instance types (sat/unsat) are inconsistent + if instanceType != extra["instanceType"]: + if correctedType is None: + # use a third solver (chuffed) to solve the instance + c_runStatus, c_runTotalTime, c_extra = minizinc_solve( + modelFile, + instFile, + "chuffed", + "-f", + None, + totalTimeLimit, + totalMemLimit, + ) + # TODO: what if chuffed fails to solve the instance? + assert c_extra["instanceType"] in [ + "sat", + "unsat", + ], "ERROR: inconsistent results between solvers or between runs of the same solvers, and the third solver (chuffed) fails to determine which one is correct" + correctedType = c_extra["instanceType"] + # if the current run is the incorrected one, mark its status as ERR + if instanceType == correctedType: + solver = info[solverType]["name"] + print( + f"WARNING: incorrect results by {solver} on {instFile} with seed {seed}. Results returned: {extra['instanceType']}, while chuffed returns {correctedType}" + ) + runStatus = "ERR" + # if the previous runs were the incorrected ones, mark their statuses as ERR + if extra["instanceType"] == correctedType: + for st in results.keys(): + for r in results[st]["runs"]: + if r["extra"]["instanceType"] == instanceType: + print( + f"WARNING: incorrect results by {info[st]['name']} on {instFile} with seed {r['seed']}. Results returned: {r['extra']['instanceType']}, while chuffed returns {correctedType}" + ) + r["status"] = "ERR" + # assign the correct type + instanceType = correctedType + + results[solverType]["runs"].append( + { + "seed": seed, + "status": runStatus, + "time": runTotalTime, + "extra": extra, + } + ) + + # if the instance is of an unwanted type, we stop immediately + if ( + len(unwantedTypes) > 0 + and instanceType + and (instanceType in unwantedTypes) + ): + print("Unwanted instance type. Quitting...") + score = SCORE_UNWANTED_TYPE + status = "unwantedType" + return score, get_results() + + # if the favoured solver cannot solve the instance on all runs, there's no need to run the base solver. We can just stop + if (solverType == "favoured") and (solved is False): + print("\nCannot be solved by favoured solver. Quitting...") + score = SCORE_FAVOURED_TOO_DIFFICULT + status = "favouredTooDifficult" + return score, get_results() + + # check if the instance is too easy for the base solver + baseAvgTime = sum([r["time"] for r in results["base"]["runs"]]) / nEvaluations + solvedByAllBaseRuns = True + for r in results["base"]["runs"]: + if r["status"] != "C": + solvedByAllBaseRuns = False + break + print(solvedByAllBaseRuns) + if solvedByAllBaseRuns and (baseAvgTime < baseMinTime): + print("\nInstance is too easy for the base solver. Quitting...") + score = SCORE_BASE_TOO_EASY + status = "baseTooEasy" + return score, get_results() + + problemType = get_minizinc_problem_type(modelFile) + + # if one of the two solvers is ortools and the instance is solved to optimality by ortools, use it to check correctness of the other solver in term of objective values before calculating borda score + if instanceType == "sat": + correctResults = toCheckResults = None + toCheckSolver = None + if info["base"]["name"] == "ortools": + correctResults = results["base"]["runs"] + toCheckResults = results["favoured"]["runs"] + toCheckSolver = info["favoured"]["name"] + elif info["favoured"]["name"] == "ortools": + correctResults = results["favoured"]["runs"] + toCheckResults = results["base"]["runs"] + toCheckSolver = info["base"]["name"] + if correctResults: + optimal = None + for r in correctResults: + if r["status"] == "C": + optimal = r["extra"]["objs"][-1][1] + break + if optimal: + for r in toCheckResults: + if r["status"] in ["S", "C"]: + assert len(r["extra"]["objs"]) > 0 + if has_better_objective( + r["extra"]["objs"][-1][1], optimal, problemType + ): + print( + f"WARNING: incorrect objective value by {toCheckSolver} on {instFile}. Best objective returned: {r['extra']['objs'][-1][1]}, while ortools returns {optimal}" + ) + r["status"] = "ERR" + + # calculate minizinc score of each solver per run + baseScores = [] + favouredScores = [] + print("\nBorda score for base and favoured solvers: ") + for i in range(nEvaluations): + baseResults = results["base"]["runs"][i] + favouredResults = results["favoured"]["runs"][i] + + bordaScores = calculate_minizinc_borda_scores( + baseResults["status"], + favouredResults["status"], + baseResults["time"], + favouredResults["time"], + problemType, + baseResults["extra"]["objs"], + favouredResults["extra"]["objs"], + True, + ) + print(bordaScores) + sc = bordaScores[scoringMethod] + baseScores.append(sc[0]) + favouredScores.append(sc[1]) + + # summarise over all runs + baseSum = sum(baseScores) + favouredSum = sum(favouredScores) + + # we want to maximise favouredSum / baseSum + if favouredSum == 0: + score = 0 + elif baseSum == 0: + assert ( + favouredSum == nEvaluations + ) # the best type of instance we can achieve, where the base solver fails and the favoured solver succeeds + score = -99999 + else: + score = ( + -favouredSum / baseSum + ) # when both solvers succeeds at solving the instance, we maximise the ratio of their scores + + status = "ok" + return score, get_results() + + +def evaluate_mzn_instance_graded( + modelFile: str, + instFile: str, + unwantedTypes: list = [], + nEvaluations: int = 1, + solver: str = "ortools", + solverFlags: str = "-f", + solverType: str = "complete", + minTime: int = 10, + timeLimit: int = 1200, + initSeed: int = None, + oracleSolver: str = None, + oracleSolverFlags: str = "-f", + oracleSolverTimeLimit: int = 3600, + memLimit=8192, +): + """ + Evaluate a mzn instance under the gradedness criteria + """ + # define constants for scores + SCORE_UNWANTED_TYPE = 0 + SCORE_TOO_EASY = 0 + SCORE_TOO_DIFFICULT = 0 + SCORE_INCORRECT_ANSWER = 0 + SCORE_GRADED = -1 + + # check validity of input + if len(unwantedTypes) > 0: + for s in unwantedTypes: + assert s in [ + "sat", + "unsat", + ], "ERROR: elements of unwantedTypes must be in {'sat','unsat'}" + assert nEvaluations > 0 + assert solverType in [ + "complete", + "incomplete", + ], "ERROR: solver type must be either complete or incomplete" + if solverType == "incomplete": + assert ( + oracleSolver != None + ), "ERROR: for incomplete solver, an oracle solver must be used" + assert ( + minTime < timeLimit + ), "ERROR: min solving time must be less than total time limit" + + # this is used by minizinc_utils.run_comparator + problemType = get_minizinc_problem_type(modelFile) + conf.problemType = problemType + + # initialise results + results = {"main": {}, "oracle": {}} + for st in ["main", "oracle"]: + results[st]["runs"] = [] + score = status = None + instanceType = None + + def get_results(): + assert (score is not None) and ( + status is not None + ), "ERROR: score/status is missing" + rs = { + "instance": instFile, + "status": status, + "score": score, + "results": results, + } + # print("\n",rs) + return rs + + # if it's a deterministic solver, we only need to run it once + # if (nEvaluations>1) and (solver in deterministicSolvers): + # nEvaluations = 1 + # print(f"{solver} is a deterministic solver via minizinc, so we only need to run it once.") + + # TODO: if main solver is incomplete and we don't want unsat instances, it's better to run the oracle with a small amount of time to check for satisfiability first + if (solverType == "incomplete") and ("unsat" in unwantedTypes): + smallTimeLimit = 120 + oracleRunStatus, oracleRunTotalTime, oracleExtra = minizinc_solve( + modelFile, + instFile, + oracleSolver, + oracleSolverFlags, + seed, + smallTimeLimit, + memLimit, + ) + if oracleExtra["instanceType"] == "unsat": + print("Unwanted instance type (checked by oracle). Quitting...") + score = SCORE_UNWANTED_TYPE + status = "unwantedType" + # TODO: in this context, we don't really need to run the oracle to check correctness of instance type, since return scores for unwanted type and incorrect results are the same. But if we decide to have the two scores being different, we may need to use the oracle here + return score, get_results() + + # run the main solver + instanceType = None + optimalObj = None + for i in range(nEvaluations): + if initSeed: + seed = initSeed + i + else: + seed = None + + print("\n") + runStatus, runTotalTime, extra = minizinc_solve( + modelFile, instFile, solver, solverFlags, seed, timeLimit, memLimit + ) + results["main"]["runs"].append( + {"seed": seed, "status": runStatus, "time": runTotalTime, "extra": extra} + ) + + # just for testing + # extra['instanceType']='unsat' + + # update instance type & check for inconsistency + if runStatus in ["S", "C"]: + if instanceType is None: + instanceType = extra["instanceType"] + elif ( + instanceType != extra["instanceType"] + ): # inconsistent results between runs, return immediately + print("Inconsistent instance type between runs. Quitting...") + score = SCORE_INCORRECT_ANSWER + status = "inconsistentInstanceTypes" + return score, get_results() + + # update optimal objective & check for inconsistency + if (runStatus == "C") and (instanceType == "sat"): + if optimalObj is None: + assert len(extra["objs"]) > 0 + optimalObj = extra["objs"][-1][1] + elif optimalObj != extra["objs"][-1][1]: + print("Inconsistent optimal objective value between runs. Quitting...") + score = SCORE_INCORRECT_ANSWER + status = "inconsistentOptimalValues" + return score, get_results() + + # if the instance is of an unwanted type, we stop immediately + if len(unwantedTypes) > 0 and instanceType and (instanceType in unwantedTypes): + print("Unwanted instance type. Quitting...") + score = SCORE_UNWANTED_TYPE + status = "unwantedType" + # TODO: in this context, we don't really need to run the oracle to check correctness of instance type, since return scores for unwanted type and incorrect results are the same. But if we decide to have the two scores being different, we may need to use the oracle here + return score, get_results() + + # get the median run + results["main"]["runs"] = sorted( + results["main"]["runs"], key=cmp_to_key(run_comparator) + ) + nRuns = len(results["main"]["runs"]) + medianRun = results["main"]["runs"][int(nRuns / 2)] + # pprint.pprint(results['main']['runs']) + + # if the instance is too easy by the main solver, there's no need to run the oracle + if (medianRun["status"] == "C") and (medianRun["time"] < minTime): + print("Instance too easy. Quitting...") + score = SCORE_TOO_EASY + status = "tooEasy" + return score, get_results() + + # if the instance is unsolvable by the main solver, there's no need to run the oracle + if medianRun["status"] not in ["S", "C"]: + print("Instance too difficult. Quitting...") + score = SCORE_TOO_DIFFICULT + status = "tooDifficult" + return score, get_results() + + if oracleSolver: + # run the oracle + # TODO: depending on results of the main solver, we do not necessarily run the oracle until the timelimit, e.g., if the main solver returns unsat, the oracle can stop as soon as it can find a (correct) solution. That might help to save lots of computation time. + print("\nRunning the oracle") + oracleRunStatus, oracleRunTotalTime, oracleExtra = minizinc_solve( + modelFile, + instFile, + oracleSolver, + oracleSolverFlags, + seed, + oracleSolverTimeLimit, + memLimit, + ) + results["oracle"]["runs"].append( + { + "status": oracleRunStatus, + "time": oracleRunTotalTime, + "extra": oracleExtra, + } + ) + + # for testing only + # v = oracleExtra['objs'][-1] + # oracleExtra['objs'][-1] = (v[0], v[1]-1) + + if oracleRunStatus != "C": + print("Instance cannot be solved by the oracle. Quitting...") + score = SCORE_TOO_DIFFICULT + status = "tooDifficultOracle" + return score, get_results() + + # check correctness using the oracle + for r in results["main"]["runs"]: + # instance type + # print(r) + if (r["status"] in ["S", "C"]) and ( + r["extra"]["instanceType"] != oracleExtra["instanceType"] + ): + print("Incorrect results (checked by oracle). Quitting...") + score = SCORE_INCORRECT_ANSWER + status = "incorrectInstanceType" + return score, get_results() + # objective value + if (r["status"] in ["S", "C"]) and (oracleExtra["instanceType"] == "sat"): + assert len(r["extra"]["objs"]) > 0 + optimal = oracleExtra["objs"][-1][1] + for o in r["extra"]["objs"]: + if has_better_objective(o[1], optimal, problemType): + print("Incorrect results (checked by oracle). Quitting...") + score = SCORE_INCORRECT_ANSWER + status = "incorrectObjectiveValue" + return score, get_results() + if (r["status"] == "C") and (r["extra"]["objs"][-1][1] != optimal): + print("Incorrect results (checked by oracle). Quitting...") + score = SCORE_INCORRECT_ANSWER + status = "incorrectOptimalValue" + return score, get_results() + + # for incomplete solver, use oracle to determine status + if (solverType == "incomplete") and (oracleExtra["instanceType"] == "sat"): + assert medianRun["status"] in ["S", "C"] + optimal = oracleExtra["objs"][-1][1] + o = medianRun["extra"]["objs"][-1][1] + assert has_better_objective(o, optimal, problemType) is False + if o != optimal: + print("Instance too difficult. Quitting...") + score = SCORE_TOO_DIFFICULT + status = "tooDifficult" + return score, get_results() + else: + lastTime = medianRun["extra"]["objs"][-1][0] + # if the main solver is incomplete and the optimal value is reached within in less than minTime seconds, consider the instance as too easy + if lastTime < minTime: + print("Instance too easy. Quitting...") + score = SCORE_TOO_EASY + status = "tooEasy" + return score, get_results() + + status = "ok" + score = SCORE_GRADED + return score, get_results() + + +def read_args(args): + #### read arguments (following irace's wrapper input format) ### + k = 1 + configurationId = int(args[k]) + k = k + 2 # skip second argument (<1>) + seed = int(args[k]) + k = k + 2 # skip 4th argument (dummy instance name) + params = args[k:] + paramDict = {} # generator parameter values suggested by irace + for i in range(0, len(params), 2): + paramDict[params[i][1:]] = params[i + 1] + + log(" ".join(args)) + + return configurationId, seed, paramDict + + +def read_setting(settingFile): + if os.path.isfile(settingFile) is False: + print("ERROR: setting file " + settingFile + " is missing.") + sys.exit(1) + with open(settingFile) as f: + setting = json.load(f) + + # split setting options into groups + c = {"generalSettings": {}, "generatorSettings": {}, "evaluationSettings": {}} + + c["generalSettings"]["experimentType"] = setting["instanceSetting"] + c["generalSettings"]["modelFile"] = setting["problemModel"] + c["generalSettings"]["generatorFile"] = setting["generatorModel"] + c["generalSettings"]["runDir"] = setting["runDir"] + + c["generatorSettings"]["genSRTimeLimit"] = setting["genSRTimeLimit"] + c["generatorSettings"]["genSRFlags"] = setting["genSRFlags"] + c["generatorSettings"]["genSolver"] = setting["genSolver"] + c["generatorSettings"]["genSolverTimeLimit"] = setting["genSolverTimeLimit"] + c["generatorSettings"]["genSolverFlags"] = setting["genSolverFlags"] + + c["evaluationSettings"]["nEvaluations"] = setting["nRunsPerInstance"] + c["evaluationSettings"]["gradedTypes"] = setting["instanceValidTypes"] + if setting["instanceSetting"] == "graded": + c["evaluationSettings"]["solver"] = setting["solver"] + print(setting) + if setting["solver"] in ["yuck"]: + c["evaluationSettings"]["solverType"] = "incomplete" + else: + c["evaluationSettings"]["solverType"] = "complete" + c["evaluationSettings"]["minTime"] = setting["minSolverTime"] + c["evaluationSettings"]["solverFlags"] = setting["solverFlags"] + c["evaluationSettings"]["totalTimeLimit"] = setting["maxSolverTime"] + else: + c["evaluationSettings"][ + "scoringMethod" + ] = "complete" # NOTE: incomplete scoring method is also supported by the code + baseSolverSettings = { + "name": setting["baseSolver"], + "solverMinTime": setting["minSolverTime"], + "totalTimeLimit": setting["maxSolverTime"], + "solverFlags": setting["baseSolverFlags"], + } + favouredSolverSettings = { + "name": setting["favouredSolver"], + "totalTimeLimit": setting["maxSolverTime"], + "solverFlags": setting["favouredSolverFlags"], + } + + c["evaluationSettings"]["baseSolver"] = baseSolverSettings + c["evaluationSettings"]["favouredSolver"] = favouredSolverSettings + + return c + def main(): startTime = time.time() @@ -53,7 +948,6 @@ def main(): # read all setting setting = read_setting("./config.json") - print(setting) # initialise run results @@ -63,7 +957,8 @@ def main(): def print_results(): assert (score is not None) and (status is not None) assert results["genResults"] != {} - + # if results['genResults']['status']=='sat': + # assert results['instanceResults']!={} totalWrapperTime = time.time() - startTime results["totalTime"] = totalWrapperTime results["status"] = status @@ -111,10 +1006,10 @@ def print_results(): "ERROR: invalid experimentType " + experimentType ) - modelType = os.path.basename(setting["generalSettings"]["modelFile"]).split(".")[-1] assert modelType in ["essence", "mzn"], ( "ERROR: " + + modelFile + ": model type not recognised (must be either .essence or .mzn)" ) @@ -134,50 +1029,10 @@ def get_unwanted_types(): # evaluate the generated instance if modelType == "essence": - es = setting["evaluationSettings"] - - # Case for graded - if experimentType == "graded": - oracleSolver = oracleSolverFlags = oracleSolverTimeLimit = None - # only called for incomplete solvers, which aren't actually allowed yet - if es["solverType"] == "incomplete": - oracleSolver = "ortools" - oracleSolverFlags = "-f" - oracleSolverTimeLimit = 3600 - - score, instanceResults = evaluate_essence_instance_graded( - modelFile="problem.essence", - instFile=instFile, - unwantedTypes=get_unwanted_types(), - nEvaluations=es["nEvaluations"], - solver=es["solver"], - solverFlags=es["solverFlags"], - solverType=es["solverType"], - minTime=es["minTime"], - timeLimit=es["totalTimeLimit"], - SRTimeLimit=es["SRTimeLimit"], - initSeed=seed, - oracleSolver=oracleSolver, - oracleSolverFlags=oracleSolverFlags, - oracleSolverTimeLimit=oracleSolverTimeLimit, - ) - # Case for discriminating - else: - score, instanceResults = evaluate_essence_instance_discriminating( - modelFile="problem.essence", - instFile=instFile, - scoringMethod=es["scoringMethod"], - unwantedTypes=get_unwanted_types(), - nEvaluations=es['nEvaluations'], - baseSolver=es["baseSolver"]["name"], - baseSolverFlags=es["baseSolver"]["solverFlags"], - baseMinTime=es["baseSolver"]["solverMinTime"], - favouredSolver=es["favouredSolver"]["name"], - favouredSolverFlags=es["favouredSolver"]["solverFlags"], - totalTimeLimit=es["baseSolver"]["totalTimeLimit"], - initSeed=seed, - gradedTypes=es["gradedTypes"], - ) + evaluationFunctionName = "evaluate_" + modelType + "_instance_" + experimentType + score, instanceResults = globals()[evaluationFunctionName]( + instFile, seed, setting["evaluationSettings"] + ) else: # convert the generated instance into .dzn mznInstFile = instFile.replace(".param", ".dzn") @@ -185,10 +1040,8 @@ def get_unwanted_types(): # start the evaluation es = setting["evaluationSettings"] - if es["nEvaluations"] == 1: seed = None - # Case for discriminating instance generation if experimentType == "discriminating": assert ( es["baseSolver"]["totalTimeLimit"] @@ -209,7 +1062,6 @@ def get_unwanted_types(): initSeed=seed, ) - # Case for graded instance generation else: oracleSolver = oracleSolverFlags = oracleSolverTimeLimit = None if es["solverType"] == "incomplete": @@ -232,9 +1084,7 @@ def get_unwanted_types(): oracleSolverTimeLimit=oracleSolverTimeLimit, ) - results["instanceResults"] = instanceResults - print("instance results *******", instanceResults) status = instanceResults["status"] # add the generated instance into generator's minion negative table, so that next time when we solve this generator instance again we don't re-generate the same instance @@ -243,6 +1093,7 @@ def get_unwanted_types(): # print out score and exit print_results() + main() # scoring for graded instances (single solver) diff --git a/scripts/wrapper_helpers.py b/scripts/wrapper_helpers.py deleted file mode 100644 index bc6b0f0d..00000000 --- a/scripts/wrapper_helpers.py +++ /dev/null @@ -1,77 +0,0 @@ -from utils import log, read_file, search_string, run_cmd, delete_file -import os -import sys -import json - - -def read_args(args): - #### read arguments (following irace's wrapper input format) ### - k = 1 - configurationId = int(args[k]) - k = k + 2 # skip second argument (<1>) - seed = int(args[k]) - k = k + 2 # skip 4th argument (dummy instance name) - params = args[k:] - paramDict = {} # generator parameter values suggested by irace - for i in range(0, len(params), 2): - paramDict[params[i][1:]] = params[i + 1] - - log(" ".join(args)) - - return configurationId, seed, paramDict - - -def read_setting(settingFile): - if os.path.isfile(settingFile) is False: - print("ERROR: setting file " + settingFile + " is missing.") - sys.exit(1) - with open(settingFile) as f: - setting = json.load(f) - - # split setting options into groups - c = {"generalSettings": {}, "generatorSettings": {}, "evaluationSettings": {}} - - c["generalSettings"]["experimentType"] = setting["instanceSetting"] - c["generalSettings"]["modelFile"] = setting["problemModel"] - c["generalSettings"]["generatorFile"] = setting["generatorModel"] - c["generalSettings"]["runDir"] = setting["runDir"] - - c["generatorSettings"]["genSRTimeLimit"] = setting["genSRTimeLimit"] - c["generatorSettings"]["genSRFlags"] = setting["genSRFlags"] - c["generatorSettings"]["genSolver"] = setting["genSolver"] - c["generatorSettings"]["genSolverTimeLimit"] = setting["genSolverTimeLimit"] - c["generatorSettings"]["genSolverFlags"] = setting["genSolverFlags"] - - c["evaluationSettings"]["nEvaluations"] = setting["nRunsPerInstance"] - c["evaluationSettings"]["gradedTypes"] = setting["instanceValidTypes"] - if setting["instanceSetting"] == "graded": - c["evaluationSettings"]["solver"] = setting["solver"] - print(setting) - if setting["solver"] in ["yuck"]: - c["evaluationSettings"]["solverType"] = "incomplete" - else: - c["evaluationSettings"]["solverType"] = "complete" - c["evaluationSettings"]["minTime"] = setting["minSolverTime"] - c["evaluationSettings"]["solverFlags"] = setting["solverFlags"] - c["evaluationSettings"]["SRTimeLimit"] = setting["SRTimeLimit"] - c["evaluationSettings"]["totalTimeLimit"] = setting["maxSolverTime"] - else: - c["evaluationSettings"][ - "scoringMethod" - ] = "complete" # NOTE: incomplete scoring method is also supported by the code - baseSolverSettings = { - "name": setting["baseSolver"], - "solverMinTime": setting["minSolverTime"], - "totalTimeLimit": setting["maxSolverTime"], - "solverFlags": setting["baseSolverFlags"], - } - favouredSolverSettings = { - "name": setting["favouredSolver"], - "totalTimeLimit": setting["maxSolverTime"], - "solverFlags": setting["favouredSolverFlags"], - } - - c["evaluationSettings"]["baseSolver"] = baseSolverSettings - c["evaluationSettings"]["favouredSolver"] = favouredSolverSettings - - return c \ No newline at end of file