From 931818551a80345a4bdfd099e2c52f81de522902 Mon Sep 17 00:00:00 2001 From: MaxBille Date: Wed, 3 Sep 2025 19:53:59 +0200 Subject: [PATCH 1/5] added progress_reporter.py: reports elapsed time, estimated time remaining and warns, when estimated time is more than max time (in seconds). Could be extended to write a checkpoint when t_elapsed > t_max. But sim.write_checkpoint can't be found... --- lettuce/ext/_reporter/progress_reporter.py | 109 +++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 lettuce/ext/_reporter/progress_reporter.py diff --git a/lettuce/ext/_reporter/progress_reporter.py b/lettuce/ext/_reporter/progress_reporter.py new file mode 100644 index 00000000..6b6db9cd --- /dev/null +++ b/lettuce/ext/_reporter/progress_reporter.py @@ -0,0 +1,109 @@ +import os +import datetime +from timeit import default_timer as timer +from ... import Reporter, Simulation + +def append_txt_file(filename, line: str): + ''' append a line to a file with an added linebreak''' + file = open(filename, "a") + file.write(line + "\n") + file.close() + +class ProgressReporter(Reporter): + ''' + Progress reporter that logs: current wall time, elapsed wall time, elapsed steps and estimates wall time + remaining. + Option to write a checkpoint file, when t_max is reached. (Sim. can be restarted from checkpoint) + + (!) This reporter does not export other reporters observable values etc., + so make sure you save them in other ways, if sim. is stopped by host system (e.g. HPC cluster)! + + ''' + + def __init__(self, interval=1000, t_max=None, i_target=None, outdir=None, print_message=False, ckeckpoint=False): + ## initialize local attributes... + self.t_max = t_max + self.i_start = 0 + self.i_target = i_target # should be equivalent to sim.num_steps + self.outdir= outdir + self.print_message = print_message + self.checkpoint = ckeckpoint + try: + os.makedirs(self.outdir) + except FileExistsError: + # directory already exists + pass + self.running = False + self.t_start = 0 # mutability of this up for discussion for sims. started from a checkpoint + self.t_elapsed = 0 # mutability of this up for discussion for sims. started from a checkpoint + + + ## check output directory for file (if path is not NONE) OR write to sys.out (print) + super().__init__(interval) + + def __call__(self, simulation: 'Simulation'): + if not self.running: + self.start_timer() + elif simulation.flow.i % self.interval == 0: + timestamp = datetime.datetime.now() + timestamp_str = timestamp.strftime("%y%m%d_%H%M%S") + + t_now = timer() + t_elapsed = t_now - self.t_start + t_per_step = t_elapsed / (simulation.flow.i - self.i_start) + i_remaining = self.i_target - simulation.flow.i + t_remaining_estimate = t_per_step * i_remaining + datetime_finish_estimate = timestamp + datetime.timedelta(seconds=t_remaining_estimate) + t_total_estimate = t_elapsed + t_remaining_estimate + + # write DATA and warn if t_total_estimate > t_max + base_message = (timestamp_str.ljust(13) + + " " + str(simulation.flow.i).rjust(10) + + " " + "{:.2f}".format(t_now).rjust(10) + + " " + "{:.2f}".format(t_elapsed).rjust(10) + + " " + "{:.6f}".format(t_per_step).rjust(10) + + " " + "{:.2f}".format(t_remaining_estimate).rjust(15) + + " " + "{:.2f}".format(t_total_estimate).rjust(15) + + " " + str(datetime_finish_estimate.strftime('%Y-%m-%d %H:%M:%S')).ljust(20)) + + if t_total_estimate > self.t_max: + message = base_message + " WARNING t_total>t_max=" + str(self.t_max) + else: + message = base_message + + append_txt_file(self.outdir + "/watchdog_log.txt", message) + if self.print_message: + print(message) + + # write checkpoint if t_elapsed > t_max + + if self. checkpoint and self.t_elapsed > self.t_max: + # checkpointin seems to be missing in current master... + simulation.save_checkpoint(self.outdir + "/" + timestamp_str + "_f_" + str(simulation.flow.i) + ".cpt") + + def start_timer(self): + self.running = True + self.t_start = timer() + #print("starting timer") + print("-> WATCHDOG_REPORTER ACTIVE:\nt_start: " + str(self.t_start) + + ", interval: " + str(self.interval) + ", i_target: " + str(self.i_target)) + + table_header = ("timestamp ".center(13) + +"|"+"step".center(10) + +"|"+"t_now".center(10) + +"|"+"t_elapsed".center(10) + +"|"+"t_per_step".center(10) + +"|"+"t_remain(est)".center(15) + +"|"+"t_total(est)".center(15) + +"|"+"DATE_FINISH(est)".center(20) + +"|"+" T WARNING") + if self.print_message: + print(table_header) + else: + print(f"print_message == False: see '{self.outdir}/watchdog_log.txt' for output") + + append_txt_file(self.outdir+"/watchdog_log.txt", + "t_start: "+str(self.t_start)+", interval: "+str(self.interval) + +", i_target: "+str(self.i_target)) + append_txt_file(self.outdir+"/watchdog_log.txt", table_header) + # sizes: 13, 10, 7+2.(rjust10), 7+2.(rjust10), 1+6.(rjust10), 7+2.(rjust10), 7+2.(rjust15), 27 \ No newline at end of file From 0402bd91175ab83de50bb03290dc4438683c6f8d Mon Sep 17 00:00:00 2001 From: MaxBille Date: Thu, 4 Sep 2025 17:38:09 +0200 Subject: [PATCH 2/5] fixed and tested progress_reporter.py. Can output to sys.out and/or to file. Example under examples/simple_flows/example_progress_reporter_based_on_simplest_TGV.py --- ...progress_reporter_based_on_simplest_TGV.py | 25 ++++++++++++++++ lettuce/ext/_reporter/progress_reporter.py | 29 ++++++++++--------- 2 files changed, 40 insertions(+), 14 deletions(-) create mode 100644 examples/simple_flows/example_progress_reporter_based_on_simplest_TGV.py diff --git a/examples/simple_flows/example_progress_reporter_based_on_simplest_TGV.py b/examples/simple_flows/example_progress_reporter_based_on_simplest_TGV.py new file mode 100644 index 00000000..647259b1 --- /dev/null +++ b/examples/simple_flows/example_progress_reporter_based_on_simplest_TGV.py @@ -0,0 +1,25 @@ +""" +This file showcases the simplicity of the lettuce code. +The following code will run a two-dimensional Taylor-Green vortex on GPU. +""" + +import torch +import lettuce as lt +from lettuce.ext._reporter.progress_reporter import ProgressReporter + +flow = lt.TaylorGreenVortex( + lt.Context(dtype=torch.float64, use_native=False), # for running on cpu: device='cpu' + resolution=128, + reynolds_number=100, + mach_number=0.05, + stencil=lt.D2Q9 +) + +progress_reporter = ProgressReporter(interval=10000, t_max=40, i_target=100000, print_message=True, outdir="./data/") + +simulation = lt.Simulation( + flow=flow, + collision=lt.BGKCollision(tau=flow.units.relaxation_parameter_lu), + reporter=[progress_reporter]) +mlups = simulation(num_steps=100000) +print("Performance in MLUPS:", mlups) diff --git a/lettuce/ext/_reporter/progress_reporter.py b/lettuce/ext/_reporter/progress_reporter.py index 6b6db9cd..79dc4dac 100644 --- a/lettuce/ext/_reporter/progress_reporter.py +++ b/lettuce/ext/_reporter/progress_reporter.py @@ -1,7 +1,7 @@ import os import datetime from timeit import default_timer as timer -from ... import Reporter, Simulation +from lettuce import Reporter, Simulation def append_txt_file(filename, line: str): ''' append a line to a file with an added linebreak''' @@ -20,19 +20,20 @@ class ProgressReporter(Reporter): ''' - def __init__(self, interval=1000, t_max=None, i_target=None, outdir=None, print_message=False, ckeckpoint=False): + def __init__(self, interval=1000, t_max=0, i_target=0, outdir=None, print_message=False, ckeckpoint=False): ## initialize local attributes... self.t_max = t_max self.i_start = 0 - self.i_target = i_target # should be equivalent to sim.num_steps - self.outdir= outdir + self.i_target = i_target # should be equivalent to sim.num_steps + self.outdir= str(outdir) self.print_message = print_message self.checkpoint = ckeckpoint - try: - os.makedirs(self.outdir) - except FileExistsError: - # directory already exists - pass + if self.outdir is not None: + try: + os.makedirs(self.outdir) + except FileExistsError: + # directory already exists + pass self.running = False self.t_start = 0 # mutability of this up for discussion for sims. started from a checkpoint self.t_elapsed = 0 # mutability of this up for discussion for sims. started from a checkpoint @@ -71,7 +72,7 @@ def __call__(self, simulation: 'Simulation'): else: message = base_message - append_txt_file(self.outdir + "/watchdog_log.txt", message) + append_txt_file(self.outdir + "/progress_reporter_log.txt", message) if self.print_message: print(message) @@ -85,7 +86,7 @@ def start_timer(self): self.running = True self.t_start = timer() #print("starting timer") - print("-> WATCHDOG_REPORTER ACTIVE:\nt_start: " + str(self.t_start) + print("-> PROGRESS_REPORTER ACTIVE:\nt_start: " + str(self.t_start) + ", interval: " + str(self.interval) + ", i_target: " + str(self.i_target)) table_header = ("timestamp ".center(13) @@ -100,10 +101,10 @@ def start_timer(self): if self.print_message: print(table_header) else: - print(f"print_message == False: see '{self.outdir}/watchdog_log.txt' for output") + print(f"print_message == False: see '{self.outdir}/progress_reporter_log.txt' for output") - append_txt_file(self.outdir+"/watchdog_log.txt", + append_txt_file(self.outdir+"/progress_reporter_log.txt", "t_start: "+str(self.t_start)+", interval: "+str(self.interval) +", i_target: "+str(self.i_target)) - append_txt_file(self.outdir+"/watchdog_log.txt", table_header) + append_txt_file(self.outdir+"/progress_reporter_log.txt", table_header) # sizes: 13, 10, 7+2.(rjust10), 7+2.(rjust10), 1+6.(rjust10), 7+2.(rjust10), 7+2.(rjust15), 27 \ No newline at end of file From c8d2b63c9c4a19f71b122767e3f55b7239bc987e Mon Sep 17 00:00:00 2001 From: MaxBille Date: Thu, 4 Sep 2025 17:43:06 +0200 Subject: [PATCH 3/5] added progress_reporter to lettuce/ext/reporter/__init__.py --- lettuce/ext/_reporter/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lettuce/ext/_reporter/__init__.py b/lettuce/ext/_reporter/__init__.py index 68502f78..03cbbe08 100644 --- a/lettuce/ext/_reporter/__init__.py +++ b/lettuce/ext/_reporter/__init__.py @@ -2,3 +2,4 @@ from .observable_reporter import * from .vtk_reporter import * from .write_image import * +from .progress_reporter import * \ No newline at end of file From 6d830522437704e05a3de17d0c1d7b4efe0b8ca0 Mon Sep 17 00:00:00 2001 From: mbille Date: Tue, 9 Dec 2025 15:48:57 +0100 Subject: [PATCH 4/5] minor changes for review --- ...progress_reporter_based_on_simplest_TGV.py | 0 lettuce/ext/_reporter/progress_reporter.py | 66 ++++++++++++------- 2 files changed, 43 insertions(+), 23 deletions(-) rename examples/{simple_flows => development}/example_progress_reporter_based_on_simplest_TGV.py (100%) diff --git a/examples/simple_flows/example_progress_reporter_based_on_simplest_TGV.py b/examples/development/example_progress_reporter_based_on_simplest_TGV.py similarity index 100% rename from examples/simple_flows/example_progress_reporter_based_on_simplest_TGV.py rename to examples/development/example_progress_reporter_based_on_simplest_TGV.py diff --git a/lettuce/ext/_reporter/progress_reporter.py b/lettuce/ext/_reporter/progress_reporter.py index 79dc4dac..edd0e40e 100644 --- a/lettuce/ext/_reporter/progress_reporter.py +++ b/lettuce/ext/_reporter/progress_reporter.py @@ -11,35 +11,42 @@ def append_txt_file(filename, line: str): class ProgressReporter(Reporter): ''' - Progress reporter that logs: current wall time, elapsed wall time, elapsed steps and estimates wall time - remaining. - Option to write a checkpoint file, when t_max is reached. (Sim. can be restarted from checkpoint) + Progress reporter that logs: current wall time, elapsed wall time, + elapsed steps and estimates wall time remaining. + future feature: Option to write a checkpoint file, + when t_max is reached. + (Sim. can be restarted from checkpoint) - (!) This reporter does not export other reporters observable values etc., - so make sure you save them in other ways, if sim. is stopped by host system (e.g. HPC cluster)! + (!) This reporter does not export other reporters observable values + etc., + so make sure you save them in other ways, + if sim. is stopped by host system (e.g. HPC cluster)! ''' - def __init__(self, interval=1000, t_max=0, i_target=0, outdir=None, print_message=False, ckeckpoint=False): + def __init__(self, interval=1000, t_max=0, i_target=0, i_start=0, + outdir=None, print_message=False, checkpoint=False): ## initialize local attributes... self.t_max = t_max - self.i_start = 0 + self.i_start = i_start self.i_target = i_target # should be equivalent to sim.num_steps self.outdir= str(outdir) self.print_message = print_message - self.checkpoint = ckeckpoint + self.checkpoint = checkpoint + + ## check output directory for file (if path is not NONE) + # ...OR write to sys.out (print) if self.outdir is not None: - try: - os.makedirs(self.outdir) - except FileExistsError: - # directory already exists - pass + if not os.path.exists(self.outdir): + os.mkdir(self.outdir) self.running = False - self.t_start = 0 # mutability of this up for discussion for sims. started from a checkpoint - self.t_elapsed = 0 # mutability of this up for discussion for sims. started from a checkpoint - + self.t_start = 0 + # mutability of this up for discussion for sims that are + # ...started from a checkpoint + self.t_elapsed = 0 + # mutability of this up for discussion for sims that are + # ...started from a checkpoint - ## check output directory for file (if path is not NONE) OR write to sys.out (print) super().__init__(interval) def __call__(self, simulation: 'Simulation'): @@ -51,7 +58,10 @@ def __call__(self, simulation: 'Simulation'): t_now = timer() t_elapsed = t_now - self.t_start - t_per_step = t_elapsed / (simulation.flow.i - self.i_start) + if simulation.flow.i == self.i_start: + t_per_step = t_elapsed / self.interval + else: + t_per_step = t_elapsed / (simulation.flow.i - self.i_start) i_remaining = self.i_target - simulation.flow.i t_remaining_estimate = t_per_step * i_remaining datetime_finish_estimate = timestamp + datetime.timedelta(seconds=t_remaining_estimate) @@ -65,7 +75,10 @@ def __call__(self, simulation: 'Simulation'): + " " + "{:.6f}".format(t_per_step).rjust(10) + " " + "{:.2f}".format(t_remaining_estimate).rjust(15) + " " + "{:.2f}".format(t_total_estimate).rjust(15) - + " " + str(datetime_finish_estimate.strftime('%Y-%m-%d %H:%M:%S')).ljust(20)) + + " " + str(datetime_finish_estimate.strftime( + '%Y-%m-%d %H:%M:%S')).ljust(20)) + # sizes: + # 13, 10, 7+2.(rjust10), 7+2.(rjust10), 1+6.(rjust10), 7+2.(rjust10), 7+2.(rjust15), 27 if t_total_estimate > self.t_max: message = base_message + " WARNING t_total>t_max=" + str(self.t_max) @@ -78,9 +91,15 @@ def __call__(self, simulation: 'Simulation'): # write checkpoint if t_elapsed > t_max - if self. checkpoint and self.t_elapsed > self.t_max: - # checkpointin seems to be missing in current master... - simulation.save_checkpoint(self.outdir + "/" + timestamp_str + "_f_" + str(simulation.flow.i) + ".cpt") + if self.checkpoint and self.t_elapsed > self.t_max: + # checkpointing seems to be missing in current master... + print("PROGRESS REPORTER: checkpoint was requested, " + "but current version of lettuce does not support " + "checkpointing... sorry :(") + # TO BE IMPLEMENTED IN THE FUTURE: + # simulation.save_checkpoint(self.outdir + # + "/" + timestamp_str + "_f_" + # + str(simulation.flow.i) + ".cpt") def start_timer(self): self.running = True @@ -101,7 +120,8 @@ def start_timer(self): if self.print_message: print(table_header) else: - print(f"print_message == False: see '{self.outdir}/progress_reporter_log.txt' for output") + print(f"print_message == False: see " + f"'{self.outdir}/progress_reporter_log.txt' for output") append_txt_file(self.outdir+"/progress_reporter_log.txt", "t_start: "+str(self.t_start)+", interval: "+str(self.interval) From e29da310db0512b52c5c4c9ccca8802f3c4003f1 Mon Sep 17 00:00:00 2001 From: mbille Date: Thu, 18 Dec 2025 13:03:01 +0100 Subject: [PATCH 5/5] made self.t_elapsed the variable used by call(); and some additional code reformatting and comments, minor changes in printed messages --- ...progress_reporter_based_on_simplest_TGV.py | 3 +- lettuce/ext/_reporter/progress_reporter.py | 28 ++++++++++++------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/examples/development/example_progress_reporter_based_on_simplest_TGV.py b/examples/development/example_progress_reporter_based_on_simplest_TGV.py index 647259b1..9961ced4 100644 --- a/examples/development/example_progress_reporter_based_on_simplest_TGV.py +++ b/examples/development/example_progress_reporter_based_on_simplest_TGV.py @@ -15,7 +15,8 @@ stencil=lt.D2Q9 ) -progress_reporter = ProgressReporter(interval=10000, t_max=40, i_target=100000, print_message=True, outdir="./data/") +progress_reporter = ProgressReporter(interval=10000, t_max=40, i_target=100000, + print_message=True, outdir="./data/") simulation = lt.Simulation( flow=flow, diff --git a/lettuce/ext/_reporter/progress_reporter.py b/lettuce/ext/_reporter/progress_reporter.py index edd0e40e..01121eba 100644 --- a/lettuce/ext/_reporter/progress_reporter.py +++ b/lettuce/ext/_reporter/progress_reporter.py @@ -23,6 +23,9 @@ class ProgressReporter(Reporter): if sim. is stopped by host system (e.g. HPC cluster)! ''' + # TODO (future improvements): + # - implement checkpointing functionality in lettuce + # - make progress reporter able to end simulation (similar to failureReporter) def __init__(self, interval=1000, t_max=0, i_target=0, i_start=0, outdir=None, print_message=False, checkpoint=False): @@ -50,6 +53,10 @@ def __init__(self, interval=1000, t_max=0, i_target=0, i_start=0, super().__init__(interval) def __call__(self, simulation: 'Simulation'): + """start internal timer, if timer is not running; + get timestamp, calculate elapsed time, estimate time remaining etc. + write data to file. + """ if not self.running: self.start_timer() elif simulation.flow.i % self.interval == 0: @@ -57,27 +64,27 @@ def __call__(self, simulation: 'Simulation'): timestamp_str = timestamp.strftime("%y%m%d_%H%M%S") t_now = timer() - t_elapsed = t_now - self.t_start + self.t_elapsed = t_now - self.t_start if simulation.flow.i == self.i_start: - t_per_step = t_elapsed / self.interval + t_per_step = self.t_elapsed / self.interval else: - t_per_step = t_elapsed / (simulation.flow.i - self.i_start) + t_per_step = self.t_elapsed / (simulation.flow.i - self.i_start) i_remaining = self.i_target - simulation.flow.i t_remaining_estimate = t_per_step * i_remaining datetime_finish_estimate = timestamp + datetime.timedelta(seconds=t_remaining_estimate) - t_total_estimate = t_elapsed + t_remaining_estimate + t_total_estimate = self.t_elapsed + t_remaining_estimate # write DATA and warn if t_total_estimate > t_max base_message = (timestamp_str.ljust(13) + " " + str(simulation.flow.i).rjust(10) + " " + "{:.2f}".format(t_now).rjust(10) - + " " + "{:.2f}".format(t_elapsed).rjust(10) + + " " + "{:.2f}".format(self.t_elapsed).rjust(10) + " " + "{:.6f}".format(t_per_step).rjust(10) + " " + "{:.2f}".format(t_remaining_estimate).rjust(15) + " " + "{:.2f}".format(t_total_estimate).rjust(15) + " " + str(datetime_finish_estimate.strftime( '%Y-%m-%d %H:%M:%S')).ljust(20)) - # sizes: + # (for reference) table column widths: # 13, 10, 7+2.(rjust10), 7+2.(rjust10), 1+6.(rjust10), 7+2.(rjust10), 7+2.(rjust15), 27 if t_total_estimate > self.t_max: @@ -89,8 +96,7 @@ def __call__(self, simulation: 'Simulation'): if self.print_message: print(message) - # write checkpoint if t_elapsed > t_max - + # (NOT IMPLEMENTET YET) write checkpoint if t_elapsed > t_max if self.checkpoint and self.t_elapsed > self.t_max: # checkpointing seems to be missing in current master... print("PROGRESS REPORTER: checkpoint was requested, " @@ -102,10 +108,12 @@ def __call__(self, simulation: 'Simulation'): # + str(simulation.flow.i) + ".cpt") def start_timer(self): + """ start the internal timer and write table header to output file""" self.running = True self.t_start = timer() + self.t_elapsed = 0 #print("starting timer") - print("-> PROGRESS_REPORTER ACTIVE:\nt_start: " + str(self.t_start) + print("(!) PROGRESS REPORTER ACTIVE:\nt_start: " + str(self.t_start) + ", interval: " + str(self.interval) + ", i_target: " + str(self.i_target)) table_header = ("timestamp ".center(13) @@ -120,7 +128,7 @@ def start_timer(self): if self.print_message: print(table_header) else: - print(f"print_message == False: see " + print(f"(ProgressReporter) print_message == False: see " f"'{self.outdir}/progress_reporter_log.txt' for output") append_txt_file(self.outdir+"/progress_reporter_log.txt",