diff --git a/CMakeLists.txt b/CMakeLists.txt index b4c486fa..dd26c388 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -340,8 +340,9 @@ if(USE_HDF5) message(FATAL_ERROR "HDF5 Parallel support is required: ${HDF5_IS_PARALLEL}") endif() add_definitions(-DVPIC_ENABLE_HDF5) + set(VPIC_CXX_FLAGS "${VPIC_CXX_FLAGS} -DVPIC_ENABLE_HDF5") string(REPLACE ";" " " string_libraries "${HDF5_C_LIBRARIES}") - set(VPIC_CXX_LIBRARIES "${VPIC_CXX_LIBRARIES} ${string_libraries}") + set(VPIC_CXX_LIBRARIES "${VPIC_CXX_LIBRARIES} -I${HDF5_INCLUDE_DIRS} ${string_libraries}") include_directories(${HDF5_INCLUDE_DIRS}) endif(USE_HDF5) diff --git a/arch/reference-Debug b/arch/reference-Debug index 2d5699ba..97a1550d 100755 --- a/arch/reference-Debug +++ b/arch/reference-Debug @@ -20,4 +20,5 @@ cmake \ -DENABLE_UNIT_TESTS=ON \ -DCMAKE_C_FLAGS="-rdynamic" \ -DCMAKE_CXX_FLAGS="-rdynamic" \ + -DUSE_HDF5=ON \ $src_dir diff --git a/deck/main.cc b/deck/main.cc index f9f7fb1b..d607b827 100644 --- a/deck/main.cc +++ b/deck/main.cc @@ -13,17 +13,16 @@ // The simulation variable is set up this way so both the checkpt // service and main can see it. This allows main to find where // the restored objects are after a restore. -vpic_simulation * simulation = NULL; - +vpic_simulation *simulation = NULL; /** * @brief Function to checkout main simulation object for restarting * * @param _simulation Simulation object to checkpoint */ -void checkpt_main(vpic_simulation** _simulation) +void checkpt_main(vpic_simulation **_simulation) { - CHECKPT_PTR( simulation ); + CHECKPT_PTR(simulation); } /** @@ -32,9 +31,9 @@ void checkpt_main(vpic_simulation** _simulation) * * @return Returns a double pointer (**) to the simulation object */ -vpic_simulation** restore_main(void) +vpic_simulation **restore_main(void) { - RESTORE_PTR( simulation ); + RESTORE_PTR(simulation); return &simulation; } @@ -44,13 +43,15 @@ vpic_simulation** restore_main(void) * @param fbase File name base for dumping * @param tag File tag to label what this checkpoint is (often used: time step) */ -void checkpt(const char* fbase, int tag) +void checkpt(const char *fbase, int tag) { char fname[256]; - if( !fbase ) ERROR(( "NULL filename base" )); - sprintf( fname, "%s.%i.%i", fbase, tag, world_rank ); - if( world_rank==0 ) log_printf( "*** Checkpointing to \"%s\"\n", fbase ); - checkpt_objects( fname ); + if (!fbase) + ERROR(("NULL filename base")); + sprintf(fname, "%s.%i.%i", fbase, tag, world_rank); + if (world_rank == 0) + log_printf("*** Checkpointing to \"%s\"\n", fbase); + checkpt_objects(fname); } /** @@ -61,17 +62,17 @@ void checkpt(const char* fbase, int tag) * * @return Application error code */ -int main(int argc, char** argv) +int main(int argc, char **argv) { // Initialize underlying threads and services - boot_services( &argc, &argv ); + boot_services(&argc, &argv); // TODO: this would be better if it was bool-like in nature - const char * fbase = strip_cmdline_string(&argc, &argv, "--restore", NULL); + const char *fbase = strip_cmdline_string(&argc, &argv, "--restore", NULL); // Detect if we should perform a restore as per the user request - if( fbase ) + if (fbase) { // We are restoring from a checkpoint. Determine checkpt file @@ -80,71 +81,84 @@ int main(int argc, char** argv) // that communication within reanimate functions is safe), // reanimate all the objects and issue a final barrier to // so that all processes come of a restore together. - if( world_rank==0 ) log_printf( "*** Restoring from \"%s\"\n", fbase ); + if (world_rank == 0) + log_printf("*** Restoring from \"%s\"\n", fbase); char fname[256]; - sprintf( fname, "%s.%i", fbase, world_rank ); - restore_objects( fname ); + sprintf(fname, "%s.%i", fbase, world_rank); + restore_objects(fname); mp_barrier(); reanimate_objects(); mp_barrier(); - + // std::cout << "simulation->dump_strategy_id = " << simulation->dump_strategy_id << "\n"; } else // We are initializing from scratch. { // Perform basic initialization - if( world_rank==0 ) + if (world_rank == 0) { - log_printf( "*** Initializing\n" ); + log_printf("*** Initializing\n"); } simulation = new vpic_simulation(); - simulation->initialize( argc, argv ); - REGISTER_OBJECT( &simulation, checkpt_main, restore_main, NULL ); - } + simulation->initialize(argc, argv); - // Do any post init/restore simulation modifications + // do post init setup to consume deck values + // which includes setting dump starts steps, as we didn't know it sooner + // TODO: make this use sane functions + simulation->dump_strategy = new_dump_strategy(simulation->dump_strategy_id); + simulation->dump_strategy->num_step = simulation->num_step; + // simulation->dump_strategy->num_step = vpic->num_step; + std::cout << "simulation->dump_strategy_id = " << simulation->dump_strategy_id << "\n"; + + REGISTER_OBJECT(&simulation, checkpt_main, restore_main, NULL); + } // Detect if the "modify" option is passed, which allows users to change // options (such as quota, num_step, etc) when restoring - fbase = strip_cmdline_string( &argc, &argv, "--modify", NULL ); - if( fbase ) + fbase = strip_cmdline_string(&argc, &argv, "--modify", NULL); + if (fbase) { - if( world_rank==0 ) log_printf( "*** Modifying from \"%s\"\n", fbase ); - simulation->modify( fbase ); + if (world_rank == 0) + log_printf("*** Modifying from \"%s\"\n", fbase); + simulation->modify(fbase); } // Perform the main simulation - if( world_rank==0 ) log_printf( "*** Advancing\n" ); + if (world_rank == 0) + log_printf("*** Advancing\n"); double elapsed = wallclock(); // Call the actual advance until it's done // TODO: Can we make this into a bounded loop - while( simulation->advance() ); + while (simulation->advance()) + ; elapsed = wallclock() - elapsed; // Report run time information on rank 0 - if( world_rank==0 ) + if (world_rank == 0) { // Calculate time info - int s = (int)elapsed, m = s/60, h = m/60, d = h/24, w = d/ 7; - s -= m*60; - m -= h*60; - h -= d*24; - d -= w*7; - - log_printf( "*** Done (%gs / %iw:%id:%ih:%im:%is elapsed)\n", - elapsed, w, d, h, m, s ); + int s = (int)elapsed, m = s / 60, h = m / 60, d = h / 24, w = d / 7; + s -= m * 60; + m -= h * 60; + h -= d * 24; + d -= w * 7; + + log_printf("*** Done (%gs / %iw:%id:%ih:%im:%is elapsed)\n", + elapsed, w, d, h, m, s); } - if( world_rank==0 ) log_printf( "*** Cleaning up\n" ); + if (world_rank == 0) + log_printf("*** Cleaning up\n"); // Perform Clean up, including de-registering objects - UNREGISTER_OBJECT( &simulation ); + UNREGISTER_OBJECT(&simulation); simulation->finalize(); delete simulation; // Check everything went well - if( world_rank==0 ) log_printf( "normal exit\n" ); + if (world_rank == 0) + log_printf("normal exit\n"); halt_services(); return 0; diff --git a/doc/src/hdf5.rst b/doc/src/hdf5.rst new file mode 100644 index 00000000..764a6101 --- /dev/null +++ b/doc/src/hdf5.rst @@ -0,0 +1,270 @@ +====================== +HDF5 Dump Code Usage +====================== +1. How to get the HDF5 dump code for VPIC? + Before the merge to https://github.com/lanl/vpic: + > git clone https://github.com/brtnfld/vpic.git + > git check dump_strategy + + Note: please check status of merge at the Pull request here: + https://github.com/lanl/vpic/pull/144 + After the merge, you may get it from https://github.com/lanl/vpic + +2. How to compile the HDF5 dump code for VPIC? + General build step can be found here: https://github.com/lanl/vpic + Here we use the reference-Debug and as example + + > mkdir build + > ../arch/reference-Debug + > make + > cp ../sample/harrisHDF5 ./ + > ./bin/vpic harrisHDF5 + + Trouble shooting for test cases: + + Error: Undefined symbols for architecture x86_64: "_H5Dclose" + Solution: you may manually add below flag for HDF5 to bin/vpic file + this is because the cmake sometimes can not find proper HDF5 file and link libraries + + -I/Users/dbin/work/soft/hdf5-1.10.7/build/include -L/Users/dbin/work/soft/hdf5-1.10.7/build/lib -lhdf5 -lhdf5_hl -lz + + +3. How to run VPIC with the HDF5 dump code ? + > ./harrisHDF5.Darwin + *** Initializing + Booting with 1 threads (pipelines) and 1 (MPI) ranks + ./harrisHDF5(64)[0]: Defaulting to mass_ratio of 1 and seed of 0 + ./harrisHDF5(65)[0]: For Custom Usage: ./harrisHDF5.Darwin mass_ratio seed + ./harrisHDF5(75)[0]: Computing simulation parameters + + ... ... + + user_particle_injection | 0% 0.0e+00 4.0e+00 0.0e+00 | 0% 4.3e-05 4.8e+02 9.0e-08 + user_current_injection | 0% 0.0e+00 4.0e+00 0.0e+00 | 0% 2.7e-05 4.8e+02 5.7e-08 + user_field_injection | 0% 0.0e+00 4.0e+00 0.0e+00 | 0% 2.0e-05 4.8e+02 4.1e-08 + user_diagnostics | 46% 1.4e-01 5.0e+00 2.7e-02 | 15% 2.9e+00 4.8e+02 5.9e-03 + + normal exit + +4. How to inspect the results? + +Particle data is stored in "particle_hdf5" directory. +Within the "particle_hdf5" directory, it has time steps, from T.0 , T.1, .... +Within each time step, it has two files electron_[time step].h5 and ion_[time step].h5 +> tree particle_hdf5 +particle_hdf5 +├── T.0 +│   ├── electron_0.h5 +│   └── ion_0.h5 + ... .... + +├── T.480 + ├── electron_480.h5 + └── ion_480.h5 + +For each particle file, you can use h5dump to inspect its results: + +> h5dump -A particle_hdf5/T.0/electron_0.h5 +HDF5 "particle_hdf5/T.0/electron_0.h5" { +GROUP "/" { + GROUP "Timestep_0" { + DATASET "dX" { + DATATYPE H5T_IEEE_F32LE + DATASPACE SIMPLE { ( 131072 ) / ( 131072 ) } + } + + ... ... + + DATASET "uz" { + DATATYPE H5T_IEEE_F32LE + DATASPACE SIMPLE { ( 131072 ) / ( 131072 ) } +}}}} + + +Hydro data is stored in "hydro_hdf5" directory. +It has the same structure as the "particle_hdf5" directory +By default, it also has two extra files "hydro-electron.xdmf" and "hydro-ion.xdmf" +The "hydro-electron.xdmf" and "hydro-ion.xdmf" are used to plot results of hydro data +One can also use h5dump to inspect the results of each file. + +> tree hydro_hdf5 +hydro_hdf5 +├── T.0 +│   ├── hydro_electron_0.h5 +│   └── hydro_ion_0.h5 + + ... ... + +├── T.480 +│   ├── hydro_electron_480.h5 +│   └── hydro_ion_480.h5 +├── hydro-electron.xdmf +└── hydro-ion.xdmf + + +Hydro data is stored in "field_hdf5" directory. +It has the same structure as the "hydro_hdf5" directory +By default, it also has two extra files "hdf5_field.xdmf". +The "hdf5_field.xdmf" is used to plot results of field data +One can also use h5dump to inspect the results of each file. + + +> tree field_hdf5 +field_hdf5 +├── T.0 +│   └── fields_0.h5 + + ... ... + +├── T.480 +│   └── fields_480.h5 +└── hdf5_field.xdmf + + +5. Other Notes on the HDF5 dump code (ongoing work) + * How to choose different output structure? + * How to select different attributes of fields for dump? + + The src/vpic/dump_strategy.h comes with different option to + explore different output structures. + + By default, below lines are commented out. + You can remove the comment symbols to use compound data structure for field/hydro/particle data. + But you may need to re-compile the vpic and the deck file + // #define HAS_FIELD_COMP 1 + // #define HAS_HYDRO_COMP 1 + // #define HAS_PARTICLE_COMP 1 + + One can also try the async I/O in HDF5 to dump data (not tested recently) + //#define H5_ASYNC 1 + #ifdef H5_ASYNC + #include "h5_vol_external_async_native.h" + #endif + +======================================== +HDF5 Dump Code's XDMF file for ParaView +======================================== +1. XDMF file + One can use the ParaView to open "hdf5_field.xdmf" to view the filed data. + One can use the ParaView to open "hydro-electron.xdmf" and "hydro-ion.xdmf" to view the hydro data. + +========================== +HDF5 Dump Code Test Cases +========================== +1. Test cases covered by the HDF5 Dump Code + +The HDF5 have below test case for HDF5 dump code +//Filed related test cases +- hdf5_fields +- hdf5_fields_restart +- hdf5_fields_readhdf5 +- hdf5_fields_location +- hdf5_fields_location_restart +- hdf5_fields_location_readhdf5 +- hdf5_fields_many_steps +- hdf5_fields_many_steps_restart +- hdf5_fields_many_steps_readhdf5 + +//Hydro related test cases +- hdf5_hydro +- hdf5_hydro_readhdf5 + +//Particle related test cases +- buffered_particle_dump_readhdf5 + +You can run all these test cases with below command +> make test + +Running tests... +Test project /Users/dbin/work/soft/vpic-doc/vpic/build + Start 1: rng + 1/37 Test #1: rng ............................... Passed 0.03 sec + Start 2: array_index + 2/37 Test #2: array_index ....................... Passed 0.07 sec + Start 3: buffered_particle_dump + 3/37 Test #3: buffered_particle_dump ............ Passed 0.09 sec + Start 4: buffered_particle_dump_restart + 4/37 Test #4: buffered_particle_dump_restart .... Passed 0.09 sec + Start 5: buffered_particle_dump_readhdf5 + 5/37 Test #5: buffered_particle_dump_readhdf5 ... Passed 3.53 sec + Start 6: hdf5_fields + 6/37 Test #6: hdf5_fields ....................... Passed 0.11 sec + Start 7: hdf5_fields_restart + 7/37 Test #7: hdf5_fields_restart ............... Passed 0.09 sec + Start 8: hdf5_fields_readhdf5 + 8/37 Test #8: hdf5_fields_readhdf5 .............. Passed 0.16 sec + Start 9: hdf5_fields_location + 9/37 Test #9: hdf5_fields_location .............. Passed 0.11 sec + Start 10: hdf5_fields_location_restart +10/37 Test #10: hdf5_fields_location_restart ...... Passed 0.09 sec + Start 11: hdf5_fields_location_readhdf5 +11/37 Test #11: hdf5_fields_location_readhdf5 ..... Passed 0.16 sec + Start 12: hdf5_fields_many_steps +12/37 Test #12: hdf5_fields_many_steps ............ Passed 0.12 sec + Start 13: hdf5_fields_many_steps_restart +13/37 Test #13: hdf5_fields_many_steps_restart .... Passed 0.11 sec + Start 14: hdf5_fields_many_steps_readhdf5 +14/37 Test #14: hdf5_fields_many_steps_readhdf5 ... Passed 0.18 sec + Start 15: hdf5_hydro +15/37 Test #15: hdf5_hydro ........................ Passed 0.09 sec + Start 16: hdf5_hydro_readhdf5 +16/37 Test #16: hdf5_hydro_readhdf5 ............... Passed 0.16 sec + Start 17: accel +17/37 Test #17: accel ............................. Passed 0.06 sec + Start 18: cyclo +18/37 Test #18: cyclo ............................. Passed 0.06 sec + Start 19: inbndj +19/37 Test #19: inbndj ............................ Passed 0.07 sec + Start 20: interpe +20/37 Test #20: interpe ........................... Passed 0.06 sec + Start 21: outbndj +21/37 Test #21: outbndj ........................... Passed 1.28 sec + Start 22: pcomm +22/37 Test #22: pcomm ............................. Passed 0.10 sec + Start 23: simple +23/37 Test #23: simple ............................ Passed 0.21 sec + Start 24: dump +24/37 Test #24: dump .............................. Passed 0.07 sec + Start 25: reconnection_test +25/37 Test #25: reconnection_test ................. Passed 0.69 sec + Start 26: parallel +26/37 Test #26: parallel .......................... Passed 0.22 sec + Start 27: threaded +27/37 Test #27: threaded .......................... Passed 0.28 sec + Start 28: generate_restore +28/37 Test #28: generate_restore .................. Passed 0.07 sec + Start 29: perform_restore +29/37 Test #29: perform_restore ................... Passed 0.06 sec + Start 30: test_collision +30/37 Test #30: test_collision .................... Passed 0.38 sec + Start 31: test_collision_script +31/37 Test #31: test_collision_script ............. Passed 0.13 sec + Start 32: array_syntax +32/37 Test #32: array_syntax ...................... Passed 0.07 sec + Start 33: weibel_driver +33/37 Test #33: weibel_driver ..................... Passed 0.75 sec + Start 34: 3d_test +34/37 Test #34: 3d_test ........................... Passed 1.15 sec + Start 35: weibel_driver_non_vaccum +35/37 Test #35: weibel_driver_non_vaccum .......... Passed 0.74 sec + Start 36: 3d_test_non_vaccum +36/37 Test #36: 3d_test_non_vaccum ................ Passed 1.11 sec + Start 37: 3d_test_non_vaccum_threaded +37/37 Test #37: 3d_test_non_vaccum_threaded ....... Passed 0.29 sec + +100% tests passed, 0 tests failed out of 37 + +Total Test time (real) = 13.07 sec + + + +2. Trouble shooting for test cases +Error 1: ModuleNotFoundError: No module named 'h5py' +Solution: pip install h5py + +========================== +Todo +========================== + + + diff --git a/sample/harrisHDF5 b/sample/harrisHDF5 new file mode 100644 index 00000000..f1434cfe --- /dev/null +++ b/sample/harrisHDF5 @@ -0,0 +1,486 @@ +// Magnetic reconnection in a Harris equilibrium thin current sheet +// +// This input deck reproduces the PIC simulations found in: +// William Daughton. "Nonlinear dynamics of thin current sheets." Phys. +// Plasmas. 9(9): 3668-3678. September 2002. +// +// This input deck was written by: +// Kevin J Bowers, Ph.D. +// Plasma Physics Group (X-1) +// Applied Physics Division +// Los Alamos National Lab +// August 2003 - original version +// October 2003 - heavily revised to utilize input deck syntactic sugar +// March/April 2004 - rewritten for domain decomposition V4PIC + +// If you want to use global variables (for example, to store the dump +// intervals for your diagnostics section), it must be done in the globals +// section. Variables declared the globals section will be preserved across +// restart dumps. For example, if the globals section is: +// begin_globals { +// double variable; +// } end_globals +// the double "variable" will be visible to other input deck sections as +// "global->variable". Note: Variables declared in the globals section are set +// to zero before the user's initialization block is executed. Up to 16K +// of global variables can be defined. + + +// Deck only works if VPIC was build with HDF support. Check for that: +#ifndef VPIC_ENABLE_HDF5 +#error "VPIC_ENABLE_HDF5" is required +#endif + +begin_globals { + double energies_interval; + double fields_interval; + double ehydro_interval; + double ihydro_interval; + double eparticle_interval; + double iparticle_interval; + double restart_interval; +}; + +begin_initialization { + // At this point, there is an empty grid and the random number generator is + // seeded with the rank. The grid, materials, species need to be defined. + // Then the initial non-zero fields need to be loaded at time level 0 and the + // particles (position and momentum both) need to be loaded at time level 0. + + + // Example of how to call / set dumping + //field_dump_flag.disableEMAT(); + + + double input_mass_ratio; + int input_seed; + + // Arguments can be passed from the command line to the input deck + if( num_cmdline_arguments!=3 ) { + // Set sensible defaults + input_mass_ratio = 1.0; + input_seed = 0; + + sim_log( "Defaulting to mass_ratio of " << input_mass_ratio << " and seed of " << input_seed ); + sim_log( "For Custom Usage: " << cmdline_argument[0] << " mass_ratio seed" ); + } + else { + input_mass_ratio = atof(cmdline_argument[1]); // Ion mass / electron mass + input_seed = atof(cmdline_argument[2]); // Ion mass / electron mass + sim_log( "Detected input mass_ratio of " << input_mass_ratio << " and seed of " << input_seed ); + } + seed_entropy( input_seed ); + + // Diagnostic messages can be passed written (usually to stderr) + sim_log( "Computing simulation parameters"); + + // Define the system of units for this problem (natural units) + double L = 1; // Length normalization (sheet thickness) + double ec = 1; // Charge normalization + double me = 1; // Mass normalization + double c = 1; // Speed of light + double eps0 = 1; // Permittivity of space + + // Physics parameters + double mi_me = input_mass_ratio; // Ion mass / electron mass + double rhoi_L = 1; // Ion thermal gyroradius / Sheet thickness + double Ti_Te = 1; // Ion temperature / electron temperature + double wpe_wce = 3; // Electron plasma freq / electron cycltron freq + double theta = 0; // Orientation of the simulation wrt current sheet + double taui = 100; // Simulation wci's to run + + // Numerical parameters + double Lx = 16*L; // How big should the box be in the x direction + double Ly = 16*L; // How big should the box be in the y direction + double Lz = 16*L; // How big should the box be in the z direction + double nx = 64; // Global resolution in the x direction + double ny = 64; // Global resolution in the y direction + double nz = 1; // Global resolution in the z direction + double nppc = 64; // Average number of macro particles per cell (both species combined!) + double cfl_req = 0.99; // How close to Courant should we try to run + double wpedt_max = 0.36; // How big a timestep is allowed if Courant is not too restrictive + double damp = 0.001; // Level of radiation damping + + // Derived quantities + double mi = me*mi_me; // Ion mass + double kTe = me*c*c/(2*wpe_wce*wpe_wce*(1+Ti_Te)); // Electron temperature + double kTi = kTe*Ti_Te; // Ion temperature + double vthe = sqrt(2*kTe/me); // Electron thermal velocity (B.D. convention) + double vthi = sqrt(2*kTi/mi); // Ion thermal velocity (B.D. convention) + double wci = vthi/(rhoi_L*L); // Ion cyclotron frequency + double wce = wci*mi_me; // Electron cyclotron frequency + double wpe = wce*wpe_wce; // Electron plasma frequency + double wpi = wpe/sqrt(mi_me); // Ion plasma frequency + double vdre = c*c*wce/(wpe*wpe*L*(1+Ti_Te)); // Electron drift velocity + double vdri = -Ti_Te*vdre; // Ion drift velocity + double b0 = me*wce/ec; // Asymptotic magnetic field strength + double n0 = me*eps0*wpe*wpe/(ec*ec); // Peak electron density (also peak ion density) + double Npe = 2*n0*Ly*Lz*L*tanh(0.5*Lx/L); // Number of physical electrons in box + double Npi = Npe; // Number of physical ions in box + double Ne = 0.5*nppc*nx*ny*nz; // Total macro electrons in box + Ne = trunc_granular(Ne,nproc()); // Make it divisible by number of processors + double Ni = Ne; // Total macro ions in box + double we = Npe/Ne; // Weight of a macro electron + double wi = Npi/Ni; // Weight of a macro ion + double gdri = 1/sqrt(1-vdri*vdri/(c*c)); // gamma of ion drift frame + double gdre = 1/sqrt(1-vdre*vdre/(c*c)); // gamma of electron drift frame + double udri = vdri*gdri; // 4-velocity of ion drift frame + double udre = vdre*gdre; // 4-velocity of electron drift frame + double uthi = sqrt(kTi/mi)/c; // Normalized ion thermal velocity (K.B. convention) + double uthe = sqrt(kTe/me)/c; // Normalized electron thermal velocity (K.B. convention) + double cs = cos(theta); + double sn = sin(theta); + + // Determine the timestep + double dg = courant_length(Lx,Ly,Lz,nx,ny,nz); // Courant length + double dt = cfl_req*dg/c; // Courant limited time step + if( wpe*dt>wpedt_max ) dt=wpedt_max/wpe; // Override time step if plasma frequency limited + + //////////////////////////////////////// + // Setup high level simulation parmeters + + num_step = int(0.2*taui/(wci*dt)); + status_interval = int(1./(wci*dt)); + field_interval = status_interval; + hydro_interval = status_interval; + sync_shared_interval = status_interval; + clean_div_e_interval = status_interval; + clean_div_b_interval = status_interval; + + global->energies_interval = status_interval; + global->fields_interval = status_interval; + global->ehydro_interval = status_interval; + global->ihydro_interval = status_interval; + global->eparticle_interval = status_interval; + global->iparticle_interval = status_interval; + global->restart_interval = status_interval; + + /////////////////////////// + // Setup the space and time + + // Setup basic grid parameters + define_units( c, eps0 ); + define_timestep( dt ); + + // Parition a periodic box among the processors sliced uniformly along y + define_periodic_grid( -0.5*Lx, 0, 0, // Low corner + 0.5*Lx, Ly, Lz, // High corner + nx, ny, nz, // Resolution + 1, nproc(), 1 ); // Topology + + // Override some of the boundary conditions to put a particle reflecting + // perfect electrical conductor on the -x and +x boundaries + set_domain_field_bc( BOUNDARY(-1,0,0), pec_fields ); + set_domain_field_bc( BOUNDARY( 1,0,0), pec_fields ); + set_domain_particle_bc( BOUNDARY(-1,0,0), reflect_particles ); + set_domain_particle_bc( BOUNDARY( 1,0,0), reflect_particles ); + + define_material( "vacuum", 1 ); + // Note: define_material defaults to isotropic materials with mu=1,sigma=0 + // Tensor electronic, magnetic and conductive materials are supported + // though. See "shapes" for how to define them and assign them to regions. + // Also, space is initially filled with the first material defined. + + // If you pass NULL to define field array, the standard field array will + // be used (if damp is not provided, no radiation damping will be used). + define_field_array( NULL, damp ); + + //////////////////// + // Setup the species + + // Allow 50% more local_particles in case of non-uniformity + // VPIC will pick the number of movers to use for each species + // Both species use out-of-place sorting + species_t * ion = define_species( "ion", ec, mi, 1.5*Ni/nproc(), -1, 40, 1 ); + species_t * electron = define_species( "electron", -ec, me, 1.5*Ne/nproc(), -1, 20, 1 ); + + /////////////////////////////////////////////////// + // Log diagnostic information about this simulation + + sim_log( "" ); + sim_log( "System of units" ); + sim_log( "L = " << L ); + sim_log( "ec = " << ec ); + sim_log( "me = " << me ); + sim_log( "c = " << c ); + sim_log( "eps0 = " << eps0 ); + sim_log( "" ); + sim_log( "Physics parameters" ); + sim_log( "rhoi/L = " << rhoi_L ); + sim_log( "Ti/Te = " << Ti_Te ); + sim_log( "wpe/wce = " << wpe_wce ); + sim_log( "mi/me = " << mi_me ); + sim_log( "theta = " << theta ); + sim_log( "taui = " << taui ); + sim_log( "" ); + sim_log( "Numerical parameters" ); + sim_log( "num_step = " << num_step ); + sim_log( "dt = " << dt ); + sim_log( "Lx = " << Lx << ", Lx/L = " << Lx/L ); + sim_log( "Ly = " << Ly << ", Ly/L = " << Ly/L ); + sim_log( "Lz = " << Lz << ", Lz/L = " << Lz/L ); + sim_log( "nx = " << nx << ", dx = " << Lx/nx << ", L/dx = " << L*nx/Lx ); + sim_log( "ny = " << ny << ", dy = " << Ly/ny << ", L/dy = " << L*ny/Ly ); + sim_log( "nz = " << nz << ", dz = " << Lz/nz << ", L/dz = " << L*nz/Lz ); + sim_log( "nppc = " << nppc ); + sim_log( "courant = " << c*dt/dg ); + sim_log( "damp = " << damp ); + sim_log( "" ); + sim_log( "Ion parameters" ); + sim_log( "qpi = " << ec << ", mi = " << mi << ", qpi/mi = " << ec/mi ); + sim_log( "vthi = " << vthi << ", vthi/c = " << vthi/c << ", kTi = " << kTi ); + sim_log( "vdri = " << vdri << ", vdri/c = " << vdri/c ); + sim_log( "wpi = " << wpi << ", wpi dt = " << wpi*dt << ", n0 = " << n0 ); + sim_log( "wci = " << wci << ", wci dt = " << wci*dt ); + sim_log( "rhoi = " << vthi/wci << ", L/rhoi = " << L/(vthi/wci) << ", dx/rhoi = " << (Lx/nx)/(vthi/wci) ); + sim_log( "debyei = " << vthi/wpi << ", L/debyei = " << L/(vthi/wpi) << ", dx/debyei = " << (Lx/nx)/(vthi/wpi) ); + sim_log( "Npi = " << Npi << ", Ni = " << Ni << ", Npi/Ni = " << Npi/Ni << ", wi = " << wi ); + sim_log( "" ); + sim_log( "Electron parameters" ); + sim_log( "qpe = " << -ec << ", me = " << me << ", qpe/me = " << -ec/me ); + sim_log( "vthe = " << vthe << ", vthe/c = " << vthe/c << ", kTe = " << kTe ); + sim_log( "vdre = " << vdre << ", vdre/c = " << vdre/c ); + sim_log( "wpe = " << wpe << ", wpe dt = " << wpe*dt << ", n0 = " << n0 ); + sim_log( "wce = " << wce << ", wce dt = " << wce*dt ); + sim_log( "rhoe = " << vthe/wce << ", L/rhoe = " << L/(vthe/wce) << ", dx/rhoe = " << (Lx/nx)/(vthe/wce) ); + sim_log( "debyee = " << vthe/wpe << ", L/debyee = " << L/(vthe/wpe) << ", dx/debyee = " << (Lx/nx)/(vthe/wpe) ); + sim_log( "Npe = " << Npe << ", Ne = " << Ne << ", Npe/Ne = " << Npe/Ne << ", we = " << we ); + sim_log( "" ); + sim_log( "Miscellaneous" ); + sim_log( "nptotal = " << Ni + Ne ); + sim_log( "nproc = " << nproc() ); + sim_log( "" ); + + //////////////////////////// + // Load fields and particles + + sim_log( "Loading fields" ); + + set_region_field( everywhere, 0, 0, 0, // Electric field + 0, -sn*b0*tanh(x/L), cs*b0*tanh(x/L) ); // Magnetic field + // Note: everywhere is a region that encompasses the entire simulation + // In general, regions are specied as logical equations (i.e. x>0 && x+y<2) + + sim_log( "Loading particles" ); + + double ymin = rank()*Ly/nproc(), ymax = (rank()+1)*Ly/nproc(); + + repeat( Ni/nproc() ) { + double x, y, z, ux, uy, uz, d0; + + // Pick an appropriately distributed random location for the pair + do { + x = L*atanh( uniform( rng(0), -1, 1 ) ); + } while( x<=-0.5*Lx || x>=0.5*Lx ); + y = uniform( rng(0), ymin, ymax ); + z = uniform( rng(0), 0, Lz ); + + // For the ion, pick an isothermal normalized momentum in the drift frame + // (this is a proper thermal equilibrium in the non-relativistic limit), + // boost it from the drift frame to the frame with the magnetic field + // along z and then rotate it into the lab frame. Then load the particle. + // Repeat the process for the electron. + + ux = normal( rng(0), 0, uthi ); + uy = normal( rng(0), 0, uthi ); + uz = normal( rng(0), 0, uthi ); + d0 = gdri*uy + sqrt(ux*ux+uy*uy+uz*uz+1)*udri; + uy = d0*cs - uz*sn; + uz = d0*sn + uz*cs; + inject_particle( ion, x, y, z, ux, uy, uz, wi, 0, 0 ); + + ux = normal( rng(0), 0, uthe ); + uy = normal( rng(0), 0, uthe ); + uz = normal( rng(0), 0, uthe ); + d0 = gdre*uy + sqrt(ux*ux+uy*uy+uz*uz+1)*udre; + uy = d0*cs - uz*sn; + uz = d0*sn + uz*cs; + inject_particle( electron, x, y, z, ux, uy, uz, we, 0, 0 ); + } + + // Upon completion of the initialization, the following occurs: + // - The synchronization error (tang E, norm B) is computed between domains + // and tang E / norm B are synchronized by averaging where discrepancies + // are encountered. + // - The initial divergence error of the magnetic field is computed and + // one pass of cleaning is done (for good measure) + // - The bound charge density necessary to give the simulation an initially + // clean divergence e is computed. + // - The particle momentum is uncentered from u_0 to u_{-1/2} + // - The user diagnostics are called on the initial state + // - The physics loop is started + // + // The physics loop consists of: + // - Advance particles from x_0,u_{-1/2} to x_1,u_{1/2} + // - User particle injection at x_{1-age}, u_{1/2} (use inject_particles) + // - User current injection (adjust field(x,y,z).jfx, jfy, jfz) + // - Advance B from B_0 to B_{1/2} + // - Advance E from E_0 to E_1 + // - User field injection to E_1 (adjust field(x,y,z).ex,ey,ez,cbx,cby,cbz) + // - Advance B from B_{1/2} to B_1 + // - (periodically) Divergence clean electric field + // - (periodically) Divergence clean magnetic field + // - (periodically) Synchronize shared tang e and norm b + // - Increment the time step + // - Call user diagnostics + // - (periodically) Print a status message + + // Explicitly enable HDF5 backend for IO dump + // WARNING: Call this after you have set `num_step` (for now.. soon fixed) + + enable_hdf5_dump(); +} + +begin_diagnostics { + +# define should_dump(x) (global->x##_interval>0 && remainder(step(),global->x##_interval)==0) + + if( step()==-10 ) { + // A grid dump contains all grid parameters, field boundary conditions, + // particle boundary conditions and domain connectivity information. This + // is stored in a binary format. Each rank makes a grid dump + dump_grid("grid"); + + // A materials dump contains all the materials parameters. This is in a + // text format. Only rank 0 makes the materials dump + dump_materials("materials"); + + // A species dump contains the physics parameters of a species. This is in + // a text format. Only rank 0 makes the species dump + dump_species("species"); + } + + // Energy dumps store all the energies in various directions of E and B + // and the total kinetic (not including rest mass) energies of each species + // species in a simple text format. By default, the energies are appended to + // the file. However, if a "0" is added to the dump_energies call, a new + // energies dump file will be created. The energies are in the units of the + // problem and are all time centered appropriately. Note: When restarting a + // simulation from a restart dump made at a prior time step to the last + // energies dump, the energies file will have a "hiccup" of intervening + // time levels. This "hiccup" will not occur if the simulation is aborted + // immediately following a restart dump. Energies dumps are in a text + // format and the layout is documented at the top of the file. Only rank 0 + // makes makes an energies dump. + //MSB if( should_dump(energies) ) dump_energies( "energies", step()==0 ? 0 : 1 ); + + // Field dumps store the raw electromagnetic fields, sources and material + // placement and a number of auxilliary fields. E, B and RHOB are + // timecentered, JF and TCA are half a step old. Material fields are static + // and the remaining fields (DIV E ERR, DIV B ERR and RHOF) are for + // debugging purposes. By default, field dump filenames are tagged with + // step(). However, if a "0" is added to the call, the filename will not be + // tagged. The JF that gets stored is accumulated with a charge-conserving + // algorithm. As a result, JF is not valid until at least one timestep has + // been completed. Field dumps are in a binary format. Each rank makes a + // field dump. +#if 1 + if( step()==-10 ) { + double el1 = uptime(); + dump_fields("fields"); // Get first valid total J + el1 = uptime() - el1; + sim_log("\033[1;35m dump_fields: " << el1 << " s\033[0m"); + } + if( should_dump(fields) ) { + double el1 = uptime(); + dump_fields("fields"); + el1 = uptime() - el1; + sim_log("\033[1;35m dump_fields: " << el1 << " s\033[0m"); + } +#endif + // Hydro dumps store particle charge density, current density and + // stress-energy tensor. All these quantities are known at the time + // t = time(). All these quantities are accumulated trilinear + // node-centered. By default, species dump filenames are tagged with + // step(). However, if a "0" is added to the call, the filename will not + // be tagged. Note that the current density accumulated by this routine is + // purely diagnostic. It is not used by the simulation and it is not + // accumulated using a self-consistent charge-conserving method. Hydro dumps + // are in a binary format. Each rank makes a hydro dump. +#if 1 + if(should_dump(ehydro) ) { + double el1 = uptime(); + dump_hydro("electron","ehydro"); + el1 = uptime() - el1; + sim_log("\033[1;31m dump_ehydro: " << el1 << " s\033[0m"); + } + if(should_dump(ihydro) ) { + double el1 = uptime(); + dump_hydro("ion", "ihydro"); + el1 = uptime() - el1; + sim_log("\033[1;31m dump_ihydro: " << el1 << " s\033[0m"); + } +#endif + // Particle dumps store the particle data for a given species. The data + // written is known at the time t = time(). By default, particle dumps + // are tagged with step(). However, if a "0" is added to the call, the + // filename will not be tagged. Particle dumps are in a binary format. + // Each rank makes a particle dump. +#if 1 + if( should_dump(eparticle) ) { + double el1 = uptime(); + dump_particles("electron","eparticle"); + el1 = uptime() - el1; + sim_log("\033[1;33m dump_eparticle: " << el1 << " s\033[0m"); + } +#endif +#if 1 + if( should_dump(iparticle) ){ + double el1 = uptime(); + dump_particles("ion", "iparticle"); + el1 = uptime() - el1; + sim_log("\033[1;33m dump_iparticle: " << el1 << " s\033[0m"); + } +#endif + + // A checkpt is made by calling checkpt( fbase, tag ) where fname is a string + // and tag is an integer. A typical usage is: + // checkpt( "checkpt", step() ). + // This will cause each process to write their simulation state to a file + // whose name is based on fbase, tag and the node's rank. For the above + // usage, if called on step 314 on a 4 process run, the four files: + // checkpt.314.0, checkpt.314.1, checkpt.314.2, checkpt.314.3 + // to be written. The simulation can then be restarted from this point by + // invoking the application with "--restore checkpt.314". checkpt must be + // the _VERY_ LAST_ diagnostic called. If not, diagnostics performed after + // the checkpt but before the next timestep will be missed on restore. + // Restart dumps are in a binary format unique to the each simulation. + + //MSB if( should_dump(restart) ) checkpt( "checkpt", step() ); + + // If you want to write a checkpt after a certain amount of simulation time, + // use uptime() in conjunction with checkpt. For example, this will cause + // the simulation state to be written after 7.5 hours of running to the + // same file every time (useful for dealing with quotas on big machines). + //if( uptime()>=27000 ) { + // checkpt( "timeout", 0 ); + // abort(0); + //} + +# undef should_dump + +} + +begin_particle_injection { + + // No particle injection for this simulation + +} + +begin_current_injection { + + // No current injection for this simulation + +} + +begin_field_injection { + + // No field injection for this simulation + +} + +begin_particle_collisions{ + + // No collisions for this simulation + +} diff --git a/sample/harrisOpenPMD b/sample/harrisOpenPMD new file mode 100644 index 00000000..697e0250 --- /dev/null +++ b/sample/harrisOpenPMD @@ -0,0 +1,455 @@ +// Magnetic reconnection in a Harris equilibrium thin current sheet +// +// This input deck reproduces the PIC simulations found in: +// William Daughton. "Nonlinear dynamics of thin current sheets." Phys. +// Plasmas. 9(9): 3668-3678. September 2002. +// +// This input deck was written by: +// Kevin J Bowers, Ph.D. +// Plasma Physics Group (X-1) +// Applied Physics Division +// Los Alamos National Lab +// August 2003 - original version +// October 2003 - heavily revised to utilize input deck syntactic sugar +// March/April 2004 - rewritten for domain decomposition V4PIC + +// If you want to use global variables (for example, to store the dump +// intervals for your diagnostics section), it must be done in the globals +// section. Variables declared the globals section will be preserved across +// restart dumps. For example, if the globals section is: +// begin_globals { +// double variable; +// } end_globals +// the double "variable" will be visible to other input deck sections as +// "global->variable". Note: Variables declared in the globals section are set +// to zero before the user's initialization block is executed. Up to 16K +// of global variables can be defined. + + +// Deck only works if VPIC was build with HDF support. Check for that: +#ifndef VPIC_ENABLE_OPENPMD +#error "VPIC_ENABLE_OPENPMD" is required +#endif + +begin_globals { + double energies_interval; + double fields_interval; + double ehydro_interval; + double ihydro_interval; + double eparticle_interval; + double iparticle_interval; + double restart_interval; +}; + +begin_initialization { + + enable_openpmd_dump(); + + // TODO: this should be done through a setter once we have a common options + // interface + //dump_strategy->file_type = ".bp"; + + // At this point, there is an empty grid and the random number generator is + // seeded with the rank. The grid, materials, species need to be defined. + // Then the initial non-zero fields need to be loaded at time level 0 and the + // particles (position and momentum both) need to be loaded at time level 0. + + // Example of how to call / set dumping + //field_dump_flag.disableEMAT(); + + double input_mass_ratio; + int input_seed; + + // Arguments can be passed from the command line to the input deck + if( num_cmdline_arguments!=3 ) { + // Set sensible defaults + input_mass_ratio = 1.0; + input_seed = 0; + + sim_log( "Defaulting to mass_ratio of " << input_mass_ratio << " and seed of " << input_seed ); + sim_log( "For Custom Usage: " << cmdline_argument[0] << " mass_ratio seed" ); + } + else { + input_mass_ratio = atof(cmdline_argument[1]); // Ion mass / electron mass + input_seed = atof(cmdline_argument[2]); // Ion mass / electron mass + sim_log( "Detected input mass_ratio of " << input_mass_ratio << " and seed of " << input_seed ); + } + seed_entropy( input_seed ); + + // Diagnostic messages can be passed written (usually to stderr) + sim_log( "Computing simulation parameters"); + + // Define the system of units for this problem (natural units) + double L = 1; // Length normalization (sheet thickness) + double ec = 1; // Charge normalization + double me = 1; // Mass normalization + double c = 1; // Speed of light + double eps0 = 1; // Permittivity of space + + // Physics parameters + double mi_me = input_mass_ratio; // Ion mass / electron mass + double rhoi_L = 1; // Ion thermal gyroradius / Sheet thickness + double Ti_Te = 1; // Ion temperature / electron temperature + double wpe_wce = 3; // Electron plasma freq / electron cycltron freq + double theta = 0; // Orientation of the simulation wrt current sheet + double taui = 100; // Simulation wci's to run + + // Numerical parameters + double Lx = 16*L; // How big should the box be in the x direction + double Ly = 16*L; // How big should the box be in the y direction + double Lz = 16*L; // How big should the box be in the z direction + double nx = 64; // Global resolution in the x direction + double ny = 64; // Global resolution in the y direction + double nz = 1; // Global resolution in the z direction + double nppc = 64; // Average number of macro particles per cell (both species combined!) + double cfl_req = 0.99; // How close to Courant should we try to run + double wpedt_max = 0.36; // How big a timestep is allowed if Courant is not too restrictive + double damp = 0.001; // Level of radiation damping + + // Derived quantities + double mi = me*mi_me; // Ion mass + double kTe = me*c*c/(2*wpe_wce*wpe_wce*(1+Ti_Te)); // Electron temperature + double kTi = kTe*Ti_Te; // Ion temperature + double vthe = sqrt(2*kTe/me); // Electron thermal velocity (B.D. convention) + double vthi = sqrt(2*kTi/mi); // Ion thermal velocity (B.D. convention) + double wci = vthi/(rhoi_L*L); // Ion cyclotron frequency + double wce = wci*mi_me; // Electron cyclotron frequency + double wpe = wce*wpe_wce; // Electron plasma frequency + double wpi = wpe/sqrt(mi_me); // Ion plasma frequency + double vdre = c*c*wce/(wpe*wpe*L*(1+Ti_Te)); // Electron drift velocity + double vdri = -Ti_Te*vdre; // Ion drift velocity + double b0 = me*wce/ec; // Asymptotic magnetic field strength + double n0 = me*eps0*wpe*wpe/(ec*ec); // Peak electron density (also peak ion density) + double Npe = 2*n0*Ly*Lz*L*tanh(0.5*Lx/L); // Number of physical electrons in box + double Npi = Npe; // Number of physical ions in box + double Ne = 0.5*nppc*nx*ny*nz; // Total macro electrons in box + Ne = trunc_granular(Ne,nproc()); // Make it divisible by number of processors + double Ni = Ne; // Total macro ions in box + double we = Npe/Ne; // Weight of a macro electron + double wi = Npi/Ni; // Weight of a macro ion + double gdri = 1/sqrt(1-vdri*vdri/(c*c)); // gamma of ion drift frame + double gdre = 1/sqrt(1-vdre*vdre/(c*c)); // gamma of electron drift frame + double udri = vdri*gdri; // 4-velocity of ion drift frame + double udre = vdre*gdre; // 4-velocity of electron drift frame + double uthi = sqrt(kTi/mi)/c; // Normalized ion thermal velocity (K.B. convention) + double uthe = sqrt(kTe/me)/c; // Normalized electron thermal velocity (K.B. convention) + double cs = cos(theta); + double sn = sin(theta); + + // Determine the timestep + double dg = courant_length(Lx,Ly,Lz,nx,ny,nz); // Courant length + double dt = cfl_req*dg/c; // Courant limited time step + if( wpe*dt>wpedt_max ) dt=wpedt_max/wpe; // Override time step if plasma frequency limited + + //////////////////////////////////////// + // Setup high level simulation parmeters + + num_step = int(0.2*taui/(wci*dt)); + status_interval = int(1./(wci*dt)); + field_interval = status_interval; + hydro_interval = status_interval; + sync_shared_interval = status_interval; + clean_div_e_interval = status_interval; + clean_div_b_interval = status_interval; + + global->energies_interval = status_interval; + global->fields_interval = status_interval; + global->ehydro_interval = status_interval; + global->ihydro_interval = status_interval; + global->eparticle_interval = status_interval; + global->iparticle_interval = status_interval; + global->restart_interval = status_interval; + + /////////////////////////// + // Setup the space and time + + // Setup basic grid parameters + define_units( c, eps0 ); + define_timestep( dt ); + + // Parition a periodic box among the processors sliced uniformly along y + define_periodic_grid( -0.5*Lx, 0, 0, // Low corner + 0.5*Lx, Ly, Lz, // High corner + nx, ny, nz, // Resolution + 1, nproc(), 1 ); // Topology + + // Override some of the boundary conditions to put a particle reflecting + // perfect electrical conductor on the -x and +x boundaries + set_domain_field_bc( BOUNDARY(-1,0,0), pec_fields ); + set_domain_field_bc( BOUNDARY( 1,0,0), pec_fields ); + set_domain_particle_bc( BOUNDARY(-1,0,0), reflect_particles ); + set_domain_particle_bc( BOUNDARY( 1,0,0), reflect_particles ); + + define_material( "vacuum", 1 ); + // Note: define_material defaults to isotropic materials with mu=1,sigma=0 + // Tensor electronic, magnetic and conductive materials are supported + // though. See "shapes" for how to define them and assign them to regions. + // Also, space is initially filled with the first material defined. + + // If you pass NULL to define field array, the standard field array will + // be used (if damp is not provided, no radiation damping will be used). + define_field_array( NULL, damp ); + + //////////////////// + // Setup the species + + // Allow 50% more local_particles in case of non-uniformity + // VPIC will pick the number of movers to use for each species + // Both species use out-of-place sorting + species_t * ion = define_species( "ion", ec, mi, 1.5*Ni/nproc(), -1, 40, 1 ); + species_t * electron = define_species( "electron", -ec, me, 1.5*Ne/nproc(), -1, 20, 1 ); + + /////////////////////////////////////////////////// + // Log diagnostic information about this simulation + + sim_log( "" ); + sim_log( "System of units" ); + sim_log( "L = " << L ); + sim_log( "ec = " << ec ); + sim_log( "me = " << me ); + sim_log( "c = " << c ); + sim_log( "eps0 = " << eps0 ); + sim_log( "" ); + sim_log( "Physics parameters" ); + sim_log( "rhoi/L = " << rhoi_L ); + sim_log( "Ti/Te = " << Ti_Te ); + sim_log( "wpe/wce = " << wpe_wce ); + sim_log( "mi/me = " << mi_me ); + sim_log( "theta = " << theta ); + sim_log( "taui = " << taui ); + sim_log( "" ); + sim_log( "Numerical parameters" ); + sim_log( "num_step = " << num_step ); + sim_log( "dt = " << dt ); + sim_log( "Lx = " << Lx << ", Lx/L = " << Lx/L ); + sim_log( "Ly = " << Ly << ", Ly/L = " << Ly/L ); + sim_log( "Lz = " << Lz << ", Lz/L = " << Lz/L ); + sim_log( "nx = " << nx << ", dx = " << Lx/nx << ", L/dx = " << L*nx/Lx ); + sim_log( "ny = " << ny << ", dy = " << Ly/ny << ", L/dy = " << L*ny/Ly ); + sim_log( "nz = " << nz << ", dz = " << Lz/nz << ", L/dz = " << L*nz/Lz ); + sim_log( "nppc = " << nppc ); + sim_log( "courant = " << c*dt/dg ); + sim_log( "damp = " << damp ); + sim_log( "" ); + sim_log( "Ion parameters" ); + sim_log( "qpi = " << ec << ", mi = " << mi << ", qpi/mi = " << ec/mi ); + sim_log( "vthi = " << vthi << ", vthi/c = " << vthi/c << ", kTi = " << kTi ); + sim_log( "vdri = " << vdri << ", vdri/c = " << vdri/c ); + sim_log( "wpi = " << wpi << ", wpi dt = " << wpi*dt << ", n0 = " << n0 ); + sim_log( "wci = " << wci << ", wci dt = " << wci*dt ); + sim_log( "rhoi = " << vthi/wci << ", L/rhoi = " << L/(vthi/wci) << ", dx/rhoi = " << (Lx/nx)/(vthi/wci) ); + sim_log( "debyei = " << vthi/wpi << ", L/debyei = " << L/(vthi/wpi) << ", dx/debyei = " << (Lx/nx)/(vthi/wpi) ); + sim_log( "Npi = " << Npi << ", Ni = " << Ni << ", Npi/Ni = " << Npi/Ni << ", wi = " << wi ); + sim_log( "" ); + sim_log( "Electron parameters" ); + sim_log( "qpe = " << -ec << ", me = " << me << ", qpe/me = " << -ec/me ); + sim_log( "vthe = " << vthe << ", vthe/c = " << vthe/c << ", kTe = " << kTe ); + sim_log( "vdre = " << vdre << ", vdre/c = " << vdre/c ); + sim_log( "wpe = " << wpe << ", wpe dt = " << wpe*dt << ", n0 = " << n0 ); + sim_log( "wce = " << wce << ", wce dt = " << wce*dt ); + sim_log( "rhoe = " << vthe/wce << ", L/rhoe = " << L/(vthe/wce) << ", dx/rhoe = " << (Lx/nx)/(vthe/wce) ); + sim_log( "debyee = " << vthe/wpe << ", L/debyee = " << L/(vthe/wpe) << ", dx/debyee = " << (Lx/nx)/(vthe/wpe) ); + sim_log( "Npe = " << Npe << ", Ne = " << Ne << ", Npe/Ne = " << Npe/Ne << ", we = " << we ); + sim_log( "" ); + sim_log( "Miscellaneous" ); + sim_log( "nptotal = " << Ni + Ne ); + sim_log( "nproc = " << nproc() ); + sim_log( "" ); + + //////////////////////////// + // Load fields and particles + + sim_log( "Loading fields" ); + + set_region_field( everywhere, 0, 0, 0, // Electric field + 0, -sn*b0*tanh(x/L), cs*b0*tanh(x/L) ); // Magnetic field + // Note: everywhere is a region that encompasses the entire simulation + // In general, regions are specied as logical equations (i.e. x>0 && x+y<2) + + sim_log( "Loading particles" ); + + double ymin = rank()*Ly/nproc(), ymax = (rank()+1)*Ly/nproc(); + + repeat( Ni/nproc() ) { + double x, y, z, ux, uy, uz, d0; + + // Pick an appropriately distributed random location for the pair + do { + x = L*atanh( uniform( rng(0), -1, 1 ) ); + } while( x<=-0.5*Lx || x>=0.5*Lx ); + y = uniform( rng(0), ymin, ymax ); + z = uniform( rng(0), 0, Lz ); + + // For the ion, pick an isothermal normalized momentum in the drift frame + // (this is a proper thermal equilibrium in the non-relativistic limit), + // boost it from the drift frame to the frame with the magnetic field + // along z and then rotate it into the lab frame. Then load the particle. + // Repeat the process for the electron. + + ux = normal( rng(0), 0, uthi ); + uy = normal( rng(0), 0, uthi ); + uz = normal( rng(0), 0, uthi ); + d0 = gdri*uy + sqrt(ux*ux+uy*uy+uz*uz+1)*udri; + uy = d0*cs - uz*sn; + uz = d0*sn + uz*cs; + inject_particle( ion, x, y, z, ux, uy, uz, wi, 0, 0 ); + + ux = normal( rng(0), 0, uthe ); + uy = normal( rng(0), 0, uthe ); + uz = normal( rng(0), 0, uthe ); + d0 = gdre*uy + sqrt(ux*ux+uy*uy+uz*uz+1)*udre; + uy = d0*cs - uz*sn; + uz = d0*sn + uz*cs; + inject_particle( electron, x, y, z, ux, uy, uz, we, 0, 0 ); + } + + // Upon completion of the initialization, the following occurs: + // - The synchronization error (tang E, norm B) is computed between domains + // and tang E / norm B are synchronized by averaging where discrepancies + // are encountered. + // - The initial divergence error of the magnetic field is computed and + // one pass of cleaning is done (for good measure) + // - The bound charge density necessary to give the simulation an initially + // clean divergence e is computed. + // - The particle momentum is uncentered from u_0 to u_{-1/2} + // - The user diagnostics are called on the initial state + // - The physics loop is started + // + // The physics loop consists of: + // - Advance particles from x_0,u_{-1/2} to x_1,u_{1/2} + // - User particle injection at x_{1-age}, u_{1/2} (use inject_particles) + // - User current injection (adjust field(x,y,z).jfx, jfy, jfz) + // - Advance B from B_0 to B_{1/2} + // - Advance E from E_0 to E_1 + // - User field injection to E_1 (adjust field(x,y,z).ex,ey,ez,cbx,cby,cbz) + // - Advance B from B_{1/2} to B_1 + // - (periodically) Divergence clean electric field + // - (periodically) Divergence clean magnetic field + // - (periodically) Synchronize shared tang e and norm b + // - Increment the time step + // - Call user diagnostics + // - (periodically) Print a status message +} + +begin_diagnostics { + +# define should_dump(x) (global->x##_interval>0 && remainder(step(),global->x##_interval)==0) + + if( step()==-10 ) { + // A grid dump contains all grid parameters, field boundary conditions, + // particle boundary conditions and domain connectivity information. This + // is stored in a binary format. Each rank makes a grid dump + dump_grid("grid"); + + // A materials dump contains all the materials parameters. This is in a + // text format. Only rank 0 makes the materials dump + dump_materials("materials"); + + // A species dump contains the physics parameters of a species. This is in + // a text format. Only rank 0 makes the species dump + dump_species("species"); + } + + // Energy dumps store all the energies in various directions of E and B + // and the total kinetic (not including rest mass) energies of each species + // species in a simple text format. By default, the energies are appended to + // the file. However, if a "0" is added to the dump_energies call, a new + // energies dump file will be created. The energies are in the units of the + // problem and are all time centered appropriately. Note: When restarting a + // simulation from a restart dump made at a prior time step to the last + // energies dump, the energies file will have a "hiccup" of intervening + // time levels. This "hiccup" will not occur if the simulation is aborted + // immediately following a restart dump. Energies dumps are in a text + // format and the layout is documented at the top of the file. Only rank 0 + // makes makes an energies dump. + if( should_dump(energies) ) dump_energies( "energies", step()==0 ? 0 : 1 ); + + // Field dumps store the raw electromagnetic fields, sources and material + // placement and a number of auxilliary fields. E, B and RHOB are + // timecentered, JF and TCA are half a step old. Material fields are static + // and the remaining fields (DIV E ERR, DIV B ERR and RHOF) are for + // debugging purposes. By default, field dump filenames are tagged with + // step(). However, if a "0" is added to the call, the filename will not be + // tagged. The JF that gets stored is accumulated with a charge-conserving + // algorithm. As a result, JF is not valid until at least one timestep has + // been completed. Field dumps are in a binary format. Each rank makes a + // field dump. + + // TODO: passing in the field extension as part of the name doesn't work for + // the other functions, as they use it to look up species + std::string openpm_field_name = "fields.h5"; + //std::string openpm_field_name = "fields.bp"; + if( step()==-10 ) dump_fields(openpm_field_name.c_str()); // Get first valid total J + if( should_dump(fields) ) dump_fields(openpm_field_name.c_str()); + + // Hydro dumps store particle charge density, current density and + // stress-energy tensor. All these quantities are known at the time + // t = time(). All these quantities are accumulated trilinear + // node-centered. By default, species dump filenames are tagged with + // step(). However, if a "0" is added to the call, the filename will not + // be tagged. Note that the current density accumulated by this routine is + // purely diagnostic. It is not used by the simulation and it is not + // accumulated using a self-consistent charge-conserving method. Hydro dumps + // are in a binary format. Each rank makes a hydro dump. + if(should_dump(ehydro) ) dump_hydro("electron","ehydro"); + //if( should_dump(ihydro) ) dump_hydro_hdf5("ion", "ihydro"); + + // Particle dumps store the particle data for a given species. The data + // written is known at the time t = time(). By default, particle dumps + // are tagged with step(). However, if a "0" is added to the call, the + // filename will not be tagged. Particle dumps are in a binary format. + // Each rank makes a particle dump. + if( should_dump(eparticle) ) dump_particles("electron","eparticle"); + //if( should_dump(iparticle) ) dump_particles_hdf5("ion", "iparticle"); + + // A checkpt is made by calling checkpt( fbase, tag ) where fname is a string + // and tag is an integer. A typical usage is: + // checkpt( "checkpt", step() ). + // This will cause each process to write their simulation state to a file + // whose name is based on fbase, tag and the node's rank. For the above + // usage, if called on step 314 on a 4 process run, the four files: + // checkpt.314.0, checkpt.314.1, checkpt.314.2, checkpt.314.3 + // to be written. The simulation can then be restarted from this point by + // invoking the application with "--restore checkpt.314". checkpt must be + // the _VERY_ LAST_ diagnostic called. If not, diagnostics performed after + // the checkpt but before the next timestep will be missed on restore. + // Restart dumps are in a binary format unique to the each simulation. + + if( should_dump(restart) ) checkpt( "checkpt", step() ); + + // If you want to write a checkpt after a certain amount of simulation time, + // use uptime() in conjunction with checkpt. For example, this will cause + // the simulation state to be written after 7.5 hours of running to the + // same file every time (useful for dealing with quotas on big machines). + //if( uptime()>=27000 ) { + // checkpt( "timeout", 0 ); + // abort(0); + //} + +# undef should_dump + +} + +begin_particle_injection { + + // No particle injection for this simulation + +} + +begin_current_injection { + + // No current injection for this simulation + +} + +begin_field_injection { + + // No field injection for this simulation + +} + +begin_particle_collisions{ + + // No collisions for this simulation + +} diff --git a/sample/read_openpmd.py b/sample/read_openpmd.py new file mode 100644 index 00000000..84b19009 --- /dev/null +++ b/sample/read_openpmd.py @@ -0,0 +1,39 @@ + +import openpmd_api as api + +# example: data handling +import numpy as np + +file_name = "./fields.h5" +series = api.Series( file_name, api.Access_Type.read_only) + +print(list(series.iterations)) + +from pprint import pprint +#pprint(vars(series)) +#pprint(vars(series.iterations)) + +i = series.iterations[1]; + +print("openPMD version: ", + series.openPMD) + +# record +cB = i.meshes["B"] + +# record components +cbx = cB["x"] + +x_data = cbx.load_chunk() + +series.flush() + +extent = cbx.shape + +print( + "First values in E_x " + "of shape: ", + extent) + + +print(x_data) diff --git a/scripts/hdf5_reader/tracer_print_timestep.py b/scripts/hdf5_reader/tracer_print_timestep.py index f7fbdac8..11cf06c7 100755 --- a/scripts/hdf5_reader/tracer_print_timestep.py +++ b/scripts/hdf5_reader/tracer_print_timestep.py @@ -4,7 +4,7 @@ import numpy as np import os import sys - +os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" if len(sys.argv) < 3: sys.stderr.write("Usage: "+str(sys.argv[0])+" timestep tracers.h5p [other.h5p...]\n") sys.exit(1) diff --git a/scripts/hdf5_reader/tracer_print_trajectory.py b/scripts/hdf5_reader/tracer_print_trajectory.py index 19f01d8c..2512f56d 100755 --- a/scripts/hdf5_reader/tracer_print_trajectory.py +++ b/scripts/hdf5_reader/tracer_print_trajectory.py @@ -4,7 +4,7 @@ import numpy as np import os import sys - +os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" if len(sys.argv) < 3: sys.stderr.write("Usage: "+str(sys.argv[0])+" id tracers.h5p [other.h5p...]\n") sys.exit(1) diff --git a/scripts/hdf5_reader/tracer_sort.py b/scripts/hdf5_reader/tracer_sort.py index c2398033..23ee6f78 100755 --- a/scripts/hdf5_reader/tracer_sort.py +++ b/scripts/hdf5_reader/tracer_sort.py @@ -4,7 +4,7 @@ import numpy as np import os import sys - +os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" if len(sys.argv) != 2: sys.stderr.write("Usage: "+str(sys.argv[0])+" tracers.h5p\n") sys.exit(1) diff --git a/src/grid/grid.h b/src/grid/grid.h index e376bd32..d4813eff 100644 --- a/src/grid/grid.h +++ b/src/grid/grid.h @@ -1,4 +1,4 @@ -/* +/* * Written by: * Kevin J. Bowers, Ph.D. * Plasma Physics Group (X-1) @@ -46,7 +46,7 @@ enum grid_enums { // B_tang -> Symmetric | B_tang -> Anti-symmetric // E_norm -> Symmetric | E_norm -> Anti-symmetric (see note) // div B -> Symmetric | div B -> Anti-symmetric - // + // // Note: B_norm is tricky. For a symmetry plane, B_norm on the // boundary must be zero as there are no magnetic charges (a // non-zero B_norm would imply an infinitesimal layer of magnetic @@ -80,7 +80,7 @@ typedef struct grid { int64_t step; // Current timestep double t0; // Simulation time corresponding to step 0 - // Phase 2 grid data structures + // Phase 2 grid data structures float x0, y0, z0; // Min corner local domain (must be coherent) float x1, y1, z1; // Max corner local domain (must be coherent) int nx, ny, nz; // Local voxel mesh resolution. Voxels are @@ -99,6 +99,7 @@ typedef struct grid { // 0 ... nproc-1 ... comm boundary condition // <0 ... locally applied boundary condition + int gpx, gpy, gpz = -1; // Store global processor decomposition to let us figure // out where we are in the global decomposition @@ -138,6 +139,20 @@ typedef struct grid { #define VOXEL(x,y,z, nx,ny,nz) ((x) + ((nx)+2)*((y) + ((ny)+2)*(z))) +// TODO: make the asymmetry in how nx+2 is handled more obvious +#define UNVOXEL(rank, ix, iy, iz, nx, ny, nz) BEGIN_PRIMITIVE { \ + int _ix, _iy, _iz; \ + _ix = (rank); /* ix = ix+gpx*( iy+gpy*iz ) */ \ + _iy = _ix/int(nx); /* iy = iy+gpy*iz */ \ + _ix -= _iy*int(nx); /* ix = ix */ \ + _iz = _iy/int(ny); /* iz = iz */ \ + _iy -= _iz*int(ny); /* iy = iy */ \ + (ix) = _ix; \ + (iy) = _iy; \ + (iz) = _iz; \ + } END_PRIMITIVE + + // Advance the voxel mesh index (v) and corresponding voxel mesh // coordinates (x,y,z) in a region with min- and max-corners of // (xl,yl,zl) and (xh,yh,zh) of a (nx,ny,nz) resolution voxel mesh in @@ -150,7 +165,7 @@ typedef struct grid { // inner loops.) // // This is written with seeming extraneously if tests in order to get -// the compiler to generate branceless conditional move and add +// the compiler to generate branceless conditional move and add // instructions (none of the branches below are actual branches in // assembly). @@ -314,7 +329,7 @@ end_send_port( int i, // x port coord ([-1,0,1]) // ordering (e.g. inner loop increments x-index). // // jobs are indexed from 0 to n_job-1. jobs are _always_ have the -// number of voxels an integer multiple of the bundle size. If job +// number of voxels an integer multiple of the bundle size. If job // is set to n_job, this function will determine the parameters of // the final incomplete bundle. diff --git a/src/sf_interface/hydro_array.cc b/src/sf_interface/hydro_array.cc index 837db71b..624c2964 100644 --- a/src/sf_interface/hydro_array.cc +++ b/src/sf_interface/hydro_array.cc @@ -11,17 +11,18 @@ #include "sf_interface.h" static int -ha_n_pipeline( void ) +ha_n_pipeline(void) { -#if defined(VPIC_USE_PTHREADS) // Pthreads case. - int n = serial.n_pipeline; - if ( n < thread.n_pipeline ) n = thread.n_pipeline; +#if defined(VPIC_USE_PTHREADS) // Pthreads case. + int n = serial.n_pipeline; + if (n < thread.n_pipeline) + n = thread.n_pipeline; -#elif defined(VPIC_USE_OPENMP) // OpenMP case. - int n = omp_helper.n_pipeline; +#elif defined(VPIC_USE_OPENMP) // OpenMP case. + int n = omp_helper.n_pipeline; -#else // Error case. - #error "VPIC_USE_OPENMP or VPIC_USE_PTHREADS must be specified" +#else // Error case. +#error "VPIC_USE_OPENMP or VPIC_USE_PTHREADS must be specified" #endif @@ -31,104 +32,101 @@ ha_n_pipeline( void ) // Though the checkpt / restore functions are not part of the public // API, they must not be declared as static. -void -checkpt_hydro_array( const hydro_array_t * ha ) +void checkpt_hydro_array(const hydro_array_t *ha) { - CHECKPT( ha, 1 ); + CHECKPT(ha, 1); - CHECKPT_ALIGNED( ha->h, - (size_t) ( ha->n_pipeline + 1 ) * (size_t) ha->stride, - 128 ); + CHECKPT_ALIGNED(ha->h, + (size_t)(ha->n_pipeline + 1) * (size_t)ha->stride, + 128); - CHECKPT_PTR( ha->g ); + CHECKPT_PTR(ha->g); } hydro_array_t * -restore_hydro_array( void ) +restore_hydro_array(void) { - hydro_array_t * ha; + hydro_array_t *ha; - RESTORE( ha ); + RESTORE(ha); - RESTORE_ALIGNED( ha->h ); + RESTORE_ALIGNED(ha->h); - RESTORE_PTR( ha->g ); + RESTORE_PTR(ha->g); - if ( ha->n_pipeline != ha_n_pipeline() ) + if (ha->n_pipeline != ha_n_pipeline()) { - ERROR( ( "Number of pipelines restored is not the same as the number of " - "pipelines checkpointed. Did you change the number of threads " - "per process between checkpt and restore?" ) ); + ERROR(("Number of pipelines restored is not the same as the number of " + "pipelines checkpointed. Did you change the number of threads " + "per process between checkpt and restore?")); } return ha; } hydro_array_t * -new_hydro_array( grid_t * g ) +new_hydro_array(grid_t *g) { - hydro_array_t * ha; + hydro_array_t *ha; - if ( ! g ) + if (!g) { - ERROR( ( "NULL grid." ) ); + ERROR(("NULL grid.")); } - MALLOC( ha, 1 ); + MALLOC(ha, 1); ha->n_pipeline = ha_n_pipeline(); - ha->stride = POW2_CEIL( g->nv, 2 ); - ha->g = g; + ha->stride = POW2_CEIL(g->nv, 2); + ha->g = g; - MALLOC_ALIGNED( ha->h, - (size_t) ( ha->n_pipeline + 1 ) * (size_t) ha->stride, - 128 ); + MALLOC_ALIGNED(ha->h, + (size_t)(ha->n_pipeline + 1) * (size_t)ha->stride, + 128); - CLEAR( ha->h, - (size_t) ( ha->n_pipeline + 1 ) * (size_t) ha->stride ); + CLEAR(ha->h, + (size_t)(ha->n_pipeline + 1) * (size_t)ha->stride); - REGISTER_OBJECT( ha, - checkpt_hydro_array, - restore_hydro_array, - NULL ); + REGISTER_OBJECT(ha, + checkpt_hydro_array, + restore_hydro_array, + NULL); return ha; } -void -delete_hydro_array( hydro_array_t * ha ) +void delete_hydro_array(hydro_array_t *ha) { - if ( ! ha ) + if (!ha) { return; } - UNREGISTER_OBJECT( ha ); + UNREGISTER_OBJECT(ha); - FREE_ALIGNED( ha->h ); + FREE_ALIGNED(ha->h); - FREE( ha ); + FREE(ha); } -#define hydro( x, y, z ) h0[ VOXEL( x, y, z, nx, ny, nz ) ] +#define hydro(x, y, z) h0[VOXEL(x, y, z, nx, ny, nz)] // Generic looping. -#define XYZ_LOOP( xl, xh, yl, yh, zl, zh ) \ - for( z = zl; z <= zh; z++ ) \ - for( y = yl; y <= yh; y++ ) \ - for( x = xl; x <= xh; x++ ) +#define XYZ_LOOP(xl, xh, yl, yh, zl, zh) \ + for (z = zl; z <= zh; z++) \ + for (y = yl; y <= yh; y++) \ + for (x = xl; x <= xh; x++) // x_NODE_LOOP => Loop over all non-ghost nodes at plane x. -#define x_NODE_LOOP( x ) XYZ_LOOP( x, x, 1, ny+1, 1, nz+1 ) +#define x_NODE_LOOP(x) XYZ_LOOP(x, x, 1, ny + 1, 1, nz + 1) -#define y_NODE_LOOP( y ) XYZ_LOOP( 1, nx+1, y, y, 1, nz+1 ) +#define y_NODE_LOOP(y) XYZ_LOOP(1, nx + 1, y, y, 1, nz + 1) -#define z_NODE_LOOP( z ) XYZ_LOOP( 1, nx+1, 1, ny+1, z, z ) +#define z_NODE_LOOP(z) XYZ_LOOP(1, nx + 1, 1, ny + 1, z, z) -void -synchronize_hydro_array( hydro_array_t * ha ) +void synchronize_hydro_array(hydro_array_t *ha) { int size, face, bc, x, y, z, nx, ny, nz; @@ -138,19 +136,19 @@ synchronize_hydro_array( hydro_array_t * ha ) grid_t *g; - if ( ! ha ) + if (!ha) { - ERROR( ( "NULL hydro array." ) ); + ERROR(("NULL hydro array.")); } // First reduce the pipelines. - reduce_hydro_array( ha ); + reduce_hydro_array(ha); // Now begin to synchronize. h0 = ha->h; - g = ha->g; + g = ha->g; nx = g->nx; ny = g->ny; @@ -161,151 +159,153 @@ synchronize_hydro_array( hydro_array_t * ha ) // diagnostic, correct the hydro along local boundaries to account // for accumulations over partial cell volumes. - #define ADJUST_HYDRO( i, j, k, X, Y, Z ) \ - do \ - { \ - bc = g->bc[ BOUNDARY( i, j, k ) ]; \ - if ( bc < 0 || bc >= world_size ) \ - { \ - face = ( i + j + k ) < 0 ? 1 : n##X + 1; \ - X##_NODE_LOOP( face ) \ - { \ - h = &hydro( x, y, z ); \ - h->jx *= 2; \ - h->jy *= 2; \ - h->jz *= 2; \ - h->rho *= 2; \ - h->px *= 2; \ - h->py *= 2; \ - h->pz *= 2; \ - h->ke *= 2; \ - h->txx *= 2; \ - h->tyy *= 2; \ - h->tzz *= 2; \ - h->tyz *= 2; \ - h->tzx *= 2; \ - h->txy *= 2; \ - } \ - } \ - } while( 0 ) - - ADJUST_HYDRO( (-1), 0, 0, x, y, z ); - ADJUST_HYDRO( 0, (-1), 0, y, z, x ); - ADJUST_HYDRO( 0, 0, (-1), z, x, y ); - ADJUST_HYDRO( 1, 0, 0, x, y, z ); - ADJUST_HYDRO( 0, 1, 0, y, z, x ); - ADJUST_HYDRO( 0, 0, 1, z, x, y ); - - #undef ADJUST_HYDRO - - #define BEGIN_RECV( i, j, k, X, Y, Z ) \ - begin_recv_port( i, j, k, ( 1 + 14 * ( n##Y + 1 ) * ( n##Z + 1 ) ) * sizeof(float), g ) - - #define BEGIN_SEND( i, j, k, X, Y, Z ) \ - BEGIN_PRIMITIVE \ - { \ - size = ( 1 + 14 * (n##Y+1) * (n##Z+1) ) * sizeof(float); \ - p = (float *) size_send_port( i, j, k, size, g ); \ - if ( p ) \ - { \ - ( *(p++) ) = g->d##X; \ - face = ( i + j + k ) < 0 ? 1 : n##X+1; \ - X##_NODE_LOOP( face ) \ - { \ - h = &hydro( x, y, z ); \ - ( *(p++) ) = h->jx; \ - ( *(p++) ) = h->jy; \ - ( *(p++) ) = h->jz; \ - ( *(p++) ) = h->rho; \ - ( *(p++) ) = h->px; \ - ( *(p++) ) = h->py; \ - ( *(p++) ) = h->pz; \ - ( *(p++) ) = h->ke; \ - ( *(p++) ) = h->txx; \ - ( *(p++) ) = h->tyy; \ - ( *(p++) ) = h->tzz; \ - ( *(p++) ) = h->tyz; \ - ( *(p++) ) = h->tzx; \ - ( *(p++) ) = h->txy; \ - } \ - begin_send_port( i, j, k, size, g ); \ - } \ - } END_PRIMITIVE - - #define END_RECV( i, j, k, X, Y, Z ) \ - BEGIN_PRIMITIVE \ - { \ - p = (float *) end_recv_port( i, j, k, g ); \ - if ( p ) \ - { \ - rw = ( *(p++) ); /* Remote g->d##X */ \ - lw = rw + g->d##X; \ - rw /= lw; \ - lw = g->d##X / lw; \ - lw += lw; \ - rw += rw; \ - face = ( i + j + k ) < 0 ? n##X+1 : 1; /* Twice weighted sum */ \ - X##_NODE_LOOP( face ) \ - { \ - h = &hydro( x, y, z ); \ - h->jx = lw * h->jx + rw * ( *(p++) ); \ - h->jy = lw * h->jy + rw * ( *(p++) ); \ - h->jz = lw * h->jz + rw * ( *(p++) ); \ - h->rho = lw * h->rho + rw * ( *(p++) ); \ - h->px = lw * h->px + rw * ( *(p++) ); \ - h->py = lw * h->py + rw * ( *(p++) ); \ - h->pz = lw * h->pz + rw * ( *(p++) ); \ - h->ke = lw * h->ke + rw * ( *(p++) ); \ - h->txx = lw * h->txx + rw * ( *(p++) ); \ - h->tyy = lw * h->tyy + rw * ( *(p++) ); \ - h->tzz = lw * h->tzz + rw * ( *(p++) ); \ - h->tyz = lw * h->tyz + rw * ( *(p++) ); \ - h->tzx = lw * h->tzx + rw * ( *(p++) ); \ - h->txy = lw * h->txy + rw * ( *(p++) ); \ - } \ - } \ - } END_PRIMITIVE - - #define END_SEND( i, j, k, X, Y, Z ) end_send_port( i, j, k, g ) +#define ADJUST_HYDRO(i, j, k, X, Y, Z) \ + do \ + { \ + bc = g->bc[BOUNDARY(i, j, k)]; \ + if (bc < 0 || bc >= world_size) \ + { \ + face = (i + j + k) < 0 ? 1 : n##X + 1; \ + X##_NODE_LOOP(face) \ + { \ + h = &hydro(x, y, z); \ + h->jx *= 2; \ + h->jy *= 2; \ + h->jz *= 2; \ + h->rho *= 2; \ + h->px *= 2; \ + h->py *= 2; \ + h->pz *= 2; \ + h->ke *= 2; \ + h->txx *= 2; \ + h->tyy *= 2; \ + h->tzz *= 2; \ + h->tyz *= 2; \ + h->tzx *= 2; \ + h->txy *= 2; \ + } \ + } \ + } while (0) + + ADJUST_HYDRO((-1), 0, 0, x, y, z); + ADJUST_HYDRO(0, (-1), 0, y, z, x); + ADJUST_HYDRO(0, 0, (-1), z, x, y); + ADJUST_HYDRO(1, 0, 0, x, y, z); + ADJUST_HYDRO(0, 1, 0, y, z, x); + ADJUST_HYDRO(0, 0, 1, z, x, y); + +#undef ADJUST_HYDRO + +#define BEGIN_RECV(i, j, k, X, Y, Z) \ + begin_recv_port(i, j, k, (1 + 14 * (n##Y + 1) * (n##Z + 1)) * sizeof(float), g) + +#define BEGIN_SEND(i, j, k, X, Y, Z) \ + BEGIN_PRIMITIVE \ + { \ + size = (1 + 14 * (n##Y + 1) * (n##Z + 1)) * sizeof(float); \ + p = (float *)size_send_port(i, j, k, size, g); \ + if (p) \ + { \ + (*(p++)) = g->d##X; \ + face = (i + j + k) < 0 ? 1 : n##X + 1; \ + X##_NODE_LOOP(face) \ + { \ + h = &hydro(x, y, z); \ + (*(p++)) = h->jx; \ + (*(p++)) = h->jy; \ + (*(p++)) = h->jz; \ + (*(p++)) = h->rho; \ + (*(p++)) = h->px; \ + (*(p++)) = h->py; \ + (*(p++)) = h->pz; \ + (*(p++)) = h->ke; \ + (*(p++)) = h->txx; \ + (*(p++)) = h->tyy; \ + (*(p++)) = h->tzz; \ + (*(p++)) = h->tyz; \ + (*(p++)) = h->tzx; \ + (*(p++)) = h->txy; \ + } \ + begin_send_port(i, j, k, size, g); \ + } \ + } \ + END_PRIMITIVE + +#define END_RECV(i, j, k, X, Y, Z) \ + BEGIN_PRIMITIVE \ + { \ + p = (float *)end_recv_port(i, j, k, g); \ + if (p) \ + { \ + rw = (*(p++)); /* Remote g->d##X */ \ + lw = rw + g->d##X; \ + rw /= lw; \ + lw = g->d##X / lw; \ + lw += lw; \ + rw += rw; \ + face = (i + j + k) < 0 ? n##X + 1 : 1; /* Twice weighted sum */ \ + X##_NODE_LOOP(face) \ + { \ + h = &hydro(x, y, z); \ + h->jx = lw * h->jx + rw * (*(p++)); \ + h->jy = lw * h->jy + rw * (*(p++)); \ + h->jz = lw * h->jz + rw * (*(p++)); \ + h->rho = lw * h->rho + rw * (*(p++)); \ + h->px = lw * h->px + rw * (*(p++)); \ + h->py = lw * h->py + rw * (*(p++)); \ + h->pz = lw * h->pz + rw * (*(p++)); \ + h->ke = lw * h->ke + rw * (*(p++)); \ + h->txx = lw * h->txx + rw * (*(p++)); \ + h->tyy = lw * h->tyy + rw * (*(p++)); \ + h->tzz = lw * h->tzz + rw * (*(p++)); \ + h->tyz = lw * h->tyz + rw * (*(p++)); \ + h->tzx = lw * h->tzx + rw * (*(p++)); \ + h->txy = lw * h->txy + rw * (*(p++)); \ + } \ + } \ + } \ + END_PRIMITIVE + +#define END_SEND(i, j, k, X, Y, Z) end_send_port(i, j, k, g) // Exchange x-faces. - BEGIN_SEND( (-1), 0, 0, x, y, z ); - BEGIN_SEND( 1, 0, 0, x, y, z ); - BEGIN_RECV( (-1), 0, 0, x, y, z ); - BEGIN_RECV( 1, 0, 0, x, y, z ); + BEGIN_SEND((-1), 0, 0, x, y, z); + BEGIN_SEND(1, 0, 0, x, y, z); + BEGIN_RECV((-1), 0, 0, x, y, z); + BEGIN_RECV(1, 0, 0, x, y, z); - END_RECV( (-1), 0, 0, x, y, z ); - END_RECV( 1, 0, 0, x, y, z ); - END_SEND( (-1), 0, 0, x, y, z ); - END_SEND( 1, 0, 0, x, y, z ); + END_RECV((-1), 0, 0, x, y, z); + END_RECV(1, 0, 0, x, y, z); + END_SEND((-1), 0, 0, x, y, z); + END_SEND(1, 0, 0, x, y, z); // Exchange y-faces. - BEGIN_SEND( 0, (-1), 0, y, z, x ); - BEGIN_SEND( 0, 1, 0, y, z, x ); - BEGIN_RECV( 0, (-1), 0, y, z, x ); - BEGIN_RECV( 0, 1, 0, y, z, x ); + BEGIN_SEND(0, (-1), 0, y, z, x); + BEGIN_SEND(0, 1, 0, y, z, x); + BEGIN_RECV(0, (-1), 0, y, z, x); + BEGIN_RECV(0, 1, 0, y, z, x); - END_RECV( 0, (-1), 0, y, z, x ); - END_RECV( 0, 1, 0, y, z, x ); - END_SEND( 0, (-1), 0, y, z, x ); - END_SEND( 0, 1, 0, y, z, x ); + END_RECV(0, (-1), 0, y, z, x); + END_RECV(0, 1, 0, y, z, x); + END_SEND(0, (-1), 0, y, z, x); + END_SEND(0, 1, 0, y, z, x); // Exchange z-faces. - BEGIN_SEND( 0, 0, (-1), z, x, y ); - BEGIN_SEND( 0, 0, 1, z, x, y ); - BEGIN_RECV( 0, 0, (-1), z, x, y ); - BEGIN_RECV( 0, 0, 1, z, x, y ); + BEGIN_SEND(0, 0, (-1), z, x, y); + BEGIN_SEND(0, 0, 1, z, x, y); + BEGIN_RECV(0, 0, (-1), z, x, y); + BEGIN_RECV(0, 0, 1, z, x, y); - END_RECV( 0, 0, (-1), z, x, y ); - END_RECV( 0, 0, 1, z, x, y ); - END_SEND( 0, 0, (-1), z, x, y ); - END_SEND( 0, 0, 1, z, x, y ); + END_RECV(0, 0, (-1), z, x, y); + END_RECV(0, 0, 1, z, x, y); + END_SEND(0, 0, (-1), z, x, y); + END_SEND(0, 0, 1, z, x, y); - #undef BEGIN_RECV - #undef BEGIN_SEND - #undef END_RECV - #undef END_SEND +#undef BEGIN_RECV +#undef BEGIN_SEND +#undef END_RECV +#undef END_SEND } diff --git a/src/sf_interface/sf_interface.h b/src/sf_interface/sf_interface.h index 7202dc78..f96f1c79 100644 --- a/src/sf_interface/sf_interface.h +++ b/src/sf_interface/sf_interface.h @@ -28,19 +28,19 @@ defined(USE_V16_AVX512) #define PAD_SIZE_INTERPOLATOR 14 -#define PAD_SIZE_ACCUMULATOR 4 -#define PAD_SIZE_HYDRO 2 +#define PAD_SIZE_ACCUMULATOR 4 +#define PAD_SIZE_HYDRO 2 //----------------------------------------------------------------------------// // 32-byte align #elif defined(USE_V8_PORTABLE) || \ - defined(USE_V8_AVX) || \ - defined(USE_V8_AVX2) + defined(USE_V8_AVX) || \ + defined(USE_V8_AVX2) #define PAD_SIZE_INTERPOLATOR 6 -#define PAD_SIZE_ACCUMULATOR 4 -#define PAD_SIZE_HYDRO 2 +#define PAD_SIZE_ACCUMULATOR 4 +#define PAD_SIZE_HYDRO 2 //----------------------------------------------------------------------------// // 16-byte align @@ -48,7 +48,7 @@ #else #define PAD_SIZE_INTERPOLATOR 2 -#define PAD_SIZE_HYDRO 2 +#define PAD_SIZE_HYDRO 2 #endif @@ -75,8 +75,8 @@ typedef struct interpolator typedef struct interpolator_array { - interpolator_t * ALIGNED(128) i; - grid_t * g; + interpolator_t *ALIGNED(128) i; + grid_t *g; } interpolator_array_t; BEGIN_C_DECLS @@ -84,10 +84,9 @@ BEGIN_C_DECLS // In interpolator_array.cc interpolator_array_t * -new_interpolator_array( grid_t * g ); +new_interpolator_array(grid_t *g); -void -delete_interpolator_array( interpolator_array_t * ALIGNED(128) ia ); +void delete_interpolator_array(interpolator_array_t *ALIGNED(128) ia); // Going into load_interpolator, the field array f contains the // current information such that the fields can be interpolated to @@ -96,9 +95,8 @@ delete_interpolator_array( interpolator_array_t * ALIGNED(128) ia ); // inside the local domain suitable for use by the particle update // functions. -void -load_interpolator_array( /**/ interpolator_array_t * RESTRICT ia, - const field_array_t * RESTRICT fa ); +void load_interpolator_array(/**/ interpolator_array_t *RESTRICT ia, + const field_array_t *RESTRICT fa); END_C_DECLS @@ -114,20 +112,20 @@ END_C_DECLS typedef struct accumulator { - float jx[4]; // jx0@(0,-1,-1),jx1@(0,1,-1),jx2@(0,-1,1),jx3@(0,1,1) - float jy[4]; // jy0@(-1,0,-1),jy1@(-1,0,1),jy2@(1,0,-1),jy3@(1,0,1) - float jz[4]; // jz0@(-1,-1,0),jz1@(1,-1,0),jz2@(-1,1,0),jz3@(1,1,0) - #if defined PAD_SIZE_ACCUMULATOR + float jx[4]; // jx0@(0,-1,-1),jx1@(0,1,-1),jx2@(0,-1,1),jx3@(0,1,1) + float jy[4]; // jy0@(-1,0,-1),jy1@(-1,0,1),jy2@(1,0,-1),jy3@(1,0,1) + float jz[4]; // jz0@(-1,-1,0),jz1@(1,-1,0),jz2@(-1,1,0),jz3@(1,1,0) +#if defined PAD_SIZE_ACCUMULATOR float pad2[PAD_SIZE_ACCUMULATOR]; // Padding for 32 and 64-byte align - #endif +#endif } accumulator_t; typedef struct accumulator_array { - accumulator_t * ALIGNED(128) a; + accumulator_t *ALIGNED(128) a; int n_pipeline; // Number of pipelines supported by this accumulator int stride; // Stride be each pipeline's accumulator array - grid_t * g; + grid_t *g; } accumulator_array_t; BEGIN_C_DECLS @@ -135,17 +133,15 @@ BEGIN_C_DECLS // In accumulator_array.cc accumulator_array_t * -new_accumulator_array( grid_t * g ); +new_accumulator_array(grid_t *g); -void -delete_accumulator_array( accumulator_array_t * a ); +void delete_accumulator_array(accumulator_array_t *a); // In clear_array.cc // This zeros out all the accumulator arrays in a pipelined fashion. -void -clear_accumulator_array( accumulator_array_t * RESTRICT a ); +void clear_accumulator_array(accumulator_array_t *RESTRICT a); // In reduce_array.cc @@ -155,8 +151,7 @@ clear_accumulator_array( accumulator_array_t * RESTRICT a ); // accumulator with a pipelined horizontal reduction (a deterministic // reduction). -void -reduce_accumulator_array( accumulator_array_t * RESTRICT a ); +void reduce_accumulator_array(accumulator_array_t *RESTRICT a); // In unload_accumulator.cc @@ -169,9 +164,8 @@ reduce_accumulator_array( accumulator_array_t * RESTRICT a ); // local field array jf. unload_accumulator assumes all the pipeline // accumulators have been reduced into the host accumulator. -void -unload_accumulator_array( /**/ field_array_t * RESTRICT fa, - const accumulator_array_t * RESTRICT aa ); +void unload_accumulator_array(/**/ field_array_t *RESTRICT fa, + const accumulator_array_t *RESTRICT aa); END_C_DECLS @@ -184,19 +178,19 @@ END_C_DECLS typedef struct hydro { - float jx, jy, jz, rho; // Current and charge density => , - float px, py, pz, ke; // Momentum and K.E. density => , - float txx, tyy, tzz; // Stress diagonal => , i==j - float tyz, tzx, txy; // Stress off-diagonal => , i!=j + float jx, jy, jz, rho; // Current and charge density => , + float px, py, pz, ke; // Momentum and K.E. density => , + float txx, tyy, tzz; // Stress diagonal => , i==j + float tyz, tzx, txy; // Stress off-diagonal => , i!=j float _pad[PAD_SIZE_HYDRO]; // 16, 32 and 64-byte align } hydro_t; typedef struct hydro_array { - hydro_t * ALIGNED(128) h; + hydro_t *ALIGNED(128) h; int n_pipeline; // Number of pipelines supported by this hydro int stride; // Stride be each pipeline's hydro array - grid_t * g; + grid_t *g; } hydro_array_t; BEGIN_C_DECLS @@ -206,20 +200,18 @@ BEGIN_C_DECLS // Construct a hydro array suitable for the grid hydro_array_t * -new_hydro_array( grid_t * g ); +new_hydro_array(grid_t *g); // Destruct a hydro array -void -delete_hydro_array( hydro_array_t * ha ); +void delete_hydro_array(hydro_array_t *ha); // In clear_array.cc // Zero out the hydro array. Use before accumulating species to // a hydro array. -void -clear_hydro_array( hydro_array_t * ha ); +void clear_hydro_array(hydro_array_t *ha); // In reduce_array.cc @@ -227,8 +219,7 @@ clear_hydro_array( hydro_array_t * ha ); // synchronize_hydro_array and does not typically need to be otherwise // called. -void -reduce_hydro_array( hydro_array_t * ha ); +void reduce_hydro_array(hydro_array_t *ha); // In hydro_array.cc @@ -236,8 +227,7 @@ reduce_hydro_array( hydro_array_t * ha ); // the hydro array with local boundary conditions and neighboring // processes. Use after all species have been accumulated to the hydro array. -void -synchronize_hydro_array( hydro_array_t * ha ); +void synchronize_hydro_array(hydro_array_t *ha); END_C_DECLS diff --git a/src/species_advance/standard/pipeline/hydro_p_pipeline_v16.cc b/src/species_advance/standard/pipeline/hydro_p_pipeline_v16.cc index a984ce55..de088bec 100644 --- a/src/species_advance/standard/pipeline/hydro_p_pipeline_v16.cc +++ b/src/species_advance/standard/pipeline/hydro_p_pipeline_v16.cc @@ -170,10 +170,6 @@ accumulate_hydro_p_pipeline_v16( accumulate_hydro_p_pipeline_args_t * args, uy = fma( fms( v02, cbx, v00 * cbz ), v04, uy ); uz = fma( fms( v00, cby, v01 * cbx ), v04, uz ); - ux += hax; - uy += hay; - uz += haz; - //-------------------------------------------------------------------------- // Compute velocity. //-------------------------------------------------------------------------- diff --git a/src/species_advance/standard/pipeline/hydro_p_pipeline_v4.cc b/src/species_advance/standard/pipeline/hydro_p_pipeline_v4.cc index 9c82eee3..4cf5b953 100644 --- a/src/species_advance/standard/pipeline/hydro_p_pipeline_v4.cc +++ b/src/species_advance/standard/pipeline/hydro_p_pipeline_v4.cc @@ -153,10 +153,6 @@ accumulate_hydro_p_pipeline_v4( accumulate_hydro_p_pipeline_args_t * args, uy = fma( fms( v02, cbx, v00 * cbz ), v04, uy ); uz = fma( fms( v00, cby, v01 * cbx ), v04, uz ); - ux += hax; - uy += hay; - uz += haz; - //-------------------------------------------------------------------------- // Compute velocity. //-------------------------------------------------------------------------- diff --git a/src/species_advance/standard/pipeline/hydro_p_pipeline_v8.cc b/src/species_advance/standard/pipeline/hydro_p_pipeline_v8.cc index 27ca3714..3dbbc813 100644 --- a/src/species_advance/standard/pipeline/hydro_p_pipeline_v8.cc +++ b/src/species_advance/standard/pipeline/hydro_p_pipeline_v8.cc @@ -151,10 +151,6 @@ accumulate_hydro_p_pipeline_v8( accumulate_hydro_p_pipeline_args_t * args, uy = fma( fms( v02, cbx, v00 * cbz ), v04, uy ); uz = fma( fms( v00, cby, v01 * cbx ), v04, uz ); - ux += hax; - uy += hay; - uz += haz; - //-------------------------------------------------------------------------- // Compute velocity. //-------------------------------------------------------------------------- diff --git a/src/vpic/dump.cc b/src/vpic/dump.cc index efc7d4a9..ed9fc64d 100644 --- a/src/vpic/dump.cc +++ b/src/vpic/dump.cc @@ -9,22 +9,18 @@ * snell - revised to add strided dumps, time history dumps, others 20080404 */ +#include +#include + +#include "dumpmacros.h" #include "vpic.h" #include "../util/io/FileUtils.h" -// FIXME: This implies we want to be merged into a branch that supports HDF5 -#ifdef VPIC_ENABLE_HDF5 -#include "hdf5.h" -#endif - -// FIXME: Should this be user facing? At run time or cmake? Or do we just make -// a decision in a meeting at some point? -/* 1 means convert cell index to global */ -#define OUTPUT_CONVERT_GLOBAL_ID 1 - /* -1 means no ranks talk */ #define VERBOSE_rank -1 +#define OUTPUT_CONVERT_GLOBAL_ID 1 + #ifdef VPIC_PARTICLE_ANNOTATION typedef VPIC_PARTICLE_ANNOTATION annotation_t; #endif @@ -33,406 +29,258 @@ typedef VPIC_PARTICLE_ANNOTATION annotation_t; // COMPATIBLE WITH EXISTING EXTERNAL 3RD PARTY VISUALIZATION SOFTWARE. // IN THE LONG RUN, THIS EXTERNAL SOFTWARE WILL NEED TO BE UPDATED. -int vpic_simulation::dump_mkdir(const char * dname) { - return FileUtils::makeDirectory(dname); +std::array global_particle_index(int local_i, grid_t *grid, int rank) +{ + int ix, iy, iz, rx, ry, rz; + // Convert rank to local x/y/z + UNVOXEL(rank, rx, ry, rz, grid->gpx, grid->gpy, grid->gpz); + + // Calculate local ix/iy/iz + UNVOXEL(local_i, ix, iy, iz, grid->nx + 2, grid->ny + 2, grid->nz + 2); + + // Account for the "first" ghost cell + ix = ix - 1; + iy = iy - 1; + iz = iz - 1; + + // Convert ix/iy/iz to global + int gix = ix + (grid->nx * (rx)); + int giy = iy + (grid->ny * (ry)); + int giz = iz + (grid->nz * (rz)); + + // calculate global grid sizes + int gnx = grid->nx * grid->gpx; + int gny = grid->ny * grid->gpy; + int gnz = grid->nz * grid->gpz; + + // TODO: find a better way to account for the hard coded ghosts in VOXEL + int global_i = VOXEL(gix, giy, giz, gnx - 2, gny - 2, gnz - 2); + + return {global_i, gix, giy, giz}; +} + +int vpic_simulation::dump_mkdir(const char *dname) +{ + return FileUtils::makeDirectory(dname); } // dump_mkdir -int vpic_simulation::dump_cwd(char * dname, size_t size) { - return FileUtils::getCurrentWorkingDirectory(dname, size); +int vpic_simulation::dump_cwd(char *dname, size_t size) +{ + return FileUtils::getCurrentWorkingDirectory(dname, size); } // dump_mkdir /***************************************************************************** * ASCII dump IO *****************************************************************************/ -// Do not even define these functions when there are no IDs so that input decks that rely on them don't compile. -#ifdef VPIC_GLOBAL_PARTICLE_ID -int vpic_simulation::predicate_count(species_t* sp, std::function f) +void vpic_simulation::enable_binary_dump() { - if ((f == nullptr) || (!sp->has_ids)) { - return sp->np; - } else { - return std::count_if( sp->p_id, sp->p_id + sp->np, f); - } + // dump_strategy = std::unique_ptr(new BinaryDump( rank(), nproc() )); + // dump_strategy = new BinaryDump(rank(), nproc()); + dump_strategy_id = DUMP_STRATEGY_BINARY; } -int vpic_simulation::predicate_count(species_t* sp, std::function f) + +#ifdef VPIC_ENABLE_HDF5 +void vpic_simulation::enable_hdf5_dump() { - if (f != nullptr) return std::count_if( sp->p, sp->p + sp->np, f); - else return sp->np; + if (rank() == 0) + std::cout << "Enabling HDF5 IO backend" << std::endl; + // dump_strategy = std::unique_ptr(new HDF5Dump(rank(), nproc())); + // dump_strategy = new HDF5Dump(rank(), nproc()); + dump_strategy_id = DUMP_STRATEGY_HDF5; } +#endif -void vpic_simulation::predicate_copy(species_t* sp_from, species_t* sp_to, std::function f) +#ifdef VPIC_ENABLE_OPENPMD +void vpic_simulation::enable_openpmd_dump() { - if (f != nullptr) { - // std::copy_if( sp_from->p, sp_from->p + sp_from->np, sp_to->p , f); - // std::copy_if( sp_from->p_id, sp_from->p_id + sp_from->np, sp_to->p_id, f); - // Manually loop over particles to do the 'cross copy' of p_id as well - - int next = 0; // track where we fill - for (int i = 0; i < sp_from->np; i++) - { - if ( f(sp_from->p[i]) ) - { - // copy i (inherently serial..) - sp_to->p[next] = sp_from->p[i]; - if(sp_from->has_ids && sp_to->has_ids) { - sp_to->p_id[next] = sp_from->p_id[i]; - } - #ifdef VPIC_PARTICLE_ANNOTATION - if(sp_from->has_annotation && sp_to->has_annotation) { - for(int a = 0; a < sp_from->has_annotation && a < sp_to->has_annotation; a++) { - sp_to->p_annotation[next*sp_to->has_annotation + a] = sp_from->p_annotation[next*sp_from->has_annotation + a]; - } - } - #endif - next++; - } - - } - std::cout << "copied " << next << std::endl; - } + if (rank() == 0) + std::cout << "Enabling openPMD IO backend" << std::endl; + // dump_strategy = std::unique_ptr(new OpenPMDDump(rank(), nproc())); + // dump_strategy = new OpenPMDDump(rank(), nproc()); + dump_strategy_id = DUMP_STRATEGY_OPENPMD; } -void vpic_simulation::predicate_copy(species_t* sp_from, species_t* sp_to, std::function f) +#endif + +void vpic_simulation::dump_particles(const char *sp_name, + const char *fbase, + int ftag) { - if ((f != nullptr) && (sp_from->has_ids) ) - { - //std::copy_if( sp->p_id, sp->p_id + sp->np, _sp.p, f); - // Manually loop over particles to do the 'cross copy' from p_id->p + species_t *sp = find_species_name(sp_name, species_list); + dump_strategy->dump_particles( + fbase, + sp, + grid, + step(), + interpolator_array, + ftag); +} - int next = 0; // track where we fill - for (int i = 0; i < sp_from->np; i++) - { - int this_id = sp_from->p_id[i]; - if ( f(this_id) ) - { - // copy i (inherently serial..) - sp_to->p[next] = sp_from->p[i]; - sp_to->p_id[next] = sp_from->p_id[i]; - #ifdef VPIC_PARTICLE_ANNOTATION - if(sp_from->has_annotation && sp_to->has_annotation) { - for(int a = 0; a < sp_from->has_annotation && a < sp_to->has_annotation; a++) { - sp_to->p_annotation[next*sp_to->has_annotation + a] = sp_from->p_annotation[next*sp_from->has_annotation + a]; - } - } - #endif - next++; - } +void vpic_simulation::dump_fields(const char *fbase, int ftag) +{ + dump_strategy->dump_fields( + fbase, + step(), grid, + field_array, + ftag); +} - } - std::cout << "copied " << next << std::endl; - } +void vpic_simulation::dump_hydro(const char *sp_name, const char *fbase, int ftag) +{ + species_t *sp = find_species_name(sp_name, species_list); + dump_strategy->dump_hydro( + fbase, + step(), + hydro_array, + sp, + interpolator_array, + grid, + ftag); } -#endif -void -vpic_simulation::dump_energies( const char *fname, - int append ) { +void vpic_simulation::dump_energies(const char *fname, + int append) +{ double en_f[6], en_p; species_t *sp; FileIO fileIO; FileIOStatus status(fail); - if( !fname ) ERROR(("Invalid file name")); + if (!fname) + ERROR(("Invalid file name")); - if( rank()==0 ) { + if (rank() == 0) + { status = fileIO.open(fname, append ? io_append : io_write); - if( status==fail ) ERROR(( "Could not open \"%s\".", fname )); - else { - if( append==0 ) { - fileIO.print( "%% Layout\n%% step ex ey ez bx by bz" ); - LIST_FOR_EACH(sp,species_list) - fileIO.print( " \"%s\"", sp->name ); - fileIO.print( "\n" ); - fileIO.print( "%% timestep = %e\n", grid->dt ); + if (status == fail) + ERROR(("Could not open \"%s\".", fname)); + else + { + if (append == 0) + { + fileIO.print("%% Layout\n%% step ex ey ez bx by bz"); + LIST_FOR_EACH(sp, species_list) + fileIO.print(" \"%s\"", sp->name); + fileIO.print("\n"); + fileIO.print("%% timestep = %e\n", grid->dt); } - fileIO.print( "%li ", (long)step() ); + fileIO.print("%li ", (long)step()); } } - field_array->kernel->energy_f( en_f, field_array ); - if( rank()==0 && status!=fail ) - fileIO.print( "%e %e %e %e %e %e", - en_f[0], en_f[1], en_f[2], - en_f[3], en_f[4], en_f[5] ); + field_array->kernel->energy_f(en_f, field_array); + if (rank() == 0 && status != fail) + fileIO.print("%e %e %e %e %e %e", + en_f[0], en_f[1], en_f[2], + en_f[3], en_f[4], en_f[5]); - LIST_FOR_EACH(sp,species_list) { - en_p = energy_p( sp, interpolator_array ); - if( rank()==0 && status!=fail ) fileIO.print( " %e", en_p ); + LIST_FOR_EACH(sp, species_list) + { + en_p = energy_p(sp, interpolator_array); + if (rank() == 0 && status != fail) + fileIO.print(" %e", en_p); } - if( rank()==0 && status!=fail ) { - fileIO.print( "\n" ); - if( fileIO.close() ) ERROR(("File close failed on dump energies!!!")); + if (rank() == 0 && status != fail) + { + fileIO.print("\n"); + if (fileIO.close()) + ERROR(("File close failed on dump energies!!!")); } } // Note: dump_species/materials assume that names do not contain any \n! -void -vpic_simulation::dump_species( const char *fname ) { +void vpic_simulation::dump_species(const char *fname) +{ species_t *sp; FileIO fileIO; - if( rank() ) return; - if( !fname ) ERROR(( "Invalid file name" )); - MESSAGE(( "Dumping species to \"%s\"", fname )); + if (rank()) + return; + if (!fname) + ERROR(("Invalid file name")); + MESSAGE(("Dumping species to \"%s\"", fname)); FileIOStatus status = fileIO.open(fname, io_write); - if( status==fail ) ERROR(( "Could not open \"%s\".", fname )); - LIST_FOR_EACH( sp, species_list ) - fileIO.print( "%s %i %e %e", sp->name, sp->id, sp->q, sp->m ); - if( fileIO.close() ) ERROR(( "File close failed on dump species!!!" )); + if (status == fail) + ERROR(("Could not open \"%s\".", fname)); + LIST_FOR_EACH(sp, species_list) + fileIO.print("%s %i %e %e", sp->name, sp->id, sp->q, sp->m); + if (fileIO.close()) + ERROR(("File close failed on dump species!!!")); } -void -vpic_simulation::dump_materials( const char *fname ) { +void vpic_simulation::dump_materials(const char *fname) +{ FileIO fileIO; material_t *m; - if( rank() ) return; - if( !fname ) ERROR(( "Invalid file name" )); - MESSAGE(( "Dumping materials to \"%s\"", fname )); + if (rank()) + return; + if (!fname) + ERROR(("Invalid file name")); + MESSAGE(("Dumping materials to \"%s\"", fname)); FileIOStatus status = fileIO.open(fname, io_write); - if( status==fail ) ERROR(( "Could not open \"%s\"", fname )); - LIST_FOR_EACH( m, material_list ) - fileIO.print( "%s\n%i\n%e %e %e\n%e %e %e\n%e %e %e\n", - m->name, m->id, - m->epsx, m->epsy, m->epsz, - m->mux, m->muy, m->muz, - m->sigmax, m->sigmay, m->sigmaz ); - if( fileIO.close() ) ERROR(( "File close failed on dump materials!!!" )); + if (status == fail) + ERROR(("Could not open \"%s\"", fname)); + LIST_FOR_EACH(m, material_list) + fileIO.print("%s\n%i\n%e %e %e\n%e %e %e\n%e %e %e\n", + m->name, m->id, + m->epsx, m->epsy, m->epsz, + m->mux, m->muy, m->muz, + m->sigmax, m->sigmay, m->sigmaz); + if (fileIO.close()) + ERROR(("File close failed on dump materials!!!")); } /***************************************************************************** * Binary dump IO *****************************************************************************/ -/* -enum dump_types { - grid_dump = 0, - field_dump = 1, - hydro_dump = 2, - particle_dump = 3, - restart_dump = 4 -}; -*/ - -void -vpic_simulation::dump_grid( const char *fbase ) { +void vpic_simulation::dump_grid(const char *fbase) +{ char fname[256]; FileIO fileIO; int dim[4]; - if( !fbase ) ERROR(( "Invalid filename" )); - if( rank()==0 ) MESSAGE(( "Dumping grid to \"%s\"", fbase )); + if (!fbase) + ERROR(("Invalid filename")); + if (rank() == 0) + MESSAGE(("Dumping grid to \"%s\"", fbase)); - sprintf( fname, "%s.%i", fbase, rank() ); + sprintf(fname, "%s.%i", fbase, rank()); FileIOStatus status = fileIO.open(fname, io_write); - if( status==fail ) ERROR(( "Could not open \"%s\".", fname )); + if (status == fail) + ERROR(("Could not open \"%s\".", fname)); /* IMPORTANT: these values are written in WRITE_HEADER_V0 */ - nxout = grid->nx; - nyout = grid->ny; - nzout = grid->nz; - dxout = grid->dx; - dyout = grid->dy; - dzout = grid->dz; + size_t nxout = grid->nx; + size_t nyout = grid->ny; + size_t nzout = grid->nz; + float dxout = grid->dx; + float dyout = grid->dy; + float dzout = grid->dz; - WRITE_HEADER_V0( dump_type::grid_dump, -1, 0, step(), fileIO ); + WRITE_HEADER_V0(dump_type::grid_dump, -1, 0, fileIO, step(), rank(), nproc()); dim[0] = 3; dim[1] = 3; dim[2] = 3; - WRITE_ARRAY_HEADER( grid->bc, 3, dim, fileIO ); - fileIO.write( grid->bc, dim[0]*dim[1]*dim[2] ); + WRITE_ARRAY_HEADER(grid->bc, 3, dim, fileIO); + fileIO.write(grid->bc, dim[0] * dim[1] * dim[2]); - dim[0] = nproc()+1; - WRITE_ARRAY_HEADER( grid->range, 1, dim, fileIO ); - fileIO.write( grid->range, dim[0] ); + dim[0] = nproc() + 1; + WRITE_ARRAY_HEADER(grid->range, 1, dim, fileIO); + fileIO.write(grid->range, dim[0]); dim[0] = 6; - dim[1] = grid->nx+2; - dim[2] = grid->ny+2; - dim[3] = grid->nz+2; - WRITE_ARRAY_HEADER( grid->neighbor, 4, dim, fileIO ); - fileIO.write( grid->neighbor, dim[0]*dim[1]*dim[2]*dim[3] ); - - if( fileIO.close() ) ERROR(( "File close failed on dump grid!!!" )); -} - -void -vpic_simulation::dump_fields( const char *fbase, - int ftag, - field_t *f ) -{ - char fname[256]; - FileIO fileIO; - int dim[3]; - - if( !fbase ) ERROR(( "Invalid filename" )); - - if( rank()==0 ) MESSAGE(( "Dumping fields to \"%s\"", fbase )); - - if( ftag ) sprintf( fname, "%s.%li.%i", fbase, (long)step(), rank() ); - else sprintf( fname, "%s.%i", fbase, rank() ); - - FileIOStatus status = fileIO.open(fname, io_write); - if( status==fail ) ERROR(( "Could not open \"%s\".", fname )); - - // default is to use VPIC native field data - if ( f == NULL ) f = field_array->f; - - /* IMPORTANT: these values are written in WRITE_HEADER_V0 */ - nxout = grid->nx; - nyout = grid->ny; - nzout = grid->nz; - dxout = grid->dx; - dyout = grid->dy; - dzout = grid->dz; - - WRITE_HEADER_V0( dump_type::field_dump, -1, 0, step(), fileIO ); - - dim[0] = grid->nx+2; - dim[1] = grid->ny+2; - dim[2] = grid->nz+2; - WRITE_ARRAY_HEADER( f, 3, dim, fileIO ); - fileIO.write( f, dim[0]*dim[1]*dim[2] ); - if( fileIO.close() ) ERROR(( "File close failed on dump fields." )); -} - -void -vpic_simulation::dump_hydro( const char *sp_name, - const char *fbase, - int ftag, - hydro_t *h ) -{ - species_t *sp; - char fname[256]; - FileIO fileIO; - int dim[3]; - - sp = find_species_name( sp_name, species_list ); - if( !sp ) ERROR(( "Invalid species \"%s\"", sp_name )); - - // default behavior - do regular VPIC hydro dump - - if ( h == NULL) - { - h = hydro_array->h; - clear_hydro_array( hydro_array ); - accumulate_hydro_p( hydro_array, sp, interpolator_array ); - synchronize_hydro_array( hydro_array ); - } - - if( !fbase ) ERROR(( "Invalid filename" )); - - if( rank()==0 ) - MESSAGE(("Dumping \"%s\" hydro fields to \"%s\"",sp->name,fbase)); - - if( ftag ) sprintf( fname, "%s.%li.%i", fbase, (long)step(), rank() ); - else sprintf( fname, "%s.%i", fbase, rank() ); - FileIOStatus status = fileIO.open(fname, io_write); - if( status==fail) ERROR(( "Could not open \"%s\".", fname )); - - /* IMPORTANT: these values are written in WRITE_HEADER_V0 */ - nxout = grid->nx; - nyout = grid->ny; - nzout = grid->nz; - dxout = grid->dx; - dyout = grid->dy; - dzout = grid->dz; - - WRITE_HEADER_V0( dump_type::hydro_dump, sp->id, sp->q / sp->m, step(), fileIO ); - - dim[0] = grid->nx+2; - dim[1] = grid->ny+2; - dim[2] = grid->nz+2; - WRITE_ARRAY_HEADER( h, 3, dim, fileIO ); - fileIO.write( h, dim[0]*dim[1]*dim[2] ); - if( fileIO.close() ) ERROR(( "File close failed on dump hydro." )); -} - -void -vpic_simulation::dump_particles( const char *sp_name, - const char *fbase, - int ftag ) -{ - species_t *sp; - char fname[256]; - FileIO fileIO; - int dim[2], buf_start; - static particle_t * ALIGNED(128) p_buf = NULL; -# define PBUF_SIZE 32768 // 1MB of particles - - sp = find_species_name( sp_name, species_list ); - if( !sp ) ERROR(( "Invalid species name \"%s\".", sp_name )); - - if( !fbase ) ERROR(( "Invalid filename" )); - - if( !p_buf ) MALLOC_ALIGNED( p_buf, PBUF_SIZE, 128 ); - - if( rank()==0 ) - MESSAGE(("Dumping \"%s\" particles to \"%s\"",sp->name,fbase)); - - if( ftag ) sprintf( fname, "%s.%li.%i", fbase, (long)step(), rank() ); - else sprintf( fname, "%s.%i", fbase, rank() ); - FileIOStatus status = fileIO.open(fname, io_write); - if( status==fail ) ERROR(( "Could not open \"%s\"", fname )); - - /* IMPORTANT: these values are written in WRITE_HEADER_V0 */ - nxout = grid->nx; - nyout = grid->ny; - nzout = grid->nz; - dxout = grid->dx; - dyout = grid->dy; - dzout = grid->dz; - - WRITE_HEADER_V0( dump_type::particle_dump, sp->id, sp->q / sp->m, step(), fileIO ); - - dim[0] = sp->np; - WRITE_ARRAY_HEADER( p_buf, 1, dim, fileIO ); - - // Copy a PBUF_SIZE hunk of the particle list into the particle - // buffer, timecenter it and write it out. This is done this way to - // guarantee the particle list unchanged while not requiring too - // much memory. - - // FIXME: WITH A PIPELINED CENTER_P, PBUF NOMINALLY SHOULD BE QUITE - // LARGE. - - particle_t * sp_p = sp->p; sp->p = p_buf; - int sp_np = sp->np; sp->np = 0; - int sp_max_np = sp->max_np; sp->max_np = PBUF_SIZE; - for( buf_start=0; buf_startnp = sp_np-buf_start; if( sp->np > PBUF_SIZE ) sp->np = PBUF_SIZE; - COPY( sp->p, &sp_p[buf_start], sp->np ); - center_p( sp, interpolator_array ); - fileIO.write( sp->p, sp->np ); - } - sp->p = sp_p; - sp->np = sp_np; - sp->max_np = sp_max_np; - - #ifdef VPIC_GLOBAL_PARTICLE_ID - // append ID array at the end of the file - if(sp->has_ids) { - dim[0] = sp->np; - WRITE_ARRAY_HEADER( sp->p_id, 1, dim, fileIO ); - // Maybe do this write in batched of PBUF_SIZE as well? - fileIO.write(sp->p_id, sp->np); - } - #endif - #ifdef VPIC_PARTICLE_ANNOTATION - // append annotation buffer at the end of the file - if(sp->has_annotation) { - dim[0] = sp->np; - dim[1] = sp->has_annotation; - WRITE_ARRAY_HEADER( sp->p_annotation, 2, dim, fileIO ); - // Maybe do this write in batched of PBUF_SIZE as well? - fileIO.write(sp->p_annotation, sp->np*sp->has_annotation); - } - #endif - - - if( fileIO.close() ) ERROR(("File close failed on dump particles!!!")); + dim[1] = grid->nx + 2; + dim[2] = grid->ny + 2; + dim[3] = grid->nz + 2; + WRITE_ARRAY_HEADER(grid->neighbor, 4, dim, fileIO); + fileIO.write(grid->neighbor, dim[0] * dim[1] * dim[2] * dim[3]); + + if (fileIO.close()) + ERROR(("File close failed on dump grid!!!")); } /*------------------------------------------------------------------------------ @@ -442,71 +290,77 @@ vpic_simulation::dump_particles( const char *sp_name, #include static FieldInfo fieldInfo[12] = { - { "Electric Field", "VECTOR", "3", "FLOATING_POINT", sizeof(float) }, - { "Electric Field Divergence Error", "SCALAR", "1", "FLOATING_POINT", - sizeof(float) }, - { "Magnetic Field", "VECTOR", "3", "FLOATING_POINT", sizeof(float) }, - { "Magnetic Field Divergence Error", "SCALAR", "1", "FLOATING_POINT", - sizeof(float) }, - { "TCA Field", "VECTOR", "3", "FLOATING_POINT", sizeof(float) }, - { "Bound Charge Density", "SCALAR", "1", "FLOATING_POINT", sizeof(float) }, - { "Free Current Field", "VECTOR", "3", "FLOATING_POINT", sizeof(float) }, - { "Charge Density", "SCALAR", "1", "FLOATING_POINT", sizeof(float) }, - { "Edge Material", "VECTOR", "3", "INTEGER", sizeof(material_id) }, - { "Node Material", "SCALAR", "1", "INTEGER", sizeof(material_id) }, - { "Face Material", "VECTOR", "3", "INTEGER", sizeof(material_id) }, - { "Cell Material", "SCALAR", "1", "INTEGER", sizeof(material_id) } -}; // fieldInfo + {"Electric Field", "VECTOR", "3", "FLOATING_POINT", sizeof(float)}, + {"Electric Field Divergence Error", "SCALAR", "1", "FLOATING_POINT", + sizeof(float)}, + {"Magnetic Field", "VECTOR", "3", "FLOATING_POINT", sizeof(float)}, + {"Magnetic Field Divergence Error", "SCALAR", "1", "FLOATING_POINT", + sizeof(float)}, + {"TCA Field", "VECTOR", "3", "FLOATING_POINT", sizeof(float)}, + {"Bound Charge Density", "SCALAR", "1", "FLOATING_POINT", sizeof(float)}, + {"Free Current Field", "VECTOR", "3", "FLOATING_POINT", sizeof(float)}, + {"Charge Density", "SCALAR", "1", "FLOATING_POINT", sizeof(float)}, + {"Edge Material", "VECTOR", "3", "INTEGER", sizeof(material_id)}, + {"Node Material", "SCALAR", "1", "INTEGER", sizeof(material_id)}, + {"Face Material", "VECTOR", "3", "INTEGER", sizeof(material_id)}, + {"Cell Material", "SCALAR", "1", "INTEGER", sizeof(material_id)}}; // fieldInfo static HydroInfo hydroInfo[5] = { - { "Current Density", "VECTOR", "3", "FLOATING_POINT", sizeof(float) }, - { "Charge Density", "SCALAR", "1", "FLOATING_POINT", sizeof(float) }, - { "Momentum Density", "VECTOR", "3", "FLOATING_POINT", sizeof(float) }, - { "Kinetic Energy Density", "SCALAR", "1", "FLOATING_POINT", - sizeof(float) }, - { "Stress Tensor", "TENSOR", "6", "FLOATING_POINT", sizeof(float) } - /* - { "STRESS_DIAGONAL", "VECTOR", "3", "FLOATING_POINT", sizeof(float) } - { "STRESS_OFFDIAGONAL", "VECTOR", "3", "FLOATING_POINT", sizeof(float) } - */ + {"Current Density", "VECTOR", "3", "FLOATING_POINT", sizeof(float)}, + {"Charge Density", "SCALAR", "1", "FLOATING_POINT", sizeof(float)}, + {"Momentum Density", "VECTOR", "3", "FLOATING_POINT", sizeof(float)}, + {"Kinetic Energy Density", "SCALAR", "1", "FLOATING_POINT", + sizeof(float)}, + {"Stress Tensor", "TENSOR", "6", "FLOATING_POINT", sizeof(float)} + /* + { "STRESS_DIAGONAL", "VECTOR", "3", "FLOATING_POINT", sizeof(float) } + { "STRESS_OFFDIAGONAL", "VECTOR", "3", "FLOATING_POINT", sizeof(float) } + */ }; // hydroInfo -void -vpic_simulation::create_field_list( char * strlist, - DumpParameters & dumpParams ) { +void vpic_simulation::create_field_list(char *strlist, + DumpParameters &dumpParams) +{ strcpy(strlist, ""); - for(size_t i(0), pass(0); i0 && pass) strcat(strlist, ", "); - else pass = 1; + for (size_t i(0), pass(0); i < total_field_groups; i++) + if (dumpParams.output_vars.bitset(field_indeces[i])) + { + if (i > 0 && pass) + strcat(strlist, ", "); + else + pass = 1; strcat(strlist, fieldInfo[i].name); } } -void -vpic_simulation::create_hydro_list( char * strlist, - DumpParameters & dumpParams ) { +void vpic_simulation::create_hydro_list(char *strlist, + DumpParameters &dumpParams) +{ strcpy(strlist, ""); - for(size_t i(0), pass(0); i0 && pass) strcat(strlist, ", "); - else pass = 1; + for (size_t i(0), pass(0); i < total_hydro_groups; i++) + if (dumpParams.output_vars.bitset(hydro_indeces[i])) + { + if (i > 0 && pass) + strcat(strlist, ", "); + else + pass = 1; strcat(strlist, hydroInfo[i].name); } } -void -vpic_simulation::print_hashed_comment( FileIO & fileIO, - const char * comment) { +void vpic_simulation::print_hashed_comment(FileIO &fileIO, + const char *comment) +{ fileIO.print("################################################################################\n"); fileIO.print("# %s\n", comment); fileIO.print("################################################################################\n"); } -void -vpic_simulation::global_header( const char * base, - std::vector dumpParams ) { - if( rank() ) return; +void vpic_simulation::global_header(const char *base, + std::vector dumpParams) +{ + if (rank()) + return; // Open the file for output char filename[256]; @@ -516,7 +370,8 @@ vpic_simulation::global_header( const char * base, FileIOStatus status; status = fileIO.open(filename, io_write); - if(status == fail) ERROR(("Failed opening file: %s", filename)); + if (status == fail) + ERROR(("Failed opening file: %s", filename)); print_hashed_comment(fileIO, "Header version information"); fileIO.print("VPIC_HEADER_VERSION 1.0.0\n\n"); @@ -574,15 +429,15 @@ vpic_simulation::global_header( const char * base, size_t numvars = std::min(dumpParams[0]->output_vars.bitsum(field_indeces, total_field_groups), total_field_groups); - size_t * varlist = new size_t[numvars]; - for(size_t v(0), c(0); voutput_vars.bitset(field_indeces[v])) + size_t *varlist = new size_t[numvars]; + for (size_t v(0), c(0); v < total_field_groups; v++) + if (dumpParams[0]->output_vars.bitset(field_indeces[v])) varlist[c++] = v; // output variable list fileIO.print("FIELD_DATA_VARIABLES %d\n", numvars); - for(size_t v(0); voutput_vars.bitsum(hydro_indeces, total_hydro_groups), total_hydro_groups); @@ -611,62 +467,47 @@ vpic_simulation::global_header( const char * base, fileIO.print("HYDRO_DATA_VARIABLES %d\n", numvars); varlist = new size_t[numvars]; - for(size_t v(0), c(0); voutput_vars.bitset(hydro_indeces[v])) + for (size_t v(0), c(0); v < total_hydro_groups; v++) + if (dumpParams[i]->output_vars.bitset(hydro_indeces[v])) varlist[c++] = v; - for(size_t v(0); vf - if ( f==NULL ) f = field_array->f; + if (status == fail) + ERROR(("Failed opening file: %s", filename)); // convenience const size_t istride(dumpParams.stride_x); @@ -674,25 +515,25 @@ vpic_simulation::field_dump( DumpParameters & dumpParams, const size_t kstride(dumpParams.stride_z); // Check stride values. - if(remainder(grid->nx, istride) != 0) + if (remainder(grid->nx, istride) != 0) ERROR(("x stride must be an integer factor of nx")); - if(remainder(grid->ny, jstride) != 0) + if (remainder(grid->ny, jstride) != 0) ERROR(("y stride must be an integer factor of ny")); - if(remainder(grid->nz, kstride) != 0) + if (remainder(grid->nz, kstride) != 0) ERROR(("z stride must be an integer factor of nz")); int dim[3]; /* define to do C-style indexing */ -# define f(x,y,z) f[ VOXEL(x,y,z, grid->nx,grid->ny,grid->nz) ] +#define f(x, y, z) f[VOXEL(x, y, z, grid->nx, grid->ny, grid->nz)] /* IMPORTANT: these values are written in WRITE_HEADER_V0 */ - nxout = (grid->nx)/istride; - nyout = (grid->ny)/jstride; - nzout = (grid->nz)/kstride; - dxout = (grid->dx)*istride; - dyout = (grid->dy)*jstride; - dzout = (grid->dz)*kstride; + size_t nxout = (grid->nx) / istride; + size_t nyout = (grid->ny) / jstride; + size_t nzout = (grid->nz) / kstride; + float dxout = (grid->dx) * istride; + float dyout = (grid->dy) * jstride; + float dzout = (grid->dz) * kstride; /* Banded output will write data as a single block-array as opposed to * the Array-of-Structure format that is used for native storage. @@ -702,15 +543,16 @@ vpic_simulation::field_dump( DumpParameters & dumpParams, * specified for a particular dimension, VPIC will write the boundary * plus every "stride" elements in that dimension. */ - if ( dumpParams.format == band ) + if (dumpParams.format == band) { - WRITE_HEADER_V0( dump_type::field_dump, -1, 0, dumpStep, fileIO ); - dim[0] = nxout+2; - dim[1] = nyout+2; - dim[2] = nzout+2; + WRITE_HEADER_V0(dump_type::field_dump, -1, 0, fileIO, step(), rank(), nproc()); + + dim[0] = nxout + 2; + dim[1] = nyout + 2; + dim[2] = nzout + 2; - if ( rank() == VERBOSE_rank ) + if (rank() == VERBOSE_rank) { std::cerr << "nxout: " << nxout << std::endl; std::cerr << "nyout: " << nyout << std::endl; @@ -720,138 +562,154 @@ vpic_simulation::field_dump( DumpParameters & dumpParams, std::cerr << "nz: " << grid->nz << std::endl; } - WRITE_ARRAY_HEADER(f, 3, dim, fileIO); + WRITE_ARRAY_HEADER(field_array->f, 3, dim, fileIO); // Create a variable list of field values to output. size_t numvars = std::min(dumpParams.output_vars.bitsum(), total_field_variables); - size_t * varlist = new size_t[numvars]; + size_t *varlist = new size_t[numvars]; - for(size_t i(0), c(0); i(&f(i,j,k)); + if (istride == 1 && jstride == 1 && kstride == 1) + for (size_t v(0); v < numvars; v++) + { + for (size_t k(0); k < nzout + 2; k++) + { + for (size_t j(0); j < nyout + 2; j++) + { + for (size_t i(0); i < nxout + 2; i++) + { + const uint32_t *fref = reinterpret_cast(&field_array->f(i, j, k)); fileIO.write(&fref[varlist[v]], 1); - if(rank()==VERBOSE_rank) printf("%f ", f(i,j,k).ex); - if(rank()==VERBOSE_rank) std::cout << "(" << i << " " << j << " " << k << ")" << std::endl; - } if(rank()==VERBOSE_rank) std::cout << std::endl << "ROW_BREAK " << j << " " << k << std::endl; - } if(rank()==VERBOSE_rank) std::cout << std::endl << "PLANE_BREAK " << k << std::endl; - } if(rank()==VERBOSE_rank) std::cout << std::endl << "BLOCK_BREAK" << std::endl; + if (rank() == VERBOSE_rank) + printf("%f ", field_array->f(i, j, k).ex); + if (rank() == VERBOSE_rank) + std::cout << "(" << i << " " << j << " " << k << ")" << std::endl; + } + if (rank() == VERBOSE_rank) + std::cout << std::endl + << "ROW_BREAK " << j << " " << k << std::endl; + } + if (rank() == VERBOSE_rank) + std::cout << std::endl + << "PLANE_BREAK " << k << std::endl; + } + if (rank() == VERBOSE_rank) + std::cout << std::endl + << "BLOCK_BREAK" << std::endl; } - } else - { - for(size_t v(0); vnz+1 : k*kstride-1; - for(size_t j(0); jny+1 : j*jstride-1; - for(size_t i(0); inx+1 : i*istride-1; - const uint32_t * fref = reinterpret_cast(&f(ioff,joff,koff)); + + for (size_t v(0); v < numvars; v++) + { + for (size_t k(0); k < nzout + 2; k++) + { + const size_t koff = (k == 0) ? 0 : (k == nzout + 1) ? grid->nz + 1 + : k * kstride - 1; + for (size_t j(0); j < nyout + 2; j++) + { + const size_t joff = (j == 0) ? 0 : (j == nyout + 1) ? grid->ny + 1 + : j * jstride - 1; + for (size_t i(0); i < nxout + 2; i++) + { + const size_t ioff = (i == 0) ? 0 : (i == nxout + 1) ? grid->nx + 1 + : i * istride - 1; + const uint32_t *fref = reinterpret_cast(&field_array->f(ioff, joff, koff)); fileIO.write(&fref[varlist[v]], 1); - if(rank()==VERBOSE_rank) printf("%f ", f(ioff,joff,koff).ex); - if(rank()==VERBOSE_rank) std::cout << "(" << ioff << " " << joff << " " << koff << ")" << std::endl; - } if(rank()==VERBOSE_rank) std::cout << std::endl << "ROW_BREAK " << joff << " " << koff << std::endl; - } if(rank()==VERBOSE_rank) std::cout << std::endl << "PLANE_BREAK " << koff << std::endl; - } if(rank()==VERBOSE_rank) std::cout << std::endl << "BLOCK_BREAK" << std::endl; + if (rank() == VERBOSE_rank) + printf("%f ", field_array->f(ioff, joff, koff).ex); + if (rank() == VERBOSE_rank) + std::cout << "(" << ioff << " " << joff << " " << koff << ")" << std::endl; + } + if (rank() == VERBOSE_rank) + std::cout << std::endl + << "ROW_BREAK " << joff << " " << koff << std::endl; + } + if (rank() == VERBOSE_rank) + std::cout << std::endl + << "PLANE_BREAK " << koff << std::endl; + } + if (rank() == VERBOSE_rank) + std::cout << std::endl + << "BLOCK_BREAK" << std::endl; } - } delete[] varlist; - } - - // band_interleave else - { - WRITE_HEADER_V0( dump_type::field_dump, -1, 0, dumpStep, fileIO ); + { // band_interleave - dim[0] = nxout+2; - dim[1] = nyout+2; - dim[2] = nzout+2; + WRITE_HEADER_V0(dump_type::field_dump, -1, 0, fileIO, step(), rank(), nproc()); - WRITE_ARRAY_HEADER(f, 3, dim, fileIO); + dim[0] = nxout + 2; + dim[1] = nyout + 2; + dim[2] = nzout + 2; - if ( istride == 1 && - jstride == 1 && - kstride == 1 ) - { - fileIO.write( f, dim[0] * dim[1] * dim[2] ); - } + WRITE_ARRAY_HEADER(field_array->f, 3, dim, fileIO); + if (istride == 1 && jstride == 1 && kstride == 1) + fileIO.write(field_array->f, dim[0] * dim[1] * dim[2]); else - { - for(size_t k(0); knz+1 : k*kstride-1; - for(size_t j(0); jny+1 : j*jstride-1; - for(size_t i(0); inx+1 : i*istride-1; - fileIO.write( &f( ioff, joff, koff ), 1 ); - } - } + for (size_t k(0); k < nzout + 2; k++) + { + const size_t koff = (k == 0) ? 0 : (k == nzout + 1) ? grid->nz + 1 + : k * kstride - 1; + for (size_t j(0); j < nyout + 2; j++) + { + const size_t joff = (j == 0) ? 0 : (j == nyout + 1) ? grid->ny + 1 + : j * jstride - 1; + for (size_t i(0); i < nxout + 2; i++) + { + const size_t ioff = (i == 0) ? 0 : (i == nxout + 1) ? grid->nx + 1 + : i * istride - 1; + fileIO.write(&field_array->f(ioff, joff, koff), 1); + } + } } - } } -# undef f +#undef f - if ( fileIO.close() ) ERROR(( "File close failed on field dump." )); + if (fileIO.close()) + ERROR(("File close failed on field dump!!!")); } -void -vpic_simulation::hydro_dump( const char * speciesname, - DumpParameters & dumpParams, - hydro_t *h, - int64_t userStep ) +void vpic_simulation::hydro_dump(const char *speciesname, + DumpParameters &dumpParams) { - long dumpStep = ( userStep == -1 ) ? (long) step() : userStep; // Create directory for this time step char timeDir[256]; - - sprintf( timeDir, - "%s/T.%ld", - dumpParams.baseDir, - dumpStep ); - - dump_mkdir( timeDir ); + sprintf(timeDir, "%s/T.%ld", dumpParams.baseDir, (long)step()); + dump_mkdir(timeDir); // Open the file for output - char filename[300]; - - sprintf( filename, - "%s/T.%ld/%s.%ld.%d", - dumpParams.baseDir, - dumpStep, - dumpParams.baseFileName, - dumpStep, - rank() ); + char filename[256]; + sprintf(filename, "%s/T.%ld/%s.%ld.%d", dumpParams.baseDir, (long)step(), + dumpParams.baseFileName, (long)step(), rank()); FileIO fileIO; FileIOStatus status; status = fileIO.open(filename, io_write); - if(status == fail) ERROR(("Failed opening file: %s", filename)); + if (status == fail) + ERROR(("Failed opening file: %s", filename)); - species_t * sp = find_species_name(speciesname, species_list); - if( !sp ) ERROR(( "Invalid species name: %s", speciesname )); + species_t *sp = find_species_name(speciesname, species_list); + if (!sp) + ERROR(("Invalid species name: %s", speciesname)); - // default behavior is to accumulate hydro array and then write - if ( h == NULL ) - { - h = hydro_array->h; - clear_hydro_array( hydro_array ); - accumulate_hydro_p( hydro_array, sp, interpolator_array ); - synchronize_hydro_array( hydro_array ); - } + clear_hydro_array(hydro_array); + accumulate_hydro_p(hydro_array, sp, interpolator_array); + synchronize_hydro_array(hydro_array); // convenience const size_t istride(dumpParams.stride_x); @@ -859,25 +717,22 @@ vpic_simulation::hydro_dump( const char * speciesname, const size_t kstride(dumpParams.stride_z); // Check stride values. - if(remainder(grid->nx, istride) != 0) + if (remainder(grid->nx, istride) != 0) ERROR(("x stride must be an integer factor of nx")); - if(remainder(grid->ny, jstride) != 0) + if (remainder(grid->ny, jstride) != 0) ERROR(("y stride must be an integer factor of ny")); - if(remainder(grid->nz, kstride) != 0) + if (remainder(grid->nz, kstride) != 0) ERROR(("z stride must be an integer factor of nz")); int dim[3]; - /* define to do C-style indexing */ -# define hydro(x,y,z) h[VOXEL(x,y,z, grid->nx,grid->ny,grid->nz)] - /* IMPORTANT: these values are written in WRITE_HEADER_V0 */ - nxout = (grid->nx)/istride; - nyout = (grid->ny)/jstride; - nzout = (grid->nz)/kstride; - dxout = (grid->dx)*istride; - dyout = (grid->dy)*jstride; - dzout = (grid->dz)*kstride; + size_t nxout = (grid->nx) / istride; + size_t nyout = (grid->ny) / jstride; + size_t nzout = (grid->nz) / kstride; + float dxout = (grid->dx) * istride; + float dyout = (grid->dy) * jstride; + float dzout = (grid->dz) * kstride; /* Banded output will write data as a single block-array as opposed to * the Array-of-Structure format that is used for native storage. @@ -887,87 +742,102 @@ vpic_simulation::hydro_dump( const char * speciesname, * specified for a particular dimension, VPIC will write the boundary * plus every "stride" elements in that dimension. */ - if ( dumpParams.format == band ) + if (dumpParams.format == band) { - WRITE_HEADER_V0( dump_type::hydro_dump, sp->id, sp->q/sp->m, dumpStep, fileIO ); - dim[0] = nxout+2; - dim[1] = nyout+2; - dim[2] = nzout+2; + WRITE_HEADER_V0(dump_type::hydro_dump, sp->id, sp->q / sp->m, fileIO, step(), rank(), nproc()); - WRITE_ARRAY_HEADER(h, 3, dim, fileIO); + dim[0] = nxout + 2; + dim[1] = nyout + 2; + dim[2] = nzout + 2; + + WRITE_ARRAY_HEADER(hydro_array->h, 3, dim, fileIO); /* * Create a variable list of hydro values to output. */ size_t numvars = std::min(dumpParams.output_vars.bitsum(), total_hydro_variables); - size_t * varlist = new size_t[numvars]; - for(size_t i(0), c(0); i(&hydro(i,j,k)); + for (size_t v(0); v < numvars; v++) + for (size_t k(0); k < nzout + 2; k++) + for (size_t j(0); j < nyout + 2; j++) + for (size_t i(0); i < nxout + 2; i++) + { + const uint32_t *href = reinterpret_cast(&hydro(i, j, k)); fileIO.write(&href[varlist[v]], 1); - } + } else - for(size_t v(0); vnz+1 : k*kstride-1; - for(size_t j(0); jny+1 : j*jstride-1; - for(size_t i(0); inx+1 : i*istride-1; - const uint32_t * href = reinterpret_cast(&hydro(ioff,joff,koff)); + for (size_t v(0); v < numvars; v++) + for (size_t k(0); k < nzout + 2; k++) + { + const size_t koff = (k == 0) ? 0 : (k == nzout + 1) ? grid->nz + 1 + : k * kstride - 1; + for (size_t j(0); j < nyout + 2; j++) + { + const size_t joff = (j == 0) ? 0 : (j == nyout + 1) ? grid->ny + 1 + : j * jstride - 1; + for (size_t i(0); i < nxout + 2; i++) + { + const size_t ioff = (i == 0) ? 0 : (i == nxout + 1) ? grid->nx + 1 + : i * istride - 1; + const uint32_t *href = reinterpret_cast(&hydro(ioff, joff, koff)); fileIO.write(&href[varlist[v]], 1); - } - } - } + } + } + } delete[] varlist; - } - - // band_interleave else - { - WRITE_HEADER_V0( dump_type::hydro_dump, sp->id, sp->q/sp->m, dumpStep, fileIO ); + { // band_interleave + + WRITE_HEADER_V0(dump_type::hydro_dump, sp->id, sp->q / sp->m, fileIO, step(), rank(), nproc()); dim[0] = nxout; dim[1] = nyout; dim[2] = nzout; - WRITE_ARRAY_HEADER(h, 3, dim, fileIO); + WRITE_ARRAY_HEADER(hydro_array->h, 3, dim, fileIO); - if ( istride == 1 && - jstride == 1 && - kstride == 1 ) - { - fileIO.write( h, dim[0] * dim[1] * dim[2] ); - } + if (istride == 1 && jstride == 1 && kstride == 1) + + fileIO.write(hydro_array->h, dim[0] * dim[1] * dim[2]); else - { - for(size_t k(0); knz+1 : k*kstride-1; - for(size_t j(0); jny+1 : j*jstride-1; - for(size_t i(0); inx+1 : i*istride-1; - fileIO.write( &hydro( ioff, joff, koff ), 1 ); - } - } + + for (size_t k(0); k < nzout; k++) + { + const size_t koff = (k == 0) ? 0 : (k == nzout + 1) ? grid->nz + 1 + : k * kstride - 1; + for (size_t j(0); j < nyout; j++) + { + const size_t joff = (j == 0) ? 0 : (j == nyout + 1) ? grid->ny + 1 + : j * jstride - 1; + for (size_t i(0); i < nxout; i++) + { + const size_t ioff = (i == 0) ? 0 : (i == nxout + 1) ? grid->nx + 1 + : i * istride - 1; + fileIO.write(&hydro(ioff, joff, koff), 1); + } + } } - } } -# undef hydro +#undef hydro - if( fileIO.close() ) ERROR(( "File close failed on hydro dump!!!" )); + if (fileIO.close()) + ERROR(("File close failed on hydro dump!!!")); } - /** * @brief Allocate memory inside the species to allow for buffering and later output * @@ -979,37 +849,86 @@ vpic_simulation::hydro_dump( const char * speciesname, * * @return Nothing, but allocates buffers internally. */ -void -vpic_simulation::init_buffered_particle_dump(const char * sp_name, const int N_timesteps, const double safety_factor) { - species_t *sp = find_species_name( sp_name, species_list ); - if( !sp ) ERROR(( "Invalid species \"%s\"", sp_name )); +void vpic_simulation::init_buffered_particle_dump(const char *sp_name, const int N_timesteps, const double safety_factor) +{ + species_t *sp = find_species_name(sp_name, species_list); + if (!sp) + ERROR(("Invalid species \"%s\"", sp_name)); // If any of those are still around from a previous init - if(sp->output_buffer_dx) { FREE_ALIGNED(sp->output_buffer_dx); sp->output_buffer_dx = nullptr; } - if(sp->output_buffer_dy) { FREE_ALIGNED(sp->output_buffer_dy); sp->output_buffer_dy = nullptr; } - if(sp->output_buffer_dz) { FREE_ALIGNED(sp->output_buffer_dz); sp->output_buffer_dz = nullptr; } - if(sp->output_buffer_i) { FREE_ALIGNED(sp->output_buffer_i); sp->output_buffer_i = nullptr; } - if(sp->output_buffer_ux) { FREE_ALIGNED(sp->output_buffer_ux); sp->output_buffer_ux = nullptr; } - if(sp->output_buffer_uy) { FREE_ALIGNED(sp->output_buffer_uy); sp->output_buffer_uy = nullptr; } - if(sp->output_buffer_uz) { FREE_ALIGNED(sp->output_buffer_uz); sp->output_buffer_uz = nullptr; } - if(sp->output_buffer_w) { FREE_ALIGNED(sp->output_buffer_w); sp->output_buffer_w = nullptr; } - #ifdef VPIC_GLOBAL_PARTICLE_ID - if(sp->output_buffer_id) { FREE_ALIGNED(sp->output_buffer_id); sp->output_buffer_id = nullptr; } - #endif - #ifdef VPIC_PARTICLE_ANNOTATION - if(sp->output_buffer_an) { FREE_ALIGNED(sp->output_buffer_an); sp->output_buffer_an = nullptr; } - #endif - if(sp->output_buffer_ts) { FREE_ALIGNED(sp->output_buffer_ts); sp->output_buffer_ts = nullptr; } - if(sp->buf_n_valid) { FREE_ALIGNED(sp->buf_n_valid); sp->buf_n_valid = nullptr; } + if (sp->output_buffer_dx) + { + FREE_ALIGNED(sp->output_buffer_dx); + sp->output_buffer_dx = nullptr; + } + if (sp->output_buffer_dy) + { + FREE_ALIGNED(sp->output_buffer_dy); + sp->output_buffer_dy = nullptr; + } + if (sp->output_buffer_dz) + { + FREE_ALIGNED(sp->output_buffer_dz); + sp->output_buffer_dz = nullptr; + } + if (sp->output_buffer_i) + { + FREE_ALIGNED(sp->output_buffer_i); + sp->output_buffer_i = nullptr; + } + if (sp->output_buffer_ux) + { + FREE_ALIGNED(sp->output_buffer_ux); + sp->output_buffer_ux = nullptr; + } + if (sp->output_buffer_uy) + { + FREE_ALIGNED(sp->output_buffer_uy); + sp->output_buffer_uy = nullptr; + } + if (sp->output_buffer_uz) + { + FREE_ALIGNED(sp->output_buffer_uz); + sp->output_buffer_uz = nullptr; + } + if (sp->output_buffer_w) + { + FREE_ALIGNED(sp->output_buffer_w); + sp->output_buffer_w = nullptr; + } +#ifdef VPIC_GLOBAL_PARTICLE_ID + if (sp->output_buffer_id) + { + FREE_ALIGNED(sp->output_buffer_id); + sp->output_buffer_id = nullptr; + } +#endif +#ifdef VPIC_PARTICLE_ANNOTATION + if (sp->output_buffer_an) + { + FREE_ALIGNED(sp->output_buffer_an); + sp->output_buffer_an = nullptr; + } +#endif + if (sp->output_buffer_ts) + { + FREE_ALIGNED(sp->output_buffer_ts); + sp->output_buffer_ts = nullptr; + } + if (sp->buf_n_valid) + { + FREE_ALIGNED(sp->buf_n_valid); + sp->buf_n_valid = nullptr; + } sp->buf_size = 0; - // How many annotations per particles - #ifdef VPIC_PARTICLE_ANNOTATION - sp->buf_n_annotation = sp->has_annotation; - #else - sp->buf_n_annotation = 0; - #endif +// How many annotations per particles +#ifdef VPIC_PARTICLE_ANNOTATION + sp->buf_n_annotation = sp->has_annotation; +#else + sp->buf_n_annotation = 0; +#endif // How many frames are we expected to store? sp->buf_n_frames = N_timesteps; @@ -1021,42 +940,47 @@ vpic_simulation::init_buffered_particle_dump(const char * sp_name, const int N_t sp->buf_size = sp->buf_n_particles * sp->buf_n_frames; // Allocate buffer - MALLOC_ALIGNED(sp->output_buffer_dx, sp->buf_size*sizeof(float), 128); - MALLOC_ALIGNED(sp->output_buffer_dy, sp->buf_size*sizeof(float), 128); - MALLOC_ALIGNED(sp->output_buffer_dz, sp->buf_size*sizeof(float), 128); - MALLOC_ALIGNED(sp->output_buffer_i, sp->buf_size*sizeof(int64_t), 128); - MALLOC_ALIGNED(sp->output_buffer_ux, sp->buf_size*sizeof(float), 128); - MALLOC_ALIGNED(sp->output_buffer_uy, sp->buf_size*sizeof(float), 128); - MALLOC_ALIGNED(sp->output_buffer_uz, sp->buf_size*sizeof(float), 128); - MALLOC_ALIGNED(sp->output_buffer_w, sp->buf_size*sizeof(float), 128); - #ifdef VPIC_GLOBAL_PARTICLE_ID - if(sp->has_ids) { - MALLOC_ALIGNED(sp->output_buffer_id, sp->buf_size*sizeof(size_t), 128); + MALLOC_ALIGNED(sp->output_buffer_dx, sp->buf_size * sizeof(float), 128); + MALLOC_ALIGNED(sp->output_buffer_dy, sp->buf_size * sizeof(float), 128); + MALLOC_ALIGNED(sp->output_buffer_dz, sp->buf_size * sizeof(float), 128); + MALLOC_ALIGNED(sp->output_buffer_i, sp->buf_size * sizeof(int64_t), 128); + MALLOC_ALIGNED(sp->output_buffer_ux, sp->buf_size * sizeof(float), 128); + MALLOC_ALIGNED(sp->output_buffer_uy, sp->buf_size * sizeof(float), 128); + MALLOC_ALIGNED(sp->output_buffer_uz, sp->buf_size * sizeof(float), 128); + MALLOC_ALIGNED(sp->output_buffer_w, sp->buf_size * sizeof(float), 128); +#ifdef VPIC_GLOBAL_PARTICLE_ID + if (sp->has_ids) + { + MALLOC_ALIGNED(sp->output_buffer_id, sp->buf_size * sizeof(size_t), 128); } - #endif - #ifdef VPIC_PARTICLE_ANNOTATION - if(sp->has_annotation) { - MALLOC_ALIGNED(sp->output_buffer_an, sp->buf_size*sp->buf_n_annotation*sizeof(annotation_t), 128); +#endif +#ifdef VPIC_PARTICLE_ANNOTATION + if (sp->has_annotation) + { + MALLOC_ALIGNED(sp->output_buffer_an, sp->buf_size * sp->buf_n_annotation * sizeof(annotation_t), 128); } - #endif - MALLOC_ALIGNED(sp->output_buffer_ts, sp->buf_size*sizeof(size_t), 128); - MALLOC_ALIGNED(sp->buf_n_valid, sp->buf_n_frames*sizeof(int64_t), 128); +#endif + MALLOC_ALIGNED(sp->output_buffer_ts, sp->buf_size * sizeof(size_t), 128); + MALLOC_ALIGNED(sp->buf_n_valid, sp->buf_n_frames * sizeof(int64_t), 128); // Done } #ifdef OUTPUT_CONVERT_GLOBAL_ID -# define UNVOXEL(rank, ix, iy, iz, nx, ny, nz) BEGIN_PRIMITIVE { \ -int _ix, _iy, _iz; \ -_ix = (rank); /* ix = ix+gpx*( iy+gpy*iz ) */ \ -_iy = _ix/int(nx); /* iy = iy+gpy*iz */ \ -_ix -= _iy*int(nx); /* ix = ix */ \ -_iz = _iy/int(ny); /* iz = iz */ \ -_iy -= _iz*int(ny); /* iy = iy */ \ -(ix) = _ix; \ -(iy) = _iy; \ -(iz) = _iz; \ -} END_PRIMITIVE +#define UNVOXEL(rank, ix, iy, iz, nx, ny, nz) \ + BEGIN_PRIMITIVE \ + { \ + int _ix, _iy, _iz; \ + _ix = (rank); /* ix = ix+gpx*( iy+gpy*iz ) */ \ + _iy = _ix / int(nx); /* iy = iy+gpy*iz */ \ + _ix -= _iy * int(nx); /* ix = ix */ \ + _iz = _iy / int(ny); /* iz = iz */ \ + _iy -= _iz * int(ny); /* iy = iy */ \ + (ix) = _ix; \ + (iy) = _iy; \ + (iz) = _iz; \ + } \ + END_PRIMITIVE #endif /** @@ -1067,23 +991,28 @@ _iy -= _iz*int(ny); /* iy = iy */ \ * * @return Nothing, but fills internal buffers */ -void -vpic_simulation::accumulate_buffered_particle_dump(const char * sp_name, const int frame) { - species_t *sp = find_species_name( sp_name, species_list ); - if( !sp ) ERROR(( "Invalid species \"%s\"", sp_name )); +void vpic_simulation::accumulate_buffered_particle_dump(const char *sp_name, const int frame) +{ + species_t *sp = find_species_name(sp_name, species_list); + if (!sp) + ERROR(("Invalid species \"%s\"", sp_name)); - if( !sp->buf_size ) ERROR(( "No buffer setup on species \"%s\"", sp_name)); + if (!sp->buf_size) + ERROR(("No buffer setup on species \"%s\"", sp_name)); - if(frame < 0 || frame >= sp->buf_n_frames) ERROR(( "Invalid frame %ld (has to be in (0;%ld( on species \"%s\"", frame, sp->buf_n_frames, sp_name )); + if (frame < 0 || frame >= sp->buf_n_frames) + ERROR(("Invalid frame %ld (has to be in (0;%ld( on species \"%s\"", frame, sp->buf_n_frames, sp_name)); - if(sp->np > sp->buf_n_particles) WARNING(( "We have %d particles but buffer space for only %ld in species \"%s\". You will loose information. Chose a larger safety_factor next time.", sp->np, sp->buf_n_particles, sp_name)); + if (sp->np > sp->buf_n_particles) + WARNING(("We have %d particles but buffer space for only %ld in species \"%s\". You will loose information. Chose a larger safety_factor next time.", sp->np, sp->buf_n_particles, sp_name)); const int mpi_rank = rank(); sp->buf_n_valid[frame] = 0; // Loop over particles - for(int64_t n = 0; n < sp->np && n < sp->buf_n_particles; n++) { - #ifdef OUTPUT_CONVERT_GLOBAL_ID + for (int64_t n = 0; n < sp->np && n < sp->buf_n_particles; n++) + { +#ifdef OUTPUT_CONVERT_GLOBAL_ID // Convert Cell ID to global int64_t local_i = sp->p[n].i; @@ -1092,7 +1021,7 @@ vpic_simulation::accumulate_buffered_particle_dump(const char * sp_name, const i UNVOXEL(mpi_rank, rx, ry, rz, grid->gpx, grid->gpy, grid->gpz); // Calculate local ix/iy/iz - UNVOXEL(local_i, ix, iy, iz, grid->nx+2, grid->ny+2, grid->nz+2); + UNVOXEL(local_i, ix, iy, iz, grid->nx + 2, grid->ny + 2, grid->nz + 2); // Account for the "first" ghost cell ix = ix - 1; @@ -1110,43 +1039,47 @@ vpic_simulation::accumulate_buffered_particle_dump(const char * sp_name, const i int gnz = grid->nz * grid->gpz; // TODO: find a better way to account for the hard coded ghosts in VOXEL - int64_t global_i = VOXEL(gix, giy, giz, gnx-2, gny-2, gnz-2); - #endif + int64_t global_i = VOXEL(gix, giy, giz, gnx - 2, gny - 2, gnz - 2); +#endif - const size_t index = frame*sp->buf_n_particles + n; + const size_t index = frame * sp->buf_n_particles + n; // Particle Properties sp->output_buffer_dx[index] = sp->p[n].dx; sp->output_buffer_dy[index] = sp->p[n].dy; sp->output_buffer_dz[index] = sp->p[n].dz; - #ifdef OUTPUT_CONVERT_GLOBAL_ID - sp->output_buffer_i[index] = global_i; - #else - sp->output_buffer_i[index] = sp->p[n].i; - #endif +#ifdef OUTPUT_CONVERT_GLOBAL_ID + printf("\n\n DEBUG: OUTPUT_CONVERT_GLOBAL_ID is enabled ! \n\n"); + sp->output_buffer_i[index] = global_i; +#else + sp->output_buffer_i[index] = sp->p[n].i; +#endif sp->output_buffer_ux[index] = sp->p[n].ux; sp->output_buffer_uy[index] = sp->p[n].uy; sp->output_buffer_uz[index] = sp->p[n].uz; - sp->output_buffer_w[index] = sp->p[n].w; + sp->output_buffer_w[index] = sp->p[n].w; - #ifdef VPIC_GLOBAL_PARTICLE_ID - if(sp->has_ids) { +#ifdef VPIC_GLOBAL_PARTICLE_ID + if (sp->has_ids) + { sp->output_buffer_id[index] = sp->p_id[n]; } - #endif - - // Annotations - #ifdef VPIC_PARTICLE_ANNOTATION - if(sp->has_annotation) { - for(int a = 0; a < sp->has_annotation; a++) { - const size_t out_index = sp->buf_n_particles * sp->buf_n_frames * a + sp->buf_n_particles * frame + n; - const size_t in_index = sp->has_annotation * n + a; - sp->output_buffer_an[out_index] = sp->p_annotation[in_index]; +#endif + +// Annotations +#ifdef VPIC_PARTICLE_ANNOTATION + if (sp->has_annotation) + { + for (int a = 0; a < sp->has_annotation; a++) + { + const size_t out_index = sp->buf_n_particles * sp->buf_n_frames * a + sp->buf_n_particles * frame + n; + const size_t in_index = sp->has_annotation * n + a; + sp->output_buffer_an[out_index] = sp->p_annotation[in_index]; } } - #endif +#endif sp->output_buffer_ts[index] = step(); sp->buf_n_valid[frame]++; @@ -1162,10 +1095,11 @@ vpic_simulation::accumulate_buffered_particle_dump(const char * sp_name, const i * * @return Nothing, but performs IO */ -void -vpic_simulation::write_buffered_particle_dump(const char * dname, const char * sp_name) { - species_t *sp = find_species_name( sp_name, species_list ); - if( !sp ) ERROR(( "Invalid species \"%s\"", sp_name )); +void vpic_simulation::write_buffered_particle_dump(const char *dname, const char *sp_name) +{ + species_t *sp = find_species_name(sp_name, species_list); + if (!sp) + ERROR(("Invalid species \"%s\"", sp_name)); #ifdef VPIC_ENABLE_HDF5 char strbuf[4096]; @@ -1194,21 +1128,25 @@ vpic_simulation::write_buffered_particle_dump(const char * dname, const char * s hid_t file; mp_barrier(); H5E_auto2_t func; - void* data; + void *data; H5Eget_auto2(H5E_DEFAULT, &func, &data); H5Eset_auto2(H5E_DEFAULT, NULL, NULL); hid_t state = H5Fis_hdf5(strbuf); H5Eset_auto2(H5E_DEFAULT, func, data); mp_barrier(); - if(state > 0) { + if (state > 0) + { // state > 0: valid HDF5 file file = H5Fopen(strbuf, H5F_ACC_RDWR, pl_file); - } else { + } + else + { // state == 0: not an HDF5 file // state < 0: some error, e.g. because the file doesn't exist file = H5Fcreate(strbuf, H5F_ACC_TRUNC, H5P_DEFAULT, pl_file); } - if(file<0) { + if (file < 0) + { ERROR(("Failed to open HDF5 file %s. state was %d", strbuf, state)); } @@ -1220,15 +1158,19 @@ vpic_simulation::write_buffered_particle_dump(const char * dname, const char * s mp_barrier(); state = H5Lexists(file, strbuf, pl_link); mp_barrier(); - if(state > 0) { + if (state > 0) + { // state > 0: link name exists. let's hope it's a valid group group = H5Gopen(file, strbuf, H5P_DEFAULT); - } else { + } + else + { // state == 0: link does not exists // state < 0: some error group = H5Gcreate(file, strbuf, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); } - if(group<0) { + if (group < 0) + { ERROR(("Failed to open group %s. state was %d", strbuf, state)); } @@ -1238,12 +1180,16 @@ vpic_simulation::write_buffered_particle_dump(const char * dname, const char * s mp_barrier(); state = H5Lexists(group, strbuf, pl_link); mp_barrier(); - if(state > 0) { + if (state > 0) + { subgroup = H5Gopen(group, strbuf, H5P_DEFAULT); - } else { + } + else + { subgroup = H5Gcreate(group, strbuf, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); } - if(subgroup<0) { + if (subgroup < 0) + { ERROR(("Failed to open group %s. state was %d", strbuf, state)); } @@ -1255,83 +1201,123 @@ vpic_simulation::write_buffered_particle_dump(const char * dname, const char * s offset -= local_entries; hid_t pl_xfer = H5Pcreate(H5P_DATASET_XFER); - if(total_entries > 0) { + if (total_entries > 0) + { hid_t filespace, memspace, dset; H5Pset_dxpl_mpio(pl_xfer, H5FD_MPIO_COLLECTIVE); - filespace = H5Screate_simple(1, (hsize_t*) &total_entries, NULL); - memspace = H5Screate_simple(1, (hsize_t*) &local_entries, NULL); + filespace = H5Screate_simple(1, (hsize_t *)&total_entries, NULL); + memspace = H5Screate_simple(1, (hsize_t *)&local_entries, NULL); - H5Sselect_hyperslab(filespace, H5S_SELECT_SET, (hsize_t*)&offset, NULL, (hsize_t*)&local_entries, NULL); + H5Sselect_hyperslab(filespace, H5S_SELECT_SET, (hsize_t *)&offset, NULL, (hsize_t *)&local_entries, NULL); // dX - if(H5Lexists(subgroup, "dX", pl_link)) { H5Ldelete(subgroup, "dX", pl_link); } + if (H5Lexists(subgroup, "dX", pl_link)) + { + H5Ldelete(subgroup, "dX", pl_link); + } dset = H5Dcreate(subgroup, "dX", H5T_NATIVE_FLOAT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Dwrite(dset, H5T_NATIVE_FLOAT, memspace, filespace, pl_xfer, sp->output_buffer_dx); H5Dclose(dset); // dY - if(H5Lexists(subgroup, "dY", pl_link)) { H5Ldelete(subgroup, "dY", pl_link); } + if (H5Lexists(subgroup, "dY", pl_link)) + { + H5Ldelete(subgroup, "dY", pl_link); + } dset = H5Dcreate(subgroup, "dY", H5T_NATIVE_FLOAT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Dwrite(dset, H5T_NATIVE_FLOAT, memspace, filespace, pl_xfer, sp->output_buffer_dy); H5Dclose(dset); // dZ - if(H5Lexists(subgroup, "dZ", pl_link)) { H5Ldelete(subgroup, "dZ", pl_link); } + if (H5Lexists(subgroup, "dZ", pl_link)) + { + H5Ldelete(subgroup, "dZ", pl_link); + } dset = H5Dcreate(subgroup, "dZ", H5T_NATIVE_FLOAT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Dwrite(dset, H5T_NATIVE_FLOAT, memspace, filespace, pl_xfer, sp->output_buffer_dz); H5Dclose(dset); // i - if(H5Lexists(subgroup, "i", pl_link)) { H5Ldelete(subgroup, "i", pl_link); } + if (H5Lexists(subgroup, "i", pl_link)) + { + H5Ldelete(subgroup, "i", pl_link); + } dset = H5Dcreate(subgroup, "i", H5T_NATIVE_INT64, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Dwrite(dset, H5T_NATIVE_INT64, memspace, filespace, pl_xfer, sp->output_buffer_i); H5Dclose(dset); // Ux - if(H5Lexists(subgroup, "Ux", pl_link)) { H5Ldelete(subgroup, "Ux", pl_link); } + if (H5Lexists(subgroup, "Ux", pl_link)) + { + H5Ldelete(subgroup, "Ux", pl_link); + } dset = H5Dcreate(subgroup, "Ux", H5T_NATIVE_FLOAT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Dwrite(dset, H5T_NATIVE_FLOAT, memspace, filespace, pl_xfer, sp->output_buffer_ux); H5Dclose(dset); // Uy - if(H5Lexists(subgroup, "Uy", pl_link)) { H5Ldelete(subgroup, "Uy", pl_link); } + if (H5Lexists(subgroup, "Uy", pl_link)) + { + H5Ldelete(subgroup, "Uy", pl_link); + } dset = H5Dcreate(subgroup, "Uy", H5T_NATIVE_FLOAT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Dwrite(dset, H5T_NATIVE_FLOAT, memspace, filespace, pl_xfer, sp->output_buffer_uy); H5Dclose(dset); // Uz - if(H5Lexists(subgroup, "Uz", pl_link)) { H5Ldelete(subgroup, "Uz", pl_link); } + if (H5Lexists(subgroup, "Uz", pl_link)) + { + H5Ldelete(subgroup, "Uz", pl_link); + } dset = H5Dcreate(subgroup, "Uz", H5T_NATIVE_FLOAT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Dwrite(dset, H5T_NATIVE_FLOAT, memspace, filespace, pl_xfer, sp->output_buffer_uz); H5Dclose(dset); // w - if(H5Lexists(subgroup, "w", pl_link)) { H5Ldelete(subgroup, "w", pl_link); } + if (H5Lexists(subgroup, "w", pl_link)) + { + H5Ldelete(subgroup, "w", pl_link); + } dset = H5Dcreate(subgroup, "w", H5T_NATIVE_FLOAT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Dwrite(dset, H5T_NATIVE_FLOAT, memspace, filespace, pl_xfer, sp->output_buffer_w); H5Dclose(dset); - // ID - #ifdef VPIC_GLOBAL_PARTICLE_ID - if(sp->has_ids) { - if(H5Lexists(subgroup, "q", pl_link)) { H5Ldelete(subgroup, "q", pl_link); } +// ID +#ifdef VPIC_GLOBAL_PARTICLE_ID + if (sp->has_ids) + { + if (H5Lexists(subgroup, "q", pl_link)) + { + H5Ldelete(subgroup, "q", pl_link); + } dset = H5Dcreate(subgroup, "q", H5T_NATIVE_HSIZE, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Dwrite(dset, H5T_NATIVE_HSIZE, memspace, filespace, pl_xfer, sp->output_buffer_id); H5Dclose(dset); } - #endif - // dX - #ifdef VPIC_PARTICLE_ANNOTATION - if(sp->has_annotation) { - for(int a = 0; a < sp->has_annotation; a++) { +#endif +// dX +#ifdef VPIC_PARTICLE_ANNOTATION + if (sp->has_annotation) + { + for (int a = 0; a < sp->has_annotation; a++) + { sprintf(strbuf, "Annotation_%d", a); - if(H5Lexists(subgroup, strbuf, pl_link)) { H5Ldelete(subgroup, strbuf, pl_link); } - annotation_t* data = &(sp->output_buffer_an[sp->buf_n_particles * sp->buf_n_frames * a]); - if(sizeof(VPIC_PARTICLE_ANNOTATION)==4) { - dset = H5Dcreate(subgroup, strbuf, H5T_NATIVE_FLOAT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - H5Dwrite(dset, H5T_NATIVE_FLOAT, memspace, filespace, pl_xfer, data); - } else { - dset = H5Dcreate(subgroup, strbuf, H5T_NATIVE_DOUBLE, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - H5Dwrite(dset, H5T_NATIVE_DOUBLE, memspace, filespace, pl_xfer, data); + if (H5Lexists(subgroup, strbuf, pl_link)) + { + H5Ldelete(subgroup, strbuf, pl_link); + } + annotation_t *data = &(sp->output_buffer_an[sp->buf_n_particles * sp->buf_n_frames * a]); + if (sizeof(VPIC_PARTICLE_ANNOTATION) == 4) + { + dset = H5Dcreate(subgroup, strbuf, H5T_NATIVE_FLOAT, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + H5Dwrite(dset, H5T_NATIVE_FLOAT, memspace, filespace, pl_xfer, data); + } + else + { + dset = H5Dcreate(subgroup, strbuf, H5T_NATIVE_DOUBLE, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + H5Dwrite(dset, H5T_NATIVE_DOUBLE, memspace, filespace, pl_xfer, data); } H5Dclose(dset); } } - #endif +#endif // timestep - if(H5Lexists(subgroup, "timestep", pl_link)) { H5Ldelete(subgroup, "timestep", pl_link); } + if (H5Lexists(subgroup, "timestep", pl_link)) + { + H5Ldelete(subgroup, "timestep", pl_link); + } dset = H5Dcreate(subgroup, "timestep", H5T_NATIVE_INT64, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Dwrite(dset, H5T_NATIVE_INT64, memspace, filespace, pl_xfer, sp->output_buffer_ts); H5Dclose(dset); @@ -1352,24 +1338,31 @@ vpic_simulation::write_buffered_particle_dump(const char * dname, const char * s dset_dims[1] = nproc(); hid_t filespace = H5Screate_simple(2, dset_dims, NULL); - hid_t memspace = H5Screate_simple(2, dcount, NULL); + hid_t memspace = H5Screate_simple(2, dcount, NULL); H5Sselect_hyperslab(filespace, H5S_SELECT_SET, doffset, NULL, dcount, NULL); sprintf(strbuf, "%s", "grid_meta_data"); mp_barrier(); state = H5Lexists(group, strbuf, pl_link); mp_barrier(); - if(state > 0) { + if (state > 0) + { subgroup = H5Gopen(group, strbuf, H5P_DEFAULT); - } else { + } + else + { subgroup = H5Gcreate(group, strbuf, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); } - if(subgroup<0) { + if (subgroup < 0) + { ERROR(("Failed to open group %s. state was %d", strbuf, state)); } sprintf(strbuf, "np_local_%s", sp->name); - if(H5Lexists(subgroup, strbuf, pl_link)) { H5Ldelete(subgroup, strbuf, pl_link); } + if (H5Lexists(subgroup, strbuf, pl_link)) + { + H5Ldelete(subgroup, strbuf, pl_link); + } hid_t dset = H5Dcreate(subgroup, strbuf, H5T_NATIVE_INT64, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Dwrite(dset, H5T_NATIVE_INT64, memspace, filespace, pl_xfer, sp->buf_n_valid); H5Dclose(dset); @@ -1389,9 +1382,17 @@ vpic_simulation::write_buffered_particle_dump(const char * dname, const char * s MPI_Allreduce(&grid->y0, &y0, 1, MPI_FLOAT, MPI_MIN, MPI_COMM_WORLD); MPI_Allreduce(&grid->z0, &z0, 1, MPI_FLOAT, MPI_MIN, MPI_COMM_WORLD); - if(rank() == 0 ) { + if (rank() == 0) + { sprintf(strbuf, "%s/T.%ld/tracers.h5p", dname, step()); - file = H5Fopen(strbuf, H5F_ACC_RDWR, H5P_DEFAULT); + + pl_file = H5Pcreate(H5P_FILE_ACCESS); +#if H5_VERSION_GE(1, 10, 7) + H5Pset_file_locking(pl_file, (hbool_t)false, (hbool_t)false ); +#endif + file = H5Fopen(strbuf, H5F_ACC_RDWR, pl_file); + H5Pclose(pl_file); + sprintf(strbuf, "Step#%ld", step()); hid_t group = H5Gopen(file, strbuf, H5P_DEFAULT); subgroup = H5Gopen(group, "grid_meta_data", H5P_DEFAULT); @@ -1402,10 +1403,13 @@ vpic_simulation::write_buffered_particle_dump(const char * dname, const char * s memspace = H5Screate_simple(1, &count, NULL); // Number of grid cells - intdata[0] = grid->gpx*grid->nx; - intdata[1] = grid->gpy*grid->ny; - intdata[2] = grid->gpz*grid->nz; - if(H5Lexists(subgroup, "cells", H5P_DEFAULT)) { H5Ldelete(subgroup, "cells", H5P_DEFAULT); } + intdata[0] = grid->gpx * grid->nx; + intdata[1] = grid->gpy * grid->ny; + intdata[2] = grid->gpz * grid->nz; + if (H5Lexists(subgroup, "cells", H5P_DEFAULT)) + { + H5Ldelete(subgroup, "cells", H5P_DEFAULT); + } dset = H5Dcreate(subgroup, "cells", H5T_NATIVE_INT, memspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, intdata); H5Dclose(dset); @@ -1414,7 +1418,10 @@ vpic_simulation::write_buffered_particle_dump(const char * dname, const char * s floatdata[0] = x0; floatdata[1] = y0; floatdata[2] = z0; - if(H5Lexists(subgroup, "offset", H5P_DEFAULT)) { H5Ldelete(subgroup, "offset", H5P_DEFAULT); } + if (H5Lexists(subgroup, "offset", H5P_DEFAULT)) + { + H5Ldelete(subgroup, "offset", H5P_DEFAULT); + } dset = H5Dcreate(subgroup, "offset", H5T_NATIVE_FLOAT, memspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Dwrite(dset, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, floatdata); H5Dclose(dset); @@ -1423,7 +1430,10 @@ vpic_simulation::write_buffered_particle_dump(const char * dname, const char * s floatdata[0] = grid->dx; floatdata[1] = grid->dy; floatdata[2] = grid->dz; - if(H5Lexists(subgroup, "resolution", H5P_DEFAULT)) { H5Ldelete(subgroup, "resolution", H5P_DEFAULT); } + if (H5Lexists(subgroup, "resolution", H5P_DEFAULT)) + { + H5Ldelete(subgroup, "resolution", H5P_DEFAULT); + } dset = H5Dcreate(subgroup, "resolution", H5T_NATIVE_FLOAT, memspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Dwrite(dset, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, floatdata); H5Dclose(dset); @@ -1446,38 +1456,44 @@ vpic_simulation::write_buffered_particle_dump(const char * dname, const char * s * * @return Nothing */ -void -vpic_simulation::clear_buffered_particle_dump(const char * sp_name) { - species_t *sp = find_species_name( sp_name, species_list ); - if( !sp ) ERROR(( "Invalid species \"%s\"", sp_name )); - - if(sp->buf_size) { - memset(sp->output_buffer_dx, 0, sp->buf_size*sizeof(float)); - memset(sp->output_buffer_dy, 0, sp->buf_size*sizeof(float)); - memset(sp->output_buffer_dz, 0, sp->buf_size*sizeof(float)); - memset(sp->output_buffer_i, 0, sp->buf_size*sizeof(int64_t)); - memset(sp->output_buffer_ux, 0, sp->buf_size*sizeof(float)); - memset(sp->output_buffer_uy, 0, sp->buf_size*sizeof(float)); - memset(sp->output_buffer_uz, 0, sp->buf_size*sizeof(float)); - memset(sp->output_buffer_w, 0, sp->buf_size*sizeof(float)); - #ifdef VPIC_GLOBAL_PARTICLE_ID - if(sp->has_ids) { - memset(sp->output_buffer_id, 0, sp->buf_size*sizeof(size_t)); +void vpic_simulation::clear_buffered_particle_dump(const char *sp_name) +{ + species_t *sp = find_species_name(sp_name, species_list); + if (!sp) + ERROR(("Invalid species \"%s\"", sp_name)); + + if (sp->buf_size) + { + memset(sp->output_buffer_dx, 0, sp->buf_size * sizeof(float)); + memset(sp->output_buffer_dy, 0, sp->buf_size * sizeof(float)); + memset(sp->output_buffer_dz, 0, sp->buf_size * sizeof(float)); + memset(sp->output_buffer_i, 0, sp->buf_size * sizeof(int64_t)); + memset(sp->output_buffer_ux, 0, sp->buf_size * sizeof(float)); + memset(sp->output_buffer_uy, 0, sp->buf_size * sizeof(float)); + memset(sp->output_buffer_uz, 0, sp->buf_size * sizeof(float)); + memset(sp->output_buffer_w, 0, sp->buf_size * sizeof(float)); +#ifdef VPIC_GLOBAL_PARTICLE_ID + if (sp->has_ids) + { + memset(sp->output_buffer_id, 0, sp->buf_size * sizeof(size_t)); } - #endif - #ifdef VPIC_PARTICLE_ANNOTATION - if(sp->has_annotation) { - memset(sp->output_buffer_an, 0, sp->buf_size*sp->buf_n_annotation*sizeof(annotation_t)); +#endif +#ifdef VPIC_PARTICLE_ANNOTATION + if (sp->has_annotation) + { + memset(sp->output_buffer_an, 0, sp->buf_size * sp->buf_n_annotation * sizeof(annotation_t)); } - #endif +#endif // Set all timesteps to -1 - for(int64_t n = 0; n < sp->buf_n_frames * sp->buf_n_particles; n++) { + for (int64_t n = 0; n < sp->buf_n_frames * sp->buf_n_particles; n++) + { sp->output_buffer_ts[n] = -1; } // No valid particles yet - for(int n = 0; n < sp->buf_n_frames; n++) { + for (int n = 0; n < sp->buf_n_frames; n++) + { sp->buf_n_valid[n] = 0; } } @@ -1499,43 +1515,53 @@ vpic_simulation::clear_buffered_particle_dump(const char * sp_name) { * * @return Nothing */ -void -vpic_simulation::interpolate_fields_annotation(const char * sp_name, const interpolator_array_t * ia, - int where_ex, int where_ey, int where_ez, - int where_bx, int where_by, int where_bz) { - - species_t *sp = find_species_name( sp_name, species_list ); - if( !sp ) ERROR(( "Invalid species \"%s\"", sp_name )); - if( !ia ) ERROR(( "Invalid interpolator array" )); - if(where_ex >= sp->has_annotation) { +void vpic_simulation::interpolate_fields_annotation(const char *sp_name, const interpolator_array_t *ia, + int where_ex, int where_ey, int where_ez, + int where_bx, int where_by, int where_bz) +{ + + species_t *sp = find_species_name(sp_name, species_list); + if (!sp) + ERROR(("Invalid species \"%s\"", sp_name)); + if (!ia) + ERROR(("Invalid interpolator array")); + if (where_ex >= sp->has_annotation) + { WARNING(("Attribute %d is invalid for species %s, will skip Ex", where_ex, sp_name)); where_ex = -1; } - if(where_ey >= sp->has_annotation) { + if (where_ey >= sp->has_annotation) + { WARNING(("Attribute %d is invalid for species %s, will skip Ey", where_ey, sp_name)); where_ey = -1; } - if(where_ez >= sp->has_annotation) { + if (where_ez >= sp->has_annotation) + { WARNING(("Attribute %d is invalid for species %s, will skip Ez", where_ez, sp_name)); where_ez = -1; } - if(where_bx >= sp->has_annotation) { + if (where_bx >= sp->has_annotation) + { WARNING(("Attribute %d is invalid for species %s, will skip Bx", where_bx, sp_name)); where_bx = -1; } - if(where_by >= sp->has_annotation) { + if (where_by >= sp->has_annotation) + { WARNING(("Attribute %d is invalid for species %s, will skip By", where_by, sp_name)); where_by = -1; } - if(where_bz >= sp->has_annotation) { + if (where_bz >= sp->has_annotation) + { WARNING(("Attribute %d is invalid for species %s, will skip Bz", where_bz, sp_name)); where_bz = -1; } - if(where_ex<0 && where_ey<0 && where_ez<0 && where_bx<0 && where_by<0 && where_bz<0) return; + if (where_ex < 0 && where_ey < 0 && where_ez < 0 && where_bx < 0 && where_by < 0 && where_bz < 0) + return; - const particle_t* p; + const particle_t *p; int j; - for(p=sp->p, j=0; jnp; j++, p++ ) { + for (p = sp->p, j = 0; j < sp->np; j++, p++) + { // Interpolator for this cell const int i = p->i; const interpolator_t f = ia->i[i]; @@ -1544,19 +1570,25 @@ vpic_simulation::interpolate_fields_annotation(const char * sp_name, const inter const float dy0 = p->dy; const float dz0 = p->dz; // Interpolate field values - const float ex = f.ex + dy0*f.dexdy + dz0*(f.dexdz+dy0*f.d2exdydz); - const float ey = f.ey + dz0*f.deydz + dx0*(f.deydx+dz0*f.d2eydzdx); - const float ez = f.ez + dx0*f.dezdx + dy0*(f.dezdy+dx0*f.d2ezdxdy); - const float bx = f.cbx + dx0*f.dcbxdx; - const float by = f.cby + dy0*f.dcbydy; - const float bz = f.cbz + dz0*f.dcbzdz; + const float ex = f.ex + dy0 * f.dexdy + dz0 * (f.dexdz + dy0 * f.d2exdydz); + const float ey = f.ey + dz0 * f.deydz + dx0 * (f.deydx + dz0 * f.d2eydzdx); + const float ez = f.ez + dx0 * f.dezdx + dy0 * (f.dezdy + dx0 * f.d2ezdxdy); + const float bx = f.cbx + dx0 * f.dcbxdx; + const float by = f.cby + dy0 * f.dcbydy; + const float bz = f.cbz + dz0 * f.dcbzdz; // Store - if(where_ex>=0) sp->set_annotation(j, where_ex, ex); - if(where_ey>=0) sp->set_annotation(j, where_ey, ey); - if(where_ez>=0) sp->set_annotation(j, where_ez, ez); - if(where_bx>=0) sp->set_annotation(j, where_bx, bx); - if(where_by>=0) sp->set_annotation(j, where_by, by); - if(where_bz>=0) sp->set_annotation(j, where_bz, bz); + if (where_ex >= 0) + sp->set_annotation(j, where_ex, ex); + if (where_ey >= 0) + sp->set_annotation(j, where_ey, ey); + if (where_ez >= 0) + sp->set_annotation(j, where_ez, ez); + if (where_bx >= 0) + sp->set_annotation(j, where_bx, bx); + if (where_by >= 0) + sp->set_annotation(j, where_by, by); + if (where_bz >= 0) + sp->set_annotation(j, where_bz, bz); } } @@ -1569,78 +1601,96 @@ vpic_simulation::interpolate_fields_annotation(const char * sp_name, const inter * * @return Nothing */ -void -vpic_simulation::interpolate_hydro_annotation(const char * sp_name, const hydro_array_t * ha, - int where_jx, int where_jy, int where_jz, int where_rho, - int where_px, int where_py, int where_pz, int where_ke, - int where_txx, int where_tyy, int where_tzz, - int where_tyz, int where_tzx, int where_txy) { - - species_t *sp = find_species_name( sp_name, species_list ); - if( !sp ) ERROR(( "Invalid species \"%s\"", sp_name )); - if( !ha ) ERROR(( "Invalid hydro array" )); - if(where_jx >= sp->has_annotation) { +void vpic_simulation::interpolate_hydro_annotation(const char *sp_name, const hydro_array_t *ha, + int where_jx, int where_jy, int where_jz, int where_rho, + int where_px, int where_py, int where_pz, int where_ke, + int where_txx, int where_tyy, int where_tzz, + int where_tyz, int where_tzx, int where_txy) +{ + + species_t *sp = find_species_name(sp_name, species_list); + if (!sp) + ERROR(("Invalid species \"%s\"", sp_name)); + if (!ha) + ERROR(("Invalid hydro array")); + if (where_jx >= sp->has_annotation) + { WARNING(("Attribute %d is invalid for species %s, will skip jx", where_jx, sp_name)); where_jx = -1; } - if(where_jy >= sp->has_annotation) { + if (where_jy >= sp->has_annotation) + { WARNING(("Attribute %d is invalid for species %s, will skip jy", where_jy, sp_name)); where_jy = -1; } - if(where_jz >= sp->has_annotation) { + if (where_jz >= sp->has_annotation) + { WARNING(("Attribute %d is invalid for species %s, will skip jz", where_jz, sp_name)); where_jz = -1; } - if(where_rho >= sp->has_annotation) { + if (where_rho >= sp->has_annotation) + { WARNING(("Attribute %d is invalid for species %s, will skip rho", where_rho, sp_name)); where_rho = -1; } - if(where_px >= sp->has_annotation) { + if (where_px >= sp->has_annotation) + { WARNING(("Attribute %d is invalid for species %s, will skip px", where_px, sp_name)); where_px = -1; } - if(where_py >= sp->has_annotation) { + if (where_py >= sp->has_annotation) + { WARNING(("Attribute %d is invalid for species %s, will skip py", where_py, sp_name)); where_py = -1; } - if(where_pz >= sp->has_annotation) { + if (where_pz >= sp->has_annotation) + { WARNING(("Attribute %d is invalid for species %s, will skip pz", where_pz, sp_name)); where_pz = -1; } - if(where_ke >= sp->has_annotation) { + if (where_ke >= sp->has_annotation) + { WARNING(("Attribute %d is invalid for species %s, will skip ke", where_ke, sp_name)); where_ke = -1; } - if(where_txx >= sp->has_annotation) { + if (where_txx >= sp->has_annotation) + { WARNING(("Attribute %d is invalid for species %s, will skip txx", where_txx, sp_name)); where_txx = -1; } - if(where_tyy >= sp->has_annotation) { + if (where_tyy >= sp->has_annotation) + { WARNING(("Attribute %d is invalid for species %s, will skip tyy", where_tyy, sp_name)); where_tyy = -1; } - if(where_tzz >= sp->has_annotation) { + if (where_tzz >= sp->has_annotation) + { WARNING(("Attribute %d is invalid for species %s, will skip tzz", where_tzz, sp_name)); where_tzz = -1; } - if(where_tyz >= sp->has_annotation) { + if (where_tyz >= sp->has_annotation) + { WARNING(("Attribute %d is invalid for species %s, will skip tyz", where_tyz, sp_name)); where_tyz = -1; } - if(where_tzx >= sp->has_annotation) { + if (where_tzx >= sp->has_annotation) + { WARNING(("Attribute %d is invalid for species %s, will skip tzx", where_tzx, sp_name)); where_tzx = -1; } - if(where_txy >= sp->has_annotation) { + if (where_txy >= sp->has_annotation) + { WARNING(("Attribute %d is invalid for species %s, will skip txy", where_txy, sp_name)); where_txy = -1; } - if(where_jx<0 && where_jy<0 && where_jz<0 && where_rho<0 && where_px<0 && where_py<0 && where_pz<0 && where_ke<0 && - where_txx<0 && where_tyy<0 && where_tzz<0 && where_tyz<0 && where_tzx<0 && where_txy<0 ) return; + if (where_jx < 0 && where_jy < 0 && where_jz < 0 && where_rho < 0 && where_px < 0 && where_py < 0 && where_pz < 0 && where_ke < 0 && + where_txx < 0 && where_tyy < 0 && where_tzz < 0 && where_tyz < 0 && where_tzx < 0 && where_txy < 0) + return; - const particle_t* p; + const particle_t *p; int j; - for(p=sp->p, j=0; jnp; j++, p++ ) { + for (p = sp->p, j = 0; j < sp->np; j++, p++) + { // Cell index for this cell const int i = p->i; // Load displacement in the cell @@ -1650,89 +1700,100 @@ vpic_simulation::interpolate_hydro_annotation(const char * sp_name, const hydro_ // Compute the trilinear coefficients float w0, w1, w2, w3, w4, w5, w6, w7, w8; - w0 = 0.125; /* w0 = 1/(8V) = w/8 */ - dx *= w0; /* dx = wx */ - w1 = w0+dx; /* w1 = w/8 + wx/8 = (w/8)(1+x) */ - w0 -= dx; /* w0 = w/8 - wx/8 = (w/8)(1-x) */ - w3 = 1+dy; /* w3 = 1+y */ - w2 = w0*w3; /* w2 = (w/8)(1-x)(1+y) */ - w3 *= w1; /* w3 = (w/8)(1+x)(1+y) */ - dy = 1-dy; /* dy = 1-y */ - w0 *= dy; /* w0 = (w/8)(1-x)(1-y) */ - w1 *= dy; /* w1 = (w/8)(1+x)(1-y) */ - w7 = 1+dz; /* w7 = 1+z */ - w4 = w0*w7; /* w4 = (w/8)(1-x)(1-y)(1+z) *Done */ - w5 = w1*w7; /* w5 = (w/8)(1+x)(1-y)(1+z) *Done */ - w6 = w2*w7; /* w6 = (w/8)(1-x)(1+y)(1+z) *Done */ - w7 *= w3; /* w7 = (w/8)(1+x)(1+y)(1+z) *Done */ - dz = 1-dz; /* dz = 1-z */ - w0 *= dz; /* w0 = (w/8)(1-x)(1-y)(1-z) *Done */ - w1 *= dz; /* w1 = (w/8)(1+x)(1-y)(1-z) *Done */ - w2 *= dz; /* w2 = (w/8)(1-x)(1+y)(1-z) *Done */ - w3 *= dz; /* w3 = (w/8)(1+x)(1+y)(1-z) *Done */ + w0 = 0.125; /* w0 = 1/(8V) = w/8 */ + dx *= w0; /* dx = wx */ + w1 = w0 + dx; /* w1 = w/8 + wx/8 = (w/8)(1+x) */ + w0 -= dx; /* w0 = w/8 - wx/8 = (w/8)(1-x) */ + w3 = 1 + dy; /* w3 = 1+y */ + w2 = w0 * w3; /* w2 = (w/8)(1-x)(1+y) */ + w3 *= w1; /* w3 = (w/8)(1+x)(1+y) */ + dy = 1 - dy; /* dy = 1-y */ + w0 *= dy; /* w0 = (w/8)(1-x)(1-y) */ + w1 *= dy; /* w1 = (w/8)(1+x)(1-y) */ + w7 = 1 + dz; /* w7 = 1+z */ + w4 = w0 * w7; /* w4 = (w/8)(1-x)(1-y)(1+z) *Done */ + w5 = w1 * w7; /* w5 = (w/8)(1+x)(1-y)(1+z) *Done */ + w6 = w2 * w7; /* w6 = (w/8)(1-x)(1+y)(1+z) *Done */ + w7 *= w3; /* w7 = (w/8)(1+x)(1+y)(1+z) *Done */ + dz = 1 - dz; /* dz = 1-z */ + w0 *= dz; /* w0 = (w/8)(1-x)(1-y)(1-z) *Done */ + w1 *= dz; /* w1 = (w/8)(1+x)(1-y)(1-z) *Done */ + w2 *= dz; /* w2 = (w/8)(1-x)(1+y)(1-z) *Done */ + w3 *= dz; /* w3 = (w/8)(1+x)(1+y)(1-z) *Done */ // Split cell index into components // FIXME: should that use UNVOXEL instead? const int nx2 = grid->nx + 2; - const int nx2ny2 = (grid->ny+2) * nx2; + const int nx2ny2 = (grid->ny + 2) * nx2; const int iz = i / nx2ny2; const int iy = (i % nx2ny2) / nx2; const int ix = i % nx2; // Values at particle location - annotation_t jx=0., jy=0., jz=0., rho=0., px=0., py=0., pz=0., ke=0., txx=0., tyy=0., tzz=0., tyz=0., tzx=0., txy=0.; + annotation_t jx = 0., jy = 0., jz = 0., rho = 0., px = 0., py = 0., pz = 0., ke = 0., txx = 0., tyy = 0., tzz = 0., tyz = 0., tzx = 0., txy = 0.; hydro_t hy; #ifndef VELS_RHOS #define DEFAULT_VELS_RHOS -#define VELS_RHOS(wn, i, j, k) \ - hy = ha->h[VOXEL(i,j,k, grid->nx,grid->ny,grid->nz)]; \ - jx += wn*hy.jx; \ - jy += wn*hy.jy; \ - jz += wn*hy.jz; \ - rho += wn*hy.rho; \ - px += wn*hy.px; \ - py += wn*hy.py; \ - pz += wn*hy.pz; \ - ke += wn*hy.ke; \ - txx += wn*hy.txx; \ - tyy += wn*hy.tyy; \ - tzz += wn*hy.tzz; \ - tyz += wn*hy.tyz; \ - tzx += wn*hy.tzx; \ - txy += wn*hy.txy; +#define VELS_RHOS(wn, i, j, k) \ + hy = ha->h[VOXEL(i, j, k, grid->nx, grid->ny, grid->nz)]; \ + jx += wn * hy.jx; \ + jy += wn * hy.jy; \ + jz += wn * hy.jz; \ + rho += wn * hy.rho; \ + px += wn * hy.px; \ + py += wn * hy.py; \ + pz += wn * hy.pz; \ + ke += wn * hy.ke; \ + txx += wn * hy.txx; \ + tyy += wn * hy.tyy; \ + tzz += wn * hy.tzz; \ + tyz += wn * hy.tyz; \ + tzx += wn * hy.tzx; \ + txy += wn * hy.txy; #endif // Do the interpolation - VELS_RHOS(w0, ix, iy, iz); - VELS_RHOS(w1, ix+1, iy, iz); - VELS_RHOS(w2, ix, iy+1, iz); - VELS_RHOS(w3, ix+1, iy+1, iz); - VELS_RHOS(w4, ix, iy, iz+1); - VELS_RHOS(w5, ix+1, iy, iz+1); - VELS_RHOS(w6, ix, iy+1, iz+1); - VELS_RHOS(w7, ix+1, iy+1, iz+1); + VELS_RHOS(w0, ix, iy, iz); + VELS_RHOS(w1, ix + 1, iy, iz); + VELS_RHOS(w2, ix, iy + 1, iz); + VELS_RHOS(w3, ix + 1, iy + 1, iz); + VELS_RHOS(w4, ix, iy, iz + 1); + VELS_RHOS(w5, ix + 1, iy, iz + 1); + VELS_RHOS(w6, ix, iy + 1, iz + 1); + VELS_RHOS(w7, ix + 1, iy + 1, iz + 1); #ifdef DEFAULT_VELS_RHOS #undef VELS_RHOS #endif // Store - if(where_jx>=0) sp->set_annotation(j, where_jx, jx); - if(where_jy>=0) sp->set_annotation(j, where_jy, jy); - if(where_jz>=0) sp->set_annotation(j, where_jz, jz); - if(where_rho>=0) sp->set_annotation(j, where_rho, rho); - if(where_px>=0) sp->set_annotation(j, where_px, px); - if(where_py>=0) sp->set_annotation(j, where_py, py); - if(where_pz>=0) sp->set_annotation(j, where_pz, pz); - if(where_ke>=0) sp->set_annotation(j, where_ke, ke); - if(where_txx>=0) sp->set_annotation(j, where_txx, txx); - if(where_tyy>=0) sp->set_annotation(j, where_tyy, tyy); - if(where_tzz>=0) sp->set_annotation(j, where_tzz, tzz); - if(where_tyz>=0) sp->set_annotation(j, where_tyz, tyz); - if(where_tzx>=0) sp->set_annotation(j, where_tzx, tzx); - if(where_txy>=0) sp->set_annotation(j, where_txy, txy); - + if (where_jx >= 0) + sp->set_annotation(j, where_jx, jx); + if (where_jy >= 0) + sp->set_annotation(j, where_jy, jy); + if (where_jz >= 0) + sp->set_annotation(j, where_jz, jz); + if (where_rho >= 0) + sp->set_annotation(j, where_rho, rho); + if (where_px >= 0) + sp->set_annotation(j, where_px, px); + if (where_py >= 0) + sp->set_annotation(j, where_py, py); + if (where_pz >= 0) + sp->set_annotation(j, where_pz, pz); + if (where_ke >= 0) + sp->set_annotation(j, where_ke, ke); + if (where_txx >= 0) + sp->set_annotation(j, where_txx, txx); + if (where_tyy >= 0) + sp->set_annotation(j, where_tyy, tyy); + if (where_tzz >= 0) + sp->set_annotation(j, where_tzz, tzz); + if (where_tyz >= 0) + sp->set_annotation(j, where_tyz, tyz); + if (where_tzx >= 0) + sp->set_annotation(j, where_tzx, tzx); + if (where_txy >= 0) + sp->set_annotation(j, where_txy, txy); } } #endif - - diff --git a/src/vpic/dump.h b/src/vpic/dump.h new file mode 100644 index 00000000..1d17ee8a --- /dev/null +++ b/src/vpic/dump.h @@ -0,0 +1,19 @@ +#ifndef dump_h +#define dump_h + +#include +#include "../grid/grid.h" + +// TODO: should this be an enum? +namespace dump_type { + const int grid_dump = 0; + const int field_dump = 1; + const int hydro_dump = 2; + const int particle_dump = 3; + const int restart_dump = 4; + const int history_dump = 5; +} // namespace + +// TODO: namesapce? +std::array global_particle_index(int local_i, grid_t* grid, int rank); +#endif diff --git a/src/vpic/dump_strategy.cc b/src/vpic/dump_strategy.cc new file mode 100644 index 00000000..0eef7372 --- /dev/null +++ b/src/vpic/dump_strategy.cc @@ -0,0 +1,251 @@ +// BinaryDump::BinaryDump(int _rank, int _nproc) : Dump_Strategy(_rank, _nproc) +//{ +//// empty +//} +#include "dump_strategy.h" + +void checkpt_dump_strategy(const Dump_Strategy *ds) +{ + CHECKPT(ds, 1); + // CHECKPT_SYM(ds->dump_fields); + // CHECKPT_SYM(ds->dump_hydro); + // CHECKPT_SYM(ds->dump_particles); +} + +Dump_Strategy * +restore_dump_strategy(void) +{ + Dump_Strategy *ds; + RESTORE(ds); + // RESTORE_SYM(ds->dump_fields); + // RESTORE_SYM(ds->dump_hydro); + // RESTORE_SYM(ds->dump_particles); + return ds; +} + +Dump_Strategy * +new_dump_strategy(DumpStrategyID dump_strategy_id) +{ + Dump_Strategy *ds; + int i; + MALLOC(ds, 1); + CLEAR(ds, 1); + + // Do any post init/restore simulation modifications + switch (dump_strategy_id) + { + case DUMP_STRATEGY_BINARY: + std::cout << "DUMP_STRATEGY_BINARY enabled \n"; + ds = new BinaryDump(rank(), nproc()); + break; + case DUMP_STRATEGY_HDF5: +#ifdef VPIC_ENABLE_HDF5 + std::cout << "DUMP_STRATEGY_HDF5 enabled \n"; + ds = new HDF5Dump(rank(), nproc()); +#else + std::cout << "HDF5Dump is not enabled \n"; + +#endif + break; + case DUMP_STRATEGY_OPENPMD: +#ifdef VPIC_ENABLE_OPENPMD + // std::cout << "DUMP_STRATEGY_OPENPMD \n"; + ds = new OpenPMDDump(rank(), nproc()); +#else + std::cout << "OpenPMDDump is not enabled \n"; +#endif + break; + default: + break; + } + + // REGISTER_OBJECT(ds, checkpt_dump_strategy, restore_dump_strategy, NULL); + return ds; +} + +void delete_dump_strategy(Dump_Strategy *ds) +{ + if (!ds) + return; + UNREGISTER_OBJECT(ds); + FREE(ds); +} + +void BinaryDump::dump_fields( + const char *fbase, + int step, + grid_t *grid, + field_array_t *field_array, + int ftag) +{ + char fname[256]; + FileIO fileIO; + int dim[3]; + + if (!fbase) + ERROR(("Invalid filename")); + + if (rank == 0) + MESSAGE(("Dumping fields to \"%s\"", fbase)); + + if (ftag) + sprintf(fname, "%s.%li.%i", fbase, (long)step, rank); + else + sprintf(fname, "%s.%i", fbase, rank); + + FileIOStatus status = fileIO.open(fname, io_write); + if (status == fail) + ERROR(("Could not open \"%s\".", fname)); + + /* IMPORTANT: these values are written in WRITE_HEADER_V0 */ + size_t nxout = grid->nx; + size_t nyout = grid->ny; + size_t nzout = grid->nz; + float dxout = grid->dx; + float dyout = grid->dy; + float dzout = grid->dz; + + WRITE_HEADER_V0(dump_type::field_dump, -1, 0, fileIO, step, rank, nproc); + + dim[0] = grid->nx + 2; + dim[1] = grid->ny + 2; + dim[2] = grid->nz + 2; + WRITE_ARRAY_HEADER(field_array->f, 3, dim, fileIO); + fileIO.write(field_array->f, dim[0] * dim[1] * dim[2]); + if (fileIO.close()) + ERROR(("File close failed on dump fields!!!")); +} + +void BinaryDump::dump_particles( + const char *fbase, + species_t *sp, + grid_t *grid, + int step, + interpolator_array_t *interpolator_array, + int ftag) +{ + char fname[256]; + FileIO fileIO; + int dim[1], buf_start; + static particle_t *ALIGNED(128) p_buf = NULL; + + // TODO: reconcile this with MAX_IO_CHUNK, and update Cmake option + // description to explain what backends use it +#define PBUF_SIZE 32768 // 1MB of particles + + if (!sp) + ERROR(("Invalid species name \"%s\".", sp->name)); + + if (!fbase) + ERROR(("Invalid filename")); + + if (!p_buf) + MALLOC_ALIGNED(p_buf, PBUF_SIZE, 128); + + if (rank == 0) + MESSAGE(("Dumping \"%s\" particles to \"%s\"", sp->name, fbase)); + + if (ftag) + sprintf(fname, "%s.%li.%i", fbase, (long)step, rank); + else + sprintf(fname, "%s.%i", fbase, rank); + FileIOStatus status = fileIO.open(fname, io_write); + if (status == fail) + ERROR(("Could not open \"%s\"", fname)); + + /* IMPORTANT: these values are written in WRITE_HEADER_V0 */ + size_t nxout = grid->nx; + size_t nyout = grid->ny; + size_t nzout = grid->nz; + float dxout = grid->dx; + float dyout = grid->dy; + float dzout = grid->dz; + + WRITE_HEADER_V0(dump_type::particle_dump, sp->id, sp->q / sp->m, fileIO, step, rank, nproc); + + dim[0] = sp->np; + WRITE_ARRAY_HEADER(p_buf, 1, dim, fileIO); + + // Copy a PBUF_SIZE hunk of the particle list into the particle + // buffer, timecenter it and write it out. This is done this way to + // guarantee the particle list unchanged while not requiring too + // much memory. + + // FIXME: WITH A PIPELINED CENTER_P, PBUF NOMINALLY SHOULD BE QUITE + // LARGE. + + particle_t *sp_p = sp->p; + sp->p = p_buf; + int sp_np = sp->np; + sp->np = 0; + int sp_max_np = sp->max_np; + sp->max_np = PBUF_SIZE; + for (buf_start = 0; buf_start < sp_np; buf_start += PBUF_SIZE) + { + sp->np = sp_np - buf_start; + if (sp->np > PBUF_SIZE) + sp->np = PBUF_SIZE; + COPY(sp->p, &sp_p[buf_start], sp->np); + center_p(sp, interpolator_array); + fileIO.write(sp->p, sp->np); + } + sp->p = sp_p; + sp->np = sp_np; + sp->max_np = sp_max_np; + + if (fileIO.close()) + ERROR(("File close failed on dump particles!!!")); +} +void BinaryDump::dump_hydro( + const char *fbase, + int step, + hydro_array_t *hydro_array, + species_t *sp, + interpolator_array_t *interpolator_array, + grid_t *grid, + int ftag) +{ + char fname[256]; + FileIO fileIO; + int dim[3]; + printf("Calling Binay dump_hydro ... \n"); + + if (!sp) + ERROR(("Invalid species \"%s\"", sp->name)); + + clear_hydro_array(hydro_array); + accumulate_hydro_p(hydro_array, sp, interpolator_array); + synchronize_hydro_array(hydro_array); + + if (!fbase) + ERROR(("Invalid filename")); + + if (rank == 0) + MESSAGE(("Dumping \"%s\" hydro fields to \"%s\"", sp->name, fbase)); + + if (ftag) + sprintf(fname, "%s.%li.%i", fbase, (long)step, rank); + else + sprintf(fname, "%s.%i", fbase, rank); + FileIOStatus status = fileIO.open(fname, io_write); + if (status == fail) + ERROR(("Could not open \"%s\".", fname)); + + /* IMPORTANT: these values are written in WRITE_HEADER_V0 */ + size_t nxout = grid->nx; + size_t nyout = grid->ny; + size_t nzout = grid->nz; + float dxout = grid->dx; + float dyout = grid->dy; + float dzout = grid->dz; + + WRITE_HEADER_V0(dump_type::hydro_dump, sp->id, sp->q / sp->m, fileIO, step, rank, nproc); + + dim[0] = grid->nx + 2; + dim[1] = grid->ny + 2; + dim[2] = grid->nz + 2; + WRITE_ARRAY_HEADER(hydro_array->h, 3, dim, fileIO); + fileIO.write(hydro_array->h, dim[0] * dim[1] * dim[2]); + if (fileIO.close()) + ERROR(("File close failed on dump hydro!!!")); +} diff --git a/src/vpic/dump_strategy.h b/src/vpic/dump_strategy.h new file mode 100644 index 00000000..010b33b8 --- /dev/null +++ b/src/vpic/dump_strategy.h @@ -0,0 +1,2195 @@ +#ifndef Dump_Strategy_h +#define Dump_Strategy_h + +#include +#include +#include + +// #define DUMP_INFO_DEBUG 1 +// #define H5_ASYNC 1 +#ifdef H5_ASYNC +#include "h5_vol_external_async_native.h" +#endif +// #define CHUNK_FLAG 1 + +// #define METADATA_COLL_WRITE 1 +// #define TRUE 1 + +// #define HAS_FIELD_COMP 1 +// #define HAS_HYDRO_COMP 1 +// #define HAS_PARTICLE_COMP 1 + +// #define HAS_INDEPENDENT_IO 1 + +#include // TODO: it would be good if this didn't have to know about MPI +#include + +// TODO: should I drop the ./src here? +#include "../util/io/FileIO.h" +#include "../util/util_base.h" +#include "../util/io/FileUtils.h" +#include "../field_advance/field_advance.h" +#include "../sf_interface/sf_interface.h" +#include "../species_advance/species_advance.h" + +#include "dump.h" +#include "dumpmacros.h" + +#ifdef VPIC_ENABLE_HDF5 +#include "hdf5.h" // from the lib +#include "hdf5_header_info.h" // from vpic +// #define daos "daos:" +#define daos "" +#endif + +#ifdef VPIC_ENABLE_OPENPMD +#include +#endif + +// #define N_FILE_N_PROCESS 1 +// #define TEST_MPIIO 1 + +// TODO: delete this +#define _LOG_PREFIX \ + __FILE__ "(" EXPAND_AND_STRINGIFY(__LINE__) ")[" << rank << "]: " +#define io_log(x) \ + do \ + { \ + if (rank == 0) \ + { \ + std::cerr << _LOG_PREFIX << x << std::endl; \ + std::cerr.flush(); \ + } \ + } while (0) + +typedef enum DumpStrategyID +{ + DUMP_STRATEGY_BINARY = 0, + DUMP_STRATEGY_HDF5 = 1, + DUMP_STRATEGY_OPENPMD = 2, +} DumpStrategyID; + +// Runtime inheritance is obviously not very "VPIC like", as we will [probably] +// incur a penalty for the vtable lookup, but given we're about to do IO this +// is very negligible. +class Dump_Strategy +{ +public: + int rank, nproc, num_step; + + Dump_Strategy(int _rank, int _nproc) : rank(_rank), + nproc(_nproc) + { + } // empty + + virtual ~Dump_Strategy(){}; + + virtual void dump_fields( + const char *fbase, + int step, + grid_t *grid, + field_array_t *field_array, + int ftag) = 0; + virtual void dump_hydro( + const char *fbase, + int step, + hydro_array_t *hydro_array, + species_t *sp, + interpolator_array_t *interpolator_array, + grid_t *grid, + int ftag) = 0; + virtual void dump_particles( + const char *fbase, + species_t *sp, + grid_t *grid, + int step, + interpolator_array_t *interpolator_array, + int ftag) = 0; +}; + +Dump_Strategy *new_dump_strategy(DumpStrategyID dump_strategy_id); +void delete_dump_strategy(Dump_Strategy *ds); + +class BinaryDump : public Dump_Strategy +{ +public: + using Dump_Strategy::Dump_Strategy; // inherit constructor + // BinaryDump(int _rank, int _nproc ) : Dump_Strategy(_rank, _nproc ){ } // empty + + // TODO: now we pass rank and step, ftag has odd semanticds + void dump_fields( + const char *fbase, + int step, + grid_t *grid, + field_array_t *field_array, + int ftag); + void dump_hydro( + const char *fbase, + int step, + hydro_array_t *hydro_array, + species_t *sp, + interpolator_array_t *interpolator_array, + grid_t *grid, + int ftag); + void dump_particles( + const char *fbase, + species_t *sp, + grid_t *grid, + int step, + interpolator_array_t *interpolator_array, + int ftag); +}; + +#ifdef VPIC_ENABLE_HDF5 + +struct field_dump_flag_t +{ + bool ex = true, ey = true, ez = true, div_e_err = true; + bool cbx = true, cby = true, cbz = true, div_b_err = true; + bool tcax = true, tcay = true, tcaz = true, rhob = true; + bool jfx = true, jfy = true, jfz = true, rhof = true; + bool ematx = true, ematy = true, ematz = true, nmat = true; + bool fmatx = true, fmaty = true, fmatz = true, cmat = true; + void disableE() + { + ex = false, ey = false, ez = false, div_e_err = false; + } + + void disableCB() + { + cbx = false, cby = false, cbz = false, div_b_err = false; + } + + void disableTCA() + { + tcax = false, tcay = false, tcaz = false, rhob = false; + } + + void disableJF() + { + jfx = false, jfy = false, jfz = false, rhof = false; + } + + void disableEMAT() + { + ematx = false, ematy = false, ematz = false, nmat = false; + } + + void disableFMAT() + { + fmatx = false, fmaty = false, fmatz = false, cmat = false; + } + + void resetToDefaults() + { + ex = true, ey = true, ez = true, div_e_err = true; + cbx = true, cby = true, cbz = true, div_b_err = true; + tcax = true, tcay = true, tcaz = true, rhob = true; + jfx = true, jfy = true, jfz = true, rhof = true; + ematx = true, ematy = true, ematz = true, nmat = true; + fmatx = true, fmaty = true, fmatz = true, cmat = true; + } + + bool enabledE() + { + return ex && ey && ez; + } + + bool enabledCB() + { + return cbx && cby && cbz; + } + + bool enabledTCA() + { + return tcax && tcay && tcaz; + } + + bool enabledJF() + { + return jfx && jfy && jfz; + } + + bool enabledEMAT() + { + return ematx && ematy && ematz; + } + + bool enabledFMAT() + { + return fmatx && fmaty && fmatz; + } +}; + +struct hydro_dump_flag_t +{ + bool jx = true, jy = true, jz = true, rho = true; + bool px = true, py = true, pz = true, ke = true; + bool txx = true, tyy = true, tzz = true; + bool tyz = true, tzx = true, txy = true; + + void disableJ() + { + jx = false, jy = false, jz = false, rho = false; + } + + void disableP() + { + px = false, py = false, pz = false, ke = false; + } + + void disableTD() // Stress diagonal + { + txx = false, tyy = false, tzz = false; + } + + void disableTOD() // Stress off-diagonal + { + tyz = false, tzx = false, txy = false; + } + void resetToDefaults() + { + jx = true, jy = true, jz = true, rho = true; + px = true, py = true, pz = true, ke = true; + txx = true, tyy = true, tzz = true; + tyz = true, tzx = true, txy = true; + } + + bool enabledJ() + { + return jx && jy && jz; + } + + bool enabledP() + { + return px && py && pz; + } + + bool enabledTD() + { + return txx && tyy && tzz; + } + + bool enabledTOD() + { + return tyz && tzx && txy; + } +}; +class HDF5Dump : public Dump_Strategy +{ + std::unordered_map tframe_map; + +public: + using Dump_Strategy::Dump_Strategy; // inherit constructor + + // TODO: replace these with a common dump interface + // Declare vars to use + hydro_dump_flag_t hydro_dump_flag; + field_dump_flag_t field_dump_flag; + +#define DUMP_DIR_FORMAT "./%s" + +// TODO: naming a macro so close to existing functions AND data is not a good +// define to do C-style indexing +#define _hydro(x, y, z) hydro_array->h[VOXEL(x, y, z, grid->nx, grid->ny, grid->nz)] + + // TODO: make function? + void dump_fields( + const char *fbase, + int step, + grid_t *grid, + field_array_t *field_array, + int ftag) + { + size_t step_for_viou = step; + int file_exist_flag = 0; + + int mpi_size, mpi_rank; + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + +#ifdef DUMP_INFO_DEBUG + printf("MPI rank = %d, size = %d \n", mpi_rank, mpi_size); + // printf("base dir for field: %s \n", fdParams.baseDir); + // printf("stride x y z = (%ld, %ld, %ld)\n", fdParams.stride_x, fdParams.stride_y, fdParams.stride_z); + printf("grid x, y z = (%d, %d, %d) \n", grid->nx, grid->ny, grid->nz); + printf("domain loc (x0, y0, z0) -> (x1, y1, z1) = (%f, %f, %f) -> (%f, %f, %f) \n", grid->x0, grid->y0, grid->z0, grid->x1, grid->y1, grid->z1); + // printf("global->topology_x, y, z = %f, %f, %f \n ", global->topology_x, global->topology_y, global->topology_z); + printf("grid -> sx, sy, sz = (%d, %d, %d), nv=%d \n", grid->sx, grid->sy, grid->sz, grid->nv); +#endif + char fname[256]; + char field_scratch[128]; + char subfield_scratch[128]; + + sprintf(field_scratch, DUMP_DIR_FORMAT, "field_hdf5"); + FileUtils::makeDirectory(field_scratch); + sprintf(subfield_scratch, "%s/T.%zu/", field_scratch, step_for_viou); + FileUtils::makeDirectory(subfield_scratch); + + sprintf(fname, "%s%s/%s_%zu.h5", daos, subfield_scratch, "fields", step_for_viou); + double el1 = uptime(); + + // int file_exist(const char *filename) + //{ + // struct stat buffer; + // return (stat(filename, &buffer) == 0); + //} + + // struct stat buffer; + // if((stat(fname, &buffer) == 0)){ + // file_exist_flag = 1; + // if(!mpi_rank) + // printf("Write original files /w HDF5! \n"); + // } + file_exist_flag = 0; + + hid_t plist_id; + hid_t file_id; + if (!file_exist_flag) + { + plist_id = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(plist_id, MPI_COMM_WORLD, MPI_INFO_NULL); + // H5Pset_alignment(plist_id, 4194304, 4194304); + /*if(H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0){ + exit(-1); + }*/ + +#ifdef METADATA_COLL_WRITE + if (!mpi_rank) + printf("Enable collective metadata write !\n"); + H5Pset_coll_metadata_write(plist_id, TRUE); +#endif + file_id = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, plist_id); + H5Pclose(plist_id); + } + else + { + file_id = H5Fopen(fname, H5F_ACC_RDWR, H5P_DEFAULT); + } + + sprintf(fname, "Timestep_%zu", step_for_viou); + hid_t group_id; + if (!file_exist_flag) + { + group_id = H5Gcreate(file_id, fname, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + } + else + { + group_id = H5Gopen(file_id, fname, H5P_DEFAULT); + } + el1 = uptime() - el1; + io_log("TimeHDF5Open): " << el1 << " s"); // Easy to handle results for scripts + double el2 = uptime(); + + /* + // Create a variable list of field values to output. + size_t numvars = std::min(global->fdParams.output_vars.bitsum(), total_field_variables); + size_t * varlist = new size_t[numvars]; + + for(size_t i(0), c(0); ifdParams.output_vars.bitset(i)) varlist[c++] = i; + + printf("\nBEGIN_OUTPUT: numvars = %zd \n", numvars);*/ + +#define fpp(x, y, z) f[VOXEL(x, y, z, grid->nx, grid->ny, grid->nz)] + + /* + typedef struct field { + float ex, ey, ez, div_e_err; // Electric field and div E error + float cbx, cby, cbz, div_b_err; // Magnetic field and div B error + float tcax, tcay, tcaz, rhob; // TCA fields and bound charge density + float jfx, jfy, jfz, rhof; // Free current and charge density + material_id ematx, ematy, ematz, nmat; // Material at edge centers and nodes + material_id fmatx, fmaty, fmatz, cmat; // Material at face and cell centers + } field_t;*/ + // Local voxel mesh resolution. Voxels are + // indexed FORTRAN style 0:nx+1,0:ny+1,0:nz+1 + // with voxels 1:nx,1:ny,1:nz being non-ghost + // voxels. + + hsize_t temp_buf_index; + hid_t dset_id; + // char *field_var_name[] = {"ex","ey","ez","div_e_err","cbx","cby","cbz","div_b_err","tcax","tcay","tcaz","rhob","jfx","jfy","jfz","rhof"}; + // Comment out for test only + + if (!file_exist_flag) + { + plist_id = H5Pcreate(H5P_DATASET_XFER); +// H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE); +#ifdef HAS_INDEPENDENT_IO + if (!mpi_rank) + printf("\n ###\n VPIC Independent I/O! \n ###\n"); + H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_INDEPENDENT); +#else + H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE); +#endif + } + else + { + plist_id = H5P_DEFAULT; + } + + // H5Sselect_hyperslab(filespace, H5S_SELECT_SET, (hsize_t *) &offset, NULL, (hsize_t *) &numparticles, NULL); + + // global->topology_x + + hsize_t field_global_size[3], field_local_size[3], global_offset[3], global_count[3]; + field_global_size[0] = (grid->nx * grid->gpx); + field_global_size[1] = (grid->ny * grid->gpy); + field_global_size[2] = (grid->nz * grid->gpz); + + field_local_size[0] = grid->nx; + field_local_size[1] = grid->ny; + field_local_size[2] = grid->nz; + + int gpx = grid->gpx; + int gpy = grid->gpy; + int gpz = grid->gpz; + + // Convert rank to local decomposition + int rx, ry, rz; + UNVOXEL(mpi_rank, rx, ry, rz, grid->gpx, grid->gpy, grid->gpz); + + int mpi_rank_x, mpi_rank_y, mpi_rank_z; + mpi_rank_x = rx; + mpi_rank_y = ry; + mpi_rank_z = rz; + + global_offset[0] = (grid->nx) * mpi_rank_x; + global_offset[1] = (grid->ny) * mpi_rank_y; + global_offset[2] = (grid->nz) * mpi_rank_z; + + global_count[0] = (grid->nx); + global_count[1] = (grid->ny); + global_count[2] = (grid->nz); +#ifdef DUMP_INFO_DEBUG + if (mpi_rank < 4) + { + printf("grid nx, ny nz = (%d, %d, %d) \n", grid->nx, grid->ny, grid->nz); + printf("mpi-rank = %d, rank index = (%d, %d, %d) \n", mpi_rank, mpi_rank_x, mpi_rank_y, mpi_rank_z); + printf("global size = %llu %llu %llu \n", field_global_size[0], field_global_size[1], field_global_size[2]); + printf("global_offset = %llu %llu %llu \n", global_offset[0], global_offset[1], global_offset[2]); + printf("global_count = %llu %llu %llu \n", global_count[0], global_count[1], global_count[2]); + fflush(stdout); + } +#endif + hid_t filespace; //= H5Screate_simple(3, field_global_size, NULL); + hid_t memspace; // = H5Screate_simple(3, field_local_size, NULL); + if (!file_exist_flag) + { + filespace = H5Screate_simple(3, field_global_size, NULL); + } + memspace = H5Screate_simple(3, field_local_size, NULL); + + hsize_t chunk_dims[3]; + chunk_dims[0] = 288; // grid->nx; //8 x 8 x 8 + chunk_dims[1] = 24; // grid->ny; // + chunk_dims[2] = 24; // grid->nz; + + hid_t dataspace_id; + hid_t dcpl_id = H5Pcreate(H5P_DATASET_CREATE); +#ifdef CHUNK_FLAG + H5Pset_chunk(dcpl_id, 3, chunk_dims); + if (!mpi_rank) + printf("Enable chunking !\n"); +#endif + + /* + typedef struct field { + float ex, ey, ez, div_e_err; // Electric field and div E error + float cbx, cby, cbz, div_b_err; // Magnetic field and div B error + float tcax, tcay, tcaz, rhob; // TCA fields and bound charge density + float jfx, jfy, jfz, rhof; // Free current and charge density + material_id ematx, ematy, ematz, nmat; // Material at edge centers and nodes + material_id fmatx, fmaty, fmatz, cmat; // Material at face and cell centers + } field_t;*/ + +#if HAS_FIELD_COMP + + if (!mpi_rank) + printf("Using Field Compund type !\n"); + hid_t field_comp_type_it = H5Tcreate(H5T_COMPOUND, sizeof(field_t)); + H5Tinsert(field_comp_type_it, "ex", HOFFSET(field_t, ex), H5T_NATIVE_FLOAT); + H5Tinsert(field_comp_type_it, "ey", HOFFSET(field_t, ey), H5T_NATIVE_FLOAT); + H5Tinsert(field_comp_type_it, "ez", HOFFSET(field_t, ez), H5T_NATIVE_FLOAT); + H5Tinsert(field_comp_type_it, "div_e_err", HOFFSET(field_t, div_e_err), H5T_NATIVE_FLOAT); + + H5Tinsert(field_comp_type_it, "cbx", HOFFSET(field_t, cbx), H5T_NATIVE_FLOAT); + H5Tinsert(field_comp_type_it, "cby", HOFFSET(field_t, cby), H5T_NATIVE_FLOAT); + H5Tinsert(field_comp_type_it, "cbz", HOFFSET(field_t, cbz), H5T_NATIVE_FLOAT); + H5Tinsert(field_comp_type_it, "div_b_err", HOFFSET(field_t, div_b_err), H5T_NATIVE_FLOAT); + + H5Tinsert(field_comp_type_it, "tcax", HOFFSET(field_t, tcax), H5T_NATIVE_FLOAT); + H5Tinsert(field_comp_type_it, "tcay", HOFFSET(field_t, tcay), H5T_NATIVE_FLOAT); + H5Tinsert(field_comp_type_it, "tcaz", HOFFSET(field_t, tcaz), H5T_NATIVE_FLOAT); + H5Tinsert(field_comp_type_it, "rhob", HOFFSET(field_t, rhob), H5T_NATIVE_FLOAT); + + H5Tinsert(field_comp_type_it, "jfx", HOFFSET(field_t, jfx), H5T_NATIVE_FLOAT); + H5Tinsert(field_comp_type_it, "jfy", HOFFSET(field_t, jfy), H5T_NATIVE_FLOAT); + H5Tinsert(field_comp_type_it, "jfz", HOFFSET(field_t, jfz), H5T_NATIVE_FLOAT); + H5Tinsert(field_comp_type_it, "rhof", HOFFSET(field_t, rhof), H5T_NATIVE_FLOAT); + + H5Tinsert(field_comp_type_it, "ematx", HOFFSET(field_t, ematx), H5T_NATIVE_SHORT); + H5Tinsert(field_comp_type_it, "ematy", HOFFSET(field_t, ematy), H5T_NATIVE_SHORT); + H5Tinsert(field_comp_type_it, "ematz", HOFFSET(field_t, ematz), H5T_NATIVE_SHORT); + H5Tinsert(field_comp_type_it, "nmat", HOFFSET(field_t, nmat), H5T_NATIVE_SHORT); + + H5Tinsert(field_comp_type_it, "fmatx", HOFFSET(field_t, fmatx), H5T_NATIVE_SHORT); + H5Tinsert(field_comp_type_it, "fmaty", HOFFSET(field_t, fmaty), H5T_NATIVE_SHORT); + H5Tinsert(field_comp_type_it, "fmatz", HOFFSET(field_t, fmatz), H5T_NATIVE_SHORT); + H5Tinsert(field_comp_type_it, "cmat", HOFFSET(field_t, cmat), H5T_NATIVE_SHORT); + + if (!file_exist_flag) + { + dset_id = H5Dcreate(group_id, "FIELD_DATA_COMP_T", field_comp_type_it, filespace, H5P_DEFAULT, dcpl_id, H5P_DEFAULT); + } + else + { + dset_id = H5Dopen(group_id, "FIELD_DATA_COMP_T", H5P_DEFAULT); + } + dataspace_id = H5Dget_space(dset_id); + H5Sselect_hyperslab(dataspace_id, H5S_SELECT_SET, global_offset, NULL, global_count, NULL); + + field_t *temp_buf = (field_t *)malloc(sizeof(field_t) * (grid->nx) * (grid->ny) * (grid->nz)); + temp_buf_index = 0; + for (size_t i(1); i < grid->nx + 1; i++) + { + for (size_t j(1); j < grid->ny + 1; j++) + { + for (size_t k(1); k < grid->nz + 1; k++) + { + temp_buf[temp_buf_index].ex = FIELD_ARRAY_NAME->fpp(i, j, k).ex; + temp_buf[temp_buf_index].ey = FIELD_ARRAY_NAME->fpp(i, j, k).ey; + temp_buf[temp_buf_index].ez = FIELD_ARRAY_NAME->fpp(i, j, k).ez; + temp_buf[temp_buf_index].div_e_err = FIELD_ARRAY_NAME->fpp(i, j, k).div_e_err; + temp_buf[temp_buf_index].cbx = FIELD_ARRAY_NAME->fpp(i, j, k).cbx; + temp_buf[temp_buf_index].cby = FIELD_ARRAY_NAME->fpp(i, j, k).cby; + temp_buf[temp_buf_index].cbz = FIELD_ARRAY_NAME->fpp(i, j, k).cbz; + temp_buf[temp_buf_index].div_b_err = FIELD_ARRAY_NAME->fpp(i, j, k).div_b_err; + temp_buf[temp_buf_index].tcax = FIELD_ARRAY_NAME->fpp(i, j, k).tcax; + temp_buf[temp_buf_index].tcay = FIELD_ARRAY_NAME->fpp(i, j, k).tcay; + temp_buf[temp_buf_index].tcaz = FIELD_ARRAY_NAME->fpp(i, j, k).tcaz; + temp_buf[temp_buf_index].rhob = FIELD_ARRAY_NAME->fpp(i, j, k).rhob; + temp_buf[temp_buf_index].jfx = FIELD_ARRAY_NAME->fpp(i, j, k).jfx; + temp_buf[temp_buf_index].jfy = FIELD_ARRAY_NAME->fpp(i, j, k).jfy; + temp_buf[temp_buf_index].jfz = FIELD_ARRAY_NAME->fpp(i, j, k).jfz; + temp_buf[temp_buf_index].rhof = FIELD_ARRAY_NAME->fpp(i, j, k).rhof; + temp_buf[temp_buf_index].ematx = FIELD_ARRAY_NAME->fpp(i, j, k).ematx; + temp_buf[temp_buf_index].ematy = FIELD_ARRAY_NAME->fpp(i, j, k).ematy; + temp_buf[temp_buf_index].ematz = FIELD_ARRAY_NAME->fpp(i, j, k).ematz; + temp_buf[temp_buf_index].nmat = FIELD_ARRAY_NAME->fpp(i, j, k).nmat; + temp_buf[temp_buf_index].fmatx = FIELD_ARRAY_NAME->fpp(i, j, k).fmatx; + temp_buf[temp_buf_index].fmaty = FIELD_ARRAY_NAME->fpp(i, j, k).fmaty; + temp_buf[temp_buf_index].fmatz = FIELD_ARRAY_NAME->fpp(i, j, k).fmatz; + temp_buf[temp_buf_index].cmat = FIELD_ARRAY_NAME->fpp(i, j, k).cmat; + temp_buf_index = temp_buf_index + 1; + } + } + } + + H5Dwrite(dset_id, field_comp_type_it, memspace, dataspace_id, plist_id, temp_buf); + H5Sclose(dataspace_id); + H5Dclose(dset_id); + H5Tclose(field_comp_type_it); + free(temp_buf); +#else + +#define DUMP_FIELD_TO_HDF5(DSET_NAME, ATTRIBUTE_NAME, ELEMENT_TYPE) \ + { \ + if (!file_exist_flag) \ + { \ + dset_id = H5Dcreate(group_id, DSET_NAME, ELEMENT_TYPE, filespace, H5P_DEFAULT, dcpl_id, H5P_DEFAULT); \ + } \ + else \ + { \ + dset_id = H5Dopen(group_id, DSET_NAME, H5P_DEFAULT); \ + dataspace_id = H5Dget_space(dset_id); \ + } \ + temp_buf_index = 0; \ + for (size_t i(1); i < grid->nx + 1; i++) \ + { \ + for (size_t j(1); j < grid->ny + 1; j++) \ + { \ + for (size_t k(1); k < grid->nz + 1; k++) \ + { \ + temp_buf[temp_buf_index] = FIELD_ARRAY_NAME->fpp(i, j, k).ATTRIBUTE_NAME; \ + temp_buf_index = temp_buf_index + 1; \ + } \ + } \ + } \ + dataspace_id = H5Dget_space(dset_id); \ + H5Sselect_hyperslab(dataspace_id, H5S_SELECT_SET, global_offset, NULL, global_count, NULL); \ + H5Dwrite(dset_id, ELEMENT_TYPE, memspace, dataspace_id, plist_id, temp_buf); \ + H5Sclose(dataspace_id); \ + H5Dclose(dset_id); \ + } + +#define DUMP_FIELD_TO_HDF5_PRINT(DSET_NAME, ATTRIBUTE_NAME, ELEMENT_TYPE) \ + { \ + if (!file_exist_flag) \ + { \ + dset_id = H5Dcreate(group_id, DSET_NAME, ELEMENT_TYPE, filespace, H5P_DEFAULT, dcpl_id, H5P_DEFAULT); \ + } \ + else \ + { \ + dset_id = H5Dopen(group_id, DSET_NAME, H5P_DEFAULT); \ + dataspace_id = H5Dget_space(dset_id); \ + } \ + temp_buf_index = 0; \ + for (size_t i(1); i < grid->nx + 1; i++) \ + { \ + for (size_t j(1); j < grid->ny + 1; j++) \ + { \ + for (size_t k(1); k < grid->nz + 1; k++) \ + { \ + printf("%s, ijk = (%d, %d, %d), value = %f \n", DSET_NAME, i, j, k, FIELD_ARRAY_NAME->fpp(i, j, k).ATTRIBUTE_NAME); \ + temp_buf[temp_buf_index] = FIELD_ARRAY_NAME->fpp(i, j, k).ATTRIBUTE_NAME; \ + temp_buf_index = temp_buf_index + 1; \ + } \ + } \ + } \ + dataspace_id = H5Dget_space(dset_id); \ + H5Sselect_hyperslab(dataspace_id, H5S_SELECT_SET, global_offset, NULL, global_count, NULL); \ + H5Dwrite(dset_id, ELEMENT_TYPE, memspace, dataspace_id, plist_id, temp_buf); \ + H5Sclose(dataspace_id); \ + H5Dclose(dset_id); \ + } + + float *temp_buf = (float *)malloc(sizeof(float) * (grid->nx) * (grid->ny) * (grid->nz)); + + // printf("Hellow 1 \n"); + // if (field_dump_flag.ex) + // DUMP_FIELD_TO_HDF5("ex", ex, H5T_NATIVE_FLOAT); + if (field_dump_flag.ex) + DUMP_FIELD_TO_HDF5_PRINT("ex", ex, H5T_NATIVE_FLOAT); + + if (field_dump_flag.ey) + DUMP_FIELD_TO_HDF5("ey", ey, H5T_NATIVE_FLOAT); + // if (field_dump_flag.ey) + // DUMP_FIELD_TO_HDF5_PRINT("ey", ey, H5T_NATIVE_FLOAT); + + if (field_dump_flag.ez) + DUMP_FIELD_TO_HDF5("ez", ez, H5T_NATIVE_FLOAT); + if (field_dump_flag.div_e_err) + DUMP_FIELD_TO_HDF5("div_e_err", div_e_err, H5T_NATIVE_FLOAT); + + if (field_dump_flag.cbx) + DUMP_FIELD_TO_HDF5("cbx", cbx, H5T_NATIVE_FLOAT); + if (field_dump_flag.cby) + DUMP_FIELD_TO_HDF5("cby", cby, H5T_NATIVE_FLOAT); + if (field_dump_flag.cbz) + DUMP_FIELD_TO_HDF5("cbz", cbz, H5T_NATIVE_FLOAT); + if (field_dump_flag.div_b_err) + DUMP_FIELD_TO_HDF5("div_b_err", div_b_err, H5T_NATIVE_FLOAT); + + if (field_dump_flag.tcax) + DUMP_FIELD_TO_HDF5("tcax", tcax, H5T_NATIVE_FLOAT); + if (field_dump_flag.tcay) + DUMP_FIELD_TO_HDF5("tcay", tcay, H5T_NATIVE_FLOAT); + if (field_dump_flag.tcaz) + DUMP_FIELD_TO_HDF5("tcaz", tcaz, H5T_NATIVE_FLOAT); + if (field_dump_flag.rhob) + DUMP_FIELD_TO_HDF5("rhob", rhob, H5T_NATIVE_FLOAT); + + if (field_dump_flag.jfx) + DUMP_FIELD_TO_HDF5("jfx", jfx, H5T_NATIVE_FLOAT); + if (field_dump_flag.jfy) + DUMP_FIELD_TO_HDF5("jfy", jfy, H5T_NATIVE_FLOAT); + if (field_dump_flag.jfz) + DUMP_FIELD_TO_HDF5("jfz", jfz, H5T_NATIVE_FLOAT); + if (field_dump_flag.rhof) + DUMP_FIELD_TO_HDF5("rhof", rhof, H5T_NATIVE_FLOAT); + + // H5T_NATIVE_SHORT for material_id (typedef int16_t material_id) + if (field_dump_flag.ematx) + DUMP_FIELD_TO_HDF5("ematx", ematx, H5T_NATIVE_SHORT); + if (field_dump_flag.ematy) + DUMP_FIELD_TO_HDF5("ematy", ematy, H5T_NATIVE_SHORT); + if (field_dump_flag.ematz) + DUMP_FIELD_TO_HDF5("ematz", ematz, H5T_NATIVE_SHORT); + if (field_dump_flag.nmat) + DUMP_FIELD_TO_HDF5("nmat", nmat, H5T_NATIVE_SHORT); + + if (field_dump_flag.fmatx) + DUMP_FIELD_TO_HDF5("fmatx", fmatx, H5T_NATIVE_SHORT); + if (field_dump_flag.fmaty) + DUMP_FIELD_TO_HDF5("fmaty", fmaty, H5T_NATIVE_SHORT); + if (field_dump_flag.fmatz) + DUMP_FIELD_TO_HDF5("fmatz", fmatz, H5T_NATIVE_SHORT); + if (field_dump_flag.cmat) + DUMP_FIELD_TO_HDF5("cmat", cmat, H5T_NATIVE_SHORT); + + free(temp_buf); +#endif + + /* + H5D_mpio_actual_io_mode_t actual_io_mode; + H5Pget_mpio_actual_io_mode(plist_id, &actual_io_mode); + switch (actual_io_mode) + { + case H5D_MPIO_NO_COLLECTIVE: + io_log("H5Pget_mpio_actual_io_mode: H5D_MPIO_NO_COLLECTIVE: "); + break; + case H5D_MPIO_CHUNK_INDEPENDENT: + io_log("H5Pget_mpio_actual_io_mode: H5D_MPIO_CHUNK_INDEPENDENT: "); + break; + case H5D_MPIO_CHUNK_COLLECTIVE: + io_log("H5Pget_mpio_actual_io_mode: H5D_MPIO_CHUNK_COLLECTIVE: "); + break; + case H5D_MPIO_CHUNK_MIXED: + io_log("H5Pget_mpio_actual_io_mode: H5D_MPIO_CHUNK_MIXED: "); + break; + case H5D_MPIO_CONTIGUOUS_COLLECTIVE: + io_log("H5Pget_mpio_actual_io_mode: H5D_MPIO_CONTIGUOUS_COLLECTIVE: "); + break; + default: + io_log("H5Pget_mpio_actual_io_mode: None returend: "); + break; + } + + H5D_mpio_actual_chunk_opt_mode_t actual_chunk_opt_mode; + H5Pget_mpio_actual_chunk_opt_mode(plist_id, &actual_chunk_opt_mode); + switch (actual_chunk_opt_mode) + { + case H5D_MPIO_NO_CHUNK_OPTIMIZATION: + io_log("H5Pget_mpio_actual_chunk_opt_mode: H5D_MPIO_NO_CHUNK_OPTIMIZATION: "); + break; + case H5D_MPIO_MULTI_CHUNK: + io_log("H5Pget_mpio_actual_chunk_opt_mode: H5D_MPIO_MULTI_CHUNK: "); + break; + // case H5D_MPIO_MULTI_CHUNK_NO_OPT: + // io_log("H5Pget_mpio_actual_chunk_opt_mode: H5D_MPIO_MULTI_CHUNK_NO_OPT: "); + // break; + case H5D_MPIO_LINK_CHUNK: + io_log("H5Pget_mpio_actual_chunk_opt_mode: H5D_MPIO_LINK_CHUNK: "); + break; + default: + io_log("H5Pget_mpio_actual_chunk_opt_mode: None returend: "); + break; + } + + uint32_t local_no_collective_cause, global_no_collective_cause; + H5Pget_mpio_no_collective_cause(plist_id, &local_no_collective_cause, &global_no_collective_cause); + + switch (local_no_collective_cause) + { + case H5D_MPIO_COLLECTIVE: + io_log("local_no_collective_cause: H5D_MPIO_COLLECTIVE: "); + break; + case H5D_MPIO_SET_INDEPENDENT: + io_log("local_no_collective_cause: H5D_MPIO_SET_INDEPENDENT: "); + break; + case H5D_MPIO_DATA_TRANSFORMS: + io_log("local_no_collective_cause: H5D_MPIO_DATA_TRANSFORMS: "); + break; + // case H5D_MPIO_SET_MPIPOSIX: + // io_log("local_no_collective_cause: H5D_MPIO_SET_MPIPOSIX: "); + // break; + case H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES: + io_log("local_no_collective_cause: H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES: "); + break; + // case H5D_MPIO_POINT_SELECTIONS: + // io_log("local_no_collective_cause: H5D_MPIO_POINT_SELECTIONS: "); + // break; + // case H5D_MPIO_FILTERS: + // io_log("local_no_collective_cause: H5D_MPIO_FILTERS: "); + // break; + default: + io_log("local_no_collective_cause: None returend: "); + break; + } + + switch (global_no_collective_cause) + { + case H5D_MPIO_COLLECTIVE: + io_log("global_no_collective_cause: H5D_MPIO_COLLECTIVE: "); + break; + case H5D_MPIO_SET_INDEPENDENT: + io_log("global_no_collective_cause: H5D_MPIO_SET_INDEPENDENT: "); + break; + case H5D_MPIO_DATA_TRANSFORMS: + io_log("global_no_collective_cause: H5D_MPIO_DATA_TRANSFORMS: "); + break; + // case H5D_MPIO_SET_MPIPOSIX: + // io_log("global_no_collective_cause: H5D_MPIO_SET_MPIPOSIX: "); + // break; + case H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES: + io_log("global_no_collective_cause: H5D_MPIO_NOT_SIMPLE_OR_SCALAR_DATASPACES: "); + break; + // case H5D_MPIO_POINT_SELECTIONS: + // io_log("global_no_collective_cause: H5D_MPIO_POINT_SELECTIONS: "); + // break; + // case H5D_MPIO_FILTERS: + // io_log("global_no_collective_cause: H5D_MPIO_FILTERS: "); + // break; + default: + io_log("global_no_collective_cause: None returend: "); + break; + } + */ + el2 = uptime() - el2; + io_log("TimeHDF5Write: " << el2 << " s"); + + double el3 = uptime(); + + /* + //Write metadata (geo original and geo dx/dy/dz) for ArrayUDF + float attr_data[2][3]; + attr_data[0][0] = grid->x0; + attr_data[0][1] = grid->y0; + attr_data[0][2] = grid->z0; + attr_data[1][0] = grid->dx; + attr_data[1][1] = grid->dy; + attr_data[1][2] = grid->dz; + hsize_t dims[2]; + dims[0] = 2; + dims[1] = 3; + if(!file_exist_flag){ + hid_t va_geo_dataspace_id = H5Screate_simple(2, dims, NULL); + hid_t va_geo_attribute_id = H5Acreate2(file_id, "VPIC-ArrayUDF-GEO", H5T_IEEE_F32BE, va_geo_dataspace_id, H5P_DEFAULT, H5P_DEFAULT); + H5Awrite(va_geo_attribute_id, H5T_NATIVE_FLOAT, attr_data); + H5Sclose(va_geo_dataspace_id); + H5Aclose(va_geo_attribute_id); + } + */ + if (!file_exist_flag) + H5Sclose(filespace); + H5Sclose(memspace); + H5Pclose(plist_id); + H5Gclose(group_id); + H5Fclose(file_id); + + el3 = uptime() - el3; + io_log("TimeHDF5Close: " << el3 << " s"); + + if (mpi_rank == 0) + { + char const *output_xml_file = "./field_hdf5/hdf5_field.xdmf"; + char dimensions_3d[128]; + sprintf(dimensions_3d, "%llu %llu %llu", field_global_size[0], field_global_size[1], field_global_size[2]); + char dimensions_4d[128]; + sprintf(dimensions_4d, "%llu %llu %llu %d", field_global_size[0], field_global_size[1], field_global_size[2], 3); + char orignal[128]; + sprintf(orignal, "%f %f %f", grid->x0, grid->y0, grid->z0); + char dxdydz[128]; + sprintf(dxdydz, "%f %f %f", grid->dx, grid->dy, grid->dz); + + // TODO: remove or let the user set + int field_interval = 1; + + // TODO: remove this dependence on number of steps + std::cout << "num_step " << num_step << std::endl; + + int nframes = num_step / field_interval + 1; + static int field_tframe = 0; + +#ifdef DUMP_INFO_DEBUG + printf(" meta file : %s \n", output_xml_file); + printf(" array dims per var: %s \n", dimensions_3d); + printf("array dims all vars: %s \n", dimensions_4d); + printf(" orignal: %s \n", orignal); + printf(" dxdydz: %s \n", dxdydz); + printf(" nframes: %d \n", nframes); + printf(" field_interval: %d \n", field_interval); + printf(" current step: %zd \n", step_for_viou); + printf(" current step: %zd \n", step_for_viou); + + // printf(" Simulation time: %f \n", grid->t0); + printf(" tframe: %d \n", field_tframe); +#endif + + // TODO: this footer dumping is more likely better done in a + // destructor, rather than hoping a multiple division works out + if (field_tframe >= 1) + { + if (field_tframe == (nframes - 1)) + { + invert_field_xml_item(output_xml_file, "fields", step_for_viou, dimensions_4d, dimensions_3d, 1); + } + else + { + invert_field_xml_item(output_xml_file, "fields", step_for_viou, dimensions_4d, dimensions_3d, 0); + } + } + else + { + create_file_with_header(output_xml_file, dimensions_3d, orignal, dxdydz, nframes, field_interval); + if (field_tframe == (nframes - 1)) + { + invert_field_xml_item(output_xml_file, "fields", step_for_viou, dimensions_4d, dimensions_3d, 1); + } + else + { + invert_field_xml_item(output_xml_file, "fields", step_for_viou, dimensions_4d, dimensions_3d, 0); + } + } + field_tframe++; + } + } + void dump_particles( + const char *fbase, + species_t *sp, + grid_t *grid, + int step, + interpolator_array_t *interpolator_array, + int ftag) + { + static int file_index = 0; + file_index++; + int mpi_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + double dump_particles_uptime = uptime(); + time_t seconds = time(NULL); + // printf("Atrank = %d, file_index = %d, dump_particles_uptime = %f, epoch_seconds = %ld \n ", mpi_rank, file_index, dump_particles_uptime, seconds); + + size_t step_for_viou = step; + char fname[256]; + char group_name[256]; + char particle_scratch[128]; + char subparticle_scratch[128]; + + int np_local; + + float *Pf; + int *Pi; + + // get the total number of particles. in this example, output only electrons + // sp = species_list; + sprintf(particle_scratch, DUMP_DIR_FORMAT, "particle_hdf5"); + FileUtils::makeDirectory(particle_scratch); + sprintf(subparticle_scratch, "%s/T.%ld/", particle_scratch, step_for_viou); + FileUtils::makeDirectory(subparticle_scratch); + + // TODO: Allow the user to set this + int stride_particle_dump = 1; + + np_local = (sp->np + stride_particle_dump - 1) / stride_particle_dump; + + // make a copy of the part of particle data to be dumped + double ec1 = uptime(); + + int sp_np = sp->np; + int sp_max_np = sp->max_np; + particle_t *ALIGNED(128) p_buf = NULL; + if (!p_buf) + MALLOC_ALIGNED(p_buf, np_local, 128); + particle_t *sp_p = sp->p; + sp->p = p_buf; + sp->np = np_local; + sp->max_np = np_local; + + for (long long iptl = 0, i = 0; iptl < sp_np; iptl += stride_particle_dump, ++i) + { + COPY(&sp->p[i], &sp_p[iptl], 1); + } + + center_p(sp, interpolator_array); + + ec1 = uptime() - ec1; + + // if(!mpi_rank || mpi_rank == 2047 ) + // std::cout << "on mpi_rank: " << mpi_rank << ", time in copying particle data: " << ec1 << " s" << ", np_local = " << np_local << std::endl; + +#ifndef N_FILE_N_PROCESS + int np_local_max, np_local_min; + MPI_Reduce(&np_local, &np_local_max, 1, MPI_INT, MPI_MAX, 0, MPI_COMM_WORLD); + MPI_Reduce(&np_local, &np_local_min, 1, MPI_INT, MPI_MIN, 0, MPI_COMM_WORLD); +// io_log("on mpi_rank: " << mpi_rank << ", time in copying particle data: " << ec1 << " s" << ", np_local = " << np_local << ",np_local_max = " << np_local_max << ", local_min = "<< np_local_min); +#endif + + Pf = (float *)sp->p; + Pi = (int *)sp->p; + +// open HDF5 file in "particle/T./" subdirectory +// filename: eparticle.h5p +#ifndef N_FILE_N_PROCESS + sprintf(fname, "%s%s/%s_%ld.h5", daos, subparticle_scratch, sp->name, step_for_viou); +#else + sprintf(fname, "%s%s/%s_%ld_p%d.h5", daos, subparticle_scratch, sp->name, step_for_viou, mpi_rank); +#endif + + sprintf(group_name, "/Timestep_%ld", step_for_viou); + double el1 = uptime(); + + long long total_particles, offset; + long long numparticles = np_local; + +#ifndef N_FILE_N_PROCESS + MPI_Allreduce(&numparticles, &total_particles, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD); + MPI_Scan(&numparticles, &offset, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD); + offset -= numparticles; +#else + total_particles = np_local; + offset = 0; +#endif + + hid_t file_plist_id = H5Pcreate(H5P_FILE_ACCESS); + +#ifndef N_FILE_N_PROCESS + H5Pset_fapl_mpio(file_plist_id, MPI_COMM_WORLD, MPI_INFO_NULL); +#endif + +#ifdef H5_ASYNC + if (!mpi_rank) + printf("Enable async on particle data"); + + assert(H5Pset_vol_async(file_plist_id)); +#endif + +#ifdef METADATA_COLL_WRITE + if (!mpi_rank) + printf("Enable collective metadata write !\n"); + H5Pset_coll_metadata_write(file_plist_id, TRUE); +#endif + + hid_t file_id = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, file_plist_id); + // if(!mpi_rank ) + // io_log("++Particle H5Fcreate) "); + + hid_t group_id = H5Gcreate(file_id, group_name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + // if(!mpi_rank ) + // io_log("++Particle H5Gcreate) "); + +#ifdef HAS_PARTICLE_COMP + if (!mpi_rank) + printf("Using Particle Compound type !\n"); + hid_t particle_comp_type_it = H5Tcreate(H5T_COMPOUND, sizeof(particle_t)); + H5Tinsert(particle_comp_type_it, "dx", HOFFSET(particle_t, dx), H5T_NATIVE_FLOAT); + H5Tinsert(particle_comp_type_it, "dy", HOFFSET(particle_t, dy), H5T_NATIVE_FLOAT); + H5Tinsert(particle_comp_type_it, "dz", HOFFSET(particle_t, dz), H5T_NATIVE_FLOAT); + + H5Tinsert(particle_comp_type_it, "i", HOFFSET(particle_t, i), H5T_NATIVE_INT); + + H5Tinsert(particle_comp_type_it, "ux", HOFFSET(particle_t, ux), H5T_NATIVE_FLOAT); + H5Tinsert(particle_comp_type_it, "uy", HOFFSET(particle_t, uy), H5T_NATIVE_FLOAT); + H5Tinsert(particle_comp_type_it, "uz", HOFFSET(particle_t, uz), H5T_NATIVE_FLOAT); + H5Tinsert(particle_comp_type_it, "w", HOFFSET(particle_t, w), H5T_NATIVE_FLOAT); +#endif + + // MSBprintf("%d: offset %ld %ld %ld \n",mpi_rank,offset,total_particles, numparticles); + hid_t filespace = H5Screate_simple(1, (hsize_t *)&total_particles, NULL); + H5Sselect_hyperslab(filespace, H5S_SELECT_SET, (hsize_t *)&offset, NULL, (hsize_t *)&numparticles, NULL); + + // if(!mpi_rank ) + // io_log("++Particle H5Sselect_hyperslab) "); + + // plist_id = H5P_DEFAULT; + hid_t io_plist_id = H5Pcreate(H5P_DATASET_XFER); + +#ifndef N_FILE_N_PROCESS +#ifdef HAS_INDEPENDENT_IO + if (!mpi_rank) + printf("\n ###\n VPIC Independent I/O! \n ###\n"); + H5Pset_dxpl_mpio(io_plist_id, H5FD_MPIO_INDEPENDENT); +#else + H5Pset_dxpl_mpio(io_plist_id, H5FD_MPIO_COLLECTIVE); +#endif +#endif + +#ifdef H5_ASYNC + H5Pset_dxpl_async(io_plist_id, true); +#endif + // hsize_t linearspace_count_temp = numparticles; + // hid_t linearspace = H5Screate_simple(1, &linearspace_count_temp, NULL); + + hsize_t memspace_count_temp; + hid_t memspace; +#ifdef HAS_PARTICLE_COMP + memspace_count_temp = numparticles; + memspace = H5Screate_simple(1, &memspace_count_temp, NULL); +#else + memspace_count_temp = numparticles * 8; + memspace = H5Screate_simple(1, &memspace_count_temp, NULL); + // MSB printf("%d: hyperslab %ld\n",mpi_rank, np_local); + hsize_t memspace_start = 0, memspace_stride = 8, memspace_count = np_local; + H5Sselect_hyperslab(memspace, H5S_SELECT_SET, &memspace_start, &memspace_stride, &memspace_count, NULL); +#endif + el1 = uptime() - el1; + // if(!mpi_rank || mpi_rank == 2047 ) + // io_log("Particle TimeHDF5Open): " << el1 << " s"); //Easy to handle results for scripts + double el2 = uptime(); + int ierr; + +#define WRITE_H5_FILE(group_id_p, data_buf_p, type_p, dname_p) \ + { \ + hid_t dset_id = H5Dcreate(group_id_p, dname_p, type_p, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); \ + H5Dwrite(dset_id, type_p, memspace, filespace, io_plist_id, data_buf_p); \ + H5Dclose(dset_id); \ + } + +// MPI_Info_set(info, "romio_cb_write", "disable"); +#define WRITE_MPI_FILE(dname_p, offset_p, data_buf_p, count_p, type_p) \ + { \ + MPI_File fh; \ + MPI_Status status; \ + sprintf(fname, "%s%s/%s_%ld_%s.h5", daos, subparticle_scratch, sp->name, step_for_viou, dname_p); \ + if (mpi_rank == 0) \ + printf("fname= %s \n", fname); \ + MPI_Info info; \ + MPI_Info_create(&info); \ + MPI_File_open(MPI_COMM_WORLD, fname, MPI_MODE_WRONLY | MPI_MODE_CREATE, info, &fh); \ + MPI_File_write_at(fh, offset_p, data_buf_p, count_p, type_p, &status); \ + MPI_Info_free(&info); \ + MPI_File_close(&fh); \ + } + +#ifdef HAS_PARTICLE_COMP + hid_t dset_id = H5Dcreate(group_id, "ParticleComp", particle_comp_type_it, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + H5Dwrite(dset_id, particle_comp_type_it, memspace, filespace, io_plist_id, sp->p); + H5Dclose(dset_id); +#else +#ifdef TEST_MPIIO + // Here we don't use the stripe but just for performance test + if (!mpi_rank) + printf("Test MPI-IO\n"); + WRITE_MPI_FILE("dX", offset * sizeof(float), Pf, numparticles, MPI_FLOAT); + WRITE_MPI_FILE("dY", offset * sizeof(float), Pf, numparticles, MPI_FLOAT); + WRITE_MPI_FILE("dZ", offset * sizeof(float), Pf, numparticles, MPI_FLOAT); + WRITE_MPI_FILE("i", offset * sizeof(int), Pf, numparticles, MPI_INT); + WRITE_MPI_FILE("ux", offset * sizeof(float), Pf, numparticles, MPI_FLOAT); + WRITE_MPI_FILE("uy", offset * sizeof(float), Pf, numparticles, MPI_FLOAT); + WRITE_MPI_FILE("uz", offset * sizeof(float), Pf, numparticles, MPI_FLOAT); + WRITE_MPI_FILE("q", offset * sizeof(float), Pf, numparticles, MPI_FLOAT); +#else +#ifndef N_FILE_N_PROCESS + if (!mpi_rank) + printf("Test HDF5-IO Single \n"); +#else + if (!mpi_rank) + printf("Test HDF5-IO N Files N Process\n"); +#endif + // if(!mpi_rank ) + // io_log("++Particle Starting to write ) "); + WRITE_H5_FILE(group_id, Pf, H5T_NATIVE_FLOAT, "dX") + WRITE_H5_FILE(group_id, Pf + 1, H5T_NATIVE_FLOAT, "dY") + WRITE_H5_FILE(group_id, Pf + 2, H5T_NATIVE_FLOAT, "dZ") + WRITE_H5_FILE(group_id, Pi + 3, H5T_NATIVE_INT, "i") + WRITE_H5_FILE(group_id, Pf + 4, H5T_NATIVE_FLOAT, "ux") + WRITE_H5_FILE(group_id, Pf + 5, H5T_NATIVE_FLOAT, "uy") + WRITE_H5_FILE(group_id, Pf + 6, H5T_NATIVE_FLOAT, "uz") + WRITE_H5_FILE(group_id, Pf + 7, H5T_NATIVE_FLOAT, "q") +#endif +#endif + el2 = uptime() - el2; + io_log("Particle TimeHDF5Write: " << el2 << " s"); + + double el3 = uptime(); + H5Sclose(memspace); + H5Sclose(filespace); + H5Pclose(file_plist_id); + H5Pclose(io_plist_id); + H5Gclose(group_id); + + H5Fclose(file_id); + +#ifdef H5_ASYNC + H5VLasync_finalize(); +#endif + el3 = uptime() - el3; + io_log("Particle TimeHDF5Close: " << el3 << " s"); + } + + void dump_hydro( + const char *fbase, + int step, + hydro_array_t *hydro_array, + species_t *sp, + interpolator_array_t *interpolator_array, + grid_t *grid, + int ftag) + { + size_t step_for_viou = step; + + // #define DUMP_INFO_DEBUG 1 + int mpi_size, mpi_rank; + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + printf("Calling HDF5 dump_hydro ... \n"); + if (!sp) + { + ERROR(("Invalid species")); + } + + clear_hydro_array(hydro_array); + accumulate_hydro_p(hydro_array, sp, interpolator_array); + synchronize_hydro_array(hydro_array); + + char hname[256]; + char hydro_scratch[128]; + char subhydro_scratch[128]; + + sprintf(hydro_scratch, "./%s", "hydro_hdf5"); + FileUtils::makeDirectory(hydro_scratch); + sprintf(subhydro_scratch, "%s/T.%zu/", hydro_scratch, step_for_viou); + FileUtils::makeDirectory(subhydro_scratch); + + sprintf(hname, "%s%s/hydro_%s_%zu.h5", daos, subhydro_scratch, sp->name, step_for_viou); + double el1 = uptime(); + hid_t plist_id = H5Pcreate(H5P_FILE_ACCESS); + + /* + if(H5Pset_libver_bounds(plist_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0){ + exit(-1); + }*/ + // if((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, plist_id)) < 0) + // ERROR_RETURN; + + H5Pset_fapl_mpio(plist_id, MPI_COMM_WORLD, MPI_INFO_NULL); +#ifdef METADATA_COLL_WRITE + if (!mpi_rank) + printf("Enable collective metadata write !\n"); + H5Pset_coll_metadata_write(plist_id, TRUE); +#endif + hid_t file_id = H5Fcreate(hname, H5F_ACC_TRUNC, H5P_DEFAULT, plist_id); + H5Pclose(plist_id); + + sprintf(hname, "Timestep_%zu", step_for_viou); + hid_t group_id = H5Gcreate(file_id, hname, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + + el1 = uptime() - el1; + io_log("TimeHDF5Open: " << el1 << " s"); // Easy to handle results for scripts + double el2 = uptime(); + + // Create a variable list of field values to output. + // size_t numvars = std::min(global->fdParams.output_vars.bitsum(), total_field_variables); + // size_t *varlist = new size_t[numvars]; + + // for (size_t i(0), c(0); i < total_field_variables; i++) + // if (global->fdParams.output_vars.bitset(i)) + // varlist[c++] = i; + + // printf("\nBEGIN_OUTPUT: numvars = %zu \n", numvars); + + // typedef struct hydro { + // float jx, jy, jz, rho; // Current and charge density => , + // float px, py, pz, ke; // Momentum and K.E. density => , + // float txx, tyy, tzz; // Stress diagonal => , i==j + // float tyz, tzx, txy; // Stress off-diagonal => , i!=j + // float _pad[2]; // 16-byte align + // } hydro_t; + + // typedef struct hydro_array { + // hydro_t * ALIGNED(128) h; + // grid_t * g; + // } hydro_array_t; + + hsize_t temp_buf_index; + hid_t dset_id; + // char *field_var_name[] = {"ex","ey","ez","div_e_err","cbx","cby","cbz","div_b_err","tcax","tcay","tcaz","rhob","jfx","jfy","jfz","rhof"}; + plist_id = H5Pcreate(H5P_DATASET_XFER); + // Comment out for test only + H5Pset_dxpl_mpio(plist_id, H5FD_MPIO_COLLECTIVE); + // H5Sselect_hyperslab(filespace, H5S_SELECT_SET, (hsize_t *) &offset, NULL, (hsize_t *) &numparticles, NULL); + + // global->topology_x + + hsize_t hydro_global_size[3], hydro_local_size[3], global_offset[3], global_count[3]; + hydro_global_size[0] = (grid->nx * grid->gpx); + hydro_global_size[1] = (grid->ny * grid->gpy); + hydro_global_size[2] = (grid->nz * grid->gpz); + + hydro_local_size[0] = grid->nx; + hydro_local_size[1] = grid->ny; + hydro_local_size[2] = grid->nz; + + int mpi_rank_x, mpi_rank_y, mpi_rank_z; + UNVOXEL(mpi_rank, mpi_rank_x, mpi_rank_y, mpi_rank_z, grid->gpx, grid->gpy, grid->gpz); + + global_offset[0] = (grid->nx) * mpi_rank_x; + global_offset[1] = (grid->ny) * mpi_rank_y; + global_offset[2] = (grid->nz) * mpi_rank_z; + + global_count[0] = (grid->nx); + global_count[1] = (grid->ny); + global_count[2] = (grid->nz); + +#ifdef DUMP_INFO_DEBUG + printf("global size = %llu %llu %llu \n", hydro_global_size[0], hydro_global_size[1], hydro_global_size[2]); + printf("global_offset = %llu %llu %llu \n", global_offset[0], global_offset[1], global_offset[2]); + printf("global_count = %llu %llu %llu \n", global_count[0], global_count[1], global_count[2]); + printf("mpi-rank = %d, rank index = (%d, %d, %d) \n", mpi_rank, mpi_rank_x, mpi_rank_y, mpi_rank_z); + fflush(stdout); +#endif + + hid_t filespace = H5Screate_simple(3, hydro_global_size, NULL); + hid_t memspace = H5Screate_simple(3, hydro_local_size, NULL); + hid_t dataspace_id; + /* + typedef struct hydro { + float jx, jy, jz, rho; // Current and charge density => , + float px, py, pz, ke; // Momentum and K.E. density => , + float txx, tyy, tzz; // Stress diagonal => , i==j + float tyz, tzx, txy; // Stress off-diagonal => , i!=j + float _pad[2]; // 16-byte align + } hydro_t; + */ + +#if HAS_HYDRO_COMP + + if (!mpi_rank) + printf("Using Hydro Compund type !\n"); + + hid_t hydro_comp_type_it = H5Tcreate(H5T_COMPOUND, sizeof(hydro_t)); + + H5Tinsert(hydro_comp_type_it, "jx", HOFFSET(hydro_t, jx), H5T_NATIVE_FLOAT); + H5Tinsert(hydro_comp_type_it, "jy", HOFFSET(hydro_t, jy), H5T_NATIVE_FLOAT); + H5Tinsert(hydro_comp_type_it, "jz", HOFFSET(hydro_t, jz), H5T_NATIVE_FLOAT); + H5Tinsert(hydro_comp_type_it, "rho", HOFFSET(hydro_t, rho), H5T_NATIVE_FLOAT); + + H5Tinsert(hydro_comp_type_it, "px", HOFFSET(hydro_t, px), H5T_NATIVE_FLOAT); + H5Tinsert(hydro_comp_type_it, "py", HOFFSET(hydro_t, py), H5T_NATIVE_FLOAT); + H5Tinsert(hydro_comp_type_it, "pz", HOFFSET(hydro_t, pz), H5T_NATIVE_FLOAT); + H5Tinsert(hydro_comp_type_it, "ke", HOFFSET(hydro_t, ke), H5T_NATIVE_FLOAT); + + H5Tinsert(hydro_comp_type_it, "txx", HOFFSET(hydro_t, txx), H5T_NATIVE_FLOAT); + H5Tinsert(hydro_comp_type_it, "tyy", HOFFSET(hydro_t, tyy), H5T_NATIVE_FLOAT); + H5Tinsert(hydro_comp_type_it, "tzz", HOFFSET(hydro_t, tzz), H5T_NATIVE_FLOAT); + + H5Tinsert(hydro_comp_type_it, "tyz", HOFFSET(hydro_t, tyz), H5T_NATIVE_FLOAT); + H5Tinsert(hydro_comp_type_it, "tzx", HOFFSET(hydro_t, tzx), H5T_NATIVE_FLOAT); + H5Tinsert(hydro_comp_type_it, "txy", HOFFSET(hydro_t, txy), H5T_NATIVE_FLOAT); + + // if(!file_exist_flag){ + dset_id = H5Dcreate(group_id, "HYDRO_DATA_COMP_T", hydro_comp_type_it, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); // dcpd_id MSB + // }else{ + // dset_id = H5Dopen(group_id, "HYDRO_DATA_COMP_T", H5P_DEFAULT); + // } + dataspace_id = H5Dget_space(dset_id); + + hydro_t *temp_buf = (hydro_t *)malloc(sizeof(hydro_t) * (grid->nx) * (grid->ny) * (grid->nz)); + temp_buf_index = 0; + for (size_t i(1); i < grid->nx + 1; i++) + { + for (size_t j(1); j < grid->ny + 1; j++) + { + for (size_t k(1); k < grid->nz + 1; k++) + { + temp_buf[temp_buf_index].jx = _hydro(i, j, k).jx; + temp_buf[temp_buf_index].jy = _hydro(i, j, k).jy; + temp_buf[temp_buf_index].jz = _hydro(i, j, k).jz; + temp_buf[temp_buf_index].rho = _hydro(i, j, k).rho; + temp_buf[temp_buf_index].px = _hydro(i, j, k).px; + temp_buf[temp_buf_index].py = _hydro(i, j, k).py; + temp_buf[temp_buf_index].pz = _hydro(i, j, k).pz; + temp_buf[temp_buf_index].ke = _hydro(i, j, k).ke; + temp_buf[temp_buf_index].txx = _hydro(i, j, k).txx; + temp_buf[temp_buf_index].tyy = _hydro(i, j, k).tyy; + temp_buf[temp_buf_index].tzz = _hydro(i, j, k).tzz; + temp_buf[temp_buf_index].tyz = _hydro(i, j, k).tyz; + temp_buf[temp_buf_index].tzx = _hydro(i, j, k).tzx; + temp_buf[temp_buf_index].txy = _hydro(i, j, k).txy; + temp_buf_index = temp_buf_index + 1; + } + } + } + H5Sselect_hyperslab(dataspace_id, H5S_SELECT_SET, global_offset, NULL, global_count, NULL); + H5Dwrite(dset_id, hydro_comp_type_it, memspace, dataspace_id, plist_id, temp_buf); + H5Sclose(dataspace_id); + H5Dclose(dset_id); + H5Tclose(hydro_comp_type_it); +#else + float *temp_buf = (float *)malloc(sizeof(float) * (grid->nx) * (grid->ny) * (grid->nz)); + +#define DUMP_HYDRO_TO_HDF5(DSET_NAME, ATTRIBUTE_NAME, ELEMENT_TYPE) \ + { \ + dset_id = H5Dcreate(group_id, DSET_NAME, ELEMENT_TYPE, filespace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); \ + temp_buf_index = 0; \ + for (size_t i(1); i < grid->nx + 1; i++) \ + { \ + for (size_t j(1); j < grid->ny + 1; j++) \ + { \ + for (size_t k(1); k < grid->nz + 1; k++) \ + { \ + temp_buf[temp_buf_index] = _hydro(i, j, k).ATTRIBUTE_NAME; \ + temp_buf_index = temp_buf_index + 1; \ + } \ + } \ + } \ + dataspace_id = H5Dget_space(dset_id); \ + H5Sselect_hyperslab(dataspace_id, H5S_SELECT_SET, global_offset, NULL, global_count, NULL); \ + H5Dwrite(dset_id, ELEMENT_TYPE, memspace, dataspace_id, plist_id, temp_buf); \ + H5Sclose(dataspace_id); \ + H5Dclose(dset_id); \ + } + + if (hydro_dump_flag.jx) + DUMP_HYDRO_TO_HDF5("jx", jx, H5T_NATIVE_FLOAT); + if (hydro_dump_flag.jy) + DUMP_HYDRO_TO_HDF5("jy", jy, H5T_NATIVE_FLOAT); + if (hydro_dump_flag.jz) + DUMP_HYDRO_TO_HDF5("jz", jz, H5T_NATIVE_FLOAT); + if (hydro_dump_flag.rho) + DUMP_HYDRO_TO_HDF5("rho", rho, H5T_NATIVE_FLOAT); + + if (hydro_dump_flag.px) + DUMP_HYDRO_TO_HDF5("px", px, H5T_NATIVE_FLOAT); + if (hydro_dump_flag.py) + DUMP_HYDRO_TO_HDF5("py", py, H5T_NATIVE_FLOAT); + if (hydro_dump_flag.pz) + DUMP_HYDRO_TO_HDF5("pz", pz, H5T_NATIVE_FLOAT); + if (hydro_dump_flag.ke) + DUMP_HYDRO_TO_HDF5("ke", ke, H5T_NATIVE_FLOAT); + + if (hydro_dump_flag.txx) + DUMP_HYDRO_TO_HDF5("txx", txx, H5T_NATIVE_FLOAT); + if (hydro_dump_flag.tyy) + DUMP_HYDRO_TO_HDF5("tyy", tyy, H5T_NATIVE_FLOAT); + if (hydro_dump_flag.tzz) + DUMP_HYDRO_TO_HDF5("tzz", tzz, H5T_NATIVE_FLOAT); + + if (hydro_dump_flag.tyz) + DUMP_HYDRO_TO_HDF5("tyz", tyz, H5T_NATIVE_FLOAT); + if (hydro_dump_flag.tzx) + DUMP_HYDRO_TO_HDF5("tzx", tzx, H5T_NATIVE_FLOAT); + if (hydro_dump_flag.txy) + DUMP_HYDRO_TO_HDF5("txy", txy, H5T_NATIVE_FLOAT); +#endif + + el2 = uptime() - el2; + io_log("TimeHDF5Write: " << el2 << " s"); + + double el3 = uptime(); + + // Write metadata (geo original and geo dx/dy/dz) for ArrayUDF + /* + float attr_data[2][3]; + attr_data[0][0] = grid->x0; + attr_data[0][1] = grid->y0; + attr_data[0][2] = grid->z0; + attr_data[1][0] = grid->dx; + attr_data[1][1] = grid->dy; + attr_data[1][2] = grid->dz; + hsize_t dims[2]; + dims[0] = 2; + dims[1] = 3; + hid_t va_geo_dataspace_id = H5Screate_simple(2, dims, NULL); + hid_t va_geo_attribute_id = H5Acreate2(file_id, "VPIC-ArrayUDF-GEO", H5T_IEEE_F32BE, va_geo_dataspace_id, H5P_DEFAULT, H5P_DEFAULT); + H5Awrite(va_geo_attribute_id, H5T_NATIVE_FLOAT, attr_data); + H5Sclose(va_geo_dataspace_id); + H5Aclose(va_geo_attribute_id);*/ + + free(temp_buf); + H5Sclose(filespace); + H5Sclose(memspace); + H5Pclose(plist_id); + H5Gclose(group_id); + H5Fclose(file_id); + + el3 = uptime() - el3; + io_log("TimeHDF5Close: " << el3 << " s"); + + if (mpi_rank == 0) + { + char output_xml_file[128]; + sprintf(output_xml_file, "./%s/%s%s%s", "hydro_hdf5", "hydro-", sp->name, ".xdmf"); + char dimensions_3d[128]; + sprintf(dimensions_3d, "%llu %llu %llu", hydro_global_size[0], hydro_global_size[1], hydro_global_size[2]); + char dimensions_4d[128]; + sprintf(dimensions_4d, "%llu %llu %llu %d", hydro_global_size[0], hydro_global_size[1], hydro_global_size[2], 3); + char orignal[128]; + sprintf(orignal, "%f %f %f", grid->x0, grid->y0, grid->z0); + char dxdydz[128]; + sprintf(dxdydz, "%f %f %f", grid->dx, grid->dy, grid->dz); + + // TODO: remove or let user set + int hydro_interval = 1; + + // TODO: remove this dependence on number of steps + int nframes = num_step / hydro_interval + 1; + + const int tframe = tframe_map[sp->id]; + +#ifdef DUMP_INFO_DEBUG + printf(" meta file : %s \n", output_xml_file); + printf(" array dims per var: %s \n", dimensions_3d); + printf("array dims all vars: %s \n", dimensions_4d); + printf(" orignal: %s \n", orignal); + printf(" dxdydz: %s \n", dxdydz); + printf(" nframes: %d \n", nframes); + printf(" hydro_fields_interval: %d \n", hydro_interval); + printf(" current step: %zu \n", step_for_viou); + printf(" Simulation time: %f \n", grid->t0); + printf(" tframe: %d \n", tframe); +#endif + + // TODO: why doesnt this just use the cstr? + char speciesname_new[128]; + sprintf(speciesname_new, "hydro_%s", sp->name); + if (tframe >= 1) + { + if (tframe == (nframes - 1)) + { + invert_hydro_xml_item(output_xml_file, speciesname_new, step_for_viou, dimensions_4d, dimensions_3d, 1); + } + else + { + invert_hydro_xml_item(output_xml_file, speciesname_new, step_for_viou, dimensions_4d, dimensions_3d, 0); + } + } + else + { + create_file_with_header(output_xml_file, dimensions_3d, orignal, dxdydz, nframes, hydro_interval); + if (tframe == (nframes - 1)) + { + invert_hydro_xml_item(output_xml_file, speciesname_new, step_for_viou, dimensions_4d, dimensions_3d, 1); + } + else + { + invert_hydro_xml_item(output_xml_file, speciesname_new, step_for_viou, dimensions_4d, dimensions_3d, 0); + } + } + tframe_map[sp->id]++; + } + } +}; +#endif + +#ifdef VPIC_ENABLE_OPENPMD +class OpenPMDDump : public Dump_Strategy +{ +public: + // openPMD::Series* series; + using Dump_Strategy::Dump_Strategy; // inherit constructor + + // std::string file_type = ".h5"; + std::string file_type = ".bp"; + + void dump_fields( + const char *fbase, + int step, + grid_t *grid, + field_array_t *field_array, + int ftag) + { + std::cout << "Writing openPMD data" << std::endl; + + // if (series == nullptr) { + std::cout << "init series" << std::endl; + openPMD::Series series = openPMD::Series( + fbase, + openPMD::AccessType::CREATE, + MPI_COMM_WORLD); + //} + + std::cout << "Writing iteration " << step << std::endl; + auto i = series.iterations[step]; + // TODO: it would be nice to set these... + // series.setAuthor( "Axel Huebl "); + // series.setMachine( "Hall Probe 5000, Model 3"); + i.setAttribute("vacuum", true); + + auto cB = i.meshes["B"]; + auto E = i.meshes["E"]; + auto J = i.meshes["J"]; + auto Tca = i.meshes["Tca"]; + auto Emat = i.meshes["Emat"]; + auto Fmat = i.meshes["Fmat"]; + auto Rho = i.meshes["Rho"]; + auto DivErr = i.meshes["DivErr"]; + + // record components + auto Cbx = cB["x"]; + auto Cby = cB["y"]; + auto Cbz = cB["z"]; + + auto Ex = E["x"]; + auto Ey = E["y"]; + auto Ez = E["z"]; + + auto Jx = J["x"]; + auto Jy = J["y"]; + auto Jz = J["z"]; + + auto Tcax = Tca["x"]; + auto Tcay = Tca["y"]; + auto Tcaz = Tca["z"]; + + auto Ematx = Emat["x"]; + auto Ematy = Emat["y"]; + auto Ematz = Emat["z"]; + + auto Fmatx = Fmat["x"]; + auto Fmaty = Fmat["y"]; + auto Fmatz = Fmat["z"]; + + auto RhoB = Rho["B"]; + auto RhoF = Rho["F"]; + + auto DivEErr = DivErr["E"]; + auto DivBErr = DivErr["B"]; + + // TODO: set unitDimension so the anaylsis software knows what fields + // things are + // + // // TODO: add timers for the convert and for the write + + size_t gnx = (grid->nx * grid->gpx); + size_t gny = (grid->ny * grid->gpy); + size_t gnz = (grid->nz * grid->gpz); + openPMD::Extent global_extent = {gnx, gny, gnz}; + + openPMD::Datatype datatype = openPMD::determineDatatype(); + openPMD::Dataset dataset = openPMD::Dataset(datatype, global_extent); + + Cbx.resetDataset(dataset); + Cby.resetDataset(dataset); + Cbz.resetDataset(dataset); + + Ex.resetDataset(dataset); + Ey.resetDataset(dataset); + Ez.resetDataset(dataset); + + Jx.resetDataset(dataset); + Jy.resetDataset(dataset); + Jz.resetDataset(dataset); + + Tcax.resetDataset(dataset); + Tcay.resetDataset(dataset); + Tcaz.resetDataset(dataset); + + Ematx.resetDataset(dataset); + Ematy.resetDataset(dataset); + Ematz.resetDataset(dataset); + + Fmatx.resetDataset(dataset); + Fmaty.resetDataset(dataset); + Fmatz.resetDataset(dataset); + + RhoB.resetDataset(dataset); + RhoF.resetDataset(dataset); + + DivEErr.resetDataset(dataset); + DivBErr.resetDataset(dataset); + + // TODO: hoist this conversion code, as is it used elsewhere + // Convert rank to local x/y/z + int rx, ry, rz; + UNVOXEL(rank, rx, ry, rz, grid->gpx, grid->gpy, grid->gpz); + + size_t nx = grid->nx; + size_t ny = grid->ny; + size_t nz = grid->nz; + + // NOTE: this assumes a static mesh decomposition in nx/ny/nz + size_t global_offset_x = (nx)*rx; + size_t global_offset_y = (ny)*ry; + size_t global_offset_z = (nz)*rz; + + openPMD::Offset chunk_offset = {global_offset_x, global_offset_y, global_offset_z}; + openPMD::Extent chunk_extent = {nx, ny, nz}; + + std::cout << "Local offset " + << " x: " << global_offset_x << " y: " << global_offset_y << " z: " << global_offset_z << std::endl; + + // Store a local copy of the data which we pull out of the AoS + std::vector cbx_data; + std::vector cby_data; + std::vector cbz_data; + + std::vector ex_data; + std::vector ey_data; + std::vector ez_data; + + std::vector jx_data; + std::vector jy_data; + std::vector jz_data; + + std::vector tcax_data; + std::vector tcay_data; + std::vector tcaz_data; + + // TODO: these are material_id (ints not floats) + std::vector ematx_data; + std::vector ematy_data; + std::vector ematz_data; + + std::vector fmatx_data; + std::vector fmaty_data; + std::vector fmatz_data; + // end todo + + std::vector rhob_data; + std::vector rhof_data; + + std::vector divb_data; + std::vector dive_data; + + size_t nv = nx * ny * nz; + + // TODO: resize here will zero out the data which we don't need, we + // could change to a different semantic to avoid this + cbx_data.resize(nv); + cby_data.resize(nv); + cbz_data.resize(nv); + + ex_data.resize(nv); + ey_data.resize(nv); + ez_data.resize(nv); + + jx_data.resize(nv); + jy_data.resize(nv); + jz_data.resize(nv); + + tcax_data.resize(nv); + tcay_data.resize(nv); + tcaz_data.resize(nv); + + ematx_data.resize(nv); + ematy_data.resize(nv); + ematz_data.resize(nv); + + fmatx_data.resize(nv); + fmaty_data.resize(nv); + fmatz_data.resize(nv); + + rhob_data.resize(nv); + rhof_data.resize(nv); + + divb_data.resize(nv); + dive_data.resize(nv); + + // TODO: make this AoS to SoA conversion a function + + // We could do 1D here, but we don't really care about the ghosts, and we + // can thread over nz/ny (collapsed?) + // Go over non-ghosts and grab just that data into a dense array + for (size_t k = 1; k < grid->nz + 1; k++) + { + for (size_t j = 1; j < grid->ny + 1; j++) + { + for (size_t i = 1; i < grid->nx + 1; i++) + { + int local_index = VOXEL(i - 1, j - 1, k - 1, grid->nx - 2, grid->ny - 2, grid->nz - 2); + int global_index = VOXEL(i, j, k, grid->nx, grid->ny, grid->nz); + + cbx_data[local_index] = field_array->f[global_index].cbx; + cby_data[local_index] = field_array->f[global_index].cby; + cbz_data[local_index] = field_array->f[global_index].cbz; + + ex_data[local_index] = field_array->f[global_index].ex; + ey_data[local_index] = field_array->f[global_index].ey; + ez_data[local_index] = field_array->f[global_index].ez; + + jx_data[local_index] = field_array->f[global_index].jfx; + jy_data[local_index] = field_array->f[global_index].jfy; + jz_data[local_index] = field_array->f[global_index].jfz; + + tcax_data[local_index] = field_array->f[global_index].tcax; + tcay_data[local_index] = field_array->f[global_index].tcay; + tcaz_data[local_index] = field_array->f[global_index].tcaz; + + ematx_data[local_index] = field_array->f[global_index].ematx; + ematy_data[local_index] = field_array->f[global_index].ematy; + ematz_data[local_index] = field_array->f[global_index].ematz; + + fmatx_data[local_index] = field_array->f[global_index].fmatx; + fmaty_data[local_index] = field_array->f[global_index].fmaty; + fmatz_data[local_index] = field_array->f[global_index].fmatz; + + rhob_data[local_index] = field_array->f[global_index].rhob; + rhof_data[local_index] = field_array->f[global_index].rhof; + + dive_data[local_index] = field_array->f[global_index].div_e_err; + divb_data[local_index] = field_array->f[global_index].div_b_err; + } + } + } + + Cbx.storeChunk(cbx_data, chunk_offset, chunk_extent); + Cby.storeChunk(cby_data, chunk_offset, chunk_extent); + Cbz.storeChunk(cbz_data, chunk_offset, chunk_extent); + + Ex.storeChunk(ex_data, chunk_offset, chunk_extent); + Ey.storeChunk(ey_data, chunk_offset, chunk_extent); + Ez.storeChunk(ez_data, chunk_offset, chunk_extent); + + Jx.storeChunk(jx_data, chunk_offset, chunk_extent); + Jy.storeChunk(jy_data, chunk_offset, chunk_extent); + Jz.storeChunk(jz_data, chunk_offset, chunk_extent); + + Tcax.storeChunk(tcax_data, chunk_offset, chunk_extent); + Tcay.storeChunk(tcay_data, chunk_offset, chunk_extent); + Tcaz.storeChunk(tcaz_data, chunk_offset, chunk_extent); + + Ematx.storeChunk(ematx_data, chunk_offset, chunk_extent); + Ematy.storeChunk(ematy_data, chunk_offset, chunk_extent); + Ematz.storeChunk(ematz_data, chunk_offset, chunk_extent); + + Fmatx.storeChunk(fmatx_data, chunk_offset, chunk_extent); + Fmaty.storeChunk(fmaty_data, chunk_offset, chunk_extent); + Fmatz.storeChunk(fmatz_data, chunk_offset, chunk_extent); + + RhoB.storeChunk(rhob_data, chunk_offset, chunk_extent); + RhoF.storeChunk(rhof_data, chunk_offset, chunk_extent); + + DivEErr.storeChunk(dive_data, chunk_offset, chunk_extent); + DivBErr.storeChunk(divb_data, chunk_offset, chunk_extent); + + series.flush(); + } + + void dump_particles( + const char *fbase, + species_t *sp, + grid_t *grid, + int step, + interpolator_array_t *interpolator_array, + int ftag) + { + std::string full_file_name = fbase + file_type; + + std::cout << "writing particles to " << full_file_name << std::endl; + + // if (series == nullptr) { + openPMD::Series series = openPMD::Series( + full_file_name, + openPMD::AccessType::CREATE, + MPI_COMM_WORLD); + //} + + auto i = series.iterations[step]; + + // TODO: set these + i.setTime((float)step); + i.setDt(1.0); + i.setTimeUnitSI(1.0); + + auto &p = i.particles[sp->name]; + + const int np = sp->np; + + // TODO: this could be a function call as it's used elsewhere (in hdf5) + unsigned long long total_particles, offset; + unsigned long long numparticles = np; + MPI_Allreduce(&numparticles, &total_particles, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD); + MPI_Scan(&numparticles, &offset, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD); + offset -= numparticles; + + openPMD::Extent global_extent = {total_particles}; + openPMD::Datatype datatype = openPMD::determineDatatype(); + openPMD::Dataset dataset = openPMD::Dataset(datatype, global_extent); + + auto px = p["position"]["x"]; + auto pxo = p["positionOffset"]["x"]; + + auto py = p["position"]["y"]; + auto pyo = p["positionOffset"]["y"]; + + auto pz = p["position"]["z"]; + auto pzo = p["positionOffset"]["z"]; + + auto ux = p["velocity"]["x"]; + auto uy = p["velocity"]["y"]; + auto uz = p["velocity"]["z"]; + + px.resetDataset(dataset); + pxo.resetDataset(dataset); + + py.resetDataset(dataset); + pyo.resetDataset(dataset); + + pz.resetDataset(dataset); + pzo.resetDataset(dataset); + + ux.resetDataset(dataset); + uy.resetDataset(dataset); + uz.resetDataset(dataset); + // convert data to SoA, allowing the user to chunk the operation + + // TODO: Add code the convert to global offsets +#ifndef PMD_MAX_IO_CHUNK // in particles +#define PMD_MAX_IO_CHUNK 16777216; // 512MB total write +#endif + const int max_chunk = PMD_MAX_IO_CHUNK; + + // Loop over all particles in chunks + for (int i = 0; i < np; i += max_chunk) + { + // We have to be careful as the last chunk may not be full + // Find how many are left and do that many + size_t to_write = std::min(np - i, max_chunk); + + // Convert the chunk ready to write + std::vector x_pos; + std::vector x_off; + x_pos.resize(to_write); + x_off.resize(to_write); + + std::vector y_pos; + std::vector y_off; + y_pos.resize(to_write); + y_off.resize(to_write); + + std::vector z_pos; + std::vector z_off; + z_pos.resize(to_write); + z_off.resize(to_write); + + std::vector ux_pos; + ux_pos.resize(to_write); + + std::vector uy_pos; + uy_pos.resize(to_write); + + std::vector uz_pos; + uz_pos.resize(to_write); + + for (int j = 0; j < to_write; j++) + { + // TODO: do I need to center the particles? + auto &particle = sp->p[i + j]; + + x_pos[j] = particle.dx; + y_pos[j] = particle.dy; + z_pos[j] = particle.dz; + + ux_pos[j] = particle.ux; + uy_pos[j] = particle.uy; + uz_pos[j] = particle.uz; + + std::array gi = global_particle_index(particle.i, grid, rank); + x_off[j] = (float)gi[1]; + y_off[j] = (float)gi[2]; + z_off[j] = (float)gi[3]; + } + + // Base offset plus i to account for chunks + auto o = openPMD::Offset{offset + i}; + auto e = openPMD::Extent{to_write}; + px.storeChunk(x_pos, o, e); + pxo.storeChunk(x_off, o, e); + + py.storeChunk(y_pos, o, e); + pyo.storeChunk(y_off, o, e); + + pz.storeChunk(z_pos, o, e); + pzo.storeChunk(z_off, o, e); + + ux.storeChunk(ux_pos, o, e); + uy.storeChunk(uy_pos, o, e); + uz.storeChunk(uz_pos, o, e); + + series.flush(); + } + } + void dump_hydro( + const char *fbase, + int step, + hydro_array_t *hydro_array, + species_t *sp, + interpolator_array_t *interpolator_array, + grid_t *grid, + int ftag) + { + std::string full_file_name = fbase + file_type; + + std::cout << "OpenPMD dumping hydro to " << full_file_name << std::endl; + + // if (series == nullptr) { + openPMD::Series series = openPMD::Series( + full_file_name, + openPMD::AccessType::CREATE, + MPI_COMM_WORLD); + //} + + auto i = series.iterations[step]; + + // TODO: set these + i.setTime((float)step); + i.setDt(1.0); + i.setTimeUnitSI(1.0); + + if (!sp) + ERROR(("Invalid species \"%s\"", sp->name)); + + // TODO: do we want each backend to have to explicitly call these + // manually? Or, as it is common, should we hoist it to the VPIC + // call-site + clear_hydro_array(hydro_array); + accumulate_hydro_p(hydro_array, sp, interpolator_array); + synchronize_hydro_array(hydro_array); + + if (!fbase) + ERROR(("Invalid filename")); + + if (rank == 0) + MESSAGE(("Dumping \"%s\" hydro fields to \"%s\"", sp->name, fbase)); + + // Write data + // float jx, jy, jz, rho; // Current and charge density => , + // float px, py, pz, ke; // Momentum and K.E. density => , + // float txx, tyy, tzz; // Stress diagonal => , i==j + // float tyz, tzx, txy; // Stress off-diagonal => , i!=j + auto J = i.meshes["J"]; + auto P = i.meshes["P"]; + auto T = i.meshes["T"]; + auto _Ke = i.meshes["Ke"]; + auto _Rho = i.meshes["Rho"]; + + auto Jx = J["x"]; + auto Jy = J["y"]; + auto Jz = J["z"]; + + auto Px = P["x"]; + auto Py = P["y"]; + auto Pz = P["z"]; + + auto Txx = T["xx"]; + auto Tyy = T["yy"]; + auto Tzz = T["zz"]; + auto Tyz = T["yz"]; + auto Tzx = T["zx"]; + auto Txy = T["xy"]; + + auto Rho = _Rho["rho"]; // TODO: bad name.. + auto Ke = _Ke["ke"]; // TODO: bad name.. + + size_t gnx = (grid->nx * grid->gpx); + size_t gny = (grid->ny * grid->gpy); + size_t gnz = (grid->nz * grid->gpz); + openPMD::Extent global_extent = {gnx, gny, gnz}; + + openPMD::Datatype datatype = openPMD::determineDatatype(); + openPMD::Dataset dataset = openPMD::Dataset(datatype, global_extent); + + Jx.resetDataset(dataset); + Jy.resetDataset(dataset); + Jz.resetDataset(dataset); + + Px.resetDataset(dataset); + Py.resetDataset(dataset); + Pz.resetDataset(dataset); + + Txx.resetDataset(dataset); + Tyy.resetDataset(dataset); + Tzz.resetDataset(dataset); + Tyz.resetDataset(dataset); + Tzx.resetDataset(dataset); + Txy.resetDataset(dataset); + + Rho.resetDataset(dataset); + Ke.resetDataset(dataset); + + // TODO: hoist this conversion code, as is it used elsewhere + // Convert rank to local x/y/z + int rx, ry, rz; + UNVOXEL(rank, rx, ry, rz, grid->gpx, grid->gpy, grid->gpz); + + size_t nx = grid->nx; + size_t ny = grid->ny; + size_t nz = grid->nz; + + // NOTE: this assumes a static mesh decomposition in nx/ny/nz + size_t global_offset_x = (nx)*rx; + size_t global_offset_y = (ny)*ry; + size_t global_offset_z = (nz)*rz; + + openPMD::Offset chunk_offset = {global_offset_x, global_offset_y, global_offset_z}; + openPMD::Extent chunk_extent = {nx, ny, nz}; + + std::cout << "Local offset " + << " x: " << global_offset_x << " y: " << global_offset_y << " z: " << global_offset_z << std::endl; + + std::vector jx_data; + std::vector jy_data; + std::vector jz_data; + + std::vector px_data; + std::vector py_data; + std::vector pz_data; + + std::vector txx_data; + std::vector tyy_data; + std::vector tzz_data; + std::vector tyz_data; + std::vector tzx_data; + std::vector txy_data; + + std::vector rho_data; + std::vector ke_data; + + size_t nv = nx * ny * nz; + + jx_data.resize(nv); + jy_data.resize(nv); + jz_data.resize(nv); + + px_data.resize(nv); + py_data.resize(nv); + pz_data.resize(nv); + + txx_data.resize(nv); + tyy_data.resize(nv); + tzz_data.resize(nv); + tyz_data.resize(nv); + tzx_data.resize(nv); + txy_data.resize(nv); + + rho_data.resize(nv); + ke_data.resize(nv); + + // Transpose AoS to SoAs + for (size_t k = 1; k < grid->nz + 1; k++) + { + for (size_t j = 1; j < grid->ny + 1; j++) + { + for (size_t i = 1; i < grid->nx + 1; i++) + { + int local_index = VOXEL(i - 1, j - 1, k - 1, grid->nx - 2, grid->ny - 2, grid->nz - 2); + int global_index = VOXEL(i, j, k, grid->nx, grid->ny, grid->nz); + + jx_data[local_index] = hydro_array->h[global_index].jx; + jy_data[local_index] = hydro_array->h[global_index].jy; + jz_data[local_index] = hydro_array->h[global_index].jz; + + px_data[local_index] = hydro_array->h[global_index].px; + py_data[local_index] = hydro_array->h[global_index].py; + pz_data[local_index] = hydro_array->h[global_index].pz; + + txx_data[local_index] = hydro_array->h[global_index].txx; + tyy_data[local_index] = hydro_array->h[global_index].tyy; + tzz_data[local_index] = hydro_array->h[global_index].tzz; + tyz_data[local_index] = hydro_array->h[global_index].tyz; + tzx_data[local_index] = hydro_array->h[global_index].tzx; + txy_data[local_index] = hydro_array->h[global_index].txy; + + rho_data[local_index] = hydro_array->h[global_index].rho; + ke_data[local_index] = hydro_array->h[global_index].ke; + } + } + } + + Jx.storeChunk(jx_data, chunk_offset, chunk_extent); + Jy.storeChunk(jy_data, chunk_offset, chunk_extent); + Jz.storeChunk(jz_data, chunk_offset, chunk_extent); + + Px.storeChunk(px_data, chunk_offset, chunk_extent); + Py.storeChunk(py_data, chunk_offset, chunk_extent); + Pz.storeChunk(pz_data, chunk_offset, chunk_extent); + + Txx.storeChunk(txx_data, chunk_offset, chunk_extent); + Tyy.storeChunk(tyy_data, chunk_offset, chunk_extent); + Tzz.storeChunk(tzz_data, chunk_offset, chunk_extent); + Tyz.storeChunk(tyz_data, chunk_offset, chunk_extent); + Tzx.storeChunk(tzx_data, chunk_offset, chunk_extent); + Txy.storeChunk(txy_data, chunk_offset, chunk_extent); + + Rho.storeChunk(rho_data, chunk_offset, chunk_extent); + Ke.storeChunk(ke_data, chunk_offset, chunk_extent); + + series.flush(); + } +}; +#endif + +/* + template + struct IODump : private Policy { + using Policy::dump_particles; + using Policy::dump_fields; + using Policy::dump_hydro; + }; + */ + +#endif diff --git a/src/vpic/dumpmacros.h b/src/vpic/dumpmacros.h index 2f619352..a5f27a21 100644 --- a/src/vpic/dumpmacros.h +++ b/src/vpic/dumpmacros.h @@ -4,50 +4,50 @@ /* FIXME: WHEN THESE MACROS WERE HOISTED AND VARIOUS HACKS DONE TO THEM THEY BECAME _VERY_ _DANGEROUS. */ -#define WRITE_HEADER_V0(dump_type,sp_id,q_m,cstep,fileIO) do { \ - /* Binary compatibility information */ \ - WRITE( char, CHAR_BIT, fileIO ); \ - WRITE( char, sizeof(short int), fileIO ); \ - WRITE( char, sizeof(int), fileIO ); \ - WRITE( char, sizeof(float), fileIO ); \ - WRITE( char, sizeof(double), fileIO ); \ - WRITE( unsigned short, 0xcafe, fileIO ); \ - WRITE( unsigned int, 0xdeadbeef, fileIO ); \ - WRITE( float, 1.0, fileIO ); \ - WRITE( double, 1.0, fileIO ); \ - /* Dump type and header format version */ \ - WRITE( int, 0 /* Version */, fileIO ); \ - WRITE( int, dump_type, fileIO ); \ - /* High level information */ \ - WRITE( int, cstep, fileIO ); \ - WRITE( int, nxout, fileIO ); \ - WRITE( int, nyout, fileIO ); \ - WRITE( int, nzout, fileIO ); \ - WRITE( float, grid->dt, fileIO ); \ - WRITE( float, dxout, fileIO ); \ - WRITE( float, dyout, fileIO ); \ - WRITE( float, dzout, fileIO ); \ - WRITE( float, grid->x0, fileIO ); \ - WRITE( float, grid->y0, fileIO ); \ - WRITE( float, grid->z0, fileIO ); \ - WRITE( float, grid->cvac, fileIO ); \ - WRITE( float, grid->eps0, fileIO ); \ - WRITE( float, 0 /* damp */, fileIO ); \ - WRITE( int, rank(), fileIO ); \ - WRITE( int, nproc(), fileIO ); \ - /* Species parameters */ \ - WRITE( int, sp_id, fileIO ); \ - WRITE( float, q_m, fileIO ); \ +#define WRITE_HEADER_V0(dump_type,sp_id,q_m,fileIO,cstep,rank,nproc) do { \ + /* Binary compatibility information */ \ + WRITE( char, CHAR_BIT, fileIO ); \ + WRITE( char, sizeof(short int), fileIO ); \ + WRITE( char, sizeof(int), fileIO ); \ + WRITE( char, sizeof(float), fileIO ); \ + WRITE( char, sizeof(double), fileIO ); \ + WRITE( short int, 0xcafe, fileIO ); \ + WRITE( int, 0xdeadbeef, fileIO ); \ + WRITE( float, 1.0, fileIO ); \ + WRITE( double, 1.0, fileIO ); \ + /* Dump type and header format version */ \ + WRITE( int, 0 /* Version */, fileIO ); \ + WRITE( int, dump_type, fileIO ); \ + /* High level information */ \ + WRITE( int, cstep, fileIO ); \ + WRITE( int, nxout, fileIO ); \ + WRITE( int, nyout, fileIO ); \ + WRITE( int, nzout, fileIO ); \ + WRITE( float, grid->dt, fileIO ); \ + WRITE( float, dxout, fileIO ); \ + WRITE( float, dyout, fileIO ); \ + WRITE( float, dzout, fileIO ); \ + WRITE( float, grid->x0, fileIO ); \ + WRITE( float, grid->y0, fileIO ); \ + WRITE( float, grid->z0, fileIO ); \ + WRITE( float, grid->cvac, fileIO ); \ + WRITE( float, grid->eps0, fileIO ); \ + WRITE( float, 0 /* damp */, fileIO ); \ + WRITE( int, rank, fileIO ); \ + WRITE( int, nproc, fileIO ); \ + /* Species parameters */ \ + WRITE( int, sp_id, fileIO ); \ + WRITE( float, q_m, fileIO ); \ } while(0) - + // Note dim _MUST_ be a pointer to an int - + #define WRITE_ARRAY_HEADER(p,ndim,dim,fileIO) do { \ WRITE( int, sizeof(p[0]), fileIO ); \ WRITE( int, ndim, fileIO ); \ fileIO.write( dim, ndim ); \ } while(0) - + // The WRITE macro copies the output "value" into a temporary variable // of the requested output "type" so that the write to the "file" // occurs from a known binary data type. For example, if grid.dx were @@ -60,12 +60,12 @@ // single precision write copies. However, specialty types could be // created so that the type cast __WRITE_tmp = (type)(value) // automatically does the underlying conversion in C++ - + #define WRITE(type,value,fileIO) do { \ type __WRITE_tmp = (type)(value); \ fileIO.write( &__WRITE_tmp, 1 ); \ } while(0) - + // Note: strlen does not include the terminating \0 #define WRITE_STRING(string,fileIO) do { \ int __WRITE_STRING_len = 0; \ @@ -74,103 +74,102 @@ if( __WRITE_STRING_len>0 ) \ fileIO.write( string, __WRITE_STRING_len ); \ } while(0) - + #define READ(type,value,fileIO) do { \ type __READ_tmp; \ fileIO.read(&__READ_tmp, 1 ); \ (value) = __READ_tmp; \ } while(0) -#define F_WRITE_HEADER_V0(dump_type,sp_id,q_m,fileIO) do { \ - /* Binary compatibility information */ \ - F_WRITE( char, CHAR_BIT, fileIO ); \ - F_WRITE( char, sizeof(short int), fileIO ); \ - F_WRITE( char, sizeof(int), fileIO ); \ - F_WRITE( char, sizeof(float), fileIO ); \ - F_WRITE( char, sizeof(double), fileIO ); \ - F_WRITE( short int, 0xcafe, fileIO ); \ - F_WRITE( int, 0xdeadbeef, fileIO ); \ - F_WRITE( float, 1.0, fileIO ); \ - F_WRITE( double, 1.0, fileIO ); \ - /* Dump type and header format version */ \ - F_WRITE( int, 0 /* Version */, fileIO ); \ - F_WRITE( int, dump_type, fileIO ); \ - /* High level information */ \ - F_WRITE( int, step(), fileIO ); \ - F_WRITE( int, imxstr-2, fileIO ); \ - F_WRITE( int, jmxstr-2, fileIO ); \ - F_WRITE( int, kmxstr-2, fileIO ); \ - F_WRITE( float, grid->dt, fileIO ); \ - F_WRITE( float, dxstr, fileIO ); \ - F_WRITE( float, dystr, fileIO ); \ - F_WRITE( float, dzstr, fileIO ); \ - F_WRITE( float, grid->x0, fileIO ); \ - F_WRITE( float, grid->y0, fileIO ); \ - F_WRITE( float, grid->z0, fileIO ); \ - F_WRITE( float, grid->cvac, fileIO ); \ - F_WRITE( float, grid->eps0, fileIO ); \ - F_WRITE( float, 0 /*damp*/, fileIO ); \ - F_WRITE( int, rank(), fileIO ); \ - F_WRITE( int, nproc(), fileIO ); \ - /* Species parameters */ \ - F_WRITE( int, sp_id, fileIO ); \ - F_WRITE( float, q_m, fileIO ); \ - } while(0) - -#define F_WRITE_HEADER_PAR(dump_type,sp_id,q_m,fileIO) do { \ - /* Binary compatibility information */ \ - F_WRITE( char, CHAR_BIT, fileIO ); \ - F_WRITE( char, sizeof(short int), fileIO ); \ - F_WRITE( char, sizeof(int), fileIO ); \ - F_WRITE( char, sizeof(float), fileIO ); \ - F_WRITE( char, sizeof(double), fileIO ); \ - F_WRITE( short int, 0xcafe, fileIO ); \ - F_WRITE( int, 0xdeadbeef, fileIO ); \ - F_WRITE( float, 1.0, fileIO ); \ - F_WRITE( double, 1.0, fileIO ); \ - /* Dump type and header format version */ \ - F_WRITE( int, 0 /* Version */, fileIO ); \ - F_WRITE( int, dump_type, fileIO ); \ - /* High level information */ \ - F_WRITE( int, step(), fileIO ); \ - F_WRITE( int, grid->nx, fileIO ); \ - F_WRITE( int, grid->ny, fileIO ); \ - F_WRITE( int, grid->nz, fileIO ); \ - F_WRITE( float, grid->dt, fileIO ); \ - F_WRITE( float, grid->dx, fileIO ); \ - F_WRITE( float, grid->dy, fileIO ); \ - F_WRITE( float, grid->dz, fileIO ); \ - F_WRITE( float, grid->x0, fileIO ); \ - F_WRITE( float, grid->y0, fileIO ); \ - F_WRITE( float, grid->z0, fileIO ); \ - F_WRITE( float, grid->cvac, fileIO ); \ - F_WRITE( float, grid->eps0, fileIO ); \ - F_WRITE( float, 0 /*damp*/, fileIO ); \ - F_WRITE( int, rank(), fileIO ); \ - F_WRITE( int, nproc(), fileIO ); \ - /* Species parameters */ \ - F_WRITE( int, sp_id, fileIO ); \ - F_WRITE( float, q_m, fileIO ); \ - } while(0) - +//#define F_WRITE_HEADER_V0(dump_type,sp_id,q_m,fileIO) do { \ + ///* Binary compatibility information */ \ + //F_WRITE( char, CHAR_BIT, fileIO ); \ + //F_WRITE( char, sizeof(short int), fileIO ); \ + //F_WRITE( char, sizeof(int), fileIO ); \ + //F_WRITE( char, sizeof(float), fileIO ); \ + //F_WRITE( char, sizeof(double), fileIO ); \ + //F_WRITE( short int, 0xcafe, fileIO ); \ + //F_WRITE( int, 0xdeadbeef, fileIO ); \ + //F_WRITE( float, 1.0, fileIO ); \ + //F_WRITE( double, 1.0, fileIO ); \ + ///* Dump type and header format version */ \ + //F_WRITE( int, 0 /* Version */, fileIO ); \ + //F_WRITE( int, dump_type, fileIO ); \ + ///* High level information */ \ + //F_WRITE( int, step(), fileIO ); \ + //F_WRITE( int, imxstr-2, fileIO ); \ + //F_WRITE( int, jmxstr-2, fileIO ); \ + //F_WRITE( int, kmxstr-2, fileIO ); \ + //F_WRITE( float, grid->dt, fileIO ); \ + //F_WRITE( float, dxstr, fileIO ); \ + //F_WRITE( float, dystr, fileIO ); \ + //F_WRITE( float, dzstr, fileIO ); \ + //F_WRITE( float, grid->x0, fileIO ); \ + //F_WRITE( float, grid->y0, fileIO ); \ + //F_WRITE( float, grid->z0, fileIO ); \ + //F_WRITE( float, grid->cvac, fileIO ); \ + //F_WRITE( float, grid->eps0, fileIO ); \ + //F_WRITE( float, 0 /*damp*/, fileIO ); \ + //F_WRITE( int, rank(), fileIO ); \ + //F_WRITE( int, nproc(), fileIO ); \ + ///* Species parameters */ \ + //F_WRITE( int, sp_id, fileIO ); \ + //F_WRITE( float, q_m, fileIO ); \ + //} while(0) + +//#define F_WRITE_HEADER_PAR(dump_type,sp_id,q_m,fileIO) do { \ + ///* Binary compatibility information */ \ + //F_WRITE( char, CHAR_BIT, fileIO ); \ + //F_WRITE( char, sizeof(short int), fileIO ); \ + //F_WRITE( char, sizeof(int), fileIO ); \ + //F_WRITE( char, sizeof(float), fileIO ); \ + //F_WRITE( char, sizeof(double), fileIO ); \ + //F_WRITE( short int, 0xcafe, fileIO ); \ + //F_WRITE( int, 0xdeadbeef, fileIO ); \ + //F_WRITE( float, 1.0, fileIO ); \ + //F_WRITE( double, 1.0, fileIO ); \ + ///* Dump type and header format version */ \ + //F_WRITE( int, 0 /* Version */, fileIO ); \ + //F_WRITE( int, dump_type, fileIO ); \ + ///* High level information */ \ + //F_WRITE( int, step(), fileIO ); \ + //F_WRITE( int, grid->nx, fileIO ); \ + //F_WRITE( int, grid->ny, fileIO ); \ + //F_WRITE( int, grid->nz, fileIO ); \ + //F_WRITE( float, grid->dt, fileIO ); \ + //F_WRITE( float, grid->dx, fileIO ); \ + //F_WRITE( float, grid->dy, fileIO ); \ + //F_WRITE( float, grid->dz, fileIO ); \ + //F_WRITE( float, grid->x0, fileIO ); \ + //F_WRITE( float, grid->y0, fileIO ); \ + //F_WRITE( float, grid->z0, fileIO ); \ + //F_WRITE( float, grid->cvac, fileIO ); \ + //F_WRITE( float, grid->eps0, fileIO ); \ + //F_WRITE( float, 0 /*damp*/, fileIO ); \ + //F_WRITE( int, rank(), fileIO ); \ + //F_WRITE( int, nproc(), fileIO ); \ + ///* Species parameters */ \ + //F_WRITE( int, sp_id, fileIO ); \ + //F_WRITE( float, q_m, fileIO ); \ + //} while(0) + // Note dim _MUST_ be a pointer to an int - -#define F_WRITE_ARRAY_HEADER(psiz,ndim,dim,fileIO) do { \ - F_WRITE( int, psiz, fileIO ); \ - F_WRITE( int, ndim, fileIO ); \ - fileIO.write( dim, ndim ); \ - } while(0) - -#define F_WRITE(type,value,fileIO) do { \ - type __F_WRITE_tmp = (type)(value); \ - fileIO.write( &__F_WRITE_tmp, 1 ); \ - } while(0) - -#define F_READ(type,value,fileIO) do { \ - type __F_READ_tmp; \ - fileIO.read( &__F_READ_tmp, 1 ); \ - (value) = __F_READ_tmp; \ - } while(0) +//#define F_WRITE_ARRAY_HEADER(psiz,ndim,dim,fileIO) do { \ + //F_WRITE( int, psiz, fileIO ); \ + //F_WRITE( int, ndim, fileIO ); \ + //fileIO.write( dim, ndim ); \ + //} while(0) + +//#define F_WRITE(type,value,fileIO) do { \ + //type __F_WRITE_tmp = (type)(value); \ + //fileIO.write( &__F_WRITE_tmp, 1 ); \ + //} while(0) + +//#define F_READ(type,value,fileIO) do { \ + //type __F_READ_tmp; \ + //fileIO.read( &__F_READ_tmp, 1 ); \ + //(value) = __F_READ_tmp; \ + //} while(0) #define ABORT(cond) if( cond ) ERROR(( #cond )) diff --git a/src/vpic/hdf5_header_info.h b/src/vpic/hdf5_header_info.h new file mode 100644 index 00000000..e3810612 --- /dev/null +++ b/src/vpic/hdf5_header_info.h @@ -0,0 +1,125 @@ +#ifndef VPIC_HDF5_HEAD_INFO +#define VPIC_HDF5_HEAD_INFO + +#define FIELD_ARRAY_NAME field_array + +namespace VPIC_HDF { + // XML header stuff + static const char *header = "\n\n\n\t\n"; + static const char *header_topology = "\t\t\n"; + static const char *header_geom = "\t\t\n"; + static const char *header_origin = "\t\t\t \n\t\t\t%s\n"; + static const char *header_dxdydz = "\t\t\t \n\t\t\t%s\n"; + static const char *footer_geom = "\t\t\n"; + static const char *grid_line = "\t\t \n \ + \t\t\t\n"; + static const char *footer = "\t\t\n\t\n\n"; + + static const char *main_body_head = "\t\t\t \n \ + \t\t\t\t \n \ + \t\t\t\t \n"; + static const char *main_body_foot = "\t\t\t\n"; + + static const char *main_body_attributeV = "\ + \t\t\t\t \n \ + \t\t\t\t\t \n \ + \t\t\t\t\t\t T.%d/%s_%d.h5:/Timestep_%d/%s \n \ + \t\t\t\t\t\t T.%d/%s_%d.h5:/Timestep_%d/%s \n \ + \t\t\t\t\t\t T.%d/%s_%d.h5:/Timestep_%d/%s \n \ + \t\t\t\t\t \n \ + \t\t\t\t \n "; + + static const char *main_body_attributeS = "\ + \t\t\t\t \n \ + \t\t\t\t\t\t T.%d/%s_%d.h5:/Timestep_%d/%s \n \ + \t\t\t\t \n "; + +} // end namespace + +#define create_file_with_header(xml_file_name, dimensions, orignal, dxdydz, nframes, fields_interval) \ + { \ + FILE *fp; \ + fp = fopen(xml_file_name, "w"); \ + fputs(VPIC_HDF::header, fp); \ + fprintf(fp, VPIC_HDF::header_topology, dimensions); \ + fputs(VPIC_HDF::header_geom, fp); \ + fprintf(fp, VPIC_HDF::header_origin, orignal); \ + fprintf(fp, VPIC_HDF::header_dxdydz, dxdydz); \ + fputs(VPIC_HDF::footer_geom, fp); \ + fprintf(fp, VPIC_HDF::grid_line, nframes); \ + int i; \ + for (i = 0; i < nframes; i++) \ + fprintf(fp, "%d ", i*fields_interval); \ + fputs(VPIC_HDF::grid_line_footer, fp); \ + fclose(fp); \ + } +#define write_main_body_attribute(fpp, main_body_attribute_p, attribute_name, dims_4d_p, dims_3d_p, file_name_pre_p, time_step_p, a1, a2, a3) \ + { \ + fprintf(fpp, main_body_attribute_p, attribute_name, dims_4d_p, \ + dims_3d_p, time_step_p, file_name_pre_p, time_step_p, time_step_p, a1, \ + dims_3d_p, time_step_p, file_name_pre_p, time_step_p, time_step_p, a2, \ + dims_3d_p, time_step_p, file_name_pre_p, time_step_p, time_step_p, a3); \ + } + +#define invert_field_xml_item(xml_file_name, speciesname_p, time_step, dims_4d, dims_3d, add_footer_flag) \ + { \ + FILE *fp; \ + fp = fopen(xml_file_name, "a"); \ + fprintf(fp, VPIC_HDF::main_body_head, time_step); \ + if (field_dump_flag.enabledE()) \ + write_main_body_attribute(fp, VPIC_HDF::main_body_attributeV, "E", dims_4d, dims_3d, speciesname_p, time_step, "ex", "ey", "ez"); \ + if (field_dump_flag.div_e_err) \ + fprintf(fp, VPIC_HDF::main_body_attributeS, "div_e_err", dims_3d, time_step, speciesname_p, time_step, time_step, "div_e_err"); \ + if (field_dump_flag.enabledCB()) \ + write_main_body_attribute(fp, VPIC_HDF::main_body_attributeV, "B", dims_4d, dims_3d, speciesname_p, time_step, "cbx", "cby", "cbz"); \ + if (field_dump_flag.div_b_err) \ + fprintf(fp, VPIC_HDF::main_body_attributeS, "div_b_err", dims_3d, time_step, speciesname_p, time_step, time_step, "div_b_err"); \ + if (field_dump_flag.enabledTCA()) \ + write_main_body_attribute(fp, VPIC_HDF::main_body_attributeV, "TCA", dims_4d, dims_3d, speciesname_p, time_step, "tcax", "tcay", "tcaz"); \ + if (field_dump_flag.rhob) \ + fprintf(fp, VPIC_HDF::main_body_attributeS, "rhob", dims_3d, time_step, speciesname_p, time_step, time_step, "rhob"); \ + if (field_dump_flag.enabledJF()) \ + write_main_body_attribute(fp, VPIC_HDF::main_body_attributeV, "JF", dims_4d, dims_3d, speciesname_p, time_step, "jfx", "jfy", "jfz"); \ + if (field_dump_flag.rhof) \ + fprintf(fp, VPIC_HDF::main_body_attributeS, "rhof", dims_3d, time_step, speciesname_p, time_step, time_step, "rhof"); \ + if (field_dump_flag.enabledEMAT()) \ + write_main_body_attribute(fp, VPIC_HDF::main_body_attributeV, "EMAT", dims_4d, dims_3d, speciesname_p, time_step, "ematx", "ematy", "ematz"); \ + if (field_dump_flag.nmat) \ + fprintf(fp, VPIC_HDF::main_body_attributeS, "nmat", dims_3d, time_step, speciesname_p, time_step, time_step, "nmat"); \ + if (field_dump_flag.enabledFMAT()) \ + write_main_body_attribute(fp, VPIC_HDF::main_body_attributeV, "FMAT", dims_4d, dims_3d, speciesname_p, time_step, "fmatx", "fmaty", "fmatz"); \ + if (field_dump_flag.cmat) \ + fprintf(fp, VPIC_HDF::main_body_attributeS, "cmat", dims_3d, time_step, speciesname_p, time_step, time_step, "cmat"); \ + fprintf(fp, "%s", VPIC_HDF::main_body_foot); \ + if (add_footer_flag) \ + fputs(VPIC_HDF::footer, fp); \ + fclose(fp); \ + } +#define invert_hydro_xml_item(xml_file_name, speciesname_p, time_step, dims_4d, dims_3d, add_footer_flag) \ + { \ + FILE *fp; \ + fp = fopen(xml_file_name, "a"); \ + fprintf(fp, VPIC_HDF::main_body_head, time_step); \ + if (hydro_dump_flag.enabledJ()) \ + write_main_body_attribute(fp, VPIC_HDF::main_body_attributeV, "J", dims_4d, dims_3d, speciesname_p, time_step, "jx", "jy", "jz"); \ + if (hydro_dump_flag.rho) \ + fprintf(fp, VPIC_HDF::main_body_attributeS, "rho", dims_3d, time_step, speciesname_p, time_step, time_step, "rho"); \ + if (hydro_dump_flag.enabledP()) \ + write_main_body_attribute(fp, VPIC_HDF::main_body_attributeV, "P", dims_4d, dims_3d, speciesname_p, time_step, "px", "py", "pz"); \ + if (hydro_dump_flag.ke) \ + fprintf(fp, VPIC_HDF::main_body_attributeS, "ke", dims_3d, time_step, speciesname_p, time_step, time_step, "ke"); \ + if (hydro_dump_flag.enabledTD()) \ + write_main_body_attribute(fp, VPIC_HDF::main_body_attributeV, "TD", dims_4d, dims_3d, speciesname_p, time_step, "txx", "tyy", "tzz"); \ + if (hydro_dump_flag.enabledTOD()) \ + write_main_body_attribute(fp, VPIC_HDF::main_body_attributeV, "TOD", dims_4d, dims_3d, speciesname_p, time_step, "tyz", "tzx", "txy"); \ + fprintf(fp, "%s", VPIC_HDF::main_body_foot); \ + if (add_footer_flag) { \ + fputs(VPIC_HDF::footer, fp); \ + } \ + fclose(fp); \ + } + +#endif // VPIC_HDF5_HEAD_INFO diff --git a/src/vpic/initialize.cc b/src/vpic/initialize.cc index fb476272..572575f6 100644 --- a/src/vpic/initialize.cc +++ b/src/vpic/initialize.cc @@ -1,70 +1,106 @@ #include "vpic.h" #define FAK field_array->kernel -void -vpic_simulation::initialize( int argc, - char **argv ) { +void vpic_simulation::initialize(int argc, + char **argv) +{ double err; - species_t * sp; + species_t *sp; // Call the user initialize the simulation - TIC user_initialization( argc, argv ); TOC( user_initialization, 1 ); + TIC user_initialization(argc, argv); + TOC(user_initialization, 1); - // Do some consistency checks on user initialized fields + dump_strategy = new_dump_strategy(dump_strategy_id); + dump_strategy->num_step = num_step; - if( rank()==0 ) MESSAGE(( "Checking interdomain synchronization" )); - TIC err = FAK->synchronize_tang_e_norm_b( field_array ); TOC( synchronize_tang_e_norm_b, 1 ); - if( rank()==0 ) MESSAGE(( "Error = %e (arb units)", err )); + // Do some consistency checks on user initialized fields - if( rank()==0 ) MESSAGE(( "Checking magnetic field divergence" )); - TIC FAK->compute_div_b_err( field_array ); TOC( compute_div_b_err, 1 ); - TIC err = FAK->compute_rms_div_b_err( field_array ); TOC( compute_rms_div_b_err, 1 ); - if( rank()==0 ) MESSAGE(( "RMS error = %e (charge/volume)", err )); - TIC FAK->clean_div_b( field_array ); TOC( clean_div_b, 1 ); + if (rank() == 0) + MESSAGE(("Checking interdomain synchronization")); + TIC err = FAK->synchronize_tang_e_norm_b(field_array); + TOC(synchronize_tang_e_norm_b, 1); + if (rank() == 0) + MESSAGE(("Error = %e (arb units)", err)); + + if (rank() == 0) + MESSAGE(("Checking magnetic field divergence")); + TIC FAK->compute_div_b_err(field_array); + TOC(compute_div_b_err, 1); + TIC err = FAK->compute_rms_div_b_err(field_array); + TOC(compute_rms_div_b_err, 1); + if (rank() == 0) + MESSAGE(("RMS error = %e (charge/volume)", err)); + TIC FAK->clean_div_b(field_array); + TOC(clean_div_b, 1); // Load fields not initialized by the user - if( rank()==0 ) MESSAGE(( "Initializing radiation damping fields" )); - TIC FAK->compute_curl_b( field_array ); TOC( compute_curl_b, 1 ); - - if( rank()==0 ) MESSAGE(( "Initializing bound charge density" )); - TIC FAK->clear_rhof( field_array ); TOC( clear_rhof, 1 ); - LIST_FOR_EACH( sp, species_list ) TIC accumulate_rho_p( field_array, sp ); TOC( accumulate_rho_p, 1 ); - TIC FAK->synchronize_rho( field_array ); TOC( synchronize_rho, 1 ); - TIC FAK->compute_rhob( field_array ); TOC( compute_rhob, 1 ); + if (rank() == 0) + MESSAGE(("Initializing radiation damping fields")); + TIC FAK->compute_curl_b(field_array); + TOC(compute_curl_b, 1); + + if (rank() == 0) + MESSAGE(("Initializing bound charge density")); + TIC FAK->clear_rhof(field_array); + TOC(clear_rhof, 1); + LIST_FOR_EACH(sp, species_list) + TIC accumulate_rho_p(field_array, sp); + TOC(accumulate_rho_p, 1); + TIC FAK->synchronize_rho(field_array); + TOC(synchronize_rho, 1); + TIC FAK->compute_rhob(field_array); + TOC(compute_rhob, 1); // Internal sanity checks - if( rank()==0 ) MESSAGE(( "Checking electric field divergence" )); - - TIC FAK->compute_div_e_err( field_array ); TOC( compute_div_e_err, 1 ); - TIC err = FAK->compute_rms_div_e_err( field_array ); TOC( compute_rms_div_e_err, 1 ); - if( rank()==0 ) MESSAGE(( "RMS error = %e (charge/volume)", err )); - TIC FAK->clean_div_e( field_array ); TOC( clean_div_e, 1 ); - - if( rank()==0 ) MESSAGE(( "Rechecking interdomain synchronization" )); - TIC err = FAK->synchronize_tang_e_norm_b( field_array ); TOC( synchronize_tang_e_norm_b, 1 ); - if( rank()==0 ) MESSAGE(( "Error = %e (arb units)", err )); - - if( species_list ) { - if( rank()==0 ) MESSAGE(( "Uncentering particles" )); - TIC load_interpolator_array( interpolator_array, field_array ); TOC( load_interpolator, 1 ); + if (rank() == 0) + MESSAGE(("Checking electric field divergence")); + + TIC FAK->compute_div_e_err(field_array); + TOC(compute_div_e_err, 1); + TIC err = FAK->compute_rms_div_e_err(field_array); + TOC(compute_rms_div_e_err, 1); + if (rank() == 0) + MESSAGE(("RMS error = %e (charge/volume)", err)); + TIC FAK->clean_div_e(field_array); + TOC(clean_div_e, 1); + + if (rank() == 0) + MESSAGE(("Rechecking interdomain synchronization")); + TIC err = FAK->synchronize_tang_e_norm_b(field_array); + TOC(synchronize_tang_e_norm_b, 1); + if (rank() == 0) + MESSAGE(("Error = %e (arb units)", err)); + + if (species_list) + { + if (rank() == 0) + MESSAGE(("Uncentering particles")); + TIC load_interpolator_array(interpolator_array, field_array); + TOC(load_interpolator, 1); } - LIST_FOR_EACH( sp, species_list ) TIC uncenter_p( sp, interpolator_array ); TOC( uncenter_p, 1 ); + LIST_FOR_EACH(sp, species_list) + TIC uncenter_p(sp, interpolator_array); + TOC(uncenter_p, 1); - if( rank()==0 ) MESSAGE(( "Performing initial diagnostics" )); + if (rank() == 0) + MESSAGE(("Performing initial diagnostics")); // Let the user to perform diagnostics on the initial condition // field(i,j,k).jfx, jfy, jfz will not be valid at this point. - TIC user_diagnostics(); TOC( user_diagnostics, 1 ); + TIC user_diagnostics(); + TOC(user_diagnostics, 1); - if( rank()==0 ) MESSAGE(( "Initialization complete" )); - update_profile( rank()==0 ); // Let the user know how initialization went + if (rank() == 0) + MESSAGE(("Initialization complete")); + update_profile(rank() == 0); // Let the user know how initialization went } -void -vpic_simulation::finalize( void ) { +void vpic_simulation::finalize(void) +{ barrier(); - update_profile( rank()==0 ); + update_profile(rank() == 0); } diff --git a/src/vpic/vpic.cc b/src/vpic/vpic.cc index 9b1f7e8e..dbf08147 100644 --- a/src/vpic/vpic.cc +++ b/src/vpic/vpic.cc @@ -9,6 +9,7 @@ */ #include "vpic.h" +#include "dump_strategy.h" /* Note that, when a vpic_simulation is created (and thus registered with the checkpt service), it is created empty; none of the simulation @@ -21,74 +22,114 @@ to proper resize semantics so we could then create the objects during vpic_simulation construction (as opposed to after it). */ -void -checkpt_vpic_simulation( const vpic_simulation * vpic ) { - CHECKPT( vpic, 1 ); - CHECKPT_PTR( vpic->entropy ); - CHECKPT_PTR( vpic->sync_entropy ); - CHECKPT_PTR( vpic->grid ); - CHECKPT_FPTR( vpic->material_list ); - CHECKPT_FPTR( vpic->field_array ); - CHECKPT_FPTR( vpic->interpolator_array ); - CHECKPT_FPTR( vpic->accumulator_array ); - CHECKPT_FPTR( vpic->hydro_array ); - CHECKPT_FPTR( vpic->species_list ); - CHECKPT_FPTR( vpic->particle_bc_list ); - CHECKPT_FPTR( vpic->emitter_list ); - CHECKPT_FPTR( vpic->collision_op_list ); +void checkpt_vpic_simulation(const vpic_simulation *vpic) +{ + CHECKPT(vpic, 1); + // CHECKPT_PTR(vpic->dump_strategy); + CHECKPT_PTR(vpic->entropy); + CHECKPT_PTR(vpic->sync_entropy); + CHECKPT_PTR(vpic->grid); + CHECKPT_FPTR(vpic->material_list); + CHECKPT_FPTR(vpic->field_array); + CHECKPT_FPTR(vpic->interpolator_array); + CHECKPT_FPTR(vpic->accumulator_array); + CHECKPT_FPTR(vpic->hydro_array); + CHECKPT_FPTR(vpic->species_list); + CHECKPT_FPTR(vpic->particle_bc_list); + CHECKPT_FPTR(vpic->emitter_list); + CHECKPT_FPTR(vpic->collision_op_list); } vpic_simulation * -restore_vpic_simulation( void ) { - vpic_simulation * vpic; - RESTORE( vpic ); - RESTORE_PTR( vpic->entropy ); - RESTORE_PTR( vpic->sync_entropy ); - RESTORE_PTR( vpic->grid ); - RESTORE_FPTR( vpic->material_list ); - RESTORE_FPTR( vpic->field_array ); - RESTORE_FPTR( vpic->interpolator_array ); - RESTORE_FPTR( vpic->accumulator_array ); - RESTORE_FPTR( vpic->hydro_array ); - RESTORE_FPTR( vpic->species_list ); - RESTORE_FPTR( vpic->particle_bc_list ); - RESTORE_FPTR( vpic->emitter_list ); - RESTORE_FPTR( vpic->collision_op_list ); +restore_vpic_simulation(void) +{ + vpic_simulation *vpic; + RESTORE(vpic); + // RESTORE_PTR(vpic->dump_strategy); + RESTORE_PTR(vpic->entropy); + RESTORE_PTR(vpic->sync_entropy); + RESTORE_PTR(vpic->grid); + RESTORE_FPTR(vpic->material_list); + RESTORE_FPTR(vpic->field_array); + RESTORE_FPTR(vpic->interpolator_array); + RESTORE_FPTR(vpic->accumulator_array); + RESTORE_FPTR(vpic->hydro_array); + RESTORE_FPTR(vpic->species_list); + RESTORE_FPTR(vpic->particle_bc_list); + RESTORE_FPTR(vpic->emitter_list); + RESTORE_FPTR(vpic->collision_op_list); + + // printf("\n\n\n vpic->dump_strategy_id is restored ! \n\n\n"); + vpic->dump_strategy = new_dump_strategy(vpic->dump_strategy_id); + vpic->dump_strategy->num_step = vpic->num_step; + + // // Do any post init/restore simulation modifications + // switch (vpic->dump_strategy_id) + // { + // case DUMP_STRATEGY_BINARY: + // //::cout << "DUMP_STRATEGY_BINARY \n"; + // vpic->dump_strategy = new BinaryDump(rank(), nproc()); + // break; + // case DUMP_STRATEGY_HDF5: + // #ifdef VPIC_ENABLE_HDF5 + // // std::cout << "DUMP_STRATEGY_HDF5 \n"; + // vpic->dump_strategy = new HDF5Dump(rank(), nproc()); + // #else + // std::cout << "HDF5Dump is not enabled \n"; + + // #endif + // break; + // case DUMP_STRATEGY_OPENPMD: + // #ifdef VPIC_ENABLE_OPENPMD + // // std::cout << "DUMP_STRATEGY_OPENPMD \n"; + // vpic->dump_strategy = new OpenPMDDump(rank(), nproc()); + // #else + // std::cout << "OpenPMDDump is not enabled \n"; + // #endif + // break; + // default: + // break; + // } + return vpic; } -void -reanimate_vpic_simulation( vpic_simulation * vpic ) { - REANIMATE_FPTR( vpic->material_list ); - REANIMATE_FPTR( vpic->field_array ); - REANIMATE_FPTR( vpic->interpolator_array ); - REANIMATE_FPTR( vpic->accumulator_array ); - REANIMATE_FPTR( vpic->hydro_array ); - REANIMATE_FPTR( vpic->species_list ); - REANIMATE_FPTR( vpic->particle_bc_list ); - REANIMATE_FPTR( vpic->emitter_list ); - REANIMATE_FPTR( vpic->collision_op_list ); +void reanimate_vpic_simulation(vpic_simulation *vpic) +{ + REANIMATE_FPTR(vpic->material_list); + REANIMATE_FPTR(vpic->field_array); + REANIMATE_FPTR(vpic->interpolator_array); + REANIMATE_FPTR(vpic->accumulator_array); + REANIMATE_FPTR(vpic->hydro_array); + REANIMATE_FPTR(vpic->species_list); + REANIMATE_FPTR(vpic->particle_bc_list); + REANIMATE_FPTR(vpic->emitter_list); + REANIMATE_FPTR(vpic->collision_op_list); } +vpic_simulation::vpic_simulation() +{ + // TODO: why is this a good idea? + // Is this just trying to 0 initialize everything? + // CLEAR( this, 1 ); -vpic_simulation::vpic_simulation() { - CLEAR( this, 1 ); - - /* Set non-zero defaults */ - verbose = 1; - num_comm_round = 3; - num_div_e_round = 2; - num_div_b_round = 2; + // Now done in the class def / header + ///* Set non-zero defaults */ + // verbose = 1; + // num_comm_round = 3; + // num_div_e_round = 2; + // num_div_b_round = 2; -#if defined(VPIC_USE_PTHREADS) // Pthreads case. - int n_rng = serial.n_pipeline; - if ( n_rng < thread.n_pipeline ) n_rng = thread.n_pipeline; +#if defined(VPIC_USE_PTHREADS) // Pthreads case. + int n_rng = serial.n_pipeline; + if (n_rng < thread.n_pipeline) + n_rng = thread.n_pipeline; -#elif defined(VPIC_USE_OPENMP) // OpenMP case. - int n_rng = omp_helper.n_pipeline; +#elif defined(VPIC_USE_OPENMP) // OpenMP case. + int n_rng = omp_helper.n_pipeline; -#else // Error case. - #error "VPIC_USE_OPENMP or VPIC_USE_PTHREADS must be specified" +#else // Error case. +#error "VPIC_USE_OPENMP or VPIC_USE_PTHREADS must be specified" #endif @@ -101,26 +142,45 @@ vpic_simulation::vpic_simulation() { n_rng++; - entropy = new_rng_pool( n_rng, 0, 0 ); - sync_entropy = new_rng_pool( n_rng, 0, 1 ); + entropy = new_rng_pool(n_rng, 0, 0); + sync_entropy = new_rng_pool(n_rng, 0, 1); grid = new_grid(); - - REGISTER_OBJECT( this, checkpt_vpic_simulation, - restore_vpic_simulation, reanimate_vpic_simulation ); + dump_strategy = new_dump_strategy(dump_strategy_id); + + REGISTER_OBJECT(this, checkpt_vpic_simulation, + restore_vpic_simulation, reanimate_vpic_simulation); + + // Initialize the dump strategy to use the binary dumpin, assuming the user + // may overwrite this later + // dump_strategy = std::unique_ptr(new BinaryDump( rank(), nproc() )); + // enable_binary_dump(); + + // TODO: this this still makes sense now we have a dump strategy + // #ifdef VPIC_ENABLE_HDF5 + // Default init hdf5 dump flags + // field_interval = 1; + // hydro_interval = 1; + // field_dump_flag = field_dump_flag_t(); + // hydro_dump_flag = hydro_dump_flag_t(); + // #endif + + field_interval = 1; + hydro_interval = 1; } -vpic_simulation::~vpic_simulation() { - UNREGISTER_OBJECT( this ); - delete_collision_op_list( collision_op_list ); - delete_emitter_list( emitter_list ); - delete_particle_bc_list( particle_bc_list ); - delete_species_list( species_list ); - delete_hydro_array( hydro_array ); - delete_accumulator_array( accumulator_array ); - delete_interpolator_array( interpolator_array ); - delete_field_array( field_array ); - delete_material_list( material_list ); - delete_grid( grid ); - delete_rng_pool( sync_entropy ); - delete_rng_pool( entropy ); +vpic_simulation::~vpic_simulation() +{ + UNREGISTER_OBJECT(this); + delete_collision_op_list(collision_op_list); + delete_emitter_list(emitter_list); + delete_particle_bc_list(particle_bc_list); + delete_species_list(species_list); + delete_hydro_array(hydro_array); + delete_accumulator_array(accumulator_array); + delete_interpolator_array(interpolator_array); + delete_field_array(field_array); + delete_material_list(material_list); + delete_grid(grid); + delete_rng_pool(sync_entropy); + delete_rng_pool(entropy); } diff --git a/src/vpic/vpic.h b/src/vpic/vpic.h index da365840..16b58500 100644 --- a/src/vpic/vpic.h +++ b/src/vpic/vpic.h @@ -15,6 +15,7 @@ #include #include +#include // unique_ptr #include #include #include @@ -27,6 +28,7 @@ #include "../util/bitfield.h" #include "../util/checksum.h" #include "../util/system.h" +#include "dump_strategy.h" #include "dumpmacros.h" #ifndef USER_GLOBAL_SIZE @@ -40,38 +42,39 @@ typedef FileIO FILETYPE; -const uint32_t all (0xffffffff); -const uint32_t electric (1<<0 | 1<<1 | 1<<2); -const uint32_t div_e_err (1<<3); -const uint32_t magnetic (1<<4 | 1<<5 | 1<<6); -const uint32_t div_b_err (1<<7); -const uint32_t tca (1<<8 | 1<<9 | 1<<10); -const uint32_t rhob (1<<11); -const uint32_t current (1<<12 | 1<<13 | 1<<14); -const uint32_t rhof (1<<15); -const uint32_t emat (1<<16 | 1<<17 | 1<<18); -const uint32_t nmat (1<<19); -const uint32_t fmat (1<<20 | 1<<21 | 1<<22); -const uint32_t cmat (1<<23); +const uint32_t all(0xffffffff); +const uint32_t electric(1 << 0 | 1 << 1 | 1 << 2); +const uint32_t div_e_err(1 << 3); +const uint32_t magnetic(1 << 4 | 1 << 5 | 1 << 6); +const uint32_t div_b_err(1 << 7); +const uint32_t tca(1 << 8 | 1 << 9 | 1 << 10); +const uint32_t rhob(1 << 11); +const uint32_t current(1 << 12 | 1 << 13 | 1 << 14); +const uint32_t rhof(1 << 15); +const uint32_t emat(1 << 16 | 1 << 17 | 1 << 18); +const uint32_t nmat(1 << 19); +const uint32_t fmat(1 << 20 | 1 << 21 | 1 << 22); +const uint32_t cmat(1 << 23); const size_t total_field_variables(24); const size_t total_field_groups(12); // this counts vectors, tensors etc... // These bits will be tested to determine which variables to output -const size_t field_indeces[12] = { 0, 3, 4, 7, 8, 11, 12, 15, 16, 19, 20, 23 }; - -struct FieldInfo { - char name[128]; - char degree[128]; - char elements[128]; - char type[128]; - size_t size; +const size_t field_indeces[12] = {0, 3, 4, 7, 8, 11, 12, 15, 16, 19, 20, 23}; + +struct FieldInfo +{ + char name[128]; + char degree[128]; + char elements[128]; + char type[128]; + size_t size; }; // struct FieldInfo -const uint32_t current_density (1<<0 | 1<<1 | 1<<2); -const uint32_t charge_density (1<<3); -const uint32_t momentum_density (1<<4 | 1<<5 | 1<<6); -const uint32_t ke_density (1<<7); -const uint32_t stress_tensor (1<<8 | 1<<9 | 1<<10 | 1<<11 | 1<<12 | 1<<13); +const uint32_t current_density(1 << 0 | 1 << 1 | 1 << 2); +const uint32_t charge_density(1 << 3); +const uint32_t momentum_density(1 << 4 | 1 << 5 | 1 << 6); +const uint32_t ke_density(1 << 7); +const uint32_t stress_tensor(1 << 8 | 1 << 9 | 1 << 10 | 1 << 11 | 1 << 12 | 1 << 13); /* May want to use these instead const uint32_t stress_diagonal (1<<8 | 1<<9 | 1<<10); const uint32_t stress_offdiagonal (1<<11 | 1<<12 | 1<<13); @@ -80,20 +83,22 @@ const uint32_t stress_offdiagonal (1<<11 | 1<<12 | 1<<13); const size_t total_hydro_variables(14); const size_t total_hydro_groups(5); // this counts vectors, tensors etc... // These bits will be tested to determine which variables to output -const size_t hydro_indeces[5] = { 0, 3, 4, 7, 8 }; - -struct HydroInfo { - char name[128]; - char degree[128]; - char elements[128]; - char type[128]; - size_t size; +const size_t hydro_indeces[5] = {0, 3, 4, 7, 8}; + +struct HydroInfo +{ + char name[128]; + char degree[128]; + char elements[128]; + char type[128]; + size_t size; }; // struct FieldInfo /*---------------------------------------------------------------------------- * DumpFormat Enumeration ----------------------------------------------------------------------------*/ -enum DumpFormat { +enum DumpFormat +{ band = 0, band_interleave = 1 }; // enum DumpFormat @@ -101,9 +106,11 @@ enum DumpFormat { /*---------------------------------------------------------------------------- * DumpParameters Struct ----------------------------------------------------------------------------*/ -struct DumpParameters { +struct DumpParameters +{ - void output_variables(uint32_t mask) { + void output_variables(uint32_t mask) + { output_vars.set(mask); } // output_variables @@ -121,81 +128,95 @@ struct DumpParameters { }; // struct DumpParameters -// TODO: should this be an enum? -namespace dump_type { - const int grid_dump = 0; - const int field_dump = 1; - const int hydro_dump = 2; - const int particle_dump = 3; - const int restart_dump = 4; - const int history_dump = 5; -} // namespace - - -class vpic_simulation { +class vpic_simulation +{ public: vpic_simulation(); ~vpic_simulation(); - void initialize( int argc, char **argv ); - void modify( const char *fname ); - int advance( void ); - void finalize( void ); + void initialize(int argc, char **argv); + void modify(const char *fname); + int advance(void); + void finalize(void); + + // TODO: decide if I should collapse this to an enum + // An enum would stop these ifdefs being so leaky + void enable_binary_dump(); +#ifdef VPIC_ENABLE_HDF5 + void enable_hdf5_dump(); +#endif +#ifdef VPIC_ENABLE_OPENPMD + void enable_openpmd_dump(); +#endif - #ifdef VPIC_GLOBAL_PARTICLE_ID + // TODO: remake these protected + + // Very likely a user will forgot to delete this if they change the strategy, + // a smart ptr will save us from the small leak + // std::unique_ptr dump_strategy; + Dump_Strategy *dump_strategy; + int num_step = 1; // Number of steps to take + DumpStrategyID dump_strategy_id = DUMP_STRATEGY_BINARY; // 0 : binary; 1: HDF5; 2: OpenPMD +#ifdef VPIC_GLOBAL_PARTICLE_ID // TODO: move these somewhere more sensible - int predicate_count(species_t* sp, std::function f); - int predicate_count(species_t* sp, std::function f); + int predicate_count(species_t *sp, std::function f); + int predicate_count(species_t *sp, std::function f); // TOOD: those specialized in together should probably be wrapped in a class - void predicate_copy(species_t* sp_from, species_t* sp_to, std::function f); - void predicate_copy(species_t* sp_from, species_t* sp_to, std::function f); - #endif + void predicate_copy(species_t *sp_from, species_t *sp_to, std::function f); + void predicate_copy(species_t *sp_from, species_t *sp_to, std::function f); +#endif protected: - // Directly initialized by user - int verbose; // Should system be verbose - int num_step; // Number of steps to take - int num_comm_round; // Num comm round - int status_interval; // How often to print status messages - int clean_div_e_interval; // How often to clean div e - int num_div_e_round; // How many clean div e rounds per div e interval - int clean_div_b_interval; // How often to clean div b - int num_div_b_round; // How many clean div b rounds per div b interval - int sync_shared_interval; // How often to synchronize shared faces + int verbose = 1; // Should system be verbose + int num_comm_round = 3; // Num comm round + int status_interval = 0; // How often to print status messages + + int clean_div_e_interval = 0; // How often to clean div e + int num_div_e_round = 2; // How many clean div e rounds per div e interval + + int clean_div_b_interval = 0; // How often to clean div b + int num_div_b_round = 2; // How many clean div b rounds per div b interval + + int sync_shared_interval = 0; // How often to synchronize shared faces // FIXME: THESE INTERVALS SHOULDN'T BE PART OF vpic_simulation // THE BIG LIST FOLLOWING IT SHOULD BE CLEANED UP TOO - double quota; - int checkpt_interval; - int hydro_interval; - int field_interval; - int particle_interval; - - size_t nxout, nyout, nzout; - size_t px, py, pz; - float dxout, dyout, dzout; - - int ndfld; - int ndhyd; - int ndpar; - int ndhis; - int ndgrd; - int head_option; - int istride; - int jstride; - int kstride; - int stride_option; - int pstride; - int nprobe; + double quota = 0; + int checkpt_interval = 0; + int hydro_interval = 0; + int field_interval = 0; + int particle_interval = 0; + + // TODO: these can probably now be removed, as they should only be used by dump? + // TODO: check if any decks used them + // size_t nxout, nyout, nzout; + // float dxout, dyout, dzout; + + size_t px = 0; + size_t py = 0; + size_t pz = 0; + + int ndfld = 0; + int ndhyd = 0; + int ndpar = 0; + int ndhis = 0; + int ndgrd = 0; + int head_option = 0; + int istride = 0; + int jstride = 0; + int kstride = 0; + int stride_option = 0; + int pstride = 0; + int nprobe = 0; int ijkprobe[NVARHISMX][4]; float xyzprobe[NVARHISMX][3]; - int block_dump; - int stepdigit; - int rankdigit; - int ifenergies; + int block_dump = 0; + int stepdigit = 0; + int rankdigit = 0; + int ifenergies = 0; // Helper initialized by user @@ -205,21 +226,21 @@ class vpic_simulation { random numbers. Keeping the synchronous generators in sync is the generator users responsibility. */ - rng_pool_t * entropy; // Local entropy pool - rng_pool_t * sync_entropy; // Synchronous entropy pool - grid_t * grid; // define_*_grid et al - material_t * material_list; // define_material - field_array_t * field_array; // define_field_array - interpolator_array_t * interpolator_array; // define_interpolator_array - accumulator_array_t * accumulator_array; // define_accumulator_array - hydro_array_t * hydro_array; // define_hydro_array - species_t * species_list; // define_species / - // species helpers - particle_bc_t * particle_bc_list; // define_particle_bc / - // boundary helpers - emitter_t * emitter_list; // define_emitter / - // emitter helpers - collision_op_t * collision_op_list; // collision helpers + rng_pool_t *entropy; // Local entropy pool + rng_pool_t *sync_entropy; // Synchronous entropy pool + grid_t *grid; // define_*_grid et al + material_t *material_list = NULL; // define_material + field_array_t *field_array; // define_field_array + interpolator_array_t *interpolator_array; // define_interpolator_array + accumulator_array_t *accumulator_array; // define_accumulator_array + hydro_array_t *hydro_array; // define_hydro_array + species_t *species_list = NULL; // define_species / + // species helpers + particle_bc_t *particle_bc_list = NULL; // define_particle_bc / + // boundary helpers + emitter_t *emitter_list = NULL; // define_emitter / + // emitter helpers + collision_op_t *collision_op_list = NULL; // collision helpers // User defined checkpt preserved variables // Note: user_global is aliased with user_global_t (see deck_wrapper.cxx) @@ -236,208 +257,200 @@ class vpic_simulation { ---------------------------------------------------------------------------*/ #if defined(ENABLE_OPENSSL) void output_checksum_fields(); - void checksum_fields(CheckSum & cs); - void output_checksum_species(const char * species); - void checksum_species(const char * species, CheckSum & cs); + void checksum_fields(CheckSum &cs); + void output_checksum_species(const char *species); + void checksum_species(const char *species, CheckSum &cs); #endif // ENABLE_OPENSSL - void print_available_ram() { + void print_available_ram() + { SystemRAM::print_available(); } // print_available_ram /////////////// // Dump helpers - int dump_mkdir(const char * dname); - int dump_cwd(char * dname, size_t size); + static int dump_mkdir(const char *dname); + int dump_cwd(char *dname, size_t size); // Text dumps - void dump_energies( const char *fname, int append = 1 ); - void dump_materials( const char *fname ); - void dump_species( const char *fname ); + void dump_energies(const char *fname, int append = 1); + void dump_materials(const char *fname); + void dump_species(const char *fname); // Binary dumps - void dump_grid( const char *fbase ); - void dump_fields( const char *fbase, - int fname_tag = 1, - field_t *f = NULL ); - void dump_hydro( const char *sp_name, - const char *fbase, - int fname_tag = 1, - hydro_t *h = NULL ); - void dump_particles( const char *sp_name, - const char *fbase, - int fname_tag = 1 ); + void dump_grid(const char *fbase); + + void dump_fields(const char *fbase, int fname_tag = 1); + void dump_hydro(const char *sp_name, const char *fbase, + int fname_tag = 1); + + void dump_particles(const char *sp_name, const char *fbase, + int fname_tag = 1); #ifdef VPIC_GLOBAL_PARTICLE_ID -// TODO: merge back down to one function -// TODO: template out the functor type -// TODO: find a way to specify if we want to predicate on particle array, or -// particle index - template + // TODO: merge back down to one function + // TODO: template out the functor type + // TODO: find a way to specify if we want to predicate on particle array, or + // particle index + template void dump_particles_predicate( const char *sp_name, const char *fbase, int ftag, - //const std::function & f = nullptr - //const std::function & f = nullptr - const Predicate& f - ) + // const std::function & f = nullptr + // const std::function & f = nullptr + const Predicate &f) { - species_t *sp; - char fname[256]; - FileIO fileIO; - int dim[2], buf_start; - static particle_t * ALIGNED(128) p_buf = NULL; -# define PBUF_SIZE 32768 // 1MB of particles + species_t *sp; + char fname[256]; + FileIO fileIO; + int dim[2], buf_start; + static particle_t *ALIGNED(128) p_buf = NULL; +#define PBUF_SIZE 32768 // 1MB of particles + + sp = find_species_name(sp_name, species_list); + if (!sp) + ERROR(("Invalid species name \"%s\".", sp_name)); + + if (!fbase) + ERROR(("Invalid filename")); + + if (!p_buf) + MALLOC_ALIGNED(p_buf, PBUF_SIZE, 128); + + if (rank() == 0) + MESSAGE(("Dumping \"%s\" particles to \"%s\"", sp->name, fbase)); + + if (ftag) + sprintf(fname, "%s.%li.%i", fbase, (long)step(), rank()); + else + sprintf(fname, "%s.%i", fbase, rank()); + FileIOStatus status = fileIO.open(fname, io_write); + if (status == fail) + ERROR(("Could not open \"%s\"", fname)); + + int count_true = predicate_count(sp, f); + std::cout << "copying " << count_true << " of " << sp->np << std::endl; + + dim[0] = count_true; + WRITE_ARRAY_HEADER(p_buf, 1, dim, fileIO); + + // Copy a PBUF_SIZE hunk of the particle list into the particle + // buffer, timecenter it and write it out. This is done this way to + // guarantee the particle list unchanged while not requiring too + // much memory. + + // FIXME: WITH A PIPELINED CENTER_P, PBUF NOMINALLY SHOULD BE QUITE + // LARGE. + + // Make a second species array, and space to hold the IDs too + particle_t *ALIGNED(128) _p; + size_t *ALIGNED(128) _p_id; + MALLOC_ALIGNED(_p, count_true, 128); + MALLOC_ALIGNED(_p_id, count_true, 128); +#ifdef VPIC_PARTICLE_ANNOTATION + typedef VPIC_PARTICLE_ANNOTATION annotation_t; + annotation_t *ALIGNED(128) _p_annotation = nullptr; + if (sp->has_annotation) + { + MALLOC_ALIGNED(_p_annotation, count_true * sp->has_annotation, 128); + } +#endif - sp = find_species_name( sp_name, species_list ); - if( !sp ) ERROR(( "Invalid species name \"%s\".", sp_name )); + species_t _sp = *sp; // FIXME: Is this copy careful/safe enough? + _sp.p = _p; + _sp.np = count_true; + _sp.p_id = _p_id; +#ifdef VPIC_PARTICLE_ANNOTATION + _sp.has_annotation = sp->has_annotation; + _sp.p_annotation = _p_annotation; +#endif - if( !fbase ) ERROR(( "Invalid filename" )); + // TODO: why do we need to update max_np? + _sp.max_np = PBUF_SIZE; - if( !p_buf ) MALLOC_ALIGNED( p_buf, PBUF_SIZE, 128 ); + // Copy the right particles over, that meet the predicate + predicate_copy(sp, &_sp, f); - if( rank()==0 ) - MESSAGE(("Dumping \"%s\" particles to \"%s\"",sp->name,fbase)); + // Use that instead below + for (buf_start = 0; buf_start < count_true; buf_start += PBUF_SIZE) + { + _sp.np = count_true - buf_start; - if( ftag ) sprintf( fname, "%s.%li.%i", fbase, (long)step(), rank() ); - else sprintf( fname, "%s.%i", fbase, rank() ); - FileIOStatus status = fileIO.open(fname, io_write); - if( status==fail ) ERROR(( "Could not open \"%s\"", fname )); + if (_sp.np > PBUF_SIZE) + { + _sp.np = PBUF_SIZE; + } - /* IMPORTANT: these values are written in WRITE_HEADER_V0 */ - nxout = grid->nx; - nyout = grid->ny; - nzout = grid->nz; - dxout = grid->dx; - dyout = grid->dy; - dzout = grid->dz; + // COPY( _sp.p, &sp_p[buf_start], _sp.np ); - WRITE_HEADER_V0( dump_type::particle_dump, sp->id, sp->q/sp->m, step(), fileIO ); + center_p(&_sp, interpolator_array); - int count_true = predicate_count(sp, f); - std::cout << "copying " << count_true << " of " << sp->np << std::endl; + fileIO.write(_p, _sp.np); + } + // append ID array at the end of the file + if (sp->has_ids) + { dim[0] = count_true; - WRITE_ARRAY_HEADER( p_buf, 1, dim, fileIO ); - - // Copy a PBUF_SIZE hunk of the particle list into the particle - // buffer, timecenter it and write it out. This is done this way to - // guarantee the particle list unchanged while not requiring too - // much memory. - - // FIXME: WITH A PIPELINED CENTER_P, PBUF NOMINALLY SHOULD BE QUITE - // LARGE. - - // Make a second species array, and space to hold the IDs too - particle_t* ALIGNED(128) _p; - size_t* ALIGNED(128) _p_id; - MALLOC_ALIGNED( _p, count_true, 128 ); - MALLOC_ALIGNED( _p_id, count_true, 128 ); - #ifdef VPIC_PARTICLE_ANNOTATION - typedef VPIC_PARTICLE_ANNOTATION annotation_t; - annotation_t* ALIGNED(128) _p_annotation = nullptr; - if(sp->has_annotation) { - MALLOC_ALIGNED( _p_annotation, count_true*sp->has_annotation, 128); - } - #endif - - species_t _sp = *sp; // FIXME: Is this copy careful/safe enough? - _sp.p = _p; - _sp.np = count_true; - _sp.p_id = _p_id; - #ifdef VPIC_PARTICLE_ANNOTATION - _sp.has_annotation = sp->has_annotation; - _sp.p_annotation = _p_annotation; - #endif - - // TODO: why do we need to update max_np? - _sp.max_np = PBUF_SIZE; - - // Copy the right particles over, that meet the predicate - predicate_copy(sp, &_sp, f); - - // Use that instead below - for( buf_start=0; buf_start PBUF_SIZE ) { - _sp.np = PBUF_SIZE; - } - - //COPY( _sp.p, &sp_p[buf_start], _sp.np ); - - center_p( &_sp, interpolator_array ); - - fileIO.write( _p, _sp.np ); - } - - // append ID array at the end of the file - if(sp->has_ids) { - dim[0] = count_true; - WRITE_ARRAY_HEADER( sp->p_id, 1, dim, fileIO ); - // Maybe do this write in batches of PBUF_SIZE as well - fileIO.write(_sp.p_id, count_true); - } - #ifdef VPIC_PARTICLE_ANNOTATION - // append annotation buffer at the end of the file - if(sp->has_annotation) { - dim[0] = count_true; - dim[1] = sp->has_annotation; - WRITE_ARRAY_HEADER( sp->p_annotation, 2, dim, fileIO ); - // Maybe do this write in batches of PBUF_SIZE as well - fileIO.write(_sp.p_annotation, count_true*sp->has_annotation); - } - #endif + WRITE_ARRAY_HEADER(sp->p_id, 1, dim, fileIO); + // Maybe do this write in batches of PBUF_SIZE as well + fileIO.write(_sp.p_id, count_true); + } +#ifdef VPIC_PARTICLE_ANNOTATION + // append annotation buffer at the end of the file + if (sp->has_annotation) + { + dim[0] = count_true; + dim[1] = sp->has_annotation; + WRITE_ARRAY_HEADER(sp->p_annotation, 2, dim, fileIO); + // Maybe do this write in batches of PBUF_SIZE as well + fileIO.write(_sp.p_annotation, count_true * sp->has_annotation); + } +#endif - // Free the particle array and ID array (sp done by scope) - FREE_ALIGNED( _p ); - FREE_ALIGNED( _p_id ); - #ifdef VPIC_PARTICLE_ANNOTATION - FREE_ALIGNED( _p_annotation ); - #endif + // Free the particle array and ID array (sp done by scope) + FREE_ALIGNED(_p); + FREE_ALIGNED(_p_id); +#ifdef VPIC_PARTICLE_ANNOTATION + FREE_ALIGNED(_p_annotation); +#endif - if( fileIO.close() ) ERROR(("File close failed on dump particles!!!")); + if (fileIO.close()) + ERROR(("File close failed on dump particles!!!")); } #endif - - // convenience functions for simlog output - void create_field_list(char * strlist, DumpParameters & dumpParams); - void create_hydro_list(char * strlist, DumpParameters & dumpParams); - - void print_hashed_comment(FileIO & fileIO, const char * comment); - void global_header(const char * base, - std::vector dumpParams); - - void field_header(const char * fbase, DumpParameters & dumpParams); - void hydro_header(const char * speciesname, const char * hbase, - DumpParameters & dumpParams); - - void field_dump( DumpParameters & dumpParams, - field_t *f = NULL, - int64_t userStep = -1 ); - void hydro_dump( const char *speciesname, - DumpParameters & dumpParams, - hydro_t *h = NULL, - int64_t userStep = -1 ); - - void init_buffered_particle_dump(const char * speciesname, const int N_timesteps, const double safety_factor = 2.0); - void accumulate_buffered_particle_dump(const char * speciesname, const int frame); - void write_buffered_particle_dump(const char * dname, const char * speciesname); - void clear_buffered_particle_dump(const char * speciesname); - - #ifdef VPIC_PARTICLE_ANNOTATION - void interpolate_fields_annotation(const char * sp_name, const interpolator_array_t * ia, int where_ex, int where_ey, int where_ez, int where_bx, int where_by, int where_bz); - void interpolate_hydro_annotation(const char * sp_name, const hydro_array_t * ha, - int where_jx, int where_jy, int where_jz, int where_rho, - int where_px, int where_py, int where_pz, int where_ke, + void create_field_list(char *strlist, DumpParameters &dumpParams); + void create_hydro_list(char *strlist, DumpParameters &dumpParams); + + void print_hashed_comment(FileIO &fileIO, const char *comment); + void global_header(const char *base, + std::vector dumpParams); + + void field_header(const char *fbase, DumpParameters &dumpParams); + void hydro_header(const char *speciesname, const char *hbase, + DumpParameters &dumpParams); + + void field_dump(DumpParameters &dumpParams); + void hydro_dump(const char *speciesname, DumpParameters &dumpParams); + + void init_buffered_particle_dump(const char *speciesname, const int N_timesteps, const double safety_factor = 2.0); + void accumulate_buffered_particle_dump(const char *speciesname, const int frame); + void write_buffered_particle_dump(const char *dname, const char *speciesname); + void clear_buffered_particle_dump(const char *speciesname); + +#ifdef VPIC_PARTICLE_ANNOTATION + void interpolate_fields_annotation(const char *sp_name, const interpolator_array_t *ia, int where_ex, int where_ey, int where_ez, int where_bx, int where_by, int where_bz); + void interpolate_hydro_annotation(const char *sp_name, const hydro_array_t *ha, + int where_jx, int where_jy, int where_jz, int where_rho, + int where_px, int where_py, int where_pz, int where_ke, int where_txx, int where_tyy, int where_tzz, int where_tyz, int where_tzx, int where_txy); - #endif +#endif /////////////////// // Useful accessors @@ -446,57 +459,68 @@ class vpic_simulation { barrier() { mp_barrier(); } inline double - time() { - return grid->t0 + (double)grid->dt*(double)grid->step; + time() + { + return grid->t0 + (double)grid->dt * (double)grid->step; } inline int64_t & - step() { - return grid->step; + step() + { + return grid->step; } inline field_t & - field( const int v ) { - return field_array->f[ v ]; + field(const int v) + { + return field_array->f[v]; } inline int - voxel( const int ix, const int iy, const int iz ) { - return ix + grid->sy*iy + grid->sz*iz; + voxel(const int ix, const int iy, const int iz) + { + return ix + grid->sy * iy + grid->sz * iz; } inline field_t & - field( const int ix, const int iy, const int iz ) { - return field_array->f[ voxel(ix,iy,iz) ]; + field(const int ix, const int iy, const int iz) + { + return field_array->f[voxel(ix, iy, iz)]; } inline interpolator_t & - interpolator( const int v ) { - return interpolator_array->i[ v ]; + interpolator(const int v) + { + return interpolator_array->i[v]; } inline interpolator_t & - interpolator( const int ix, const int iy, const int iz ) { - return interpolator_array->i[ voxel(ix,iy,iz) ]; + interpolator(const int ix, const int iy, const int iz) + { + return interpolator_array->i[voxel(ix, iy, iz)]; } inline hydro_t & - hydro( const int v ) { - return hydro_array->h[ v ]; + hydro(const int v) + { + return hydro_array->h[v]; } inline hydro_t & - hydro( const int ix, const int iy, const int iz ) { - return hydro_array->h[ voxel(ix,iy,iz) ]; + hydro(const int ix, const int iy, const int iz) + { + return hydro_array->h[voxel(ix, iy, iz)]; } inline rng_t * - rng( const int n ) { + rng(const int n) + { return entropy->rng[n]; } inline rng_t * - sync_rng( const int n ) { + sync_rng(const int n) + { return sync_entropy->rng[n]; } @@ -504,16 +528,18 @@ class vpic_simulation { // Grid helpers inline void - define_units( float cvac, - float eps0 ) { + define_units(float cvac, + float eps0) + { grid->cvac = cvac; grid->eps0 = eps0; } inline void - define_timestep( float dt, double t0 = 0, int64_t step = 0 ) { - grid->t0 = t0; - grid->dt = (float)dt; + define_timestep(float dt, double t0 = 0, int64_t step = 0) + { + grid->t0 = t0; + grid->dt = (float)dt; grid->step = step; } @@ -521,99 +547,118 @@ class vpic_simulation { // simple boundary conditions on the edges. inline void - define_periodic_grid( double xl, double yl, double zl, - double xh, double yh, double zh, - double gnx, double gny, double gnz, - double gpx, double gpy, double gpz ) { - px = size_t(gpx); py = size_t(gpy); pz = size_t(gpz); - partition_periodic_box( grid, xl, yl, zl, xh, yh, zh, - (int)gnx, (int)gny, (int)gnz, - (int)gpx, (int)gpy, (int)gpz ); + define_periodic_grid(double xl, double yl, double zl, + double xh, double yh, double zh, + double gnx, double gny, double gnz, + double gpx, double gpy, double gpz) + { + px = size_t(gpx); + py = size_t(gpy); + pz = size_t(gpz); + partition_periodic_box(grid, xl, yl, zl, xh, yh, zh, + (int)gnx, (int)gny, (int)gnz, + (int)gpx, (int)gpy, (int)gpz); } inline void - define_absorbing_grid( double xl, double yl, double zl, - double xh, double yh, double zh, - double gnx, double gny, double gnz, - double gpx, double gpy, double gpz, int pbc ) { - px = size_t(gpx); py = size_t(gpy); pz = size_t(gpz); - partition_absorbing_box( grid, xl, yl, zl, xh, yh, zh, - (int)gnx, (int)gny, (int)gnz, - (int)gpx, (int)gpy, (int)gpz, - pbc ); + define_absorbing_grid(double xl, double yl, double zl, + double xh, double yh, double zh, + double gnx, double gny, double gnz, + double gpx, double gpy, double gpz, int pbc) + { + px = size_t(gpx); + py = size_t(gpy); + pz = size_t(gpz); + partition_absorbing_box(grid, xl, yl, zl, xh, yh, zh, + (int)gnx, (int)gny, (int)gnz, + (int)gpx, (int)gpy, (int)gpz, + pbc); } inline void - define_reflecting_grid( double xl, double yl, double zl, - double xh, double yh, double zh, - double gnx, double gny, double gnz, - double gpx, double gpy, double gpz ) { - px = size_t(gpx); py = size_t(gpy); pz = size_t(gpz); - partition_metal_box( grid, xl, yl, zl, xh, yh, zh, - (int)gnx, (int)gny, (int)gnz, - (int)gpx, (int)gpy, (int)gpz ); + define_reflecting_grid(double xl, double yl, double zl, + double xh, double yh, double zh, + double gnx, double gny, double gnz, + double gpx, double gpy, double gpz) + { + px = size_t(gpx); + py = size_t(gpy); + pz = size_t(gpz); + partition_metal_box(grid, xl, yl, zl, xh, yh, zh, + (int)gnx, (int)gny, (int)gnz, + (int)gpx, (int)gpy, (int)gpz); } // The below macros allow custom domains to be created // Creates a particle reflecting metal box in the local domain inline void - size_domain( double lnx, double lny, double lnz ) { - size_grid(grid,(int)lnx,(int)lny,(int)lnz); + size_domain(double lnx, double lny, double lnz) + { + size_grid(grid, (int)lnx, (int)lny, (int)lnz); } // Attaches a local domain boundary to another domain - inline void join_domain( int boundary, double rank ) { - join_grid( grid, boundary, (int)rank ); + inline void join_domain(int boundary, double rank) + { + join_grid(grid, boundary, (int)rank); } // Sets the field boundary condition of a local domain boundary - inline void set_domain_field_bc( int boundary, int fbc ) { - set_fbc( grid, boundary, fbc ); + inline void set_domain_field_bc(int boundary, int fbc) + { + set_fbc(grid, boundary, fbc); } // Sets the particle boundary condition of a local domain boundary - inline void set_domain_particle_bc( int boundary, int pbc ) { - set_pbc( grid, boundary, pbc ); + inline void set_domain_particle_bc(int boundary, int pbc) + { + set_pbc(grid, boundary, pbc); } /////////////////// // Material helpers inline material_t * - define_material( const char * name, - double eps, - double mu = 1, - double sigma = 0, - double zeta = 0 ) { - return append_material( material( name, - eps, eps, eps, - mu, mu, mu, - sigma, sigma, sigma, - zeta, zeta, zeta ), &material_list ); + define_material(const char *name, + double eps, + double mu = 1, + double sigma = 0, + double zeta = 0) + { + return append_material(material(name, + eps, eps, eps, + mu, mu, mu, + sigma, sigma, sigma, + zeta, zeta, zeta), + &material_list); } inline material_t * - define_material( const char * name, - double epsx, double epsy, double epsz, - double mux, double muy, double muz, - double sigmax, double sigmay, double sigmaz, - double zetax = 0 , double zetay = 0, double zetaz = 0 ) { - return append_material( material( name, - epsx, epsy, epsz, - mux, muy, muz, - sigmax, sigmay, sigmaz, - zetax, zetay, zetaz ), &material_list ); + define_material(const char *name, + double epsx, double epsy, double epsz, + double mux, double muy, double muz, + double sigmax, double sigmay, double sigmaz, + double zetax = 0, double zetay = 0, double zetaz = 0) + { + return append_material(material(name, + epsx, epsy, epsz, + mux, muy, muz, + sigmax, sigmay, sigmaz, + zetax, zetay, zetaz), + &material_list); } inline material_t * - lookup_material( const char * name ) { - return find_material_name( name, material_list ); + lookup_material(const char *name) + { + return find_material_name(name, material_list); } inline material_t * - lookup_material( material_id id ) { - return find_material_id( id, material_list ); + lookup_material(material_id id) + { + return find_material_id(id, material_list); } ////////////////////// @@ -624,36 +669,36 @@ class vpic_simulation { // optionally provided radition damping parameter. inline void - define_field_array( field_array_t * fa = NULL, double damp = 0 ) { - int nx1 = grid->nx + 1, ny1 = grid->ny+1, nz1 = grid->nz+1; + define_field_array(field_array_t *fa = NULL, double damp = 0) + { + int nx1 = grid->nx + 1, ny1 = grid->ny + 1, nz1 = grid->nz + 1; - if( grid->nx<1 || grid->ny<1 || grid->nz<1 ) - ERROR(( "Define your grid before defining the field array" )); - if( !material_list ) - ERROR(( "Define your materials before defining the field array" )); + if (grid->nx < 1 || grid->ny < 1 || grid->nz < 1) + ERROR(("Define your grid before defining the field array")); + if (!material_list) + ERROR(("Define your materials before defining the field array")); - field_array = fa ? fa : - new_standard_field_array( grid, material_list, damp ); - interpolator_array = new_interpolator_array( grid ); - accumulator_array = new_accumulator_array( grid ); - hydro_array = new_hydro_array( grid ); + field_array = fa ? fa : new_standard_field_array(grid, material_list, damp); + interpolator_array = new_interpolator_array(grid); + accumulator_array = new_accumulator_array(grid); + hydro_array = new_hydro_array(grid); // Pre-size communications buffers. This is done to get most memory // allocation over with before the simulation starts running - mp_size_recv_buffer(grid->mp,BOUNDARY(-1, 0, 0),ny1*nz1*sizeof(hydro_t)); - mp_size_recv_buffer(grid->mp,BOUNDARY( 1, 0, 0),ny1*nz1*sizeof(hydro_t)); - mp_size_recv_buffer(grid->mp,BOUNDARY( 0,-1, 0),nz1*nx1*sizeof(hydro_t)); - mp_size_recv_buffer(grid->mp,BOUNDARY( 0, 1, 0),nz1*nx1*sizeof(hydro_t)); - mp_size_recv_buffer(grid->mp,BOUNDARY( 0, 0,-1),nx1*ny1*sizeof(hydro_t)); - mp_size_recv_buffer(grid->mp,BOUNDARY( 0, 0, 1),nx1*ny1*sizeof(hydro_t)); - - mp_size_send_buffer(grid->mp,BOUNDARY(-1, 0, 0),ny1*nz1*sizeof(hydro_t)); - mp_size_send_buffer(grid->mp,BOUNDARY( 1, 0, 0),ny1*nz1*sizeof(hydro_t)); - mp_size_send_buffer(grid->mp,BOUNDARY( 0,-1, 0),nz1*nx1*sizeof(hydro_t)); - mp_size_send_buffer(grid->mp,BOUNDARY( 0, 1, 0),nz1*nx1*sizeof(hydro_t)); - mp_size_send_buffer(grid->mp,BOUNDARY( 0, 0,-1),nx1*ny1*sizeof(hydro_t)); - mp_size_send_buffer(grid->mp,BOUNDARY( 0, 0, 1),nx1*ny1*sizeof(hydro_t)); + mp_size_recv_buffer(grid->mp, BOUNDARY(-1, 0, 0), ny1 * nz1 * sizeof(hydro_t)); + mp_size_recv_buffer(grid->mp, BOUNDARY(1, 0, 0), ny1 * nz1 * sizeof(hydro_t)); + mp_size_recv_buffer(grid->mp, BOUNDARY(0, -1, 0), nz1 * nx1 * sizeof(hydro_t)); + mp_size_recv_buffer(grid->mp, BOUNDARY(0, 1, 0), nz1 * nx1 * sizeof(hydro_t)); + mp_size_recv_buffer(grid->mp, BOUNDARY(0, 0, -1), nx1 * ny1 * sizeof(hydro_t)); + mp_size_recv_buffer(grid->mp, BOUNDARY(0, 0, 1), nx1 * ny1 * sizeof(hydro_t)); + + mp_size_send_buffer(grid->mp, BOUNDARY(-1, 0, 0), ny1 * nz1 * sizeof(hydro_t)); + mp_size_send_buffer(grid->mp, BOUNDARY(1, 0, 0), ny1 * nz1 * sizeof(hydro_t)); + mp_size_send_buffer(grid->mp, BOUNDARY(0, -1, 0), nz1 * nx1 * sizeof(hydro_t)); + mp_size_send_buffer(grid->mp, BOUNDARY(0, 1, 0), nz1 * nx1 * sizeof(hydro_t)); + mp_size_send_buffer(grid->mp, BOUNDARY(0, 0, -1), nx1 * ny1 * sizeof(hydro_t)); + mp_size_send_buffer(grid->mp, BOUNDARY(0, 0, 1), nx1 * ny1 * sizeof(hydro_t)); } // Other field helpers are provided by macros in deck_wrapper.cxx @@ -663,85 +708,106 @@ class vpic_simulation { // FIXME: SILLY PROMOTIONS inline species_t * - define_species( const char *name, - double q, - double m, - double max_local_np, - double max_local_nm, - double sort_interval, - double sort_out_of_place ) { + define_species(const char *name, + double q, + double m, + double max_local_np, + double max_local_nm, + double sort_interval, + double sort_out_of_place) + { // Compute a reasonble number of movers if user did not specify // Based on the twice the number of particles expected to hit the boundary // of a wpdt=0.2 / dx=lambda species in a 3x3x3 domain - if( max_local_nm<0 ) { - max_local_nm = 2*max_local_np/25; - if( max_local_nm<16*(MAX_PIPELINE+1) ) - max_local_nm = 16*(MAX_PIPELINE+1); + if (max_local_nm < 0) + { + max_local_nm = 2 * max_local_np / 25; + if (max_local_nm < 16 * (MAX_PIPELINE + 1)) + max_local_nm = 16 * (MAX_PIPELINE + 1); } - return append_species( species( name, (float)q, (float)m, - (size_t)max_local_np, (size_t)max_local_nm, - (int)sort_interval, (int)sort_out_of_place, - grid ), &species_list ); + return append_species(species(name, (float)q, (float)m, + (size_t)max_local_np, (size_t)max_local_nm, + (int)sort_interval, (int)sort_out_of_place, + grid), + &species_list); } inline species_t * - find_species( const char *name ) { - return find_species_name( name, species_list ); + find_species(const char *name) + { + return find_species_name(name, species_list); } inline species_t * - find_species( int32_t id ) { - return find_species_id( id, species_list ); + find_species(int32_t id) + { + return find_species_id(id, species_list); } // These might need renaming before handing this to users. But it works for now. - inline species_t * make_tracers_by_percentage(species_t* parentspecies, const float percentage, const Tracertype copyormove, std::string tracername) { - if((percentage <= 0.) || (percentage > 100.)) { - ERROR(( "%f is a bad percentage to select tracers", percentage )); + inline species_t *make_tracers_by_percentage(species_t *parentspecies, const float percentage, const Tracertype copyormove, std::string tracername) + { + if ((percentage <= 0.) || (percentage > 100.)) + { + ERROR(("%f is a bad percentage to select tracers", percentage)); } // Implemented in species_advance.cc - species_t* tracerspecies = tracerspecies_by_skip(parentspecies, 100./percentage, copyormove, tracername, species_list, grid); + species_t *tracerspecies = tracerspecies_by_skip(parentspecies, 100. / percentage, copyormove, tracername, species_list, grid); return append_species(tracerspecies, &species_list); } - inline species_t * make_tracers_by_nth(species_t* parentspecies, const float nth, const Tracertype copyormove, std::string tracername) { - if(nth < 1.) { - ERROR(( "%f is a bad stride to select every nth particle as tracers", nth )); + inline species_t *make_tracers_by_nth(species_t *parentspecies, const float nth, const Tracertype copyormove, std::string tracername) + { + if (nth < 1.) + { + ERROR(("%f is a bad stride to select every nth particle as tracers", nth)); } - species_t* tracerspecies = tracerspecies_by_skip(parentspecies, nth, copyormove, tracername, species_list, grid); + species_t *tracerspecies = tracerspecies_by_skip(parentspecies, nth, copyormove, tracername, species_list, grid); return append_species(tracerspecies, &species_list); } - inline species_t * make_tracers_by_predicate(species_t* parentspecies, std::function f, const Tracertype copyormove, std::string tracername) { + inline species_t *make_tracers_by_predicate(species_t *parentspecies, std::function f, const Tracertype copyormove, std::string tracername) + { // Implemented in species_advance.cc - species_t* tracerspecies = tracerspecies_by_predicate(parentspecies, f, copyormove, tracername, species_list, grid); + species_t *tracerspecies = tracerspecies_by_predicate(parentspecies, f, copyormove, tracername, species_list, grid); return append_species(tracerspecies, &species_list); } - inline species_t * make_n_tracers(species_t* parentspecies, const float n, const Tracertype copyormove, std::string tracername) { - if(!parentspecies) ERROR(( "Invalid parent species" )); - if((n < 1.) || (n > parentspecies->np)) { - ERROR(( "%f is a bad number of tracers", n )); + inline species_t *make_n_tracers(species_t *parentspecies, const float n, const Tracertype copyormove, std::string tracername) + { + if (!parentspecies) + ERROR(("Invalid parent species")); + if ((n < 1.) || (n > parentspecies->np)) + { + ERROR(("%f is a bad number of tracers", n)); } - species_t* tracerspecies = tracerspecies_by_skip(parentspecies, parentspecies->np/n, copyormove, tracername, species_list, grid); + species_t *tracerspecies = tracerspecies_by_skip(parentspecies, parentspecies->np / n, copyormove, tracername, species_list, grid); return append_species(tracerspecies, &species_list); } // versions without user supplied name - inline species_t * make_tracers_by_percentage(species_t* parentspecies, const float percentage, const Tracertype copyormove) { - if(!parentspecies) ERROR(( "Invalid parent species" )); + inline species_t *make_tracers_by_percentage(species_t *parentspecies, const float percentage, const Tracertype copyormove) + { + if (!parentspecies) + ERROR(("Invalid parent species")); std::string name = make_tracer_name_unique(std::string(parentspecies->name) + std::string("_tracer"), species_list); return make_tracers_by_percentage(parentspecies, percentage, copyormove, name); } - inline species_t * make_tracers_by_nth(species_t* parentspecies, const float nth, const Tracertype copyormove) { - if(!parentspecies) ERROR(( "Invalid parent species" )); + inline species_t *make_tracers_by_nth(species_t *parentspecies, const float nth, const Tracertype copyormove) + { + if (!parentspecies) + ERROR(("Invalid parent species")); std::string name = make_tracer_name_unique(std::string(parentspecies->name) + std::string("_tracer"), species_list); return make_tracers_by_nth(parentspecies, nth, copyormove, name); } - inline species_t * make_tracers_by_predicate(species_t* parentspecies, std::function f, const Tracertype copyormove) { - if(!parentspecies) ERROR(( "Invalid parent species" )); + inline species_t *make_tracers_by_predicate(species_t *parentspecies, std::function f, const Tracertype copyormove) + { + if (!parentspecies) + ERROR(("Invalid parent species")); std::string name = make_tracer_name_unique(std::string(parentspecies->name) + std::string("_tracer"), species_list); return make_tracers_by_predicate(parentspecies, f, copyormove, name); } - inline species_t * make_n_tracers(species_t* parentspecies, const float n, const Tracertype copyormove) { - if(!parentspecies) ERROR(( "Invalid parent species" )); + inline species_t *make_n_tracers(species_t *parentspecies, const float n, const Tracertype copyormove) + { + if (!parentspecies) + ERROR(("Invalid parent species")); std::string name = make_tracer_name_unique(std::string(parentspecies->name) + std::string("_tracer"), species_list); return make_n_tracers(parentspecies, n, copyormove, name); } @@ -750,10 +816,10 @@ class vpic_simulation { // Defaults in the declaration below enable backwards compatibility. void - inject_particle( species_t * sp, - double x, double y, double z, - double ux, double uy, double uz, - double w, double age = 0, int update_rhob = 1 ); + inject_particle(species_t *sp, + double x, double y, double z, + double ux, double uy, double uz, + double w, double age = 0, int update_rhob = 1); // Inject particle raw is for power users! // No nannyism _at_ _all_: @@ -765,60 +831,82 @@ class vpic_simulation { // This injection is _ultra_ _fast_. inline void - inject_particle_raw( species_t * RESTRICT sp, - float dx, float dy, float dz, int32_t i, - float ux, float uy, float uz, float w ) - { - particle_t * RESTRICT p = sp->p + sp->np; - #ifdef VPIC_GLOBAL_PARTICLE_ID - if(sp->has_ids) { - size_t * RESTRICT p_id = sp->p_id + sp->np; - *p_id = sp->generate_particle_id( sp->np, sp->max_np ); + inject_particle_raw(species_t *RESTRICT sp, + float dx, float dy, float dz, int32_t i, + float ux, float uy, float uz, float w) + { + particle_t *RESTRICT p = sp->p + sp->np; +#ifdef VPIC_GLOBAL_PARTICLE_ID + if (sp->has_ids) + { + size_t *RESTRICT p_id = sp->p_id + sp->np; + *p_id = sp->generate_particle_id(sp->np, sp->max_np); } - #endif - #ifdef VPIC_PARTICLE_ANNOTATION - if(sp->has_annotation) { - for(int j = 0; j < sp->has_annotation; j++) { +#endif +#ifdef VPIC_PARTICLE_ANNOTATION + if (sp->has_annotation) + { + for (int j = 0; j < sp->has_annotation; j++) + { // Default for annotations is 0.0 sp->set_annotation(sp->np, j, 0.); } } - #endif - p->dx = dx; p->dy = dy; p->dz = dz; p->i = i; - p->ux = ux; p->uy = uy; p->uz = uz; p->w = w; +#endif + p->dx = dx; + p->dy = dy; + p->dz = dz; + p->i = i; + p->ux = ux; + p->uy = uy; + p->uz = uz; + p->w = w; sp->np++; } // This variant does a raw inject and moves the particles inline void - inject_particle_raw( species_t * RESTRICT sp, - float dx, float dy, float dz, int32_t i, - float ux, float uy, float uz, float w, - float dispx, float dispy, float dispz, - int update_rhob ) - { - particle_t * RESTRICT p = sp->p + sp->np; - particle_mover_t * RESTRICT pm = sp->pm + sp->nm; - #ifdef VPIC_GLOBAL_PARTICLE_ID - if(sp->has_ids) { - size_t * RESTRICT p_id = sp->p_id + sp->np; - *p_id = sp->generate_particle_id( sp->np, sp->max_np ); + inject_particle_raw(species_t *RESTRICT sp, + float dx, float dy, float dz, int32_t i, + float ux, float uy, float uz, float w, + float dispx, float dispy, float dispz, + int update_rhob) + { + particle_t *RESTRICT p = sp->p + sp->np; + particle_mover_t *RESTRICT pm = sp->pm + sp->nm; +#ifdef VPIC_GLOBAL_PARTICLE_ID + if (sp->has_ids) + { + size_t *RESTRICT p_id = sp->p_id + sp->np; + *p_id = sp->generate_particle_id(sp->np, sp->max_np); } - #endif - #ifdef VPIC_PARTICLE_ANNOTATION - if(sp->has_annotation) { - for(int j = 0; j < sp->has_annotation; j++) { +#endif +#ifdef VPIC_PARTICLE_ANNOTATION + if (sp->has_annotation) + { + for (int j = 0; j < sp->has_annotation; j++) + { // Default for annotations is 0.0 sp->set_annotation(sp->np, j, 0.); } } - #endif - p->dx = dx; p->dy = dy; p->dz = dz; p->i = i; - p->ux = ux; p->uy = uy; p->uz = uz; p->w = w; - pm->dispx = dispx; pm->dispy = dispy; pm->dispz = dispz; pm->i = sp->np-1; - if( update_rhob ) accumulate_rhob( field_array->f, p, grid, -sp->q ); - sp->nm += move_p( sp->p, pm, accumulator_array->a, grid, sp->q ); +#endif + p->dx = dx; + p->dy = dy; + p->dz = dz; + p->i = i; + p->ux = ux; + p->uy = uy; + p->uz = uz; + p->w = w; + pm->dispx = dispx; + pm->dispy = dispy; + pm->dispz = dispz; + pm->i = sp->np - 1; + if (update_rhob) + accumulate_rhob(field_array->f, p, grid, -sp->q); + sp->nm += move_p(sp->p, pm, accumulator_array->a, grid, sp->q); sp->np++; } @@ -835,22 +923,25 @@ class vpic_simulation { // user seeds. // FIXME: MTRAND DESPERATELY NEEDS A LARGER SEED SPACE! - inline void seed_entropy( int base ) { - seed_rng_pool( entropy, base, 0 ); - seed_rng_pool( sync_entropy, base, 1 ); + inline void seed_entropy(int base) + { + seed_rng_pool(entropy, base, 0); + seed_rng_pool(sync_entropy, base, 1); } // Uniform random number on (low,high) (open interval) // FIXME: IS THE INTERVAL STILL OPEN IN FINITE PRECISION // AND IS THE OPEN INTERVAL REALLY WHAT USERS WANT?? - inline double uniform( rng_t * rng, double low, double high ) { - double dx = drand( rng ); - return low*(1-dx) + high*dx; + inline double uniform(rng_t *rng, double low, double high) + { + double dx = drand(rng); + return low * (1 - dx) + high * dx; } // Normal random number with mean mu and standard deviation sigma - inline double normal( rng_t * rng, double mu, double sigma ) { - return mu + sigma*drandn( rng ); + inline double normal(rng_t *rng, double mu, double sigma) + { + return mu + sigma * drandn(rng); } ///////////////////////////////// @@ -873,55 +964,64 @@ class vpic_simulation { // strict C++ prevent this.) inline emitter_t * - define_emitter( emitter_t * e ) { - return append_emitter( e, &emitter_list ); + define_emitter(emitter_t *e) + { + return append_emitter(e, &emitter_list); } inline particle_bc_t * - define_particle_bc( particle_bc_t * pbc ) { - return append_particle_bc( pbc, &particle_bc_list ); + define_particle_bc(particle_bc_t *pbc) + { + return append_particle_bc(pbc, &particle_bc_list); } inline collision_op_t * - define_collision_op( collision_op_t * cop ) { - return append_collision_op( cop, &collision_op_list ); + define_collision_op(collision_op_t *cop) + { + return append_collision_op(cop, &collision_op_list); } //////////////////////// // Miscellaneous helpers - inline void abort( double code ) { - nanodelay(2000000000); mp_abort((((int)code)<<17)+1); + inline void abort(double code) + { + nanodelay(2000000000); + mp_abort((((int)code) << 17) + 1); } // Truncate "a" to the nearest integer multiple of "b" - inline double trunc_granular( double a, double b ) { return b*int(a/b); } + inline double trunc_granular(double a, double b) { return b * int(a / b); } // Compute the remainder of a/b - inline double remainder( double a, double b ) { return std::remainder(a,b); } + inline double remainder(double a, double b) { return std::remainder(a, b); } // remainder(a,b); // Compute the Courant length on a regular mesh - inline double courant_length( double lx, double ly, double lz, - double nx, double ny, double nz ) { + inline double courant_length(double lx, double ly, double lz, + double nx, double ny, double nz) + { double w0, w1 = 0; - if( nx>1 ) w0 = nx/lx, w1 += w0*w0; - if( ny>1 ) w0 = ny/ly, w1 += w0*w0; - if( nz>1 ) w0 = nz/lz, w1 += w0*w0; - return sqrt(1/w1); + if (nx > 1) + w0 = nx / lx, w1 += w0 * w0; + if (ny > 1) + w0 = ny / ly, w1 += w0 * w0; + if (nz > 1) + w0 = nz / lz, w1 += w0 * w0; + return sqrt(1 / w1); } ////////////////////////////////////////////////////////// // These friends are used by the checkpt / restore service - friend void checkpt_vpic_simulation( const vpic_simulation * vpic ); - friend vpic_simulation * restore_vpic_simulation( void ); - friend void reanimate_vpic_simulation( vpic_simulation * vpic ); + friend void checkpt_vpic_simulation(const vpic_simulation *vpic); + friend vpic_simulation *restore_vpic_simulation(void); + friend void reanimate_vpic_simulation(vpic_simulation *vpic); //////////////////////////////////////////////////////////// // User input deck provided functions (see deck_wrapper.cxx) - void user_initialization( int argc, char **argv ); + void user_initialization(int argc, char **argv); void user_particle_injection(void); void user_current_injection(void); void user_field_injection(void); @@ -929,5 +1029,4 @@ class vpic_simulation { void user_particle_collisions(void); }; - #endif // vpic_h diff --git a/test/integrated/CMakeLists.txt b/test/integrated/CMakeLists.txt index 5a409a7a..c5efd022 100644 --- a/test/integrated/CMakeLists.txt +++ b/test/integrated/CMakeLists.txt @@ -2,6 +2,10 @@ add_subdirectory(particle_push) add_subdirectory(particle_IDs) add_subdirectory(particle_annotation) add_subdirectory(buffered_particle_dump) +add_subdirectory(hdf5_fields) +add_subdirectory(hdf5_fields_location) +add_subdirectory(hdf5_fields_many_steps) +add_subdirectory(hdf5_hydro) add_subdirectory(legacy) add_subdirectory(to_completion) add_subdirectory(collision) diff --git a/test/integrated/buffered_particle_dump/CMakeLists.txt b/test/integrated/buffered_particle_dump/CMakeLists.txt index 8b1a5785..45534ffe 100644 --- a/test/integrated/buffered_particle_dump/CMakeLists.txt +++ b/test/integrated/buffered_particle_dump/CMakeLists.txt @@ -2,6 +2,8 @@ set(MPI_NUM_RANKS 2) set(TEST "buffered_particle_dump") +find_package(Python3 COMPONENTS Interpreter) + if(USE_HDF5) # Build @@ -16,5 +18,5 @@ if(USE_HDF5) add_TEST(${TEST}_restart ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MPI_NUM_RANKS} ${MPIEXEC_PREFLAGS} ./${TEST} ${MPIEXEC_POSTFLAGS} ${RESTART_ARGS}) # and check that the output is ok - add_TEST(${TEST}_readhdf5 python ${CMAKE_CURRENT_SOURCE_DIR}/readback.py ${CMAKE_CURRENT_BINARY_DIR}) + add_TEST(${TEST}_readhdf5 python3 ${CMAKE_CURRENT_SOURCE_DIR}/readback.py ${CMAKE_CURRENT_BINARY_DIR}) endif() diff --git a/test/integrated/buffered_particle_dump/readback.py b/test/integrated/buffered_particle_dump/readback.py index adf3696b..f0274cb0 100755 --- a/test/integrated/buffered_particle_dump/readback.py +++ b/test/integrated/buffered_particle_dump/readback.py @@ -3,7 +3,7 @@ import numpy as np import os.path import sys - +os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" if len(sys.argv) != 2: sys.stderr.write("Usage: "+str(sys.argv[0])+" rundir\n") sys.exit(1) @@ -66,15 +66,15 @@ sys.exit(1) if np.any (Ux[cond]!=0.5): - printf("FAIL: Ux changed") + print("FAIL: Ux changed") sys.exit(1) if np.any (Uy[cond]!=0.0): - printf("FAIL: Uy changed") + print("FAIL: Uy changed") sys.exit(1) if np.any (Uz[cond]!=0.0): - printf("FAIL: Uz changed") + print("FAIL: Uz changed") sys.exit(1) #timestep 0 @@ -88,7 +88,7 @@ cond_dzu = dZ<0.34 cond_p1 = np.logical_and(np.logical_and(np.logical_and(cond_t,cond_i),cond_dx), np.logical_and(np.logical_and(cond_dyl,cond_dyu),np.logical_and(cond_dzl,cond_dzu))) if np.sum(cond_p1) != 1: - printf("FAIL: can't find the particle in cell 0, offset 0.,0.33,0.33 in timestep 0") + print("FAIL: can't find the particle in cell 0, offset 0.,0.33,0.33 in timestep 0") sys.exit(1) #particle 2 @@ -101,7 +101,7 @@ cond_dzu = dZ<0.34 cond_p2 = np.logical_and(np.logical_and(np.logical_and(cond_t,cond_i),cond_dx), np.logical_and(np.logical_and(cond_dyl,cond_dyu),np.logical_and(cond_dzl,cond_dzu))) if np.sum(cond_p2) != 1: - printf("FAIL: can't find the particle in cell 0, offset 0.,0.66,0.33 in timestep 0") + print("FAIL: can't find the particle in cell 0, offset 0.,0.66,0.33 in timestep 0") sys.exit(1) #particle 3 @@ -114,7 +114,7 @@ cond_dzu = dZ<0.67 cond_p3 = np.logical_and(np.logical_and(np.logical_and(cond_t,cond_i),cond_dx), np.logical_and(np.logical_and(cond_dyl,cond_dyu),np.logical_and(cond_dzl,cond_dzu))) if np.sum(cond_p3) != 1: - printf("FAIL: can't find the particle in cell 1, offset 0.,0.33,0.66 in timestep 0") + print("FAIL: can't find the particle in cell 1, offset 0.,0.33,0.66 in timestep 0") sys.exit(1) #particle 3 @@ -127,7 +127,7 @@ cond_dzu = dZ<0.67 cond_p4 = np.logical_and(np.logical_and(np.logical_and(cond_t,cond_i),cond_dx), np.logical_and(np.logical_and(cond_dyl,cond_dyu),np.logical_and(cond_dzl,cond_dzu))) if np.sum(cond_p4) != 1: - printf("FAIL: can't find the particle in cell 1, offset 0.,0.66,0.66 in timestep 0") + print("FAIL: can't find the particle in cell 1, offset 0.,0.66,0.66 in timestep 0") sys.exit(1) #timestep 1 diff --git a/test/integrated/hdf5_fields/CMakeLists.txt b/test/integrated/hdf5_fields/CMakeLists.txt new file mode 100644 index 00000000..7af00356 --- /dev/null +++ b/test/integrated/hdf5_fields/CMakeLists.txt @@ -0,0 +1,22 @@ +# add the tests +set(MPI_NUM_RANKS 2) + +set(TEST "hdf5_fields") +find_package(Python3 COMPONENTS Interpreter) + + +if(USE_HDF5) + # Build + # TODO: This method of doing the tests is really bad at rebuilding them properly + build_a_vpic(${TEST} ${CMAKE_CURRENT_SOURCE_DIR}/${TEST}.deck) + + # Add TEST + add_TEST(${TEST} ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MPI_NUM_RANKS} ${MPIEXEC_PREFLAGS} ./${TEST} ${MPIEXEC_POSTFLAGS} ${ARGS}) + + # Add a restart TEST + list(APPEND RESTART_ARGS --restore ${CMAKE_CURRENT_BINARY_DIR}/restart0/restore.0) + add_TEST(${TEST}_restart ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MPI_NUM_RANKS} ${MPIEXEC_PREFLAGS} ./${TEST} ${MPIEXEC_POSTFLAGS} ${RESTART_ARGS}) + + #and check that the output is ok + add_TEST(${TEST}_readhdf5 python3 ${CMAKE_CURRENT_SOURCE_DIR}/readback.py ${CMAKE_CURRENT_BINARY_DIR}) +endif() diff --git a/test/integrated/hdf5_fields/hdf5_fields.deck b/test/integrated/hdf5_fields/hdf5_fields.deck new file mode 100644 index 00000000..816bb23a --- /dev/null +++ b/test/integrated/hdf5_fields/hdf5_fields.deck @@ -0,0 +1,104 @@ +begin_globals { +}; + +begin_initialization { + // Numerical settings + num_step = 2; + status_interval = 1000; // Basically don't print status + sync_shared_interval = status_interval; + clean_div_e_interval = status_interval; + clean_div_b_interval = status_interval; + + int gnx = nproc(); + int gny = 2; // Would love to make this 1 as well, but then I get weird NaNs... + int gnz = 1; + + int topox = nproc(); + int topoy = 1; + int topoz = 1; + + int nppc = 100; + + // use natural PIC units + float ec = 1; // Charge normalization + float me = 1; // Mass normalization + float c = 1; // Speed of light + float de = 1; // Length normalization (electron inertial length) + float eps0 = 1; // Permittivity of space + + // physics settings + float dx = 0.125; // d_e + float dt = dx * 0.95/sqrt(3.0)/c; + float Lx = dx * gnx; + float Ly = dx * gny; + float Lz = dx * gnz; + + // Setup stuff in vpic + define_units(c, eps0); + define_timestep(dt); + + define_periodic_grid(0,0,0, // Low corner + Lx,Ly,Lz, // High corner + gnx,gny,gnz, // Resolution + topox,topoy,topoz); // Topology + + // Space is by default filled with first material defined + define_material("vacuum",1.0); + + // Create the field array + define_field_array(NULL, 0.00); + + // Just a homogeneous value in Bx + set_region_field(everywhere, 5., 6., 7., 17., 18., 19.); + + // different random numbers on different ranks + seed_entropy(rank()); + + // Note: No particles. This is fine for field tests, but we need to come up + // with something better for tests of hydro output and particle dumps + + // Deck only works if VPIC was build with HDF support. Check for that: + #ifndef VPIC_ENABLE_HDF5 + #error "VPIC_ENABLE_HDF5" is required + #endif + + // Explicitly enable HDF5 backend for IO dump + enable_hdf5_dump(); +} + + +begin_diagnostics { + + // Write field output in every timestep + dump_fields("fields"); + + // Force a restart after step 1 to make sure that dump/restart works too + if(step() == 1) { + sim_log("Force restart"); + dump_mkdir("restart0"); + char fname[256]; + sprintf(fname, "%s/restore.0.%i", "restart0", world_rank ); + checkpt_objects(fname); + sim_log("Restart dump restart completed."); + mp_barrier(); + } + + // If we reach this in the last step without any fails we probably did well enough + if(step() == num_step) { + sim_log("passed"); + } +} + +begin_particle_injection { +} + +begin_current_injection { +} + +begin_field_injection { +} + +begin_particle_collisions { +} + + diff --git a/test/integrated/hdf5_fields/readback.py b/test/integrated/hdf5_fields/readback.py new file mode 100755 index 00000000..efb1e35a --- /dev/null +++ b/test/integrated/hdf5_fields/readback.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +import h5py +import numpy as np +import os.path +import sys +os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" +if len(sys.argv) != 2: + sys.stderr.write("Usage: "+str(sys.argv[0])+" rundir\n") + sys.exit(1) + +rundir = sys.argv[1] +print(rundir) +filename = rundir+"/field_hdf5/T.2/fields_2.h5" + +if not os.path.isfile(filename): + print("FAIL: "+filename+" is missing") + sys.exit(1) + +infile = h5py.File(filename, 'r') +datagroup = infile["Timestep_2"] + + +cbx = np.array(datagroup["cbx"]) +if np.all(cbx == 17) == False: + print('cbx does not contain all 17.') + sys.exit(1) + +cby = np.array(datagroup["cby"]) +if np.all(cby == 18) == False: + print('cby does not contain all 17.') + sys.exit(1) + +cbz = np.array(datagroup["cbz"]) +if np.all(cbz == 19) == False: + print('cbz does not contain all 17.') + sys.exit(1) + +ex = np.array(datagroup["ex"]) +if np.all(ex == 5) == False: + print('ex does not contain all 17.') + sys.exit(1) + +ey = np.array(datagroup["ey"]) +if np.all(ey == 6) == False: + print('ey does not contain all 17.') + sys.exit(1) + +ez = np.array(datagroup["ez"]) +if np.all(ez == 7) == False: + print('ez does not contain all 17.') + sys.exit(1) + +other_fields_name = ["cmat", "div_b_err", "div_e_err", "ematx", "ematy", "ematz", + "fmatx", "fmaty", "fmatz", "jfx", "jfy", "jfz", "nmat", "rhob", "rhof", "tcax", "tcay", "tcaz"] + +for field_name in other_fields_name: + field_data = np.array(datagroup[field_name]) + if np.all(field_data == 0) == False: + print(field_name, 'does not contain all 0.') + sys.exit(1) + +infile.close() diff --git a/test/integrated/hdf5_fields_location/CMakeLists.txt b/test/integrated/hdf5_fields_location/CMakeLists.txt new file mode 100644 index 00000000..153b028f --- /dev/null +++ b/test/integrated/hdf5_fields_location/CMakeLists.txt @@ -0,0 +1,22 @@ +# add the tests +set(MPI_NUM_RANKS 2) + +set(TEST "hdf5_fields_location") +find_package(Python3 COMPONENTS Interpreter) + + +if(USE_HDF5) + # Build + # TODO: This method of doing the tests is really bad at rebuilding them properly + build_a_vpic(${TEST} ${CMAKE_CURRENT_SOURCE_DIR}/${TEST}.deck) + + # Add TEST + add_TEST(${TEST} ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MPI_NUM_RANKS} ${MPIEXEC_PREFLAGS} ./${TEST} ${MPIEXEC_POSTFLAGS} ${ARGS}) + + # Add a restart TEST + list(APPEND RESTART_ARGS --restore ${CMAKE_CURRENT_BINARY_DIR}/restart0/restore.0) + add_TEST(${TEST}_restart ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MPI_NUM_RANKS} ${MPIEXEC_PREFLAGS} ./${TEST} ${MPIEXEC_POSTFLAGS} ${RESTART_ARGS}) + + #and check that the output is ok + add_TEST(${TEST}_readhdf5 python3 ${CMAKE_CURRENT_SOURCE_DIR}/readback.py ${CMAKE_CURRENT_BINARY_DIR}) +endif() diff --git a/test/integrated/hdf5_fields_location/hdf5_fields_location.deck b/test/integrated/hdf5_fields_location/hdf5_fields_location.deck new file mode 100644 index 00000000..b7f40855 --- /dev/null +++ b/test/integrated/hdf5_fields_location/hdf5_fields_location.deck @@ -0,0 +1,140 @@ +begin_globals { +}; + + + + +begin_initialization { + // Numerical settings + num_step = 2; + status_interval = 1000; // Basically don't print status + sync_shared_interval = status_interval; + clean_div_e_interval = status_interval; + clean_div_b_interval = status_interval; + + int gnx = nproc(); + int gny = 2; // Would love to make this 1 as well, but then I get weird NaNs... + int gnz = 1; + + int topox = nproc(); + int topoy = 1; + int topoz = 1; + + int nppc = 100; + + // use natural PIC units + float ec = 1; // Charge normalization + float me = 1; // Mass normalization + float c = 1; // Speed of light + float de = 1; // Length normalization (electron inertial length) + float eps0 = 1; // Permittivity of space + + // physics settings + float dx = 0.125; // d_e + float dt = dx * 0.95/sqrt(3.0)/c; + float Lx = dx * gnx; + float Ly = dx * gny; + float Lz = dx * gnz; + + // Setup stuff in vpic + define_units(c, eps0); + define_timestep(dt); + + define_periodic_grid(0,0,0, // Low corner + Lx,Ly,Lz, // High corner + gnx,gny,gnz, // Resolution + topox,topoy,topoz); // Topology + + // Space is by default filled with first material defined + define_material("vacuum",1.0); + + // Create the field array + define_field_array(NULL, 0.00); + + // Just a homogeneous value in Bx + set_region_field(everywhere, 5., 6., 7., 17., 18., 19.); + + // different random numbers on different ranks + seed_entropy(rank()); + + // Note: No particles. This is fine for field tests, but we need to come up + // with something better for tests of hydro output and particle dumps + + // Deck only works if VPIC was build with HDF support. Check for that: + #ifndef VPIC_ENABLE_HDF5 + #error "VPIC_ENABLE_HDF5" is required + #endif + + // Explicitly enable HDF5 backend for IO dump + enable_hdf5_dump(); +} + + +begin_diagnostics { +// The equations are only evaluated inside the mesh-mapped region +// (This is not strictly inside the region) +#define set_region_field_loc( rgn, \ + eqn_ex, eqn_ey, eqn_ez, \ + eqn_bx, eqn_by, eqn_bz ) do { \ + const double _x0 = grid->x0, _y0 = grid->y0, _z0 = grid->z0; \ + const double _dx = grid->dx, _dy = grid->dy, _dz = grid->dz; \ + const double _c = grid->cvac; \ + const int _nx = grid->nx, _ny = grid->ny, _nz = grid->nz; \ + for( int _k=0; _k<_nz+2; _k++ ) { const double _zl = _z0 + _dz*(_k-1.5), _ze = _z0 + _dz*(_k-1.0), _zc = _z0 + _dz*(_k-0.5); \ + for( int _j=0; _j<_ny+2; _j++ ) { const double _yl = _y0 + _dy*(_j-1.5), _ye = _y0 + _dy*(_j-1.0), _yc = _y0 + _dy*(_j-0.5); field_t *_f = &field(0,_j,_k); \ + for( int _i=0; _i<_nx+2; _i++ ) { const double _xl = _x0 + _dx*(_i-1.5), _xe = _x0 + _dx*(_i-1.0), _xc = _x0 + _dx*(_i-0.5); double x, y, z; \ + memset(_f, 0, sizeof(field_t)) ; \ + int _rccc, _rlcc, _rclc, _rllc, _rccl, _rlcl, _rcll; \ + x = _xc; y = _yc; z = _zc; _rccc = (rgn); \ + x = _xl; _rlcc = (rgn); \ + x = _xc; y = _yl; _rclc = (rgn); \ + x = _xl; _rllc = (rgn); \ + x = _xc; y = _yc; z = _zl; _rccl = (rgn); \ + x = _xl; _rlcl = (rgn); \ + x = _xc; y = _yl; _rcll = (rgn); \ + x = _xc; y = _ye; z = _ze; if( _rccc || _rclc || _rccl || _rcll ) _f->ex = _i + _j + _k ; \ + x = _xe; y = _yc; z = _ze; if( _rccc || _rccl || _rlcc || _rlcl ) _f->ey = _i + _j + _k ; \ + x = _xe; y = _ye; z = _zc; if( _rccc || _rlcc || _rclc || _rllc ) _f->ez = _i + _j + _k ; \ + x = _xe; y = _yc; z = _zc; if( _rccc || _rlcc ) _f->cbx = _i + _j + _k ; \ + x = _xc; y = _ye; z = _zc; if( _rccc || _rclc ) _f->cby = _i + _j + _k ; \ + x = _xc; y = _yc; z = _ze; if( _rccc || _rccl ) _f->cbz = _i + _j + _k ; \ + printf("+++ ijk = (%d, %d, %d) ex = %f, ey = %f, ez = %f , tcaz = %f +++ \n", _i, _j, _k, _f->ex, _f->ey, _f->ez, _f->tcaz); \ + _f++; \ + }}} \ + } while(0) + + + // Write field output in every timestep + set_region_field_loc(everywhere, 5., 6., 7., 17., 18., 19.); + dump_fields("fields"); + + // Force a restart after step 1 to make sure that dump/restart works too + if(step() == 1) { + sim_log("Force restart"); + dump_mkdir("restart0"); + char fname[256]; + sprintf(fname, "%s/restore.0.%i", "restart0", world_rank ); + checkpt_objects(fname); + sim_log("Restart dump restart completed."); + mp_barrier(); + } + + // If we reach this in the last step without any fails we probably did well enough + if(step() == num_step) { + sim_log("passed"); + } +} + +begin_particle_injection { +} + +begin_current_injection { +} + +begin_field_injection { +} + +begin_particle_collisions { +} + + diff --git a/test/integrated/hdf5_fields_location/readback.py b/test/integrated/hdf5_fields_location/readback.py new file mode 100755 index 00000000..d198d2c4 --- /dev/null +++ b/test/integrated/hdf5_fields_location/readback.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +import h5py +import numpy as np +import os.path +import sys +os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" +if len(sys.argv) != 2: + sys.stderr.write("Usage: "+str(sys.argv[0])+" rundir\n") + sys.exit(1) + +rundir = sys.argv[1] +print(rundir) +filename = rundir+"/field_hdf5/T.2/fields_2.h5" + +if not os.path.isfile(filename): + print("FAIL: "+filename+" is missing") + sys.exit(1) + +infile = h5py.File(filename, 'r') +datagroup = infile["Timestep_2"] + + +cbx = np.array(datagroup["cbx"]) +if np.array_equal(cbx, [3, 4, 3, 4]): + print('cbx does not contain [3, 4, 3, 4].') + sys.exit(1) + +cby = np.array(datagroup["cby"]) +if np.array_equal(cby, [3, 4, 3, 4]): + print('cby does not contain all 17.') + sys.exit(1) + +cbz = np.array(datagroup["cbz"]) +if np.array_equal(cbz, [3, 4, 3, 4]): + print('cbz does not contain all 17.') + sys.exit(1) + +ex = np.array(datagroup["ex"]) +if np.array_equal(ex, [3, 4, 3, 4]): + print('ex does not contain all 17.') + sys.exit(1) + +ey = np.array(datagroup["ey"]) +if np.array_equal(ey, [3, 4, 3, 4]): + print('ey does not contain all 17.') + sys.exit(1) + +ez = np.array(datagroup["ez"]) +if np.array_equal(ez, [3, 4, 3, 4]): + print('ez does not contain all 17.') + sys.exit(1) + +other_fields_name = ["cmat", "div_b_err", "div_e_err", "ematx", "ematy", "ematz", + "fmatx", "fmaty", "fmatz", "jfx", "jfy", "jfz", "nmat", "rhob", "rhof", "tcax", "tcay", "tcaz"] + +for field_name in other_fields_name: + field_data = np.array(datagroup[field_name]) + if np.all(field_data == 0) == False: + print(field_name, 'does not contain all 0.') + sys.exit(1) + +infile.close() diff --git a/test/integrated/hdf5_fields_many_steps/CMakeLists.txt b/test/integrated/hdf5_fields_many_steps/CMakeLists.txt new file mode 100644 index 00000000..77853c6a --- /dev/null +++ b/test/integrated/hdf5_fields_many_steps/CMakeLists.txt @@ -0,0 +1,22 @@ +# add the tests +set(MPI_NUM_RANKS 2) + +set(TEST "hdf5_fields_many_steps") +find_package(Python3 COMPONENTS Interpreter) + + +if(USE_HDF5) + # Build + # TODO: This method of doing the tests is really bad at rebuilding them properly + build_a_vpic(${TEST} ${CMAKE_CURRENT_SOURCE_DIR}/${TEST}.deck) + + # Add TEST + add_TEST(${TEST} ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MPI_NUM_RANKS} ${MPIEXEC_PREFLAGS} ./${TEST} ${MPIEXEC_POSTFLAGS} ${ARGS}) + + # Add a restart TEST + list(APPEND RESTART_ARGS --restore ${CMAKE_CURRENT_BINARY_DIR}/restart0/restore.0) + add_TEST(${TEST}_restart ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MPI_NUM_RANKS} ${MPIEXEC_PREFLAGS} ./${TEST} ${MPIEXEC_POSTFLAGS} ${RESTART_ARGS}) + + #and check that the output is ok + add_TEST(${TEST}_readhdf5 python3 ${CMAKE_CURRENT_SOURCE_DIR}/readback.py ${CMAKE_CURRENT_BINARY_DIR}) +endif() diff --git a/test/integrated/hdf5_fields_many_steps/hdf5_fields_many_steps.deck b/test/integrated/hdf5_fields_many_steps/hdf5_fields_many_steps.deck new file mode 100644 index 00000000..8e1df007 --- /dev/null +++ b/test/integrated/hdf5_fields_many_steps/hdf5_fields_many_steps.deck @@ -0,0 +1,109 @@ +begin_globals { +}; + + + + +begin_initialization { + // Numerical settings + num_step = 5; + status_interval = 1000; // Basically don't print status + sync_shared_interval = status_interval; + clean_div_e_interval = status_interval; + clean_div_b_interval = status_interval; + + int gnx = nproc(); + int gny = 2; // Would love to make this 1 as well, but then I get weird NaNs... + int gnz = 1; + + int topox = nproc(); + int topoy = 1; + int topoz = 1; + + int nppc = 100; + + // use natural PIC units + float ec = 1; // Charge normalization + float me = 1; // Mass normalization + float c = 1; // Speed of light + float de = 1; // Length normalization (electron inertial length) + float eps0 = 1; // Permittivity of space + + // physics settings + float dx = 0.125; // d_e + float dt = dx * 0.95/sqrt(3.0)/c; + float Lx = dx * gnx; + float Ly = dx * gny; + float Lz = dx * gnz; + + // Setup stuff in vpic + define_units(c, eps0); + define_timestep(dt); + + define_periodic_grid(0,0,0, // Low corner + Lx,Ly,Lz, // High corner + gnx,gny,gnz, // Resolution + topox,topoy,topoz); // Topology + + // Space is by default filled with first material defined + define_material("vacuum",1.0); + + // Create the field array + define_field_array(NULL, 0.00); + + // Just a homogeneous value in Bx + set_region_field(everywhere, 5., 6., 7., 17., 18., 19.); + + // different random numbers on different ranks + seed_entropy(rank()); + + // Note: No particles. This is fine for field tests, but we need to come up + // with something better for tests of hydro output and particle dumps + + // Deck only works if VPIC was build with HDF support. Check for that: + #ifndef VPIC_ENABLE_HDF5 + #error "VPIC_ENABLE_HDF5" is required + #endif + + // Explicitly enable HDF5 backend for IO dump + enable_hdf5_dump(); +} + + +begin_diagnostics { +// The equations are only evaluated inside the mesh-mapped region +// (This is not strictly inside the region) + + set_region_field(everywhere, 5., 6., 7., 17., 18., 19.); + dump_fields("fields"); + + // Force a restart after step 1 to make sure that dump/restart works too + if(step() == 1) { + sim_log("Force restart"); + dump_mkdir("restart0"); + char fname[256]; + sprintf(fname, "%s/restore.0.%i", "restart0", world_rank ); + checkpt_objects(fname); + sim_log("Restart dump restart completed."); + mp_barrier(); + } + + // If we reach this in the last step without any fails we probably did well enough + if(step() == num_step) { + sim_log("passed"); + } +} + +begin_particle_injection { +} + +begin_current_injection { +} + +begin_field_injection { +} + +begin_particle_collisions { +} + + diff --git a/test/integrated/hdf5_fields_many_steps/readback.py b/test/integrated/hdf5_fields_many_steps/readback.py new file mode 100755 index 00000000..d35c617e --- /dev/null +++ b/test/integrated/hdf5_fields_many_steps/readback.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +import h5py +import numpy as np +import os.path +import sys +os.environ["HDF5_USE_FILE_LOCKING"] = "FALSE" +if len(sys.argv) != 2: + sys.stderr.write("Usage: "+str(sys.argv[0])+" rundir\n") + sys.exit(1) + +rundir = sys.argv[1] +print(rundir) + +for t in range(1, 5): + file_name = rundir+"/field_hdf5/T."+str(t)+"/fields_"+str(t)+".h5" + group_name = "Timestep_"+str(t) + # print(file_name) + # print(group_name) + ##file_name = rundir+"/field_hdf5/T.2/fields_2.h5" + + if not os.path.isfile(file_name): + print("FAIL: "+file_name+" is missing") + sys.exit(1) + + infile = h5py.File(file_name, 'r') + datagroup = infile[group_name] + + cbx = np.array(datagroup["cbx"]) + if np.array_equal(cbx, [3, 4, 3, 4]): + print('cbx does not contain [3, 4, 3, 4].') + sys.exit(1) + + cby = np.array(datagroup["cby"]) + if np.array_equal(cby, [3, 4, 3, 4]): + print('cby does not contain all 17.') + sys.exit(1) + + cbz = np.array(datagroup["cbz"]) + if np.array_equal(cbz, [3, 4, 3, 4]): + print('cbz does not contain all 17.') + sys.exit(1) + + ex = np.array(datagroup["ex"]) + if np.array_equal(ex, [3, 4, 3, 4]): + print('ex does not contain all 17.') + sys.exit(1) + + ey = np.array(datagroup["ey"]) + if np.array_equal(ey, [3, 4, 3, 4]): + print('ey does not contain all 17.') + sys.exit(1) + + ez = np.array(datagroup["ez"]) + if np.array_equal(ez, [3, 4, 3, 4]): + print('ez does not contain all 17.') + sys.exit(1) + + other_fields_name = ["cmat", "div_b_err", "div_e_err", "ematx", "ematy", "ematz", + "fmatx", "fmaty", "fmatz", "jfx", "jfy", "jfz", "nmat", "rhob", "rhof", "tcax", "tcay", "tcaz"] + + for field_name in other_fields_name: + field_data = np.array(datagroup[field_name]) + if np.all(field_data == 0) == False: + print(field_name, 'does not contain all 0.') + sys.exit(1) + + infile.close() diff --git a/test/integrated/hdf5_hydro/CMakeLists.txt b/test/integrated/hdf5_hydro/CMakeLists.txt new file mode 100644 index 00000000..014482a3 --- /dev/null +++ b/test/integrated/hdf5_hydro/CMakeLists.txt @@ -0,0 +1,18 @@ +# add the tests +set(MPI_NUM_RANKS 1) + +set(TEST "hdf5_hydro") +find_package(Python3 COMPONENTS Interpreter) + + +if(USE_HDF5) + # Build + # TODO: This method of doing the tests is really bad at rebuilding them properly + build_a_vpic(${TEST} ${CMAKE_CURRENT_SOURCE_DIR}/${TEST}.deck) + + # Add TEST + add_TEST(${TEST} ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MPI_NUM_RANKS} ${MPIEXEC_PREFLAGS} ./${TEST} ${MPIEXEC_POSTFLAGS} ${ARGS}) + + #and check that the output is ok + add_TEST(${TEST}_readhdf5 python3.9 ${CMAKE_CURRENT_SOURCE_DIR}/readback.py ${CMAKE_CURRENT_BINARY_DIR}) +endif() diff --git a/test/integrated/hdf5_hydro/hdf5_hydro copy.deck b/test/integrated/hdf5_hydro/hdf5_hydro copy.deck new file mode 100644 index 00000000..42a5c69e --- /dev/null +++ b/test/integrated/hdf5_hydro/hdf5_hydro copy.deck @@ -0,0 +1,237 @@ +begin_globals{}; + +begin_initialization +{ + // Numerical settings + num_step = 2; + status_interval = 1000; // Basically don't print status + sync_shared_interval = status_interval; + clean_div_e_interval = status_interval; + clean_div_b_interval = status_interval; + + int gnx = nproc(); + int gny = 2; // Would love to make this 1 as well, but then I get weird NaNs... + int gnz = 1; + + int topox = nproc(); + int topoy = 1; + int topoz = 1; + + int nppc = 100; + + int Nspecies = 2; + + // use natural PIC units + float ec = 1; // Charge normalization + float me = 1; // Mass normalization + float c = 1; // Speed of light + float de = 1; // Length normalization (electron inertial length) + float eps0 = 1; // Permittivity of space + + // physics settings + float dx = 0.125; // d_e + float dt = dx * 0.95 / sqrt(3.0) / c; + float Lx = dx * gnx; + float Ly = dx * gny; + float Lz = dx * gnz; + + int sizex = 24; + int sizey = 24; + int sizez = 24; + + // Setup stuff in vpic + define_units(c, eps0); + define_timestep(dt); + + define_periodic_grid(0, 0, 0, // Low corner + Lx, Ly, Lz, // High corner + gnx, gny, gnz, // Resolution + topox, topoy, topoz); // Topology + + // Space is by default filled with first material defined + define_material("vacuum", 1.0); + + // Create the field array + define_field_array(NULL, 0.00); + // Create species + species_t *species[Nspecies]; + + for (int n = 0; n < Nspecies; n++) + { + char name[256]; + if (n == 0) + { + sprintf(name, "%s", "electron"); + } + else if (n == 1) + { + sprintf(name, "%s", "proton"); + } + else + { + sprintf(name, "species_%d", n); + } + + if (rank() == 0) + { + printf("Definine species '%s'\n", name); + } + + species[n] = define_species(name, // name + n % 2 ? 1. : -1., // charge + 1., // mass + 1.5 * sizex * sizey * sizez * nppc, // maximum number of local particles + -1, // number of particle movers. -1: automatic + 0, // sort interval. 0: never + 1); // sort mode. 1: out-of-place + } + + // Just a homogeneous value in Bx + set_region_field(everywhere, 5., 6., 7., 17., 18., 19.); + + // different random numbers on different ranks + seed_entropy(rank()); + +// Note: No particles. This is fine for field tests, but we need to come up +// with something better for tests of hydro output and particle dumps + +// Deck only works if VPIC was build with HDF support. Check for that: +#ifndef VPIC_ENABLE_HDF5 +#error "VPIC_ENABLE_HDF5" is required +#endif + + // Explicitly enable HDF5 backend for IO dump + enable_hdf5_dump(); +} + +begin_diagnostics +{ +// The equations are only evaluated inside the mesh-mapped region +// (This is not strictly inside the region) +#define set_region_hydro_loc(rgn, \ + eqn_ex, eqn_ey, eqn_ez, \ + eqn_bx, eqn_by, eqn_bz) \ + do \ + { \ + const double _x0 = grid->x0, _y0 = grid->y0, _z0 = grid->z0; \ + const double _dx = grid->dx, _dy = grid->dy, _dz = grid->dz; \ + const double _c = grid->cvac; \ + const int _nx = grid->nx, _ny = grid->ny, _nz = grid->nz; \ + for (int _k = 0; _k < _nz + 2; _k++) \ + { \ + const double _zl = _z0 + _dz * (_k - 1.5), _ze = _z0 + _dz * (_k - 1.0), _zc = _z0 + _dz * (_k - 0.5); \ + for (int _j = 0; _j < _ny + 2; _j++) \ + { \ + const double _yl = _y0 + _dy * (_j - 1.5), _ye = _y0 + _dy * (_j - 1.0), _yc = _y0 + _dy * (_j - 0.5); \ + hydro_t *_h = &hydro(0, _j, _k); \ + for (int _i = 0; _i < _nx + 2; _++) \ + { \ + const double _xl = _x0 + _dx * (_i - 1.5), _xe = _x0 + _dx * (_i - 1.0), _xc = _x0 + _dx * (_i - 0.5); \ + double x, y, z; \ + memset(_h, 0, sizeof(hydro_t)); \ + int _rccc, _rlcc, _rclc, _rllc, _rccl, _rlcl, _rcll; \ + x = _xc; \ + y = _yc; \ + z = _zc; \ + _rccc = (rgn); \ + x = _xl; \ + _rlcc = (rgn); \ + x = _xc; \ + y = _yl; \ + _rclc = (rgn); \ + x = _xl; \ + _rllc = (rgn); \ + x = _xc; \ + y = _yc; \ + z = _zl; \ + _rccl = (rgn); \ + x = _xl; \ + _rlcl = (rgn); \ + x = _xc; \ + y = _yl; \ + _rcll = (rgn); \ + x = _xc; \ + y = _ye; \ + z = _ze; \ + if (_rccc || _rclc || _rccl || _rcll) \ + _h->jx = _i + _j + _k; \ + x = _xe; \ + y = _yc; \ + z = _ze; \ + if (_rccc || _rccl || _rlcc || _rlcl) \ + _h->jy = _i + _j + _k; \ + x = _xe; \ + y = _ye; \ + z = _zc; \ + if (_rccc || _rlcc || _rclc || _rllc) \ + _h->jz = _i + _j + _k; \ + x = _xe; \ + y = _yc; \ + z = _zc; \ + if (_rccc || _rlcc) \ + _h->px = _i + _j + _k; \ + x = _xc; \ + y = _ye; \ + z = _zc; \ + if (_rccc || _rclc) \ + _h->py = _i + _j + _k; \ + x = _xc; \ + y = _yc; \ + z = _ze; \ + if (_rccc || _rccl) \ + _h->pz = _i + _j + _k; \ + printf("+++ ijk = (%d, %d, %d) jx = %f, jy = %f, jz = %f , txx = %f +++ \n", _i, _j, _k, _h->jx, _h->jy, _h->jz, _h->txx); \ + _h++; \ + } \ + } \ + } \ + } while (0) + + // Write field output in every timestep + set_region_hydro_loc(everywhere, 5., 6., 7., 17., 18., 19.); + + species_t *sp; + int n = 0; + LIST_FOR_EACH(sp, species_list) + { + char baseFileName[256]; + sprintf(baseFileName, "hydro_%s", sp->name); + dump_hydro(sp->name, baseFileName); + } + + //./sample/harrisHDF5: dump_hydro("electron","ehydro"); + //./sample/harrisHDF5: dump_hydro("ion", "ihydro"); + // Force a restart after step 1 to make sure that dump/restart works too + if (step() == 1) + { + sim_log("Force restart"); + dump_mkdir("restart0"); + char fname[256]; + sprintf(fname, "%s/restore.0.%i", "restart0", world_rank); + checkpt_objects(fname); + sim_log("Restart dump restart completed."); + mp_barrier(); + } + + // If we reach this in the last step without any fails we probably did well enough + if (step() == num_step) + { + sim_log("passed"); + } +} + +begin_particle_injection +{ +} + +begin_current_injection +{ +} + +begin_field_injection +{ +} + +begin_particle_collisions +{ +} diff --git a/test/integrated/hdf5_hydro/hdf5_hydro.deck b/test/integrated/hdf5_hydro/hdf5_hydro.deck new file mode 100644 index 00000000..e6cd012d --- /dev/null +++ b/test/integrated/hdf5_hydro/hdf5_hydro.deck @@ -0,0 +1,262 @@ +begin_globals{ + // No globals required in this simulation +}; + +begin_initialization +{ + +// Deck only works if VPIC was build with HDF support. Check for that: +#ifndef VPIC_ENABLE_HDF5 +#error "VPIC_ENABLE_HDF5" is required +#endif + + // Explicitly enable HDF5 backend for IO dump + enable_hdf5_dump(); + + // At this point, there is an empty grid and the random number generator is + // seeded with the rank. The grid, materials, species need to be defined. + // Then the initial non-zero fields need to be loaded at time level 0 and the + // particles (position and momentum both) need to be loaded at time level 0. + + // Diagnostic messages can be passed written (usually to stderr) + sim_log("Computing simulation parameters"); + + // Define the system of units for this problem (natural units) + double L = 1; // Length normalization + double ec = 1; // Charge normalization + double me = 1; // Mass normalization + double c = 1; // Speed of light + double eps0 = 1; // Permittivity of space + + // Numerical parameters + double Lx = 16 * L; // How big should the box be in the x direction + double Ly = 16 * L; // How big should the box be in the y direction + double Lz = 16 * L; // How big should the box be in the z direction + double nx = 64; // Global resolution in the x direction + double ny = 64; // Global resolution in the y direction + double nz = 1; // Global resolution in the z direction + double nppc = 64; // Average number of macro particles per cell (both species combined!) + double cfl_req = 0.99; // How close to Courant should we try to run + double damp = 0.0; // Level of radiation damping + + // Determine the timestep + double dt = courant_length(Lx, Ly, Lz, nx, ny, nz); // Courant length + + //////////////////////////////////////// + // Setup high level simulation parmeters + + num_step = 1; + status_interval = 1; + sync_shared_interval = status_interval; + clean_div_e_interval = status_interval; + clean_div_b_interval = status_interval; + + /////////////////////////// + // Setup the space and time + + // Setup basic grid parameters + define_units(c, eps0); + define_timestep(dt); + + // Parition a periodic box among the processors sliced uniformly along y + define_periodic_grid(0, 0, 0, // Low corner + Lx, Ly, Lz, // High corner + nx, ny, nz, // Resolution + 1, nproc(), 1); // Topology + + define_material("vacuum", 1); + // Note: define_material defaults to isotropic materials with mu=1,sigma=0 + // Tensor electronic, magnetic and conductive materials are supported + // though. See "shapes" for how to define them and assign them to regions. + // Also, space is initially filled with the first material defined. + + // If you pass NULL to define field array, the standard field array will + // be used (if damp is not provided, no radiation damping will be used). + define_field_array(NULL, damp); + + //////////////////// + // Setup the species + + const double np = 1.5 * nx * ny * nz / nproc(); // particle number + const double nm = -1; // Automatic amount of movers + const double sort_interval = 20; + const double sort_out_of_place = 1; + species_t *electron = define_species("electron", -ec, me, np, nm, sort_interval, sort_out_of_place); + + /////////////////////////////////////////////////// + // Log diagnostic information about this simulation + + sim_log(""); + sim_log("System of units"); + sim_log("L = " << L); + sim_log("ec = " << ec); + sim_log("me = " << me); + sim_log("c = " << c); + sim_log("eps0 = " << eps0); + sim_log(""); + sim_log("Numerical parameters"); + sim_log("num_step = " << num_step); + sim_log("dt = " << dt); + sim_log("Lx = " << Lx << ", Lx/L = " << Lx / L); + sim_log("Ly = " << Ly << ", Ly/L = " << Ly / L); + sim_log("Lz = " << Lz << ", Lz/L = " << Lz / L); + sim_log("nx = " << nx << ", dx = " << Lx / nx << ", L/dx = " << L * nx / Lx); + sim_log("ny = " << ny << ", dy = " << Ly / ny << ", L/dy = " << L * ny / Ly); + sim_log("nz = " << nz << ", dz = " << Lz / nz << ", L/dz = " << L * nz / Lz); + sim_log("nppc = " << nppc); + sim_log("damp = " << damp); + sim_log(""); + + //////////////////////////// + // Load fields and particles + + sim_log("Loading fields"); + + set_region_field(everywhere, 0, 0, 0, // Electric field + 0, 0, 0); // Magnetic field + + sim_log("Loading particles"); + + int nparticle = 0; + for (int i = 0; i < grid->nx; i++) + { + for (int j = 0; j < grid->ny; j++) + { + for (int k = 0; k < grid->nz; k++) + { + + float x = grid->x0 + (i + 0.5) * grid->dx; + float y = grid->y0 + (j + 0.5) * grid->dy; + float z = grid->z0 + (k + 0.5) * grid->dz; + float ux = y / Ly * 0.1 * c; + float uy = z / Lz * 0.1 * c; + float uz = x / Lx * 0.1 * c; + // std::cout << "y = " << y << " , y / Ly = " << y / Ly << ", ux = " << y / Ly * 0.1 * c << " (ijk) = " << i << ", " << j << ", " << k << "\n"; + + const double w = 1.; + const double age = 0.; + const int update_rhob = 1; + inject_particle(electron, x, y, z, ux, uy, uz, w, age, update_rhob); + nparticle++; + } + } + } + + sim_log("Loaded " << nparticle << " electrons"); + sim_log("We now have " << electron->np << " electrons"); + + // Upon completion of the initialization, the following occurs: + // - The synchronization error (tang E, norm B) is computed between domains + // and tang E / norm B are synchronized by averaging where discrepancies + // are encountered. + // - The initial divergence error of the magnetic field is computed and + // one pass of cleaning is done (for good measure) + // - The bound charge density necessary to give the simulation an initially + // clean divergence e is computed. + // - The particle momentum is uncentered from u_0 to u_{-1/2} + // - The user diagnostics are called on the initial state + // - The physics loop is started + // + // The physics loop consists of: + // - Advance particles from x_0,u_{-1/2} to x_1,u_{1/2} + // - User particle injection at x_{1-age}, u_{1/2} (use inject_particles) + // - User current injection (adjust field(x,y,z).jfx, jfy, jfz) + // - Advance B from B_0 to B_{1/2} + // - Advance E from E_0 to E_1 + // - User field injection to E_1 (adjust field(x,y,z).ex,ey,ez,cbx,cby,cbz) + // - Advance B from B_{1/2} to B_1 + // - (periodically) Divergence clean electric field + // - (periodically) Divergence clean magnetic field + // - (periodically) Synchronize shared tang e and norm b + // - Increment the time step + // - Call user diagnostics + // - (periodically) Print a status message +} + +begin_diagnostics +{ + // sim_log(" \n\n\n here is new test \n\n\n"); + + // Hydro dumps store particle charge density, current density and + // stress-energy tensor. All these quantities are known at the time + // t = time(). All these quantities are accumulated trilinear + // node-centered. By default, species dump filenames are tagged with + // step(). However, if a "0" is added to the call, the filename will not + // be tagged. Note that the current density accumulated by this routine is + // purely diagnostic. It is not used by the simulation and it is not + // accumulated using a self-consistent charge-conserving method. Hydro dumps + // are in a binary format. Each rank makes a hydro dump. + dump_hydro("electron", "ehydro"); + + species_t *sp = find_species_name("electron", species_list); + clear_hydro_array(hydro_array); + accumulate_hydro_p(hydro_array, sp, interpolator_array); + synchronize_hydro_array(hydro_array); + + float *temp_buf = (float *)malloc(sizeof(float) * (grid->nx) * (grid->ny) * (grid->nz)); + size_t temp_buf_index = 0; + +#define DUMP_HYDRO_TO_BINARY(DSET_NAME, ATTRIBUTE_NAME) \ + { \ + std::string fn; \ + fn = "step" + std::to_string(step()) + "_rank" + std::to_string(rank()) + "_" + DSET_NAME + ".bin"; \ + std::cout << fn << std::endl; \ + FILE *handle_ = fopen(fn.c_str(), "w"); \ + temp_buf_index = 0; \ + for (size_t i(1); i < grid->nx + 1; i++) \ + { \ + for (size_t j(1); j < grid->ny + 1; j++) \ + { \ + for (size_t k(1); k < grid->nz + 1; k++) \ + { \ + temp_buf[temp_buf_index] = _hydro(i, j, k).ATTRIBUTE_NAME; \ + temp_buf_index = temp_buf_index + 1; \ + } \ + } \ + } \ + fwrite(reinterpret_cast(const_cast(temp_buf)), sizeof(float), temp_buf_index, handle_); \ + fclose(handle_); \ + } + + DUMP_HYDRO_TO_BINARY("jx", jx); + DUMP_HYDRO_TO_BINARY("jy", jy); + DUMP_HYDRO_TO_BINARY("jz", jz); + DUMP_HYDRO_TO_BINARY("jz", jz); + DUMP_HYDRO_TO_BINARY("rho", rho); + + // Force a restart after step 1 to make sure that dump/restart works too + if (step() == 1) + { + sim_log("Force restart"); + dump_mkdir("restart0"); + char fname[256]; + sprintf(fname, "%s/restore.0.%i", "restart0", world_rank); + checkpt_objects(fname); + sim_log("Restart dump restart completed."); + mp_barrier(); + } +} + +begin_particle_injection +{ + + // No particle injection for this simulation +} + +begin_current_injection +{ + + // No current injection for this simulation +} + +begin_field_injection +{ + + // No field injection for this simulation +} + +begin_particle_collisions +{ + + // No collisions for this simulation +} diff --git a/test/integrated/hdf5_hydro/readback.py b/test/integrated/hdf5_hydro/readback.py new file mode 100755 index 00000000..4aaccba3 --- /dev/null +++ b/test/integrated/hdf5_hydro/readback.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python +import h5py +import numpy as np +import os.path +import sys + +if len(sys.argv) != 2: + sys.stderr.write("Usage: "+str(sys.argv[0])+" rundir\n") + sys.exit(1) + + +# print(dd) +# print(dd.shape) +# exit(0) +rundir = sys.argv[1] +# print(rundir) + + +step_names = ["0", "1"] + +for step_name in step_names: + filename = rundir+"/hydro_hdf5/T."+step_name+"/hydro_electron_" + step_name+".h5" + + if not os.path.isfile(filename): + print("FAIL: "+filename+" is missing") + sys.exit(1) + + infile = h5py.File(filename, 'r') + datagroup = infile["Timestep_"+step_name+""] + + hydro_names = ["jx", "jy", "jz", "rho"] + + for hydro_name in hydro_names: + bin_filename = "step"+step_name+"_rank0_"+hydro_name+".bin" + hydro_data_bi = np.fromfile(bin_filename, dtype=np.float32) + # print(hydro_data_bi.shape) + hydro_data_h5 = np.array(datagroup[hydro_name]).flatten() + #print("hydro_data_bi=", hydro_data_bi[1:10]) + #print("hydro_data_h5=", hydro_data_h5[1:10]) + # print(hydro_data_h5.shape) + if np.allclose(hydro_data_bi, hydro_data_h5, atol=0.0000000001) == False: + print(hydro_name, 'in ', 'hydro_data_h5', " does not contain same value as ", bin_filename) + print_max_element = 0 + print(" Binary Output", ", ", "HDF5 Output", ", ", "Difference") + for i in range(0, hydro_data_h5.shape[0]): + if hydro_data_bi[i] - hydro_data_h5[i] > 0.0000000001: + print(i, hydro_data_bi[i], ", ", hydro_data_h5[i], ", ", hydro_data_bi[i] - hydro_data_h5[i]) + print_max_element = print_max_element + 1 + if print_max_element == 10: + break + sys.exit(1) + + infile.close() diff --git a/test/unit/energy_comparison/3d_test.cc b/test/unit/energy_comparison/3d_test.cc index aafa6011..f59149f7 100644 --- a/test/unit/energy_comparison/3d_test.cc +++ b/test/unit/energy_comparison/3d_test.cc @@ -312,7 +312,7 @@ TEST_CASE( "Check if Weibel gives correct energy (within tol)", "[energy]" ) ofs.close(); // Init and run sim - vpic_simulation simulation = vpic_simulation(); + vpic_simulation simulation; // TODO: We should do this in a safer manner simulation.initialize( 0, NULL ); diff --git a/test/unit/energy_comparison/weibel_driver.cc b/test/unit/energy_comparison/weibel_driver.cc index 68ef6b47..92f18ff8 100644 --- a/test/unit/energy_comparison/weibel_driver.cc +++ b/test/unit/energy_comparison/weibel_driver.cc @@ -310,7 +310,7 @@ TEST_CASE( "Check if Weibel gives correct energy (within tol)", "[energy]" ) ofs.close(); // Init and run sim - vpic_simulation simulation = vpic_simulation(); + vpic_simulation simulation; // TODO: We should do this in a safer manner simulation.initialize( 0, NULL ); diff --git a/test/unit/grid_heating/gridHeatingTestElec.cxx b/test/unit/grid_heating/gridHeatingTestElec.cxx index dc0e1ca5..9e804588 100644 --- a/test/unit/grid_heating/gridHeatingTestElec.cxx +++ b/test/unit/grid_heating/gridHeatingTestElec.cxx @@ -249,7 +249,7 @@ begin_initialization { TEST_CASE( "Check if Weibel gives correct energy (within tol)", "[energy]" ) { // Init and run sim - vpic_simulation simulation = vpic_simulation(); + vpic_simulation simulation; // TODO: We should do this in a safer manner simulation.initialize( 0, NULL );