diff --git a/garlic/apps/fwi/default.nix b/garlic/apps/fwi/default.nix index 594e73ecc3f0fa2d2e028e8a9adf9b3cd04a39a3..5b1a9d63641c12731df29318129521b95ece545f 100644 --- a/garlic/apps/fwi/default.nix +++ b/garlic/apps/fwi/default.nix @@ -1,30 +1,41 @@ { stdenv -, nanos6 -, mpi -, tampi -, mcxx -, icc +, mpi ? null +, tampi ? null +, mcxx ? null +, cc +, gitBranch ? "garlic/tampi+send+oss+task" +, fwiInput }: +with stdenv.lib; + +assert !(tampi != null && mcxx == null); + stdenv.mkDerivation rec { + inherit gitBranch; name = "fwi"; - variant = "oss+task"; src = builtins.fetchGit { - url = "https://gitlab.com/srodrb/BSC-FWI.git"; - ref = "${variant}"; + url = "ssh://git@bscpm03.bsc.es/garlic/apps/fwi.git"; + ref = "${gitBranch}"; }; - enableParallelBuilding = true; + enableParallelBuilding = false; buildInputs = [ - nanos6 - mpi - icc - tampi - mcxx - ]; + cc + ] + ++ optional (mpi != null) mpi + ++ optional (tampi != null) tampi + ++ optional (mcxx != null) mcxx; + + # FIXME: Correct this on the Makefile so we can just type "make fwi" + # FIXME: Allow multiple MPI implementations + postPatch = '' + sed -i 's/= OPENMPI$/= INTEL/g' Makefile + sed -i 's/USE_O_DIRECT ?= NO/USE_O_DIRECT ?= YES/g' Makefile || true + ''; # FIXME: This is an ugly hack. # When using _GNU_SOURCE or any other definition used in features.h, we need @@ -33,27 +44,26 @@ stdenv.mkDerivation rec { # below, reaches the command line of the preprocessing stage with gcc. preConfigure = '' export DEFINES=-D_GNU_SOURCE - export NANOS6_CONFIG_OVERRIDE=version.debug=true + + make depend + + cp ${fwiInput}/generated_model_params.h src/ ''; # We compile the ModelGenerator using gcc *only*, as otherwise it will # be compiled with nanos6, which requires access to /sys to determine # hardware capabilities. So it will fail in the nix-build environment, # as there is no /sys mounted. - preBuild = '' - make COMPILER=GNU ModelGenerator - ''; - makeFlags = [ - "NZF=108" - "NXF=108" - "NYF=208" - "PRECISION=float" + #"COMPILER=GNU" + #"CC=${cc.cc.CC}" + "fwi" ]; installPhase = '' mkdir -p $out/bin cp fwi $out/bin - cp ModelGenerator $out/bin ''; + + programPath = "/bin/fwi"; } diff --git a/garlic/apps/fwi/input.nix b/garlic/apps/fwi/input.nix new file mode 100644 index 0000000000000000000000000000000000000000..848759f1154369f5ddecf3d146ecf46002f04008 --- /dev/null +++ b/garlic/apps/fwi/input.nix @@ -0,0 +1,53 @@ +{ + stdenv +, nz ? 200 +, nx ? 200 +, ny ? 500 +}: + +with stdenv.lib; +with builtins; + +stdenv.mkDerivation rec { + name = "fwi-input"; + + src = builtins.fetchGit { + url = "ssh://git@bscpm03.bsc.es/garlic/apps/fwi.git"; + ref = "garlic/seq"; + }; + + enableParallelBuilding = false; + + # Set the input size with the weird order (nz,nx,ny). + postPatch = '' + sed -i 1c${toString nz} SetupParams/fwi_params.txt + sed -i 2c${toString nx} SetupParams/fwi_params.txt + sed -i 3c${toString ny} SetupParams/fwi_params.txt + ''; + + # FIXME: This is an ugly hack. + # When using _GNU_SOURCE or any other definition used in features.h, we need + # to define them before mcc includes nanos6.h from the command line. So the + # only chance is by setting it at the command line with -D. Using the DEFINES + # below, reaches the command line of the preprocessing stage with gcc. + preConfigure = '' + export DEFINES=-D_GNU_SOURCE + ''; + + # We compile the ModelGenerator using gcc *only*, as otherwise it will + # be compiled with nanos6, which requires access to /sys to determine + # hardware capabilities. So it will fail in the nix-build environment, + # as there is no /sys mounted. + # Also, we need to compile it with the builder platform as target, as is going + # to be executed during the build to generate the src/generated_model_params.h + # header. + makeFlags = [ "COMPILER=GNU" "params" "input" ]; + + installPhase = '' + mkdir -p $out/ + cp src/generated_model_params.h $out/ + cp SetupParams/fwi_params.txt $out/ + cp SetupParams/fwi_frequencies.txt $out/ + cp -r InputModels $out/ + ''; +} diff --git a/garlic/apps/index.nix b/garlic/apps/index.nix index ac2ccfd864f62709f9b724566fc7a5e09d6a1b97..41da29557e1110b178b35fe71a186e7ee6900f2a 100644 --- a/garlic/apps/index.nix +++ b/garlic/apps/index.nix @@ -53,5 +53,10 @@ hpccg = callPackage ./hpccg/default.nix { }; - fwi = callPackage ./fwi/default.nix { }; + fwi = rec { + input = callPackage ./fwi/input.nix { }; + solver = callPackage ./fwi/default.nix { + fwiInput = input; + }; + }; } diff --git a/garlic/exp/fwi/data_reuse.nix b/garlic/exp/fwi/data_reuse.nix new file mode 100644 index 0000000000000000000000000000000000000000..84f0c4b80b67ded09bb5b3b58038616d55c56bf0 --- /dev/null +++ b/garlic/exp/fwi/data_reuse.nix @@ -0,0 +1,135 @@ +# This test compares a FWI version using poor data locality (+NOREUSE) versus +# the optimized version (used for all other experiments). Follows a pseudocode +# snippet illustrating the fundamental difference between version. +# +# NOREUSE +# ---------------------- +# for (y) for (x) for (z) +# computA(v[y][x][z]); +# for (y) for (x) for (z) +# computB(v[y][x][z]); +# for (y) for (x) for (z) +# computC(v[y][x][z]); +# +# Optimized version +# ---------------------- +# for (y) for (x) for (z) +# computA(v[y][x][z]); +# computB(v[y][x][z]); +# computC(v[y][x][z]); + +{ + stdenv +, stdexp +, bsc +, targetMachine +, stages +}: + +with stdenv.lib; + +let + + inherit (targetMachine) fs; + + # Initial variable configuration + varConf = { + gitBranch = [ + "garlic/mpi+send+oss+task" + "garlic/mpi+send+oss+task+NOREUSE" + ]; + + blocksize = [ 1 2 4 8 ]; + + n = [ + {nx=300; ny=2000; nz=300;} # / half node + ]; + }; + +# The c value contains something like: +# { +# n = { nx=500; ny=500; nz=500; } +# blocksize = 1; +# gitBranch = "garlic/tampi+send+oss+task"; +# } + + machineConfig = targetMachine.config; + + # Generate the complete configuration for each unit + genConf = with bsc; c: targetMachine.config // rec { + expName = "fwi"; + unitName = "${expName}-test"; + inherit (machineConfig) hw; + + cc = icc; + inherit (c) gitBranch blocksize; + + #nx = c.n.nx; + #ny = c.n.ny; + #nz = c.n.nz; + + # Same but shorter: + inherit (c.n) nx ny nz; + + fwiInput = bsc.apps.fwi.input.override { + inherit (c.n) nx ny nz; + }; + + # Repeat the execution of each unit several times + loops = 10; + #loops = 1; + + # Resources + cpusPerTask = hw.cpusPerSocket; + ntasksPerNode = 1; + nodes = 1; + qos = "debug"; + time = "02:00:00"; + jobName = unitName; + + # Enable permissions to write in the local storage + extraMounts = [ fs.local.temp ]; + + }; + + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; + }; + + exec = {nextStage, conf, ...}: stages.exec { + inherit nextStage; + pre = '' + #CDIR=$PWD + #export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf" + EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN" + mkdir -p $EXECDIR + cd $EXECDIR + ln -fs ${conf.fwiInput}/InputModels InputModels || true + ''; + argv = [ + "${conf.fwiInput}/fwi_params.txt" + "${conf.fwiInput}/fwi_frequencies.txt" + conf.blocksize + "-1" # Fordward steps + "-1" # Backward steps + "-1" # Write/read frequency + ]; + post = '' + rm -rf Results || true + #mv trace_* $CDIR + ''; + }; + + apps = bsc.garlic.apps; + + # FWI program + program = {nextStage, conf, ...}: apps.fwi.solver.override { + inherit (conf) cc gitBranch fwiInput; + }; + + pipeline = stdexp.stdPipeline ++ [ exec program ]; + +in + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/fwi/granularity.nix b/garlic/exp/fwi/granularity.nix new file mode 100644 index 0000000000000000000000000000000000000000..7773b3b426b92cecb4dd0702d70370eb52292bfc --- /dev/null +++ b/garlic/exp/fwi/granularity.nix @@ -0,0 +1,133 @@ +# Regular granularity test for FWI + +{ + stdenv +, stdexp +, bsc +, targetMachine +, stages +}: + +with stdenv.lib; + +let + + inherit (targetMachine) fs; + + # Initial variable configuration + varConf = { + gitBranch = [ +# "garlic/tampi+send+oss+task" + "garlic/tampi+isend+oss+task" +# "garlic/mpi+send+omp+task" +# "garlic/mpi+send+oss+task" +# "garlic/mpi+send+seq" +# "garlic/oss+task" +# "garlic/omp+task" +# "garlic/seq" + ]; + + blocksize = [ 1 2 4 8 16 32 64 128 256 ]; + + n = [ + {nx=100; nz=100; ny=8000; ntpn=2; nn=1;} + ]; + + }; + +# The c value contains something like: +# { +# n = { nx=500; ny=500; nz=500; } +# blocksize = 1; +# gitBranch = "garlic/tampi+send+oss+task"; +# } + + machineConfig = targetMachine.config; + + # Generate the complete configuration for each unit + genConf = with bsc; c: targetMachine.config // rec { + expName = "fwi"; + unitName = "${expName}-test"; + inherit (machineConfig) hw; + + cc = icc; + inherit (c) gitBranch blocksize; + + #nx = c.n.nx; + #ny = c.n.ny; + #nz = c.n.nz; + + # Same but shorter: + inherit (c.n) nx ny nz ntpn nn; + + fwiInput = bsc.apps.fwi.input.override { + inherit (c.n) nx ny nz; + }; + + # Other FWI parameters + ioFreq = -1; + + # Repeat the execution of each unit several times + loops = 10; + #loops = 1; + + # Resources + cpusPerTask = hw.cpusPerSocket; + ntasksPerNode = ntpn; + nodes = nn; + qos = "debug"; + time = "02:00:00"; + jobName = unitName; + + tracing = "no"; + + # Enable permissions to write in the local storage + extraMounts = [ fs.local.temp ]; + + }; + + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; + }; + + exec = {nextStage, conf, ...}: stages.exec { + inherit nextStage; + pre = '' + CDIR=$PWD + if [[ "${conf.tracing}" == "yes" ]]; then + export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf" + fi + EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN" + mkdir -p $EXECDIR + cd $EXECDIR + ln -fs ${conf.fwiInput}/InputModels InputModels || true + ''; + argv = [ + "${conf.fwiInput}/fwi_params.txt" + "${conf.fwiInput}/fwi_frequencies.txt" + conf.blocksize + "-1" # Fordward steps + "-1" # Backward steps + conf.ioFreq # Write/read frequency + ]; + post = '' + rm -rf Results || true + if [[ "${conf.tracing}" == "yes" ]]; then + mv trace_* $CDIR + fi + ''; + }; + + apps = bsc.garlic.apps; + + # FWI program + program = {nextStage, conf, ...}: apps.fwi.solver.override { + inherit (conf) cc gitBranch fwiInput; + }; + + pipeline = stdexp.stdPipeline ++ [ exec program ]; + +in + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/fwi/strong_scaling_forkjoin.nix b/garlic/exp/fwi/strong_scaling_forkjoin.nix new file mode 100644 index 0000000000000000000000000000000000000000..902eaf60ef6d40694a37773545f6d30eac9b468d --- /dev/null +++ b/garlic/exp/fwi/strong_scaling_forkjoin.nix @@ -0,0 +1,128 @@ +# Strong scaling test for FWI variants based on forkjoint. This +# experiment does not rely on block sizes. + +{ + stdenv +, stdexp +, bsc +, targetMachine +, stages +}: + +with stdenv.lib; + +let + + inherit (targetMachine) fs; + + # Initial variable configuration + varConf = { + gitBranch = [ + "garlic/mpi+send+omp+fork" + ]; + + blocksize = [ 0 ]; + + n = [ + {nx=100; nz=100; ny=8000;} + ]; + + nodes = [ 1 2 4 8 16 ]; + + }; + +# The c value contains something like: +# { +# n = { nx=500; ny=500; nz=500; } +# blocksize = 1; +# gitBranch = "garlic/tampi+send+oss+task"; +# } + + machineConfig = targetMachine.config; + + # Generate the complete configuration for each unit + genConf = with bsc; c: targetMachine.config // rec { + expName = "fwi"; + unitName = "${expName}-test"; + inherit (machineConfig) hw; + + cc = icc; + inherit (c) gitBranch blocksize; + + #nx = c.n.nx; + #ny = c.n.ny; + #nz = c.n.nz; + + # Same but shorter: + inherit (c.n) nx ny nz; + + fwiInput = bsc.apps.fwi.input.override { + inherit (c.n) nx ny nz; + }; + + # Other FWI parameters + ioFreq = -1; + + # Repeat the execution of each unit several times + loops = 10; + #loops = 1; + + # Resources + cpusPerTask = hw.cpusPerSocket; + ntasksPerNode = 2; + nodes = c.nodes; + qos = "debug"; + time = "02:00:00"; + jobName = unitName; + + tracing = "no"; + + # Enable permissions to write in the local storage + extraMounts = [ fs.local.temp ]; + + }; + + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; + }; + + exec = {nextStage, conf, ...}: stages.exec { + inherit nextStage; + pre = '' + CDIR=$PWD + if [[ "${conf.tracing}" == "yes" ]]; then + export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf" + fi + EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN" + mkdir -p $EXECDIR + cd $EXECDIR + ln -fs ${conf.fwiInput}/InputModels InputModels || true + ''; + argv = [ + "${conf.fwiInput}/fwi_params.txt" + "${conf.fwiInput}/fwi_frequencies.txt" + "-1" # Fordward steps + "-1" # Backward steps + conf.ioFreq # Write/read frequency + ]; + post = '' + rm -rf Results || true + if [[ "${conf.tracing}" == "yes" ]]; then + mv trace_* $CDIR + fi + ''; + }; + + apps = bsc.garlic.apps; + + # FWI program + program = {nextStage, conf, ...}: apps.fwi.solver.override { + inherit (conf) cc gitBranch fwiInput; + }; + + pipeline = stdexp.stdPipeline ++ [ exec program ]; + +in + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/fwi/strong_scaling_io.nix b/garlic/exp/fwi/strong_scaling_io.nix new file mode 100644 index 0000000000000000000000000000000000000000..c063659529b1335a3b60c1c0641952a6d27aa0b7 --- /dev/null +++ b/garlic/exp/fwi/strong_scaling_io.nix @@ -0,0 +1,141 @@ +# Strong scaling test for FWI variants based on tasks with and without I/O. +# This experiment solves a computationally expensive input which brings the +# storage devices to saturation when I/O is enabled. the same input us run +# without I/O for comparison purposes.. Also, the experiments are runt for a +# range of block sizes deemed as efficient according to the granularity +# experiment. + +{ + stdenv +, stdexp +, bsc +, targetMachine +, stages +}: + +with stdenv.lib; + +let + + inherit (targetMachine) fs; + + # Initial variable configuration + varConf = { + gitBranch = [ + "garlic/tampi+send+oss+task" +# "garlic/mpi+send+omp+task" +# "garlic/mpi+send+oss+task" +# "garlic/mpi+send+seq" +# "garlic/oss+task" +# "garlic/omp+task" +# "garlic/seq" + ]; + + blocksize = [ 1 2 4 8 ]; + + n = [ + {nx=500; nz=500; ny=16000;} + ]; + + nodes = [ 1 2 4 8 16 ]; + + ioFreq = [ 9999 (-1) ]; + + }; + +# The c value contains something like: +# { +# n = { nx=500; ny=500; nz=500; } +# blocksize = 1; +# gitBranch = "garlic/tampi+send+oss+task"; +# } + + machineConfig = targetMachine.config; + + # Generate the complete configuration for each unit + genConf = with bsc; c: targetMachine.config // rec { + expName = "fwi"; + unitName = "${expName}-test"; + inherit (machineConfig) hw; + + cc = icc; + inherit (c) gitBranch blocksize; + + #nx = c.n.nx; + #ny = c.n.ny; + #nz = c.n.nz; + + # Same but shorter: + inherit (c.n) nx ny nz; + + fwiInput = bsc.apps.fwi.input.override { + inherit (c.n) nx ny nz; + }; + + # Other FWI parameters + ioFreq = c.ioFreq; + + # Repeat the execution of each unit several times + loops = 10; + #loops = 1; + + # Resources + cpusPerTask = hw.cpusPerSocket; + ntasksPerNode = 2; + nodes = c.nodes; + qos = "debug"; + time = "02:00:00"; + jobName = unitName; + + tracing = "no"; + + # Enable permissions to write in the local storage + extraMounts = [ fs.local.temp ]; + + }; + + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; + }; + + exec = {nextStage, conf, ...}: stages.exec { + inherit nextStage; + pre = '' + CDIR=$PWD + if [[ "${conf.tracing}" == "yes" ]]; then + export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf" + fi + EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN" + mkdir -p $EXECDIR + cd $EXECDIR + ln -fs ${conf.fwiInput}/InputModels InputModels || true + ''; + argv = [ + "${conf.fwiInput}/fwi_params.txt" + "${conf.fwiInput}/fwi_frequencies.txt" + conf.blocksize + "-1" # Fordward steps + "-1" # Backward steps + conf.ioFreq # Write/read frequency + ]; + post = '' + rm -rf Results || true + if [[ "${conf.tracing}" == "yes" ]]; then + mv trace_* $CDIR + fi + ''; + }; + + apps = bsc.garlic.apps; + + # FWI program + program = {nextStage, conf, ...}: apps.fwi.solver.override { + inherit (conf) cc gitBranch fwiInput; + }; + + pipeline = stdexp.stdPipeline ++ [ exec program ]; + +in + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/fwi/strong_scaling_mpionly.nix b/garlic/exp/fwi/strong_scaling_mpionly.nix new file mode 100644 index 0000000000000000000000000000000000000000..94ffd8d6899a6493e0b499fabf0b7466e46ce6a5 --- /dev/null +++ b/garlic/exp/fwi/strong_scaling_mpionly.nix @@ -0,0 +1,130 @@ +# Strong scaling test for FWI variants based exclusively on MPI. This +# experiment does not rely on block sizes. An MPI process is instantiated per +# core. + +{ + stdenv +, stdexp +, bsc +, targetMachine +, stages +}: + +with stdenv.lib; + +let + + inherit (targetMachine) fs; + + # Initial variable configuration + varConf = { + gitBranch = [ + "garlic/mpi+send+seq" + ]; + + blocksize = [ 0 ]; + + n = [ + {nx=100; nz=100; ny=8000;} + ]; + + # Not enough planes for 4, 8 and 16 nodes + nodes = [ 1 2 ]; + + }; + +# The c value contains something like: +# { +# n = { nx=500; ny=500; nz=500; } +# blocksize = 1; +# gitBranch = "garlic/tampi+send+oss+task"; +# } + + machineConfig = targetMachine.config; + + # Generate the complete configuration for each unit + genConf = with bsc; c: targetMachine.config // rec { + expName = "fwi"; + unitName = "${expName}-test"; + inherit (machineConfig) hw; + + cc = icc; + inherit (c) gitBranch blocksize; + + #nx = c.n.nx; + #ny = c.n.ny; + #nz = c.n.nz; + + # Same but shorter: + inherit (c.n) nx ny nz; + + fwiInput = bsc.apps.fwi.input.override { + inherit (c.n) nx ny nz; + }; + + # Other FWI parameters + ioFreq = -1; + + # Repeat the execution of each unit several times + loops = 10; + #loops = 1; + + # Resources + cpusPerTask = 1; + ntasksPerNode = hw.cpusPerNode; + nodes = c.nodes; + qos = "debug"; + time = "02:00:00"; + jobName = unitName; + + tracing = "no"; + + # Enable permissions to write in the local storage + extraMounts = [ fs.local.temp ]; + + }; + + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; + }; + + exec = {nextStage, conf, ...}: stages.exec { + inherit nextStage; + pre = '' + CDIR=$PWD + if [[ "${conf.tracing}" == "yes" ]]; then + export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf" + fi + EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN" + mkdir -p $EXECDIR + cd $EXECDIR + ln -fs ${conf.fwiInput}/InputModels InputModels || true + ''; + argv = [ + "${conf.fwiInput}/fwi_params.txt" + "${conf.fwiInput}/fwi_frequencies.txt" + "-1" # Fordward steps + "-1" # Backward steps + conf.ioFreq # Write/read frequency + ]; + post = '' + rm -rf Results || true + if [[ "${conf.tracing}" == "yes" ]]; then + mv trace_* $CDIR + fi + ''; + }; + + apps = bsc.garlic.apps; + + # FWI program + program = {nextStage, conf, ...}: apps.fwi.solver.override { + inherit (conf) cc gitBranch fwiInput; + }; + + pipeline = stdexp.stdPipeline ++ [ exec program ]; + +in + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/fwi/strong_scaling_task.nix b/garlic/exp/fwi/strong_scaling_task.nix new file mode 100644 index 0000000000000000000000000000000000000000..4ad78338d63a12b8d43af325870dcaabc5f30fab --- /dev/null +++ b/garlic/exp/fwi/strong_scaling_task.nix @@ -0,0 +1,133 @@ +# Strong scaling test for FWI variants based on tasks. This +# experiment explores a range of block sizes deemed as efficient +# according to the granularity experiment. + +{ + stdenv +, stdexp +, bsc +, targetMachine +, stages +}: + +with stdenv.lib; + +let + + inherit (targetMachine) fs; + + # Initial variable configuration + varConf = { + gitBranch = [ + "garlic/tampi+send+oss+task" + "garlic/tampi+isend+oss+task" + "garlic/mpi+send+omp+task" + "garlic/mpi+send+oss+task" + ]; + + blocksize = [ 1 2 4 8 16 ]; + + n = [ + {nx=100; nz=100; ny=8000;} + ]; + + nodes = [ 1 2 4 8 16 ]; + + }; + +# The c value contains something like: +# { +# n = { nx=500; ny=500; nz=500; } +# blocksize = 1; +# gitBranch = "garlic/tampi+send+oss+task"; +# } + + machineConfig = targetMachine.config; + + # Generate the complete configuration for each unit + genConf = with bsc; c: targetMachine.config // rec { + expName = "fwi"; + unitName = "${expName}-test"; + inherit (machineConfig) hw; + + cc = icc; + inherit (c) gitBranch blocksize; + + #nx = c.n.nx; + #ny = c.n.ny; + #nz = c.n.nz; + + # Same but shorter: + inherit (c.n) nx ny nz; + + fwiInput = bsc.apps.fwi.input.override { + inherit (c.n) nx ny nz; + }; + + # Other FWI parameters + ioFreq = -1; + + # Repeat the execution of each unit several times + loops = 10; + #loops = 1; + + # Resources + cpusPerTask = hw.cpusPerSocket; + ntasksPerNode = 2; + nodes = c.nodes; + qos = "debug"; + time = "02:00:00"; + jobName = unitName; + + tracing = "no"; + + # Enable permissions to write in the local storage + extraMounts = [ fs.local.temp ]; + + }; + + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; + }; + + exec = {nextStage, conf, ...}: stages.exec { + inherit nextStage; + pre = '' + CDIR=$PWD + if [[ "${conf.tracing}" == "yes" ]]; then + export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf" + fi + EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN" + mkdir -p $EXECDIR + cd $EXECDIR + ln -fs ${conf.fwiInput}/InputModels InputModels || true + ''; + argv = [ + "${conf.fwiInput}/fwi_params.txt" + "${conf.fwiInput}/fwi_frequencies.txt" + conf.blocksize + "-1" # Fordward steps + "-1" # Backward steps + conf.ioFreq # Write/read frequency + ]; + post = '' + rm -rf Results || true + if [[ "${conf.tracing}" == "yes" ]]; then + mv trace_* $CDIR + fi + ''; + }; + + apps = bsc.garlic.apps; + + # FWI program + program = {nextStage, conf, ...}: apps.fwi.solver.override { + inherit (conf) cc gitBranch fwiInput; + }; + + pipeline = stdexp.stdPipeline ++ [ exec program ]; + +in + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/fwi/sync_io.nix b/garlic/exp/fwi/sync_io.nix new file mode 100644 index 0000000000000000000000000000000000000000..59e791da80da5c445a93d1e492b1cdfcbef4f630 --- /dev/null +++ b/garlic/exp/fwi/sync_io.nix @@ -0,0 +1,138 @@ +# This experiment compares the effect of not using I/O versus using O_DIRECT | +# O_DSYNC enabled I/O. This is a reduced version of the strong_scaling_io +# experiment. + +{ + stdenv +, stdexp +, bsc +, targetMachine +, stages +}: + +with stdenv.lib; + +let + + inherit (targetMachine) fs; + + # Initial variable configuration + varConf = { + gitBranch = [ + "garlic/tampi+send+oss+task" +# "garlic/mpi+send+omp+task" +# "garlic/mpi+send+oss+task" +# "garlic/mpi+send+seq" +# "garlic/oss+task" +# "garlic/omp+task" +# "garlic/seq" + ]; + + blocksize = [ 1 ]; + + n = [ + {nx=500; nz=500; ny=16000;} + ]; + + nodes = [ 4 ]; + + ioFreq = [ 9999 (-1) ]; + + }; + +# The c value contains something like: +# { +# n = { nx=500; ny=500; nz=500; } +# blocksize = 1; +# gitBranch = "garlic/tampi+send+oss+task"; +# } + + machineConfig = targetMachine.config; + + # Generate the complete configuration for each unit + genConf = with bsc; c: targetMachine.config // rec { + expName = "fwi"; + unitName = "${expName}-test"; + inherit (machineConfig) hw; + + cc = icc; + inherit (c) gitBranch blocksize; + + #nx = c.n.nx; + #ny = c.n.ny; + #nz = c.n.nz; + + # Same but shorter: + inherit (c.n) nx ny nz; + + fwiInput = bsc.apps.fwi.input.override { + inherit (c.n) nx ny nz; + }; + + # Other FWI parameters + ioFreq = c.ioFreq; + + # Repeat the execution of each unit several times + loops = 10; + #loops = 1; + + # Resources + cpusPerTask = hw.cpusPerSocket; + ntasksPerNode = 2; + nodes = c.nodes; + qos = "debug"; + time = "02:00:00"; + jobName = unitName; + + tracing = "no"; + + # Enable permissions to write in the local storage + extraMounts = [ fs.local.temp ]; + + }; + + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; + }; + + exec = {nextStage, conf, ...}: stages.exec { + inherit nextStage; + pre = '' + CDIR=$PWD + if [[ "${conf.tracing}" == "yes" ]]; then + export NANOS6_CONFIG_OVERRIDE="version.instrument=ctf" + fi + EXECDIR="${fs.local.temp}/out/$GARLIC_USER/$GARLIC_UNIT/$GARLIC_RUN" + mkdir -p $EXECDIR + cd $EXECDIR + ln -fs ${conf.fwiInput}/InputModels InputModels || true + ''; + argv = [ + "${conf.fwiInput}/fwi_params.txt" + "${conf.fwiInput}/fwi_frequencies.txt" + conf.blocksize + "-1" # Fordward steps + "-1" # Backward steps + conf.ioFreq # Write/read frequency + ]; + post = '' + rm -rf Results || true + if [[ "${conf.tracing}" == "yes" ]]; then + mv trace_* $CDIR + fi + ''; + }; + + apps = bsc.garlic.apps; + + # FWI program + program = {nextStage, conf, ...}: apps.fwi.solver.override { + inherit (conf) cc gitBranch fwiInput; + }; + + pipeline = stdexp.stdPipeline ++ [ exec program ]; + +in + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/exp/index.nix b/garlic/exp/index.nix index 3a007fa4169840acdfd3d9368ab7c8c5d61cdb19..f6a518df0e68f9dbb5aa29d8d376b7b37bf4247d 100644 --- a/garlic/exp/index.nix +++ b/garlic/exp/index.nix @@ -97,6 +97,16 @@ test = callPackage ./lulesh/test.nix { }; }; + fwi = { + granularity = callPackage ./fwi/granularity.nix { }; + strong_scaling_task = callPackage ./fwi/strong_scaling_task.nix { }; + strong_scaling_forkjoin = callPackage ./fwi/strong_scaling_forkjoin.nix { }; + strong_scaling_mpionly = callPackage ./fwi/strong_scaling_mpionly.nix { }; + data_reuse = callPackage ./fwi/data_reuse.nix { }; + strong_scaling_io = callPackage ./fwi/strong_scaling_io.nix { }; + sync_io = callPackage ./fwi/sync_io.nix { }; + }; + osu = rec { latency = callPackage ./osu/latency.nix { }; latencyShm = latency.override { interNode = false; }; diff --git a/garlic/fig/fwi/granularity.R b/garlic/fig/fwi/granularity.R new file mode 100644 index 0000000000000000000000000000000000000000..40d6f75021b0cb796e9be760167c6da7128423c3 --- /dev/null +++ b/garlic/fig/fwi/granularity.R @@ -0,0 +1,71 @@ +library(ggplot2) +library(dplyr) +library(scales) +library(jsonlite) + +args=commandArgs(trailingOnly=TRUE) + +# Read the timetable from args[1] +input_file = "input.json" +if (length(args)>0) { input_file = args[1] } + +# Load the dataset in NDJSON format +dataset = jsonlite::stream_in(file(input_file)) %>% + jsonlite::flatten() + +# We only need the nblocks and time +df = select(dataset, config.blocksize, config.gitBranch, time) %>% + rename(blocksize=config.blocksize, gitBranch=config.gitBranch) %>% + group_by(blocksize, gitBranch) %>% + mutate(mtime = median(time)) %>% + ungroup() + +df$gitBranch = as.factor(df$gitBranch) +df$blocksize = as.factor(df$blocksize) + +ppi=300 +h=5 +w=5 + +#################################################################### +### Line Graph +#################################################################### +png("mtime.png", width=w*ppi, height=h*ppi, res=ppi) + +## Create the plot with the normalized time vs nblocks +p = ggplot(df, aes(x = blocksize, y=mtime, group=gitBranch, color=gitBranch)) + + geom_point() + + geom_line() + + theme_bw() + + labs(x="Blocksize", y="Median Time (s)", title="FWI granularity", + subtitle=input_file) + + theme(plot.subtitle=element_text(size=8)) + + theme(legend.position = c(0.5, 0.88)) + +# Render the plot +print(p) + +# Save the png image +dev.off() + +#################################################################### +### Line Graph +#################################################################### +png("time.png", width=w*ppi, height=h*ppi, res=ppi) + +## Create the plot with the normalized time vs nblocks +p = ggplot(df, aes(x = blocksize, y=time, group=gitBranch, color=gitBranch)) + + geom_point() + + geom_line() + + theme_bw() + + labs(x="Blocksize", y="Time (s)", title="FWI granularity", + subtitle=input_file) + + theme(plot.subtitle=element_text(size=8)) + + theme(legend.position = c(0.5, 0.88)) + +# Render the plot +print(p) + +# Save the png image +dev.off() + diff --git a/garlic/fig/fwi/strong_scaling.R b/garlic/fig/fwi/strong_scaling.R new file mode 100644 index 0000000000000000000000000000000000000000..89d79f155f3b1373464f809bfdaf603700952f2b --- /dev/null +++ b/garlic/fig/fwi/strong_scaling.R @@ -0,0 +1,120 @@ +library(ggplot2) +library(dplyr) +library(scales) +library(jsonlite) + +args=commandArgs(trailingOnly=TRUE) + +# Read the timetable from args[1] +input_file = "input.json" +if (length(args)>0) { input_file = args[1] } + +# Load the dataset in NDJSON format +dataset = jsonlite::stream_in(file(input_file)) %>% + jsonlite::flatten() + +# Select block size to display +useBlocksize = 2 + +# We only need the nblocks and time +df = select(dataset, config.blocksize, config.gitBranch, config.nodes, time) %>% + rename( + blocksize=config.blocksize, + gitBranch=config.gitBranch, + nodes=config.nodes + ) %>% + filter(blocksize == useBlocksize | blocksize == 0) %>% + group_by(nodes, gitBranch) %>% + mutate(mtime = median(time)) %>% + mutate(nxmtime = mtime * nodes) %>% + mutate(nxtime = time * nodes) %>% + ungroup() + +df$gitBranch = as.factor(df$gitBranch) +df$blocksize = as.factor(df$blocksize) +df$nodes = as.factor(df$nodes) + +ppi=300 +h=5 +w=5 + +#################################################################### +### Line plot (time) +#################################################################### +png("time.png", width=w*ppi, height=h*ppi, res=ppi) + +p = ggplot(df, aes(x=nodes, y=time, group=gitBranch, color=gitBranch)) + + geom_point() + + geom_line() + + theme_bw() + + labs(x="Nodes", y="Time (s)", title="FWI strong scaling", + subtitle=input_file) + + theme(plot.subtitle=element_text(size=8)) + + theme(legend.position = c(0.6, 0.75)) + +# Render the plot +print(p) + +# Save the png image +dev.off() + +#################################################################### +### Line plot (time x nodes) +#################################################################### +png("nxtime.png", width=w*ppi, height=h*ppi, res=ppi) + +p = ggplot(df, aes(x=nodes, y=nxtime, group=gitBranch, color=gitBranch)) + + geom_point() + + geom_line() + + theme_bw() + + labs(x="Nodes", y="Time * Nodes (s)", title="FWI strong scaling", + subtitle=input_file) + + theme(plot.subtitle=element_text(size=8)) + + theme(legend.position = c(0.15, 0.80)) + + theme(legend.text = element_text(size = 7)) + +# Render the plot +print(p) + +# Save the png image +dev.off() + +#################################################################### +### Line plot (median time) +#################################################################### +png("mediantime.png", width=w*ppi, height=h*ppi, res=ppi) + +p = ggplot(df, aes(x=nodes, y=mtime, group=gitBranch, color=gitBranch)) + + geom_point() + + geom_line() + + theme_bw() + + labs(x="Nodes", y="Median Time (s)", title="FWI strong scaling", + subtitle=input_file) + + theme(plot.subtitle=element_text(size=8)) + + theme(legend.position = c(0.5, 0.88)) + +# Render the plot +print(p) + +# Save the png image +dev.off() + +#################################################################### +### Line plot (nodes x median time) +#################################################################### +png("nxmtime.png", width=w*ppi, height=h*ppi, res=ppi) + +p = ggplot(df, aes(x=nodes, y=nxmtime, group=gitBranch, color=gitBranch)) + + geom_point() + + geom_line() + + theme_bw() + + labs(x="Nodes", y="Median Time * Nodes (s)", title="FWI strong scaling", + subtitle=input_file) + + theme(plot.subtitle=element_text(size=8)) + + theme(legend.position = c(0.5, 0.88)) + +# Render the plot +print(p) + +# Save the png image +dev.off() diff --git a/garlic/fig/fwi/strong_scaling_io.R b/garlic/fig/fwi/strong_scaling_io.R new file mode 100644 index 0000000000000000000000000000000000000000..415bafda886a64e7980edfabba37f0fc8f2e5acb --- /dev/null +++ b/garlic/fig/fwi/strong_scaling_io.R @@ -0,0 +1,122 @@ +library(ggplot2) +library(dplyr) +library(scales) +library(jsonlite) +library(forcats) + +args=commandArgs(trailingOnly=TRUE) + +# Read the timetable from args[1] +input_file = "input.json" +if (length(args)>0) { input_file = args[1] } + +# Load the dataset in NDJSON format +dataset = jsonlite::stream_in(file(input_file)) %>% + jsonlite::flatten() + +# We only need the nblocks and time +df = select(dataset, config.blocksize, config.ioFreq, config.gitBranch, config.nodes, time) %>% + rename( + blocksize=config.blocksize, + io=config.ioFreq, + gitBranch=config.gitBranch, + nodes=config.nodes + ) %>% + filter(blocksize == 1) %>% + group_by(nodes, gitBranch, io) %>% + mutate(mtime = median(time)) %>% + mutate(nxmtime = mtime * nodes) %>% + mutate(nxtime = time * nodes) %>% + ungroup() + +df$gitBranch = as.factor(df$gitBranch) +df$io = as.factor(df$io) +df$blocksize = as.factor(df$blocksize) +df$nodes = as.factor(df$nodes) + +df$io = fct_recode(df$io, enabled = "-1", disabled = "9999") + + +ppi=300 +h=5 +w=5 + +#################################################################### +### Line plot (time) +#################################################################### +png("time.png", width=w*ppi, height=h*ppi, res=ppi) + +p = ggplot(df, aes(x=nodes, y=time, group=io, color=io)) + + geom_point() + + geom_line() + + theme_bw() + + labs(x="Nodes", y="Time (s)", title="FWI strong scaling for mpi+send+oss+task", + subtitle=input_file) + + theme(plot.subtitle=element_text(size=8)) + + theme(legend.position = c(0.5, 0.88)) + +# Render the plot +print(p) + +# Save the png image +dev.off() + +#################################################################### +### Line plot (time x nodes) +#################################################################### +png("nxtime.png", width=w*ppi, height=h*ppi, res=ppi) + +p = ggplot(df, aes(x=nodes, y=nxtime, group=io, color=io)) + + geom_point() + + geom_line() + + theme_bw() + + labs(x="Nodes", y="Time * Nodes (s)", title="FWI strong scaling for mpi+send+oss+task", + subtitle=input_file) + + theme(plot.subtitle=element_text(size=8)) + + theme(legend.position = c(0.5, 0.88)) + +# Render the plot +print(p) + +# Save the png image +dev.off() + +##################################################################### +#### Line plot (median time) +##################################################################### +#png("mediantime.png", width=w*ppi, height=h*ppi, res=ppi) +# +#p = ggplot(df, aes(x=nodes, y=mtime, group=gitBranch, color=gitBranch)) + +# geom_point() + +# geom_line() + +# theme_bw() + +# labs(x="Nodes", y="Median Time (s)", title="FWI strong scaling", +# subtitle=input_file) + +# theme(plot.subtitle=element_text(size=8)) + +# theme(legend.position = c(0.5, 0.88)) +# +## Render the plot +#print(p) +# +## Save the png image +#dev.off() +# +##################################################################### +#### Line plot (nodes x median time) +##################################################################### +#png("nxmtime.png", width=w*ppi, height=h*ppi, res=ppi) +# +#p = ggplot(df, aes(x=nodes, y=nxmtime, group=gitBranch, color=gitBranch)) + +# geom_point() + +# geom_line() + +# theme_bw() + +# labs(x="Nodes", y="Median Time * Nodes (s)", title="FWI strong scaling", +# subtitle=input_file) + +# theme(plot.subtitle=element_text(size=8)) + +# theme(legend.position = c(0.5, 0.88)) +# +## Render the plot +#print(p) +# +## Save the png image +#dev.off() diff --git a/garlic/fig/index.nix b/garlic/fig/index.nix index c36f0b44a65d309273873c696667c09d9e3f2a48..b3f77eae1429b048ce6f3526ca5e8c1bd974ef30 100644 --- a/garlic/fig/index.nix +++ b/garlic/fig/index.nix @@ -61,6 +61,15 @@ in big.granularity = stdPlot ./creams/granularity.R [ big.granularity ]; }; + fwi = with exp.fwi; { + granularity = stdPlot ./fwi/granularity.R [ granularity ]; + strong_scaling = stdPlot ./fwi/strong_scaling.R [ strong_scaling_task strong_scaling_forkjoin ]; + #strong_scaling = stdPlot ./fwi/strong_scaling.R [ strong_scaling_task strong_scaling_forkjoin strong_scaling_mpionly ]; + data_reuse = stdPlot ./fwi/granularity.R [ data_reuse ]; + strong_scaling_io = stdPlot ./fwi/strong_scaling_io.R [ strong_scaling_io ]; + sync_io = stdPlot ./fwi/strong_scaling_io.R [ sync_io ]; + }; + osu = with exp.osu; { latency = customPlot ./osu/latency.R (ds.osu.latency latency.result); latencyShm = customPlot ./osu/latency.R (ds.osu.latency latencyShm.result);