From ea66d7e4e0b5203831ab545f59e2f2919c094868 Mon Sep 17 00:00:00 2001 From: Antoni Navarro Date: Wed, 24 Mar 2021 10:12:05 +0100 Subject: [PATCH 1/5] nbody: update granularity tests --- garlic/exp/index.nix | 6 +- garlic/exp/nbody/granularity-mpi.nix | 61 ++++++++---------- garlic/exp/nbody/granularity-oss.nix | 38 ++++++------ garlic/fig/index.nix | 11 ++-- garlic/fig/nbody/granularity.R | 92 ++++++++++++++++++++++++++++ 5 files changed, 143 insertions(+), 65 deletions(-) create mode 100644 garlic/fig/nbody/granularity.R diff --git a/garlic/exp/index.nix b/garlic/exp/index.nix index 35453371..814fd4a1 100644 --- a/garlic/exp/index.nix +++ b/garlic/exp/index.nix @@ -9,6 +9,8 @@ { nbody = rec { baseline = callPackage ./nbody/nblocks.nix { }; + granularity = callPackage ./nbody/granularity-mpi.nix { }; + scaling = callPackage ./nbody/scaling.nix { }; # Experiment variants small = baseline.override { @@ -25,10 +27,6 @@ }; }; - - scaling = callPackage ./nbody/scaling.nix { - particles = 12 * 4096; - }; }; saiph = { diff --git a/garlic/exp/nbody/granularity-mpi.nix b/garlic/exp/nbody/granularity-mpi.nix index 4c1d6a85..59eae238 100644 --- a/garlic/exp/nbody/granularity-mpi.nix +++ b/garlic/exp/nbody/granularity-mpi.nix @@ -4,48 +4,42 @@ , bsc , targetMachine , stages +, garlicTools }: with stdenv.lib; +with garlicTools; let # Initial variable configuration varConf = with bsc; { blocksize = [ 128 256 512 1024 2048 4096 ]; + gitBranch = [ + "garlic/mpi+send+oss+task" + "garlic/tampi+send+oss+task" + "garlic/tampi+isend+oss+task" + ]; }; - machineConfig = targetMachine.config; - # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - expName = "nbody.test"; - unitName = "${expName}.nb-${toString nblocks}"; - - inherit (machineConfig) hw; - # nbody options - particles = 1024 * 64; + genConf = c: targetMachine.config // rec { + hw = targetMachine.config.hw; + particles = 4096 * hw.cpusPerSocket; timesteps = 10; - inherit (c) blocksize; - totalTasks = ntasksPerNode * nodes; - particlesPerTask = particles / totalTasks; - cc = icc; - mpi = impi; - gitBranch = "garlic/mpi+send"; + blocksize = c.blocksize; + gitBranch = c.gitBranch; - # Repeat the execution of each unit 30 times - loops = 10; + expName = "nbody-granularity"; + unitName = expName + "-${toString gitBranch}" + "-bs${toString blocksize}"; - # Resources - qos = "debug"; - ntasksPerNode = 48; - nodes = 1; - time = "02:00:00"; - cpuBind = "sockets,verbose"; - jobName = "bs-${toString blocksize}-${gitBranch}-nbody"; + loops = 30; - # Experiment revision: this allows a user to run again a experiment already - # executed - rev = 0; + qos = "bsc_cs"; + ntasksPerNode = 1; + nodes = 1; + time = "04:00:00"; + cpusPerTask = hw.cpusPerSocket; + jobName = unitName; }; # Compute the array of configurations @@ -53,18 +47,13 @@ let inherit varConf genConf; }; - exec = {nextStage, conf, ...}: with conf; stages.exec { + exec = {nextStage, conf, ...}: stages.exec { inherit nextStage; - argv = [ "-t" timesteps "-p" particles ]; + argv = [ "-t" conf.timesteps "-p" conf.particles ]; }; - program = {nextStage, conf, ...}: with conf; - # FIXME: This is becoming very slow: - #let - # customPkgs = stdexp.replaceMpi conf.mpi; - #in - bsc.garlic.apps.nbody.override { - inherit cc blocksize mpi gitBranch; + program = {nextStage, conf, ...}: with conf; bsc.garlic.apps.nbody.override { + inherit (conf) blocksize gitBranch; }; pipeline = stdexp.stdPipeline ++ [ exec program ]; diff --git a/garlic/exp/nbody/granularity-oss.nix b/garlic/exp/nbody/granularity-oss.nix index 1becb3ac..1b1dd364 100644 --- a/garlic/exp/nbody/granularity-oss.nix +++ b/garlic/exp/nbody/granularity-oss.nix @@ -4,36 +4,38 @@ , bsc , targetMachine , stages +, garlicTools }: with stdenv.lib; +with garlicTools; let + # Initial variable configuration - varConf = with bsc; { + varConf = { blocksize = [ 128 256 512 1024 2048 4096 ]; }; # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - # nbody options - particles = 1024 * 64; + genConf = c: targetMachine.config // rec { + hw = targetMachine.config.hw; + particles = 4096 * hw.cpusPerSocket; timesteps = 10; - inherit (c) blocksize; - cc = icc; - mpi = impi; + blocksize = c.blocksize; + gitBranch = "garlic/oss+task"; + expName = "nbody-granularity"; + unitName = expName + "-bs${toString blocksize}"; - # Repeat the execution of each unit 30 times loops = 30; - # Resources qos = "debug"; ntasksPerNode = 1; nodes = 1; time = "02:00:00"; - cpuBind = "sockets,verbose"; - jobName = "nbody-bs-${toString blocksize}-${gitBranch}"; + cpusPerTask = hw.cpusPerSocket; + jobName = unitName; }; # Compute the array of configurations @@ -41,18 +43,14 @@ let inherit varConf genConf; }; - exec = {nextStage, conf, ...}: with conf; stages.exec { + exec = {nextStage, conf, ...}: stages.exec { inherit nextStage; - argv = [ "-t" timesteps "-p" particles ]; + argv = [ "-t" conf.timesteps "-p" conf.particles ]; }; - program = {nextStage, conf, ...}: with conf; - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.nbody.override { - inherit cc blocksize mpi gitBranch; - }; + program = {nextStage, conf, ...}: with conf; bsc.garlic.apps.nbody.override { + inherit (conf) blocksize gitBranch; + }; pipeline = stdexp.stdPipeline ++ [ exec program ]; diff --git a/garlic/fig/index.nix b/garlic/fig/index.nix index 63da83eb..d9bb9bb4 100644 --- a/garlic/fig/index.nix +++ b/garlic/fig/index.nix @@ -30,11 +30,12 @@ let in { nbody = with exp.nbody; { - baseline = stdPlot ./nbody/baseline.R [ baseline ]; - small = stdPlot ./nbody/baseline.R [ small ]; - jemalloc = stdPlot ./nbody/jemalloc.R [ baseline jemalloc ]; - ctf = stdPlot ./nbody/baseline.R [ ctf ]; - scaling = stdPlot ./nbody/baseline.R [ scaling ]; + baseline = stdPlot ./nbody/baseline.R [ baseline ]; + small = stdPlot ./nbody/baseline.R [ small ]; + jemalloc = stdPlot ./nbody/jemalloc.R [ baseline jemalloc ]; + ctf = stdPlot ./nbody/baseline.R [ ctf ]; + scaling = stdPlot ./nbody/baseline.R [ scaling ]; + granularity = stdPlot ./nbody/granularity.R [ granularity ]; }; hpcg = with exp.hpcg; { diff --git a/garlic/fig/nbody/granularity.R b/garlic/fig/nbody/granularity.R new file mode 100644 index 00000000..495f0e33 --- /dev/null +++ b/garlic/fig/nbody/granularity.R @@ -0,0 +1,92 @@ +library(ggplot2) +library(dplyr, warn.conflicts = FALSE) +library(scales) +library(jsonlite) +library(viridis, warn.conflicts = FALSE) + +# Load the arguments (argv) +args = commandArgs(trailingOnly=TRUE) +if (length(args)>0) { input_file = args[1] } else { input_file = "input" } + +df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% + jsonlite::flatten() %>% + select(config.blocksize, config.gitBranch, unit, time) %>% + rename(blocksize=config.blocksize, branch=config.gitBranch) %>% + + mutate(blocksize = as.factor(blocksize)) %>% + mutate(branch = as.factor(branch)) %>% + mutate(unit = as.factor(unit)) %>% + + group_by(unit) %>% + + mutate(median.time = median(time)) %>% + mutate(normalized.time = time / median.time - 1) %>% + mutate(log.median.time = log(median.time)) %>% + + ungroup() + +dpi = 300 +h = 5 +w = 8 + +# --------------------------------------------------------------------- + +p = ggplot(df, aes(x=blocksize, y=median.time, color=branch)) + + geom_point() + + geom_line(aes(group=branch)) + + theme_bw() + + labs(x="Blocksize", y="Median time (s)", title="NBody Granularity: Median Time", + subtitle=input_file) + + theme(plot.subtitle=element_text(size=5)) + + theme(legend.position="bottom") + + theme(legend.text = element_text(size=7)) + +ggsave("median.time.png", plot=p, width=w, height=h, dpi=dpi) +ggsave("median.time.pdf", plot=p, width=w, height=h, dpi=dpi) + +# --------------------------------------------------------------------- + +p = ggplot(df, aes(x=blocksize, y=normalized.time, color=branch)) + + geom_boxplot() + + geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + + facet_wrap(~ branch) + + theme_bw() + + labs(x="Blocksize", y="Normalized Time", title="NBody Granularity: Normalized Time", + subtitle=input_file) + + theme(plot.subtitle=element_text(size=5)) + + theme(legend.position="bottom") + + theme(legend.text = element_text(size=7)) + +ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) +ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) + +# --------------------------------------------------------------------- + +p = ggplot(df, aes(x=blocksize, y=time, color=branch)) + + geom_point(shape=21, size=3) + + theme_bw() + + labs(x="Blocksize", y="Time (s)", title="NBody Granularity: Time", + subtitle=input_file) + + theme(plot.subtitle=element_text(size=5)) + + theme(legend.position="bottom") + + theme(legend.text = element_text(size=7)) + +ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) +ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) + + +# --------------------------------------------------------------------- + +p = ggplot(df, aes(x=blocksize, y=branch, fill=median.time)) + + geom_raster() + + scale_fill_viridis(option="plasma") + + coord_fixed() + + theme_bw() + + labs(x="Blocksize", y="Branch", title="NBody Granularity: Time", + subtitle=input_file) + + theme(plot.subtitle=element_text(size=5)) + + theme(legend.position="bottom") + + theme(legend.text = element_text(size=7)) + +ggsave("time.heatmap.png", plot=p, width=w, height=h, dpi=dpi) +ggsave("time.heatmap.pdf", plot=p, width=w, height=h, dpi=dpi) -- GitLab From 5815a9af093a8c96b0239ea213d5907dc43cd0f4 Mon Sep 17 00:00:00 2001 From: Antoni Navarro Date: Wed, 24 Mar 2021 10:15:58 +0100 Subject: [PATCH 2/5] nbody: move "old" experiments to another folder --- garlic/exp/nbody/{ => old}/granularity-oss.nix | 0 garlic/exp/nbody/{ => old}/nblocks.nix | 0 garlic/exp/nbody/{ => old}/scaling.nix | 0 garlic/exp/nbody/{ => old}/strong-scaling-oss.nix | 0 garlic/exp/nbody/{ => old}/weak-scaling-mpi.nix | 0 garlic/exp/nbody/{ => old}/weak-scaling-oss.nix | 0 garlic/fig/nbody/{ => old}/baseline.R | 0 garlic/fig/nbody/{ => old}/freeCpu.R | 0 garlic/fig/nbody/{ => old}/jemalloc.R | 0 garlic/fig/nbody/{ => old}/scaling.R | 0 10 files changed, 0 insertions(+), 0 deletions(-) rename garlic/exp/nbody/{ => old}/granularity-oss.nix (100%) rename garlic/exp/nbody/{ => old}/nblocks.nix (100%) rename garlic/exp/nbody/{ => old}/scaling.nix (100%) rename garlic/exp/nbody/{ => old}/strong-scaling-oss.nix (100%) rename garlic/exp/nbody/{ => old}/weak-scaling-mpi.nix (100%) rename garlic/exp/nbody/{ => old}/weak-scaling-oss.nix (100%) rename garlic/fig/nbody/{ => old}/baseline.R (100%) rename garlic/fig/nbody/{ => old}/freeCpu.R (100%) rename garlic/fig/nbody/{ => old}/jemalloc.R (100%) rename garlic/fig/nbody/{ => old}/scaling.R (100%) diff --git a/garlic/exp/nbody/granularity-oss.nix b/garlic/exp/nbody/old/granularity-oss.nix similarity index 100% rename from garlic/exp/nbody/granularity-oss.nix rename to garlic/exp/nbody/old/granularity-oss.nix diff --git a/garlic/exp/nbody/nblocks.nix b/garlic/exp/nbody/old/nblocks.nix similarity index 100% rename from garlic/exp/nbody/nblocks.nix rename to garlic/exp/nbody/old/nblocks.nix diff --git a/garlic/exp/nbody/scaling.nix b/garlic/exp/nbody/old/scaling.nix similarity index 100% rename from garlic/exp/nbody/scaling.nix rename to garlic/exp/nbody/old/scaling.nix diff --git a/garlic/exp/nbody/strong-scaling-oss.nix b/garlic/exp/nbody/old/strong-scaling-oss.nix similarity index 100% rename from garlic/exp/nbody/strong-scaling-oss.nix rename to garlic/exp/nbody/old/strong-scaling-oss.nix diff --git a/garlic/exp/nbody/weak-scaling-mpi.nix b/garlic/exp/nbody/old/weak-scaling-mpi.nix similarity index 100% rename from garlic/exp/nbody/weak-scaling-mpi.nix rename to garlic/exp/nbody/old/weak-scaling-mpi.nix diff --git a/garlic/exp/nbody/weak-scaling-oss.nix b/garlic/exp/nbody/old/weak-scaling-oss.nix similarity index 100% rename from garlic/exp/nbody/weak-scaling-oss.nix rename to garlic/exp/nbody/old/weak-scaling-oss.nix diff --git a/garlic/fig/nbody/baseline.R b/garlic/fig/nbody/old/baseline.R similarity index 100% rename from garlic/fig/nbody/baseline.R rename to garlic/fig/nbody/old/baseline.R diff --git a/garlic/fig/nbody/freeCpu.R b/garlic/fig/nbody/old/freeCpu.R similarity index 100% rename from garlic/fig/nbody/freeCpu.R rename to garlic/fig/nbody/old/freeCpu.R diff --git a/garlic/fig/nbody/jemalloc.R b/garlic/fig/nbody/old/jemalloc.R similarity index 100% rename from garlic/fig/nbody/jemalloc.R rename to garlic/fig/nbody/old/jemalloc.R diff --git a/garlic/fig/nbody/scaling.R b/garlic/fig/nbody/old/scaling.R similarity index 100% rename from garlic/fig/nbody/scaling.R rename to garlic/fig/nbody/old/scaling.R -- GitLab From 48a61dc2928e324f5f332b3818c4178dc408e193 Mon Sep 17 00:00:00 2001 From: Antoni Navarro Date: Wed, 24 Mar 2021 10:16:54 +0100 Subject: [PATCH 3/5] nbody: update indexes --- garlic/exp/index.nix | 18 ------------------ garlic/fig/index.nix | 5 ----- 2 files changed, 23 deletions(-) diff --git a/garlic/exp/index.nix b/garlic/exp/index.nix index 814fd4a1..17ace05f 100644 --- a/garlic/exp/index.nix +++ b/garlic/exp/index.nix @@ -8,25 +8,7 @@ { nbody = rec { - baseline = callPackage ./nbody/nblocks.nix { }; granularity = callPackage ./nbody/granularity-mpi.nix { }; - scaling = callPackage ./nbody/scaling.nix { }; - - # Experiment variants - small = baseline.override { - particles = 12 * 4096; - }; - - # Some experiments with traces - trace = { - # Only one unit repeated 30 times - baseline = small.override { - enableCTF = true; - loops = 30; - steps = 1; - }; - - }; }; saiph = { diff --git a/garlic/fig/index.nix b/garlic/fig/index.nix index d9bb9bb4..5ba49e5d 100644 --- a/garlic/fig/index.nix +++ b/garlic/fig/index.nix @@ -30,11 +30,6 @@ let in { nbody = with exp.nbody; { - baseline = stdPlot ./nbody/baseline.R [ baseline ]; - small = stdPlot ./nbody/baseline.R [ small ]; - jemalloc = stdPlot ./nbody/jemalloc.R [ baseline jemalloc ]; - ctf = stdPlot ./nbody/baseline.R [ ctf ]; - scaling = stdPlot ./nbody/baseline.R [ scaling ]; granularity = stdPlot ./nbody/granularity.R [ granularity ]; }; -- GitLab From 58294d4467d502adfe6aa02c3228095b82fc48bb Mon Sep 17 00:00:00 2001 From: Antoni Navarro Date: Thu, 25 Mar 2021 12:07:50 +0100 Subject: [PATCH 4/5] nbody: add "nodes or sockets" experiment --- garlic/exp/index.nix | 1 + garlic/exp/nbody/nodes-or-sockets-mpi.nix | 72 ++++++++++ garlic/fig/index.nix | 1 + garlic/fig/nbody/nodes-or-sockets.R | 168 ++++++++++++++++++++++ 4 files changed, 242 insertions(+) create mode 100644 garlic/exp/nbody/nodes-or-sockets-mpi.nix create mode 100644 garlic/fig/nbody/nodes-or-sockets.R diff --git a/garlic/exp/index.nix b/garlic/exp/index.nix index 17ace05f..5be0f681 100644 --- a/garlic/exp/index.nix +++ b/garlic/exp/index.nix @@ -9,6 +9,7 @@ { nbody = rec { granularity = callPackage ./nbody/granularity-mpi.nix { }; + nodesorsockets = callPackage ./nbody/nodes-or-sockets-mpi.nix { }; }; saiph = { diff --git a/garlic/exp/nbody/nodes-or-sockets-mpi.nix b/garlic/exp/nbody/nodes-or-sockets-mpi.nix new file mode 100644 index 00000000..44c0c16e --- /dev/null +++ b/garlic/exp/nbody/nodes-or-sockets-mpi.nix @@ -0,0 +1,72 @@ +{ + stdenv +, stdexp +, bsc +, targetMachine +, stages +, garlicTools +, numactl +}: + +with stdenv.lib; +with garlicTools; + +let + # Initial variable configuration + varConf = with bsc; { + blocksize = [ 256 512 1024 ]; + gitBranch = [ "garlic/tampi+send+oss+task" ]; + attachToSocket = [ true false ]; + numactl = [ true false ]; + }; + + # Generate the complete configuration for each unit + genConf = c: targetMachine.config // rec { + hw = targetMachine.config.hw; + particles = 4 * 4096 * hw.cpusPerSocket; + timesteps = 10; + blocksize = c.blocksize; + gitBranch = c.gitBranch; + socketAtt = c.attachToSocket; + useNumact = c.numactl; + + expName = "nbody-granularity"; + attachName = if (socketAtt) then "PerSocket" else "PerNode"; + numaName = if (useNumact) then "True" else "False"; + unitName = expName + + "-${toString gitBranch}" + + "-bs${toString blocksize}" + + "-ranks${toString attachName}" + + "-useNuma${toString numaName}"; + + loops = 30; + + qos = "debug"; + ntasksPerNode = if (socketAtt) then 2 else 1; + nodes = 4; + time = "02:00:00"; + cpusPerTask = if (socketAtt) then hw.cpusPerSocket else 2*hw.cpusPerSocket; + jobName = unitName; + }; + + # Compute the array of configurations + configs = stdexp.buildConfigs { + inherit varConf genConf; + }; + + exec = {nextStage, conf, ...}: stages.exec ({ + inherit nextStage; + argv = [ "-t" conf.timesteps "-p" conf.particles ]; + } // optionalAttrs (conf.useNumact) { + program = "${numactl}/bin/numactl --interleave=all ${stageProgram nextStage}"; + }); + + program = {nextStage, conf, ...}: with conf; bsc.garlic.apps.nbody.override { + inherit (conf) blocksize gitBranch; + }; + + pipeline = stdexp.stdPipeline ++ [ exec program ]; + +in + + stdexp.genExperiment { inherit configs pipeline; } diff --git a/garlic/fig/index.nix b/garlic/fig/index.nix index 5ba49e5d..a488c60c 100644 --- a/garlic/fig/index.nix +++ b/garlic/fig/index.nix @@ -31,6 +31,7 @@ in { nbody = with exp.nbody; { granularity = stdPlot ./nbody/granularity.R [ granularity ]; + nodesorsockets = stdPlot ./nbody/nodes-or-sockets.R [ nodesorsockets ]; }; hpcg = with exp.hpcg; { diff --git a/garlic/fig/nbody/nodes-or-sockets.R b/garlic/fig/nbody/nodes-or-sockets.R new file mode 100644 index 00000000..21b6b3fa --- /dev/null +++ b/garlic/fig/nbody/nodes-or-sockets.R @@ -0,0 +1,168 @@ +library(ggplot2) +library(dplyr, warn.conflicts = FALSE) +library(scales) +library(jsonlite) +library(viridis, warn.conflicts = FALSE) + +# Load the arguments (argv) +args = commandArgs(trailingOnly=TRUE) +if (length(args)>0) { input_file = args[1] } else { input_file = "input" } + +dfNuma = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% + jsonlite::flatten() %>% + select(config.blocksize, config.gitBranch, config.socketAtt, config.useNumact, unit, time) %>% + rename(blocksize=config.blocksize, branch=config.gitBranch, attachment=config.socketAtt, usenuma=config.useNumact) %>% + + mutate(blocksize = as.factor(blocksize)) %>% + mutate(branch = as.factor(branch)) %>% + mutate(attachment = as.factor(attachment)) %>% + mutate(usenuma = as.factor(usenuma)) %>% + mutate(unit = as.factor(unit)) %>% + + group_by(unit) %>% + + mutate(median.time = median(time)) %>% + mutate(normalized.time = time / median.time - 1) %>% + mutate(log.median.time = log(median.time)) %>% + + filter(usenuma == TRUE) %>% + + ungroup() + +dfNonuma = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% + jsonlite::flatten() %>% + select(config.blocksize, config.gitBranch, config.socketAtt, config.useNumact, unit, time) %>% + rename(blocksize=config.blocksize, branch=config.gitBranch, attachment=config.socketAtt, usenuma=config.useNumact) %>% + + mutate(blocksize = as.factor(blocksize)) %>% + mutate(branch = as.factor(branch)) %>% + mutate(attachment = as.factor(attachment)) %>% + mutate(usenuma = as.factor(usenuma)) %>% + mutate(unit = as.factor(unit)) %>% + + group_by(unit) %>% + + mutate(median.time = median(time)) %>% + mutate(normalized.time = time / median.time - 1) %>% + mutate(log.median.time = log(median.time)) %>% + + filter(usenuma == FALSE) %>% + + ungroup() + +dpi = 300 +h = 5 +w = 8 + + +# --------------------------------------------------------------------- + +p = ggplot(dfNuma, aes(x=blocksize, y=median.time, color=attachment)) + + geom_point() + + geom_line(aes(group=attachment)) + + theme_bw() + + labs(x="Blocksize", y="Median time (s)", title="NBody Granularity (tampi+send+oss+task | numactl ON | 4 Nodes): Median Time", + subtitle=input_file, color="Rank Attachment") + + theme(plot.subtitle=element_text(size=5)) + + theme(legend.position="bottom") + + scale_color_manual(labels = c("RanksPerNode", "RanksPerSocket"), values=c("blue", "red")) + +ggsave("median-numactl.time.png", plot=p, width=w, height=h, dpi=dpi) +ggsave("median-numactl.time.pdf", plot=p, width=w, height=h, dpi=dpi) + +p = ggplot(dfNonuma, aes(x=blocksize, y=median.time, color=attachment)) + + geom_point() + + geom_line(aes(group=attachment)) + + theme_bw() + + labs(x="Blocksize", y="Median time (s)", title="NBody Granularity (tampi+send+oss+task | numactl OFF | 4 Nodes): Median Time", + subtitle=input_file, color="Rank Attachment") + + theme(plot.subtitle=element_text(size=5)) + + theme(legend.position="bottom") + + scale_color_manual(labels = c("RanksPerNode", "RanksPerSocket"), values=c("blue", "red")) + +ggsave("median.time.png", plot=p, width=w, height=h, dpi=dpi) +ggsave("median.time.pdf", plot=p, width=w, height=h, dpi=dpi) +# --------------------------------------------------------------------- + +p = ggplot(dfNuma, aes(x=blocksize, y=normalized.time, color=attachment)) + + geom_boxplot() + + geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + + facet_wrap(~ attachment) + + theme_bw() + + labs(x="Blocksize", y="Normalized Time", title="NBody Granularity (tampi+send+oss+task | numactl ON | 4 Nodes): Normalized Time", + subtitle=input_file, color="Rank Attachment") + + theme(plot.subtitle=element_text(size=5)) + + theme(legend.position="bottom") + + scale_color_manual(labels = c("RanksPerNode", "RanksPerSocket"), values=c("blue", "red")) + +ggsave("normalized-numactl.time.png", plot=p, width=w, height=h, dpi=dpi) +ggsave("normalized-numactl.time.pdf", plot=p, width=w, height=h, dpi=dpi) + +p = ggplot(dfNonuma, aes(x=blocksize, y=normalized.time, color=attachment)) + + geom_boxplot() + + geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + + facet_wrap(~ attachment) + + theme_bw() + + labs(x="Blocksize", y="Normalized Time", title="NBody Granularity (tampi+send+oss+task | 4 Nodes): Normalized Time", + subtitle=input_file, color="Rank Attachment") + + theme(plot.subtitle=element_text(size=5)) + + theme(legend.position="bottom") + + scale_color_manual(labels = c("RanksPerNode", "RanksPerSocket"), values=c("blue", "red")) + +ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) +ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) +# --------------------------------------------------------------------- + +p = ggplot(dfNuma, aes(x=blocksize, y=time, color=attachment)) + + geom_point(shape=21, size=3) + + theme_bw() + + labs(x="Blocksize", y="Time (s)", title="NBody Granularity (tampi+send+oss+task | numactl ON | 4 Nodes): Time", + subtitle=input_file, color="Rank Attachment") + + theme(plot.subtitle=element_text(size=5)) + + theme(legend.position="bottom") + + scale_color_manual(labels = c("RanksPerNode", "RanksPerSocket"), values=c("blue", "red")) + +ggsave("time-numactl.png", plot=p, width=w, height=h, dpi=dpi) +ggsave("time-numactl.pdf", plot=p, width=w, height=h, dpi=dpi) + +p = ggplot(dfNonuma, aes(x=blocksize, y=time, color=attachment)) + + geom_point(shape=21, size=3) + + theme_bw() + + labs(x="Blocksize", y="Time (s)", title="NBody Granularity (tampi+send+oss+task | 4 Nodes): Time", + subtitle=input_file, color="Rank Attachment") + + theme(plot.subtitle=element_text(size=5)) + + theme(legend.position="bottom") + + scale_color_manual(labels = c("RanksPerNode", "RanksPerSocket"), values=c("blue", "red")) + + +ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) +ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) +# --------------------------------------------------------------------- + +p = ggplot(dfNuma, aes(x=blocksize, y=attachment, fill=median.time)) + + geom_raster() + + scale_fill_viridis(option="plasma") + + coord_fixed() + + theme_bw() + + labs(x="Blocksize", y="Attachment", title="NBody Granularity (tampi+send+oss+task | numactl ON | 4 Nodes): Time", + subtitle=input_file, color = "Rank Attachment") + + theme(plot.subtitle=element_text(size=5)) + + theme(legend.position="bottom") + + scale_color_manual(labels = c("RanksPerNode", "RanksPerSocket"), values=c("blue", "red")) + +ggsave("time-numactl.heatmap.png", plot=p, width=w, height=h, dpi=dpi) +ggsave("time-numactl.heatmap.pdf", plot=p, width=w, height=h, dpi=dpi) + +p = ggplot(dfNonuma, aes(x=blocksize, y=attachment, fill=median.time)) + + geom_raster() + + scale_fill_viridis(option="plasma") + + coord_fixed() + + theme_bw() + + labs(x="Blocksize", y="Attachment", title="NBody Granularity (tampi+send+oss+task | 4 Nodes): Time", + subtitle=input_file, color = "Rank Attachment") + + theme(plot.subtitle=element_text(size=5)) + + theme(legend.position="bottom") + + scale_color_manual(labels = c("RanksPerNode", "RanksPerSocket"), values=c("blue", "red")) + +ggsave("time.heatmap.png", plot=p, width=w, height=h, dpi=dpi) +ggsave("time.heatmap.pdf", plot=p, width=w, height=h, dpi=dpi) -- GitLab From 03298228e49452e2ebda7bd434d434816dd452c1 Mon Sep 17 00:00:00 2001 From: Antoni Navarro Date: Fri, 26 Mar 2021 18:16:17 +0100 Subject: [PATCH 5/5] nbody: add strong scaling experiment --- garlic/exp/index.nix | 1 + garlic/exp/nbody/strong-scaling-mpi.nix | 51 +++++++------- garlic/fig/index.nix | 1 + garlic/fig/nbody/scaling.R | 93 +++++++++++++++++++++++++ 4 files changed, 122 insertions(+), 24 deletions(-) create mode 100644 garlic/fig/nbody/scaling.R diff --git a/garlic/exp/index.nix b/garlic/exp/index.nix index 5be0f681..5477fd5c 100644 --- a/garlic/exp/index.nix +++ b/garlic/exp/index.nix @@ -10,6 +10,7 @@ nbody = rec { granularity = callPackage ./nbody/granularity-mpi.nix { }; nodesorsockets = callPackage ./nbody/nodes-or-sockets-mpi.nix { }; + scaling = callPackage ./nbody/strong-scaling-mpi.nix { }; }; saiph = { diff --git a/garlic/exp/nbody/strong-scaling-mpi.nix b/garlic/exp/nbody/strong-scaling-mpi.nix index b7e77c21..3ca70411 100644 --- a/garlic/exp/nbody/strong-scaling-mpi.nix +++ b/garlic/exp/nbody/strong-scaling-mpi.nix @@ -4,37 +4,44 @@ , bsc , targetMachine , stages +, garlicTools }: with stdenv.lib; +with garlicTools; let # Initial variable configuration varConf = with bsc; { - numProcsAndParticles = [ 1 2 4 8 16 32 48 ]; + blocksize = [ 512 ]; + nodes = [ 1 2 4 8 16 ]; + gitBranch = [ + "garlic/mpi+send+oss+task" + "garlic/tampi+send+oss+task" + "garlic/tampi+isend+oss+task" + ]; }; # Generate the complete configuration for each unit - genConf = with bsc; c: targetMachine.config // rec { - # nbody options - inherit (c) numProcsAndParticles; - particles = 1024 * numProcsAndParticles * 2; + genConf = c: targetMachine.config // rec { + hw = targetMachine.config.hw; + particles = 16 * 4096 * hw.cpusPerSocket; timesteps = 10; - blocksize = 1024; - cc = icc; - mpi = impi; - gitBranch = "garlic/mpi+send"; + blocksize = c.blocksize; + numNodes = c.nodes; + gitBranch = c.gitBranch; + + expName = "nbody-scaling"; + unitName = expName + "-${toString gitBranch}" + "-nodes${toString numNodes}"; - # Repeat the execution of each unit 30 times loops = 30; - # Resources + nodes = numNodes; qos = "debug"; - ntasksPerNode = numProcsAndParticles; - nodes = 1; + ntasksPerNode = 2; time = "02:00:00"; - cpuBind = "sockets,verbose"; - jobName = "nbody-bs-${toString numProcsAndParticles}-${gitBranch}"; + cpusPerTask = hw.cpusPerSocket; + jobName = unitName; }; # Compute the array of configurations @@ -42,18 +49,14 @@ let inherit varConf genConf; }; - exec = {nextStage, conf, ...}: with conf; stages.exec { + exec = {nextStage, conf, ...}: stages.exec { inherit nextStage; - argv = [ "-t" timesteps "-p" particles ]; + argv = [ "-t" conf.timesteps "-p" conf.particles ]; }; - program = {nextStage, conf, ...}: with conf; - let - customPkgs = stdexp.replaceMpi conf.mpi; - in - customPkgs.apps.nbody.override { - inherit cc blocksize mpi gitBranch; - }; + program = {nextStage, conf, ...}: with conf; bsc.garlic.apps.nbody.override { + inherit (conf) blocksize gitBranch; + }; pipeline = stdexp.stdPipeline ++ [ exec program ]; diff --git a/garlic/fig/index.nix b/garlic/fig/index.nix index a488c60c..263a574d 100644 --- a/garlic/fig/index.nix +++ b/garlic/fig/index.nix @@ -32,6 +32,7 @@ in nbody = with exp.nbody; { granularity = stdPlot ./nbody/granularity.R [ granularity ]; nodesorsockets = stdPlot ./nbody/nodes-or-sockets.R [ nodesorsockets ]; + scaling = stdPlot ./nbody/scaling.R [ scaling ]; }; hpcg = with exp.hpcg; { diff --git a/garlic/fig/nbody/scaling.R b/garlic/fig/nbody/scaling.R new file mode 100644 index 00000000..60a9e4a1 --- /dev/null +++ b/garlic/fig/nbody/scaling.R @@ -0,0 +1,93 @@ +library(ggplot2) +library(dplyr, warn.conflicts = FALSE) +library(scales) +library(jsonlite) +library(viridis, warn.conflicts = FALSE) + +# Load the arguments (argv) +args = commandArgs(trailingOnly=TRUE) +if (length(args)>0) { input_file = args[1] } else { input_file = "input" } + +df = jsonlite::stream_in(file(input_file), verbose=FALSE) %>% + jsonlite::flatten() %>% + select(config.blocksize, config.gitBranch, config.numNodes, unit, time) %>% + rename(nodes = config.numNodes, blocksize=config.blocksize, branch=config.gitBranch) %>% + + mutate(blocksize = as.factor(blocksize)) %>% + mutate(nodes = as.factor(nodes)) %>% + mutate(branch = as.factor(branch)) %>% + mutate(unit = as.factor(unit)) %>% + + group_by(unit) %>% + + mutate(median.time = median(time)) %>% + mutate(normalized.time = time / median.time - 1) %>% + mutate(log.median.time = log(median.time)) %>% + + ungroup() + +dpi = 300 +h = 5 +w = 8 + +# --------------------------------------------------------------------- + +p = ggplot(df, aes(x=nodes, y=median.time, color=branch)) + + geom_point() + + geom_line(aes(group=branch)) + + theme_bw() + + labs(x="Nodes", y="Median time (s)", title="NBody Scaling: Median Time", + subtitle=input_file) + + theme(plot.subtitle=element_text(size=5)) + + theme(legend.position="bottom") + + theme(legend.text = element_text(size=7)) + +ggsave("median.time.png", plot=p, width=w, height=h, dpi=dpi) +ggsave("median.time.pdf", plot=p, width=w, height=h, dpi=dpi) + +# --------------------------------------------------------------------- + +p = ggplot(df, aes(x=nodes, y=normalized.time, color=branch)) + + geom_boxplot() + + geom_hline(yintercept=c(-0.01, 0.01), linetype="dashed", color="red") + + facet_wrap(~ branch) + + theme_bw() + + labs(x="Nodes", y="Normalized time (s)", title="NBody Scaling: Normalized Time", + subtitle=input_file) + + theme(plot.subtitle=element_text(size=5)) + + theme(legend.position="bottom") + + theme(legend.text = element_text(size=7)) + +ggsave("normalized.time.png", plot=p, width=w, height=h, dpi=dpi) +ggsave("normalized.time.pdf", plot=p, width=w, height=h, dpi=dpi) + +# --------------------------------------------------------------------- + +p = ggplot(df, aes(x=nodes, y=time, color=branch)) + + geom_point(shape=21, size=3) + + theme_bw() + + labs(x="Nodes", y="Time (s)", title="NBody Scaling: Time", + subtitle=input_file) + + theme(plot.subtitle=element_text(size=5)) + + theme(legend.position="bottom") + + theme(legend.text = element_text(size=7)) + +ggsave("time.png", plot=p, width=w, height=h, dpi=dpi) +ggsave("time.pdf", plot=p, width=w, height=h, dpi=dpi) + + +# --------------------------------------------------------------------- + +p = ggplot(df, aes(x=nodes, y=branch, fill=median.time)) + + geom_raster() + + scale_fill_viridis(option="plasma") + + coord_fixed() + + theme_bw() + + labs(x="Nodes", y="Branch", title="NBody Scaling: Time", + subtitle=input_file) + + theme(plot.subtitle=element_text(size=5)) + + theme(legend.position="bottom") + + theme(legend.text = element_text(size=7)) + +ggsave("time.heatmap.png", plot=p, width=w, height=h, dpi=dpi) +ggsave("time.heatmap.pdf", plot=p, width=w, height=h, dpi=dpi) -- GitLab