Bug 1301791 - Add a script to run octane N times and aggregate results as a CSV, and another script to plot that CSV data; r=sfink
authorNick Fitzgerald <fitzgen@gmail.com>
Mon, 12 Sep 2016 09:36:26 -0700
changeset 357073 eab4e2f9adb418a65995058aa97cec5547454118
parent 357072 9ff810febb0af9cf4e08ed5c1b4c902aef3f4f5f
child 357074 5c7368370ff94ecaa3863e199da6966c36d2fada
push id1324
push usermtabara@mozilla.com
push dateMon, 16 Jan 2017 13:07:44 +0000
treeherdermozilla-release@a01c49833940 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerssfink
bugs1301791
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1301791 - Add a script to run octane N times and aggregate results as a CSV, and another script to plot that CSV data; r=sfink
js/src/devtools/octane-csv.sh
js/src/devtools/plot-octane.R
new file mode 100755
--- /dev/null
+++ b/js/src/devtools/octane-csv.sh
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+set -e -o pipefail
+
+function echo_to_stderr {
+    echo "$1" 1>&2
+}
+
+function usage_and_exit {
+    echo_to_stderr "Usage:"
+    echo_to_stderr "    $0 <path-to-js> <number-of-iterations>"
+    echo_to_stderr
+    echo_to_stderr "Run octane <number-of-iterations> times, and aggregate the results"
+    echo_to_stderr "into one CSV file, which is written to stdout."
+    echo_to_stderr
+    echo_to_stderr "See the js/src/devtools/plot-octane.R script for plotting the"
+    echo_to_stderr "results."
+    echo_to_stderr
+    echo_to_stderr "Complete example usage with plotting:"
+    echo_to_stderr
+    echo_to_stderr "    \$ ./js/src/devtools/octane-csv.sh path/to/js 20 > control.csv"
+    echo_to_stderr
+    echo_to_stderr "    Next, apply some patch you'd like to test."
+    echo_to_stderr
+    echo_to_stderr "    \$ ./js/src/devtools/octane-csv.sh path/to/js 20 > variable.csv"
+    echo_to_stderr "    \$ ./js/src/devtools/plot-octane.R control.csv variable.csv"
+    echo_to_stderr
+    echo_to_stderr "    Open Rplots.pdf to view the results."
+    exit 1
+}
+
+if [[ "$#" != "2" ]]; then
+    usage_and_exit
+fi
+
+# Get the absolute, normalized $JS path, and ensure its an executable.
+
+JS_DIR=$(dirname $1)
+if [[ ! -d "$JS_DIR" ]]; then
+    echo_to_stderr "error: no such directory $JS_DIR"
+    echo_to_stderr
+    usage_and_exit
+fi
+
+JS=$(basename $1)
+cd "$JS_DIR" > /dev/null
+JS="$(pwd)/$JS"
+if [[ ! -e "$JS" ]]; then
+    echo_to_stderr "error: '$JS' is not executable"
+    echo_to_stderr
+    usage_and_exit
+fi
+cd - > /dev/null
+
+# Go to the js/src/octane directory.
+
+cd $(dirname $0)/../octane > /dev/null
+
+# Run octane and transform the results into CSV.
+#
+# Run once as a warm up, and to grab the column headers. Then run the benchmark
+# $ITERS times, grabbing just the data rows.
+
+echo_to_stderr "Warm up"
+"$JS" ./run.js | grep -v -- "----" | cut -f 1 -d ':' | tr '\n' ','
+echo
+
+ITERS=$2
+while [[ "$ITERS" -ge "1" ]]; do
+    echo_to_stderr "Iterations left: $ITERS"
+    "$JS" ./run.js | grep -v -- "----" | cut -f 2 -d ':' | tr '\n' ','
+    echo
+    ITERS=$((ITERS - 1))
+done
+
+echo_to_stderr "All done :)"
new file mode 100755
--- /dev/null
+++ b/js/src/devtools/plot-octane.R
@@ -0,0 +1,38 @@
+#!/usr/bin/env Rscript
+
+# Usage:
+#
+#     octane.R control.csv variable.csv
+#
+# Output will be placed in Rplots.pdf
+#
+# Remember: on Octane, higher is better!
+
+library(ggplot2)
+
+args <- commandArgs(trailingOnly = TRUE)
+
+# Reading in data.
+control <- read.table(args[1], sep=",", header=TRUE)
+variable <- read.table(args[2], sep=",", header=TRUE)
+
+# Pulling out columns that we want to plot.
+# Not totally necessary.
+ctrl <- control$Score..version.9.
+var <- variable$Score..version.9.
+
+# Concatenating the values we want to plot.
+score <- c(ctrl, var)
+# Creating a vector of labels for the data points.
+label <- c(rep("control", length(ctrl)), rep("variable", length(var)))
+
+# Creating a data frame of the score and label.
+data <- data.frame(label, score)
+
+# Now plotting!
+ggplot(data, aes(label, score, color=label, pch=label)) +
+  # Adding boxplot without the outliers.
+  geom_boxplot(outlier.shape=NA) +
+  # Adding jitter plot on top of the boxplot. If you want to spread the points
+  # more, increase jitter.
+  geom_jitter(position=position_jitter(width=0.05))