servo: Merge #19244 - Submit test-perf CSV files to S3 (from asajeffrey:test-perf-submit-to-s3); r=jdm
authorAlan Jeffrey <ajeffrey@mozilla.com>
Tue, 21 Nov 2017 11:07:55 -0600
changeset 437561 e4920bac02144782ffd0c390dfc2222ad871cee0
parent 437560 d8321357c4a7577860801e5fced1e2257ec428bf
child 437562 686eee770f7f9b94c367e74080328ca0a8db0874
push id117
push userfmarier@mozilla.com
push dateTue, 28 Nov 2017 20:17:16 +0000
reviewersjdm
milestone59.0a1
servo: Merge #19244 - Submit test-perf CSV files to S3 (from asajeffrey:test-perf-submit-to-s3); r=jdm <!-- Please describe your changes on the following line: --> Submit CSV files to S3 rather than json files to Perfherder. --- <!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: --> - [X] `./mach build -d` does not report any errors - [X] `./mach test-tidy` does not report any errors - [X] These changes do not require tests because this is test infrastructure <!-- Also, please make sure that "Allow edits from maintainers" checkbox is checked, so that we can help you if you get stuck somewhere along the way.--> <!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. --> Source-Repo: https://github.com/servo/servo Source-Revision: 755fa371cb96dd89f5b3e07c09043fc138347ef1
servo/etc/ci/performance/set_s3_policy.py
servo/etc/ci/performance/submit_to_s3.py
servo/etc/ci/performance/test_all.sh
servo/etc/ci/performance/test_perf.sh
servo/python/servo/testing_commands.py
new file mode 100644
--- /dev/null
+++ b/servo/etc/ci/performance/set_s3_policy.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import boto3
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=("Set the policy of the servo-perf bucket. "
+                     "Remember to set your S3 credentials "
+                     "https://github.com/boto/boto3"))
+    parser.parse_args()
+
+    s3 = boto3.resource('s3')
+    BUCKET = 'servo-perf'
+    POLICY = """{
+  "Version":"2012-10-17",
+  "Statement":[
+    {
+      "Effect":"Allow",
+      "Principal":"*",
+      "Action":[
+        "s3:ListBucket",
+        "s3:GetBucketLocation"
+      ],
+      "Resource":"arn:aws:s3:::servo-perf"
+    },
+    {
+      "Effect":"Allow",
+      "Principal":"*",
+      "Action":[
+        "s3:GetObject",
+        "s3:GetObjectAcl"
+      ],
+      "Resource":"arn:aws:s3:::servo-perf/*"
+    }
+  ]
+}"""
+
+    s3.BucketPolicy(BUCKET).put(Policy=POLICY)
+
+    print("Done!")
+
+
+if __name__ == "__main__":
+    main()
new file mode 100644
--- /dev/null
+++ b/servo/etc/ci/performance/submit_to_s3.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import boto3
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description=("Submit Servo performance data to S3. "
+                     "Remember to set your S3 credentials "
+                     "https://github.com/boto/boto3"))
+    parser.add_argument("perf_file",
+                        help="the output CSV file from runner")
+    parser.add_argument("perf_key",
+                        help="the S3 key to upload to")
+    args = parser.parse_args()
+
+    s3 = boto3.client('s3')
+    BUCKET = 'servo-perf'
+    s3.upload_file(args.perf_file, BUCKET, args.perf_key)
+
+    print("Done!")
+
+
+if __name__ == "__main__":
+    main()
--- a/servo/etc/ci/performance/test_all.sh
+++ b/servo/etc/ci/performance/test_all.sh
@@ -43,28 +43,23 @@ fi
 echo "Starting the local server"
 python3 -m http.server > /dev/null 2>&1 &
 
 # TODO: enable the full manifest when #11087 is fixed
 # https://github.com/servo/servo/issues/11087
 # MANIFEST="page_load_test/tp5n/20160509.manifest"
 MANIFEST="page_load_test/test.manifest" # A manifest that excludes
                                         # timeout test cases
-PERF_FILE="output/perf-$(uname -s)-$(uname -m)-$(date +%s).csv"
+PERF_KEY="perf-$(uname -s)-$(uname -m)-$(date +%s).csv"
+PERF_FILE="output/${PERF_KEY}"
 
 echo "Running tests"
 python3 runner.py ${engine} --runs 4 --timeout "${timeout}" --base "${base}" \
   "${MANIFEST}" "${PERF_FILE}"
 
 if [[ "${submit:-}" ]];
 then
-    echo "Submitting to Perfherder"
-    # Perfherder SSL check will fail if time is not accurate,
-    # sync time before you submit
-    # TODO: we are using Servo's revision hash for Gecko's result to make both
-    # results appear on the same date. Use the correct result when Perfherder
-    # allows us to change the date.
-    python3 submit_to_perfherder.py \
-            "${engine}" "${PERF_FILE}" servo/revision.json
+    echo "Submitting to S3"
+    python3 submit_to_s3.py "${PERF_FILE}" "${PERF_KEY}"
 fi
 
 echo "Stopping the local server"
 trap 'kill $(jobs -pr)' SIGINT SIGTERM EXIT
--- a/servo/etc/ci/performance/test_perf.sh
+++ b/servo/etc/ci/performance/test_perf.sh
@@ -24,15 +24,15 @@ if [[ ! -f "$(dirname "${TP5N_PATH}")/tp
     echo "done"
 else
     echo "Found existing test cases, skipping download and unzip."
 fi
 
 virtualenv venv --python="$(which python3)"
 PS1="" source venv/bin/activate
 # `PS1` must be defined before activating virtualenv
-pip install "treeherder-client>=3.0.0"
+pip install "boto3>=1.4.0"
 
 mkdir -p servo
 mkdir -p output # Test result will be saved to output/perf-<timestamp>.json
 ./git_log_to_json.sh > servo/revision.json
 
 ./test_all.sh --servo ${*}
--- a/servo/python/servo/testing_commands.py
+++ b/servo/python/servo/testing_commands.py
@@ -181,22 +181,16 @@ class MachCommands(CommandBase):
         self.set_software_rendering_env(True)
 
         self.ensure_bootstrapped()
         env = self.build_env()
         cmd = ["bash", "test_perf.sh"]
         if base:
             cmd += ["--base", base]
         if submit:
-            if not ("TREEHERDER_CLIENT_ID" in os.environ and
-                    "TREEHERDER_CLIENT_SECRET" in os.environ):
-                print("Please set the environment variable \"TREEHERDER_CLIENT_ID\""
-                      " and \"TREEHERDER_CLIENT_SECRET\" to submit the performance"
-                      " test result to perfherder")
-                return 1
             cmd += ["--submit"]
         return call(cmd,
                     env=env,
                     cwd=path.join("etc", "ci", "performance"))
 
     @Command('test-unit',
              description='Run unit tests',
              category='testing')