Bug 1202265 - Add visual regression tool for Hello, r=dmose

browser/components/loop/.gitignore

@@ -2,3 +2,6 @@
 test/coverage/desktop
 test/coverage/shared_standalone
 test/node_modules
+test/visual-regression/diff
+test/visual-regression/new
+test/visual-regression/refs

browser/components/loop/test/visual-regression/README.md

@@ -0,0 +1,74 @@
+# How to use the visual regression tool
+
+Note that this is a *work-in-progress*; watch out for *rough edges*.  In
+ particular, it can generate a fair number of false positives, most
+ visually fairly obvious.  That said, for CSS rule changes that may effect
+ multiple views, it can be extremely handy for detecting subtle regressions.
+
+It works by taking a snapshot of each view in the ui-showcase, and then
+ comparing it to a previous version of the same snapshot, generating an image
+ which shows differences between the two.
+
+## Requirements:
+
+```bash
+easy_install pip
+pip install selenium
+pip install Pillow
+```
+
+## File hierarchy
+
+./loop/test/visual-regression
+  - README
+  - screenshot -- the script that you will run
+  - /refs       -- initial screenshots
+  - /new        -- screenshots after changes have been made
+  - /diff       -- combination of reference screenshots, new and the visual diff between the two
+
+You will need to create those 3 folders.
+
+## Commands
+
+```
+screenshot -h      -- provides information
+screenshot --refs  -- creates reference file
+screenshot --diffs -- creates new screenshots and the diff against the reference files
+```
+
+## Typical session, to test a patch for regression against fx-team:
+
+```bash
+git checkout fx-team
+cd browser/components/loop/test/visual-regression
+
+# create the dirs you'll need
+mkdir refs new diff
+# generate the reference images from the ui-showcase
+# take a break; this takes at least 10 mins on a 2015 i7 Retina mac
+./screenshot --refs
+
+# create a branch and apply the patch you want to check
+git checkout -b bugzilla-bug-1234
+git bz apply 1234
+
+# generate the new images, and then diff them.  take another long break.
+./screenshot --diffs
+
+cd diff
+
+# this is a mac command, other OSes will vary in how to view all the images
+open -a Preview *
+
+# diffs have the reference image on left, the diff in the middle, and the
+# new image on right, all joined into a single image so you can see them
+# together.  look for images that have red in the diff section to find potential
+# regressions
+```
+
+## Note
+
+Because Selenium launches a separate browser window you can make changes to the
+CSS or markup without having to worry that it will affect the screenshots. You
+can generate the diffs after you have finished your changes before submitting
+the patch.

browser/components/loop/test/visual-regression/screenshot

@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+
+from selenium import webdriver
+from PIL import Image
+from os import listdir
+from os.path import isfile, join
+import subprocess
+import argparse
+import re
+from os import path
+
+refs_location = path.join(path.dirname(__file__), "refs")
+newscreenshots_location = path.join(path.dirname(__file__), "new")
+diff_location = path.join(path.dirname(__file__), "diff")
+ui_showcase_url = "http://localhost:3000/ui/"
+
+def take_screenshots(dir_name):
+    fox = webdriver.Firefox()
+    # Open the UI Showcase
+    fox.get(ui_showcase_url)
+    # Find all component containers
+    elements = fox.find_elements_by_css_selector(".example")
+    regex = re.compile(r"[/\\(),':\-]")
+
+    for element in elements:
+        header = element.find_element_by_css_selector("h3")
+        component = element.find_element_by_css_selector(".comp iframe")
+        location = component.location
+        size = component.size
+        element_id = re.escape(re.sub(regex, '', header.get_attribute("id")))
+        title = path.join(dir_name, element_id + ".png")
+        print title
+
+        fox.save_screenshot(title) # saves screenshot of entire page
+
+        im = Image.open(title) # uses PIL library to open image in memory
+
+        left = location["x"]
+        top = location["y"]
+        right = location["x"] + size["width"]
+        bottom = location["y"] + size["height"]
+
+        im = im.crop((left, top, right, bottom)) # defines crop points
+        im.save(title) # saves new cropped image
+
+    fox.quit()
+
+def diff_screenshots():
+    onlyfiles = [ f for f in listdir(refs_location)
+                  if isfile(join(refs_location,f)) and f[-3:] == "png" ]
+
+    for f in onlyfiles:
+        print f
+        ref_file = path.join(refs_location, f)
+        new_file = path.join(newscreenshots_location, f)
+        diff_file = path.join(diff_location, f)
+        compare_cmd = "compare '" + ref_file + "' '" + new_file + \
+              "' '" + diff_file + "'"
+        join_cmd = "convert +append '" + ref_file + "' '" + diff_file + \
+              "' '" + new_file + "'" + " '" + diff_file + "_joined.png'"
+        remove_cmd = "rm " + diff_file
+
+        cmd = compare_cmd + " ; " + join_cmd + " ; " + remove_cmd
+        print cmd
+        subprocess.Popen(cmd, shell=True)
+
+def cleanup(clean_location):
+    files = [ f for f in listdir(clean_location)
+              if isfile(join(clean_location,f)) and f[-3:] == "png" ]
+
+    for f in files:
+        remove_cmd = "rm " + re.escape(path.join(clean_location, f))
+        subprocess.Popen(remove_cmd, shell=True)
+
+description = "Firefox Hello test for visual regressions. Run once before " + \
+              "making any changes with --refs to build the reference files " + \
+              "and once more with your patch applied using --diffs. If any " + \
+              "screenshots have changed they will show up in /test/diff."
+
+parser = argparse.ArgumentParser(description=description)
+parser.add_argument("-r", "--refs", action="store_true", help="Build the " +
+                    "reference screenshots.")
+parser.add_argument("-d", "--diffs", action="store_true", help="Compare " +
+                    "your changes against the reference screenshots.")
+parser.add_argument("-c", "--cleanup", action="store_true", help="Remove " +
+                    "all files from /diff and /new folders but not the " +
+                    "reference files")
+args = parser.parse_args()
+
+if args.refs:
+    take_screenshots(refs_location)
+if args.diffs:
+    take_screenshots(newscreenshots_location)
+    diff_screenshots()
+if args.cleanup:
+    cleanup(newscreenshots_location)
+    cleanup(diff_location)

browser/components/loop/ui/ui-showcase.js

@@ -1370,7 +1370,7 @@
             React.createElement(FramedExample, {
               dashed: true, 
               height: 254, 
-              summary: "", 
+              summary: "Desktop Room Failure View", 
               width: 298}, 
               React.createElement("div", {className: "fx-embedded"}, 
                 React.createElement(RoomFailureView, {

browser/components/loop/ui/ui-showcase.jsx

@@ -1370,7 +1370,7 @@
             <FramedExample
               dashed={true}
               height={254}
-              summary=""
+              summary="Desktop Room Failure View"
               width={298}>
               <div className="fx-embedded">
                 <RoomFailureView