From aeffe5b3998991bf59d958de6935f825923848d0 Mon Sep 17 00:00:00 2001 From: Andrei Oprea Date: Tue, 29 Sep 2015 14:58:23 -0700 Subject: [PATCH] Bug 1202265 - Add visual regression tool for Hello, r=dmose --- browser/components/loop/.gitignore | 3 + .../loop/test/visual-regression/README.md | 74 ++++++++++++++ .../loop/test/visual-regression/screenshot | 97 +++++++++++++++++++ browser/components/loop/ui/ui-showcase.js | 2 +- browser/components/loop/ui/ui-showcase.jsx | 2 +- 5 files changed, 176 insertions(+), 2 deletions(-) create mode 100644 browser/components/loop/test/visual-regression/README.md create mode 100755 browser/components/loop/test/visual-regression/screenshot diff --git a/browser/components/loop/.gitignore b/browser/components/loop/.gitignore index 30d74470d58..2c5d430816d 100644 --- a/browser/components/loop/.gitignore +++ b/browser/components/loop/.gitignore @@ -2,3 +2,6 @@ test/coverage/desktop test/coverage/shared_standalone test/node_modules +test/visual-regression/diff +test/visual-regression/new +test/visual-regression/refs diff --git a/browser/components/loop/test/visual-regression/README.md b/browser/components/loop/test/visual-regression/README.md new file mode 100644 index 00000000000..5ac4370c13d --- /dev/null +++ b/browser/components/loop/test/visual-regression/README.md @@ -0,0 +1,74 @@ +# How to use the visual regression tool + +Note that this is a *work-in-progress*; watch out for *rough edges*. In + particular, it can generate a fair number of false positives, most + visually fairly obvious. That said, for CSS rule changes that may effect + multiple views, it can be extremely handy for detecting subtle regressions. + +It works by taking a snapshot of each view in the ui-showcase, and then + comparing it to a previous version of the same snapshot, generating an image + which shows differences between the two. + +## Requirements: + +```bash +easy_install pip +pip install selenium +pip install Pillow +``` + +## File hierarchy + +./loop/test/visual-regression + - README + - screenshot -- the script that you will run + - /refs -- initial screenshots + - /new -- screenshots after changes have been made + - /diff -- combination of reference screenshots, new and the visual diff between the two + +You will need to create those 3 folders. + +## Commands + +``` +screenshot -h -- provides information +screenshot --refs -- creates reference file +screenshot --diffs -- creates new screenshots and the diff against the reference files +``` + +## Typical session, to test a patch for regression against fx-team: + +```bash +git checkout fx-team +cd browser/components/loop/test/visual-regression + +# create the dirs you'll need +mkdir refs new diff +# generate the reference images from the ui-showcase +# take a break; this takes at least 10 mins on a 2015 i7 Retina mac +./screenshot --refs + +# create a branch and apply the patch you want to check +git checkout -b bugzilla-bug-1234 +git bz apply 1234 + +# generate the new images, and then diff them. take another long break. +./screenshot --diffs + +cd diff + +# this is a mac command, other OSes will vary in how to view all the images +open -a Preview * + +# diffs have the reference image on left, the diff in the middle, and the +# new image on right, all joined into a single image so you can see them +# together. look for images that have red in the diff section to find potential +# regressions +``` + +## Note + +Because Selenium launches a separate browser window you can make changes to the +CSS or markup without having to worry that it will affect the screenshots. You +can generate the diffs after you have finished your changes before submitting +the patch. diff --git a/browser/components/loop/test/visual-regression/screenshot b/browser/components/loop/test/visual-regression/screenshot new file mode 100755 index 00000000000..81d1b042749 --- /dev/null +++ b/browser/components/loop/test/visual-regression/screenshot @@ -0,0 +1,97 @@ +#!/usr/bin/env python + +from selenium import webdriver +from PIL import Image +from os import listdir +from os.path import isfile, join +import subprocess +import argparse +import re +from os import path + +refs_location = path.join(path.dirname(__file__), "refs") +newscreenshots_location = path.join(path.dirname(__file__), "new") +diff_location = path.join(path.dirname(__file__), "diff") +ui_showcase_url = "http://localhost:3000/ui/" + +def take_screenshots(dir_name): + fox = webdriver.Firefox() + # Open the UI Showcase + fox.get(ui_showcase_url) + # Find all component containers + elements = fox.find_elements_by_css_selector(".example") + regex = re.compile(r"[/\\(),':\-]") + + for element in elements: + header = element.find_element_by_css_selector("h3") + component = element.find_element_by_css_selector(".comp iframe") + location = component.location + size = component.size + element_id = re.escape(re.sub(regex, '', header.get_attribute("id"))) + title = path.join(dir_name, element_id + ".png") + print title + + fox.save_screenshot(title) # saves screenshot of entire page + + im = Image.open(title) # uses PIL library to open image in memory + + left = location["x"] + top = location["y"] + right = location["x"] + size["width"] + bottom = location["y"] + size["height"] + + im = im.crop((left, top, right, bottom)) # defines crop points + im.save(title) # saves new cropped image + + fox.quit() + +def diff_screenshots(): + onlyfiles = [ f for f in listdir(refs_location) + if isfile(join(refs_location,f)) and f[-3:] == "png" ] + + for f in onlyfiles: + print f + ref_file = path.join(refs_location, f) + new_file = path.join(newscreenshots_location, f) + diff_file = path.join(diff_location, f) + compare_cmd = "compare '" + ref_file + "' '" + new_file + \ + "' '" + diff_file + "'" + join_cmd = "convert +append '" + ref_file + "' '" + diff_file + \ + "' '" + new_file + "'" + " '" + diff_file + "_joined.png'" + remove_cmd = "rm " + diff_file + + cmd = compare_cmd + " ; " + join_cmd + " ; " + remove_cmd + print cmd + subprocess.Popen(cmd, shell=True) + +def cleanup(clean_location): + files = [ f for f in listdir(clean_location) + if isfile(join(clean_location,f)) and f[-3:] == "png" ] + + for f in files: + remove_cmd = "rm " + re.escape(path.join(clean_location, f)) + subprocess.Popen(remove_cmd, shell=True) + +description = "Firefox Hello test for visual regressions. Run once before " + \ + "making any changes with --refs to build the reference files " + \ + "and once more with your patch applied using --diffs. If any " + \ + "screenshots have changed they will show up in /test/diff." + +parser = argparse.ArgumentParser(description=description) +parser.add_argument("-r", "--refs", action="store_true", help="Build the " + + "reference screenshots.") +parser.add_argument("-d", "--diffs", action="store_true", help="Compare " + + "your changes against the reference screenshots.") +parser.add_argument("-c", "--cleanup", action="store_true", help="Remove " + + "all files from /diff and /new folders but not the " + + "reference files") +args = parser.parse_args() + +if args.refs: + take_screenshots(refs_location) +if args.diffs: + take_screenshots(newscreenshots_location) + diff_screenshots() +if args.cleanup: + cleanup(newscreenshots_location) + cleanup(diff_location) diff --git a/browser/components/loop/ui/ui-showcase.js b/browser/components/loop/ui/ui-showcase.js index dbd62e515e5..f79e0e51a51 100644 --- a/browser/components/loop/ui/ui-showcase.js +++ b/browser/components/loop/ui/ui-showcase.js @@ -1370,7 +1370,7 @@ React.createElement(FramedExample, { dashed: true, height: 254, - summary: "", + summary: "Desktop Room Failure View", width: 298}, React.createElement("div", {className: "fx-embedded"}, React.createElement(RoomFailureView, { diff --git a/browser/components/loop/ui/ui-showcase.jsx b/browser/components/loop/ui/ui-showcase.jsx index 6cda5f82e51..b8c468759bb 100644 --- a/browser/components/loop/ui/ui-showcase.jsx +++ b/browser/components/loop/ui/ui-showcase.jsx @@ -1370,7 +1370,7 @@