sanitycheck: various fixes for device testing

Device testing has various problems:

- the main thread doesn't halt the serial monitoring thread when an
  error occurs while flashing, meaning flash errors can look like
  timeouts

- serial_line is unbound if the ser.readline() call raises an
  exception, so the following "if serial_line" would raise NameError
  in that case

- the serial monitoring thread is not a daemon, so exceptions or
  errors in the handle() thread that cause thread exit will leave the
  process sitting around until the serial monitoring thread has a
  chance to exit

Fix these issues:

- Rewrite the serial monitoring thread as a select loop over the
  serial port file descriptor and a pipe shared by the main thread and
  serial monitoring thread. If flashing fails, the main thread uses
  the pipe to signal to the serial monitoring thread that it should
  exit.

- Make sure serial_line is always bound when read.

- Mark the serial thread daemonic.

Signed-off-by: Marti Bolivar <marti@foundries.io>

scripts/sanitycheck

@@ -603,15 +603,30 @@ class DeviceHandler(Handler):
         """
         super().__init__(instance)
 
-    def monitor_serial(self, ser, harness):
+    def monitor_serial(self, ser, halt_fileno, harness):
         log_out_fp = open(self.log, "wt")
 
+        ser_fileno = ser.fileno()
+        readlist = [halt_fileno, ser_fileno]
+
         while ser.isOpen():
+            readable, _, _ = select.select(readlist, [], [], self.timeout)
+
+            if halt_fileno in readable:
+                verbose('halted')
+                ser.close()
+                break
+            if ser_fileno not in readable:
+                continue        # Timeout.
+
+            serial_line = None
             try:
                 serial_line = ser.readline()
             except TypeError:
                 pass
 
+            # Just because ser_fileno has data doesn't mean an entire line
+            # is available yet.
             if serial_line:
                 sl = serial_line.decode('utf-8', 'ignore')
                 verbose("DEVICE: {0}".format(sl.rstrip()))
@@ -619,6 +634,7 @@ class DeviceHandler(Handler):
                 log_out_fp.write(sl)
                 log_out_fp.flush()
                 harness.handle(sl.rstrip())
+
             if harness.state:
                 ser.close()
                 break
@@ -651,8 +667,10 @@ class DeviceHandler(Handler):
         harness_import = HarnessImporter(harness_name)
         harness = harness_import.instance
         harness.configure(self.instance)
+        rpipe, wpipe = os.pipe()
 
-        t = threading.Thread(target=self.monitor_serial, args=(ser, harness))
+        t = threading.Thread(target=self.monitor_serial, daemon=True,
+                             args=(ser, rpipe, harness))
         t.start()
 
         try:
@@ -661,12 +679,11 @@ class DeviceHandler(Handler):
             else:
                 subprocess.check_output(command, stderr=subprocess.PIPE)
         except subprocess.CalledProcessError:
-            pass
+            os.write(wpipe, b'x')    # halt the thread
 
         t.join(self.timeout)
         if t.is_alive():
             out_state = "timeout"
-            ser.close()
 
         if ser.isOpen():
             ser.close()