Adding labs framework. Submit gets labs_io.txt from the frontend and uses that to drive test cases. TODO: need to figure out how to compare test case stdout with expected values to evaluate success.

compile_server/app/checker.py

@@ -222,13 +222,23 @@ def run_program(request):
         return CrossDomainResponse(result)
 
     # Run the command(s) to check the program
-    commands = [
-        # Run the program
-        ["lxc", "exec", "safecontainer", "--", "su", "runner",
-         "-c",
-         "python /workspace/run.py /workspace/sessions/{} {}".format(
-            os.path.basename(tempd), mode)]
-    ]
+    if 'lab' in received_json:
+        lab = received_json['lab']
+        commands = [
+            # Run the program
+            ["lxc", "exec", "safecontainer", "--", "su", "runner",
+             "-c",
+             "python /workspace/run.py /workspace/sessions/{} {} {}".format(
+                os.path.basename(tempd), mode, lab)]
+        ]
+    else:
+        commands = [
+            # Run the program
+            ["lxc", "exec", "safecontainer", "--", "su", "runner",
+             "-c",
+             "python /workspace/run.py /workspace/sessions/{} {}".format(
+                os.path.basename(tempd), mode)]
+        ]
 
     print "\n".join(" ".join(c) for c in commands)
 

infrastructure/container_payload/run.py

@@ -24,6 +24,10 @@ DEBUG = False
 
 CLI_FILE = "cli.txt"
 
+LAB_IO_FILE = "lab_io.txt"
+
+LAB_IO_REGEX = re.compile("(in|out) (\d+): (.+)")
+
 
 COMMON_ADC = """
 pragma Restrictions (No_Specification_of_Aspect => Import);
@@ -140,7 +144,7 @@ def doctor_main_gpr(tempd, spark_mode=False):
     return main
 
 
-def safe_run(workdir, mode):
+def safe_run(workdir, mode, lab):
     def c(cl=[]):
         """Aux procedure, run the given command line and output to stdout"""
         try:
@@ -173,9 +177,9 @@ def safe_run(workdir, mode):
 
             # In "run" mode, first build, and then launch the main
             if c(["gprbuild", "-q", "-P", "main", "-gnatwa"]):
-                cli_txt = os.path.join(workdir, CLI_FILE)
 
                 # Check to see if cli.txt was sent from the front-end
+                cli_txt = os.path.join(workdir, CLI_FILE)
                 if os.path.isfile(cli_txt):
                     # If it is found, read contents into string and replace
                     #  newlines with spaces
@@ -195,6 +199,64 @@ def safe_run(workdir, mode):
                             'LD_PRELOAD=/preloader.so {} {}'.format(
                               os.path.join(workdir, main.split('.')[0]), cli)]
                     c(line)
+        elif mode == "submit":
+            main = doctor_main_gpr(workdir, False)
+
+            # In "submit" mode, first build, and then launch the main with test_cases
+            if c(["gprbuild", "-q", "-P", "main", "-gnatwa"]):
+                # Check to see if lab has IO resources
+                labio_txt = os.path.join(workdir, LAB_IO_FILE)
+                if os.path.isfile(labio_txt):
+                    # If it is found, read contents into string and replace
+                    #  newlines with spaces
+                    with open(labio_txt, 'r') as f:
+                        io_lines = f.readlines()
+
+                    # organize test instances
+                    test_cases = {}
+                    for line in io_lines:
+                        match = LAB_IO_REGEX.match(line)
+
+                        if match is not None:
+                            # found match(es)
+                            io = match.group(1)
+                            key = match.group(2)
+                            seq = match.group(3)
+
+                            if key in test_cases.keys():
+                                if io in test_cases[key].keys():
+                                    test_cases[key][io] += seq
+                                else:
+                                    test_cases[key][io] = seq
+                            else:
+                                test_cases[key] = {io: seq}
+
+                # Loop over IO resources and run all instances
+                for index, test in test_cases.iteritems():
+                    # check that this test case has defined ins and outs
+                    if "in" in test.keys() and "out" in test.keys():
+                        # We run:
+                        #  - as user 'unprivileged' that has no write access
+                        #  - under a timeout
+                        #  - with our ld preloader to prevent forks
+                        if main:
+                            line = ['sudo', '-u', 'unprivileged', 'timeout', '10s',
+                                    'bash', '-c',
+                                    'LD_PRELOAD=/preloader.so {} {}'.format(
+                                      os.path.join(workdir, main.split('.')[0]), test["in"])]
+                            c(line)
+                            # TODO: get output and put it in actual_out
+                            actual_out = ""
+                            if actual_out != test["out"]:
+                                print("Test case #{} failed. Output was {}. Expected {}.".format(index, actual_out, test["out"]))
+                                sys.exit(1)
+                            else:
+                                print("Test #{} passed.".format(index))
+
+                    else:
+                        print("Cannot run test case #{}".format(index))
+
+                print("All test cases passed. Lab completed.")
 
         elif mode == "prove":
             doctor_main_gpr(workdir, spark_mode=True)
@@ -227,12 +289,21 @@ def safe_run(workdir, mode):
 
 
 if __name__ == '__main__':
-    # Do not perform any sanity checking on args - this is not meant to
+    # perform some sanity checking on args - this is not meant to
     # be launched interactively
-    workdir = sys.argv[1]
-    mode = sys.argv[2]
+    if len(sys.argv) >= 2:
+        workdir = sys.argv[1]
+        mode = sys.argv[2]
+
+        if len(sys.argv) == 3:
+            lab = sys.argv[3]
+        else:
+            lab = None 
+    else:
+        print "Error invoking run"
+        sys.exit(1)
 
     # This is where the compiler is installed
     os.environ["PATH"] = "/gnat/bin:{}".format(os.environ["PATH"])
 
-    safe_run(workdir, mode)
+    safe_run(workdir, mode, lab)