From 9d952cc61fa7bf215e2a89f173cbe6a4b8d0008c Mon Sep 17 00:00:00 2001 From: Robert Tice Date: Mon, 1 Apr 2019 16:28:03 -0400 Subject: [PATCH 1/9] Refactoring duplicate code. --- infrastructure/container_payload/run.py | 108 ++++++++++++------------ 1 file changed, 56 insertions(+), 52 deletions(-) diff --git a/infrastructure/container_payload/run.py b/infrastructure/container_payload/run.py index 34f106b..4ae566e 100644 --- a/infrastructure/container_payload/run.py +++ b/infrastructure/container_payload/run.py @@ -26,7 +26,7 @@ CLI_FILE = "cli.txt" LAB_IO_FILE = "lab_io.txt" -LAB_IO_REGEX = re.compile("(in|out) *(\d+): *(.*)") +LAB_IO_REGEX = re.compile("(in|out) *(\d+): *(.*)") COMMON_ADC = """ @@ -174,46 +174,64 @@ def safe_run(workdir, mode, lab): traceback.print_exc() return False, output_lines + def get_build_line(extra_args=[]): + line = ["gprbuild", "-q", "-P", "main", "-gnatwa"] + line.extend(extra_args) + return line + + def build(extra_args=[]): + return c(get_build_line(extra_args)) + + def get_run_line(main, workdir, args): + # We run: + # - as user 'unprivileged' that has no write access + # - under a timeout + # - with our ld preloader to prevent forks + return ['sudo', '-u', 'unprivileged', 'timeout', '10s', + 'bash', '-c', + 'LD_PRELOAD=/preloader.so {} {}'.format( + os.path.join(workdir, main.split('.')[0]), args)] + + def run(main, workdir, args): + return c(get_run_line(main, workdir, args)) + + def get_prove_line(extra_args=[]): + return ["gnatprove", "-P", "main", "--checks-as-errors", + "--level=0", "--no-axiom-guard"].extend(extra_args) + + def prove(extra_args=[]): + return c(get_prove_line(extra_args)) + c(["echo"]) try: if mode == "run": main = doctor_main_gpr(workdir, False) # In "run" mode, first build, and then launch the main - if c(["gprbuild", "-q", "-P", "main", "-gnatwa"]): + if build(): # Check to see if cli.txt was sent from the front-end cli_txt = os.path.join(workdir, CLI_FILE) if os.path.isfile(cli_txt): # If it is found, read contents into string and replace # newlines with spaces - with open(cli_txt, 'r') as f: - cli = f.read().replace('\n', ' ') + cli = "`cat {}`".format(CLI_FILE); else: # otherwise pass no arguments to the main cli = "" - # We run: - # - as user 'unprivileged' that has no write access - # - under a timeout - # - with our ld preloader to prevent forks if main: - line = ['sudo', '-u', 'unprivileged', 'timeout', '10s', - 'bash', '-c', - 'LD_PRELOAD=/preloader.so {} {}'.format( - os.path.join(workdir, main.split('.')[0]), cli)] - c(line) + run(main, workdir, cli) elif mode == "submit": main = doctor_main_gpr(workdir, False) # In "submit" mode, first build, and then launch the main with test_cases - if c(["gprbuild", "-q", "-P", "main", "-gnatwa"]): + if build(): # Check to see if lab has IO resources labio_txt = os.path.join(workdir, LAB_IO_FILE) if os.path.isfile(labio_txt): - # If it is found, read contents into string and replace - # newlines with spaces + # If it is found, read contents with open(labio_txt, 'r') as f: io_lines = f.readlines() @@ -236,51 +254,37 @@ def safe_run(workdir, mode, lab): else: test_cases[key] = {io: seq} - # Loop over IO resources and run all instances in sorted order by test case number - for index, test in sorted(test_cases.items()): - # check that this test case has defined ins and outs - if "in" in test.keys() and "out" in test.keys(): - # We run: - # - as user 'unprivileged' that has no write access - # - under a timeout - # - with our ld preloader to prevent forks - if main: - line = ['sudo', '-u', 'unprivileged', 'timeout', '10s', - 'bash', '-c', - 'LD_PRELOAD=/preloader.so {} {}'.format( - os.path.join(workdir, main.split('.')[0]), test["in"])] - # TODO: get output and put it in actual_out - success, actual_out_list = c(line) - actual_out = " ".join(actual_out_list).replace('\n', '').replace('\r', '') - if actual_out != test["out"]: - print("Test case #{} failed. Output was {}. Expected {}.".format(index, actual_out, test["out"])) - sys.exit(1) - else: - print("Test #{} passed.".format(index)) + # Loop over IO resources and run all instances in sorted order by test case number + for index, test in sorted(test_cases.items()): + # check that this test case has defined ins and outs + if "in" in test.keys() and "out" in test.keys(): + if main: + success, actual_out_list = run(main, workdir, test["in"]) + actual_out = " ".join(actual_out_list).replace('\n', '').replace('\r', '') + if actual_out != test["out"]: + print("Test case #{} failed.\nOutput was: {}\nExpected: {}".format(index, actual_out, test["out"])) + sys.exit(1) + else: + print("Test #{} passed.".format(index)) - else: - print("Cannot run test case #{}".format(index)) - sys.exit(1) + else: + print("Cannot run test case #{}".format(index)) + sys.exit(1) - print("All test cases passed. Lab completed.") + print("All test cases passed. Lab completed.") + else: + # No lab IO resources defined. This is an error in the lab config + print("No submission criteria found for this lab. Please report this issue on https://github.com/AdaCore/learn/issues") elif mode == "prove": doctor_main_gpr(workdir, spark_mode=True) - line = ["gnatprove", "-P", "main", "--checks-as-errors", - "--level=0", "--no-axiom-guard"] - c(line) + prove() elif mode == "prove_flow": doctor_main_gpr(workdir, spark_mode=True) - line = ["gnatprove", "-P", "main", "--checks-as-errors", - "--level=0", "--no-axiom-guard", "--mode=flow"] - c(line) - + prove(["--mode=flow"]) elif mode == "prove_report_all": doctor_main_gpr(workdir, spark_mode=True) - line = ["gnatprove", "-P", "main", "--checks-as-errors", - "--level=0", "--no-axiom-guard", "--report=all"] - c(line) - + prove(["--report=all"]) else: print "mode not implemented" From 6bd6bc889f5d75c09b6f0b2712c5d45b0a2368b5 Mon Sep 17 00:00:00 2001 From: Robert Tice Date: Mon, 1 Apr 2019 16:37:50 -0400 Subject: [PATCH 2/9] Adding more error checking and refactoring sanity checks. --- infrastructure/container_payload/run.py | 27 +++++++++++++------------ 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/infrastructure/container_payload/run.py b/infrastructure/container_payload/run.py index 4ae566e..6165c8e 100644 --- a/infrastructure/container_payload/run.py +++ b/infrastructure/container_payload/run.py @@ -208,8 +208,7 @@ def safe_run(workdir, mode, lab): main = doctor_main_gpr(workdir, False) # In "run" mode, first build, and then launch the main - if build(): - + if build() and main: # Check to see if cli.txt was sent from the front-end cli_txt = os.path.join(workdir, CLI_FILE) if os.path.isfile(cli_txt): @@ -220,14 +219,15 @@ def safe_run(workdir, mode, lab): # otherwise pass no arguments to the main cli = "" - if main: - run(main, workdir, cli) + run(main, workdir, cli) + else: + print("Build failed...") elif mode == "submit": main = doctor_main_gpr(workdir, False) # In "submit" mode, first build, and then launch the main with test_cases - if build(): + if build() and main: # Check to see if lab has IO resources labio_txt = os.path.join(workdir, LAB_IO_FILE) if os.path.isfile(labio_txt): @@ -258,14 +258,13 @@ def safe_run(workdir, mode, lab): for index, test in sorted(test_cases.items()): # check that this test case has defined ins and outs if "in" in test.keys() and "out" in test.keys(): - if main: - success, actual_out_list = run(main, workdir, test["in"]) - actual_out = " ".join(actual_out_list).replace('\n', '').replace('\r', '') - if actual_out != test["out"]: - print("Test case #{} failed.\nOutput was: {}\nExpected: {}".format(index, actual_out, test["out"])) - sys.exit(1) - else: - print("Test #{} passed.".format(index)) + success, actual_out_list = run(main, workdir, test["in"]) + actual_out = " ".join(actual_out_list).replace('\n', '').replace('\r', '') + if actual_out != test["out"]: + print("Test case #{} failed.\nOutput was: {}\nExpected: {}".format(index, actual_out, test["out"])) + sys.exit(1) + else: + print("Test #{} passed.".format(index)) else: print("Cannot run test case #{}".format(index)) @@ -275,6 +274,8 @@ def safe_run(workdir, mode, lab): else: # No lab IO resources defined. This is an error in the lab config print("No submission criteria found for this lab. Please report this issue on https://github.com/AdaCore/learn/issues") + else: + print("Build failed...") elif mode == "prove": doctor_main_gpr(workdir, spark_mode=True) From 0f0f24c49777fa359ab7cb9f0c9038d17b703d7a Mon Sep 17 00:00:00 2001 From: Robert Tice Date: Mon, 1 Apr 2019 16:46:15 -0400 Subject: [PATCH 3/9] More refactoring. --- infrastructure/container_payload/run.py | 117 ++++++++++++------------ 1 file changed, 57 insertions(+), 60 deletions(-) diff --git a/infrastructure/container_payload/run.py b/infrastructure/container_payload/run.py index 6165c8e..30ce26d 100644 --- a/infrastructure/container_payload/run.py +++ b/infrastructure/container_payload/run.py @@ -196,84 +196,81 @@ def safe_run(workdir, mode, lab): return c(get_run_line(main, workdir, args)) def get_prove_line(extra_args=[]): - return ["gnatprove", "-P", "main", "--checks-as-errors", - "--level=0", "--no-axiom-guard"].extend(extra_args) + line = ["gnatprove", "-P", "main", "--checks-as-errors", + "--level=0", "--no-axiom-guard"] + line.extend(extra_args) + return line def prove(extra_args=[]): return c(get_prove_line(extra_args)) c(["echo"]) try: - if mode == "run": + if mode == "run" or mode == "submit": main = doctor_main_gpr(workdir, False) # In "run" mode, first build, and then launch the main if build() and main: - # Check to see if cli.txt was sent from the front-end - cli_txt = os.path.join(workdir, CLI_FILE) - if os.path.isfile(cli_txt): - # If it is found, read contents into string and replace - # newlines with spaces - cli = "`cat {}`".format(CLI_FILE); + if mode == "run": + # Check to see if cli.txt was sent from the front-end + cli_txt = os.path.join(workdir, CLI_FILE) + if os.path.isfile(cli_txt): + # If it is found, read contents into string and replace + # newlines with spaces + cli = "`cat {}`".format(CLI_FILE); + else: + # otherwise pass no arguments to the main + cli = "" + + run(main, workdir, cli) else: - # otherwise pass no arguments to the main - cli = "" + # mode == "submit" + # Check to see if lab has IO resources + labio_txt = os.path.join(workdir, LAB_IO_FILE) + if os.path.isfile(labio_txt): + # If it is found, read contents + with open(labio_txt, 'r') as f: + io_lines = f.readlines() - run(main, workdir, cli) - else: - print("Build failed...") + # organize test instances + test_cases = {} + for line in io_lines: + match = LAB_IO_REGEX.match(line) - elif mode == "submit": - main = doctor_main_gpr(workdir, False) + if match is not None: + # found match(es) + io = match.group(1) + key = match.group(2) + seq = match.group(3) - # In "submit" mode, first build, and then launch the main with test_cases - if build() and main: - # Check to see if lab has IO resources - labio_txt = os.path.join(workdir, LAB_IO_FILE) - if os.path.isfile(labio_txt): - # If it is found, read contents - with open(labio_txt, 'r') as f: - io_lines = f.readlines() - - # organize test instances - test_cases = {} - for line in io_lines: - match = LAB_IO_REGEX.match(line) - - if match is not None: - # found match(es) - io = match.group(1) - key = match.group(2) - seq = match.group(3) - - if key in test_cases.keys(): - if io in test_cases[key].keys(): - test_cases[key][io] += seq + if key in test_cases.keys(): + if io in test_cases[key].keys(): + test_cases[key][io] += seq + else: + test_cases[key][io] = seq else: - test_cases[key][io] = seq - else: - test_cases[key] = {io: seq} + test_cases[key] = {io: seq} - # Loop over IO resources and run all instances in sorted order by test case number - for index, test in sorted(test_cases.items()): - # check that this test case has defined ins and outs - if "in" in test.keys() and "out" in test.keys(): - success, actual_out_list = run(main, workdir, test["in"]) - actual_out = " ".join(actual_out_list).replace('\n', '').replace('\r', '') - if actual_out != test["out"]: - print("Test case #{} failed.\nOutput was: {}\nExpected: {}".format(index, actual_out, test["out"])) + # Loop over IO resources and run all instances in sorted order by test case number + for index, test in sorted(test_cases.items()): + # check that this test case has defined ins and outs + if "in" in test.keys() and "out" in test.keys(): + success, actual_out_list = run(main, workdir, test["in"]) + actual_out = " ".join(actual_out_list).replace('\n', '').replace('\r', '') + if actual_out != test["out"]: + print("Test case #{} failed.\nOutput was: {}\nExpected: {}".format(index, actual_out, test["out"])) + sys.exit(1) + else: + print("Test #{} passed.".format(index)) + + else: + print("Cannot run test case #{}".format(index)) sys.exit(1) - else: - print("Test #{} passed.".format(index)) - else: - print("Cannot run test case #{}".format(index)) - sys.exit(1) - - print("All test cases passed. Lab completed.") - else: - # No lab IO resources defined. This is an error in the lab config - print("No submission criteria found for this lab. Please report this issue on https://github.com/AdaCore/learn/issues") + print("All test cases passed. Lab completed.") + else: + # No lab IO resources defined. This is an error in the lab config + print("No submission criteria found for this lab. Please report this issue on https://github.com/AdaCore/learn/issues") else: print("Build failed...") From 227a1a74d1af49639bb1e7b0fa9fd6cf3f3fe34c Mon Sep 17 00:00:00 2001 From: Robert Tice Date: Tue, 2 Apr 2019 10:58:26 -0400 Subject: [PATCH 4/9] Fixing print messages for clarity and fixing input arg expansion problem. --- infrastructure/container_payload/run.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/infrastructure/container_payload/run.py b/infrastructure/container_payload/run.py index 30ce26d..0e88f84 100644 --- a/infrastructure/container_payload/run.py +++ b/infrastructure/container_payload/run.py @@ -26,7 +26,7 @@ CLI_FILE = "cli.txt" LAB_IO_FILE = "lab_io.txt" -LAB_IO_REGEX = re.compile("(in|out) *(\d+): *(.*)") +LAB_IO_REGEX = re.compile("(in|out) *(\d+): *(.*)") COMMON_ADC = """ @@ -158,7 +158,7 @@ def safe_run(workdir, mode, lab): while True: line = p.stdout.readline().replace(workdir, '.') if line != '': - print line + # print line output_lines.append(line) sys.stdout.flush() else: @@ -215,14 +215,16 @@ def safe_run(workdir, mode, lab): # Check to see if cli.txt was sent from the front-end cli_txt = os.path.join(workdir, CLI_FILE) if os.path.isfile(cli_txt): - # If it is found, read contents into string and replace - # newlines with spaces - cli = "`cat {}`".format(CLI_FILE); + cli = "`cat {}`".format(cli_txt); + with open(cli_txt, 'r') as f: + print("stdin: {}".format(f.read().replace('\n', ' '))) else: # otherwise pass no arguments to the main cli = "" - run(main, workdir, cli) + errno, stdout = run(main, workdir, cli) + for line in stdout: + print("stdout: {}".format(line)) else: # mode == "submit" # Check to see if lab has IO resources @@ -255,8 +257,11 @@ def safe_run(workdir, mode, lab): for index, test in sorted(test_cases.items()): # check that this test case has defined ins and outs if "in" in test.keys() and "out" in test.keys(): - success, actual_out_list = run(main, workdir, test["in"]) - actual_out = " ".join(actual_out_list).replace('\n', '').replace('\r', '') + print("stdin: {}".format(test["in"])) + errno, stdout = run(main, workdir, "`echo {}`".format(test["in"])) + for line in stdout: + print("stdout: {}".format(line)) + actual_out = " ".join(stdout).replace('\n', '').replace('\r', '') if actual_out != test["out"]: print("Test case #{} failed.\nOutput was: {}\nExpected: {}".format(index, actual_out, test["out"])) sys.exit(1) From 1933a2980808b49f2dfbd305c8922ea19e576624 Mon Sep 17 00:00:00 2001 From: Robert Tice Date: Tue, 2 Apr 2019 11:49:52 -0400 Subject: [PATCH 5/9] refactoring unneccesary functions. Increasing readability of lab output. --- infrastructure/container_payload/run.py | 39 +++++++++---------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/infrastructure/container_payload/run.py b/infrastructure/container_payload/run.py index 0e88f84..21d91bc 100644 --- a/infrastructure/container_payload/run.py +++ b/infrastructure/container_payload/run.py @@ -158,7 +158,7 @@ def safe_run(workdir, mode, lab): while True: line = p.stdout.readline().replace(workdir, '.') if line != '': - # print line + print("stdout: {}".format(line)) output_lines.append(line) sys.stdout.flush() else: @@ -174,35 +174,27 @@ def safe_run(workdir, mode, lab): traceback.print_exc() return False, output_lines - def get_build_line(extra_args=[]): + def build(extra_args): line = ["gprbuild", "-q", "-P", "main", "-gnatwa"] line.extend(extra_args) - return line + return c(line) - def build(extra_args=[]): - return c(get_build_line(extra_args)) - - def get_run_line(main, workdir, args): + def run(main, workdir, args): # We run: # - as user 'unprivileged' that has no write access # - under a timeout # - with our ld preloader to prevent forks - return ['sudo', '-u', 'unprivileged', 'timeout', '10s', + line = ['sudo', '-u', 'unprivileged', 'timeout', '10s', 'bash', '-c', 'LD_PRELOAD=/preloader.so {} {}'.format( os.path.join(workdir, main.split('.')[0]), args)] + return c(line) - def run(main, workdir, args): - return c(get_run_line(main, workdir, args)) - - def get_prove_line(extra_args=[]): + def prove(extra_args): line = ["gnatprove", "-P", "main", "--checks-as-errors", "--level=0", "--no-axiom-guard"] line.extend(extra_args) - return line - - def prove(extra_args=[]): - return c(get_prove_line(extra_args)) + return c(line) c(["echo"]) try: @@ -210,7 +202,7 @@ def safe_run(workdir, mode, lab): main = doctor_main_gpr(workdir, False) # In "run" mode, first build, and then launch the main - if build() and main: + if build([]) and main: if mode == "run": # Check to see if cli.txt was sent from the front-end cli_txt = os.path.join(workdir, CLI_FILE) @@ -222,9 +214,7 @@ def safe_run(workdir, mode, lab): # otherwise pass no arguments to the main cli = "" - errno, stdout = run(main, workdir, cli) - for line in stdout: - print("stdout: {}".format(line)) + run(main, workdir, cli) else: # mode == "submit" # Check to see if lab has IO resources @@ -239,7 +229,7 @@ def safe_run(workdir, mode, lab): for line in io_lines: match = LAB_IO_REGEX.match(line) - if match is not None: + if match: # found match(es) io = match.group(1) key = match.group(2) @@ -257,17 +247,16 @@ def safe_run(workdir, mode, lab): for index, test in sorted(test_cases.items()): # check that this test case has defined ins and outs if "in" in test.keys() and "out" in test.keys(): + print("---------------------------"); print("stdin: {}".format(test["in"])) errno, stdout = run(main, workdir, "`echo {}`".format(test["in"])) - for line in stdout: - print("stdout: {}".format(line)) actual_out = " ".join(stdout).replace('\n', '').replace('\r', '') if actual_out != test["out"]: print("Test case #{} failed.\nOutput was: {}\nExpected: {}".format(index, actual_out, test["out"])) sys.exit(1) else: print("Test #{} passed.".format(index)) - + print("---------------------------"); else: print("Cannot run test case #{}".format(index)) sys.exit(1) @@ -281,7 +270,7 @@ def safe_run(workdir, mode, lab): elif mode == "prove": doctor_main_gpr(workdir, spark_mode=True) - prove() + prove([]) elif mode == "prove_flow": doctor_main_gpr(workdir, spark_mode=True) prove(["--mode=flow"]) From 15088ef662c5903120facc87c4ec19b7799a14e0 Mon Sep 17 00:00:00 2001 From: Robert Tice Date: Tue, 2 Apr 2019 14:22:59 -0400 Subject: [PATCH 6/9] Restructuring print statements so that we can organize them on the frontend. --- infrastructure/container_payload/run.py | 71 ++++++++++++++++--------- 1 file changed, 45 insertions(+), 26 deletions(-) diff --git a/infrastructure/container_payload/run.py b/infrastructure/container_payload/run.py index 21d91bc..b5920f4 100644 --- a/infrastructure/container_payload/run.py +++ b/infrastructure/container_payload/run.py @@ -11,6 +11,7 @@ import io import re import os import codecs +import json import glob import time import sys @@ -148,38 +149,51 @@ def doctor_main_gpr(tempd, spark_mode=False): def safe_run(workdir, mode, lab): def c(cl=[]): """Aux procedure, run the given command line and output to stdout.""" - """Returns a tuple of (Boolean success, list stdout).""" - output_lines = [] + """Returns a tuple of (Boolean success, list stdout, returncode).""" + stdout_list = [] try: debug_print("running: {}".format(cl)) p = subprocess.Popen(cl, cwd=workdir, - stdout=subprocess.PIPE, shell=False) + stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) while True: - line = p.stdout.readline().replace(workdir, '.') - if line != '': - print("stdout: {}".format(line)) - output_lines.append(line) + stdout_line = p.stdout.readline().replace(workdir, '.') + stderr_line = p.stderr.readline().replace(workdir, '.') + + if stderr_line != '': + print("stderr:{}".format(stderr_line)) + sys.stderr.flush() + + if stdout_line != '': + print("stdout:{}".format(stdout_line)) + stdout_list.append(stdout_line) sys.stdout.flush() else: p.poll() break sys.stdout.flush() + sys.stderr.flush() + if p.returncode == INTERRUPT_RETURNCODE: print INTERRUPT_STRING - return True, output_lines + return True, stdout_list, p.returncode except Exception: print "ERROR when running {}".format(' '.join(cl)) traceback.print_exc() - return False, output_lines + return False, stdout_list, p.returncode def build(extra_args): + """Builds the application using static args and extra_args.""" + """Returns a tuple of (Boolean success, list stdout, returncode).""" line = ["gprbuild", "-q", "-P", "main", "-gnatwa"] line.extend(extra_args) return c(line) def run(main, workdir, args): + """Runs the application""" + """Returns a tuple of (Boolean success, list stdout, returncode).""" + # We run: # - as user 'unprivileged' that has no write access # - under a timeout @@ -191,25 +205,29 @@ def safe_run(workdir, mode, lab): return c(line) def prove(extra_args): + """Proves the application""" + """Returns a tuple of (Boolean success, list stdout, returncode).""" line = ["gnatprove", "-P", "main", "--checks-as-errors", "--level=0", "--no-axiom-guard"] line.extend(extra_args) return c(line) + # This is necessary to get the first line from the container. Otherwise + # the first line is lost. c(["echo"]) try: if mode == "run" or mode == "submit": main = doctor_main_gpr(workdir, False) - # In "run" mode, first build, and then launch the main - if build([]) and main: + # In "run" or "submit" mode, build, and then launch the main + if build([])[2] == 0 and main: if mode == "run": # Check to see if cli.txt was sent from the front-end cli_txt = os.path.join(workdir, CLI_FILE) if os.path.isfile(cli_txt): cli = "`cat {}`".format(cli_txt); with open(cli_txt, 'r') as f: - print("stdin: {}".format(f.read().replace('\n', ' '))) + print("stdin:{}".format(f.read().replace('\n', ' '))) else: # otherwise pass no arguments to the main cli = "" @@ -244,24 +262,25 @@ def safe_run(workdir, mode, lab): test_cases[key] = {io: seq} # Loop over IO resources and run all instances in sorted order by test case number + success = True for index, test in sorted(test_cases.items()): # check that this test case has defined ins and outs if "in" in test.keys() and "out" in test.keys(): - print("---------------------------"); - print("stdin: {}".format(test["in"])) - errno, stdout = run(main, workdir, "`echo {}`".format(test["in"])) - actual_out = " ".join(stdout).replace('\n', '').replace('\r', '') - if actual_out != test["out"]: - print("Test case #{} failed.\nOutput was: {}\nExpected: {}".format(index, actual_out, test["out"])) - sys.exit(1) - else: - print("Test #{} passed.".format(index)) - print("---------------------------"); - else: - print("Cannot run test case #{}".format(index)) - sys.exit(1) + print("stdin:{}".format(test["in"])) - print("All test cases passed. Lab completed.") + errno, stdout, retcode = run(main, workdir, "`echo {}`".format(test["in"])) + test["actual"] = " ".join(stdout).replace('\n', '').replace('\r', '') + + if test["actual"] != test["out"]: + test["status"] = "Failed" + success = False + else: + test["status"] = "Success" + else: + print("Malformed test IO sequence in test case #{}. Please report this issue on https://github.com/AdaCore/learn/issues".format(index)) + sys.exit(1) + lab_output = {"success": success, "test_cases": test_cases} + print("lab_output:{}".format(json.dumps(lab_output))) else: # No lab IO resources defined. This is an error in the lab config print("No submission criteria found for this lab. Please report this issue on https://github.com/AdaCore/learn/issues") From e1cd55ba1ad5fde6ec9c3a4e617d316a1883a858 Mon Sep 17 00:00:00 2001 From: Robert Tice Date: Tue, 2 Apr 2019 14:36:05 -0400 Subject: [PATCH 7/9] Removing all print statements in favor of specialized prints to ensure that all messages coming from backend are put in the correct place in the frontend. --- infrastructure/container_payload/run.py | 42 +++++++++++++++++-------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/infrastructure/container_payload/run.py b/infrastructure/container_payload/run.py index b5920f4..e479109 100644 --- a/infrastructure/container_payload/run.py +++ b/infrastructure/container_payload/run.py @@ -147,6 +147,23 @@ def doctor_main_gpr(tempd, spark_mode=False): def safe_run(workdir, mode, lab): + def prefix_print(prefix, msg): + print("{}:{}".format(prefix, msg)) + + def print_stdout(msg): + prefix_print("stdout", msg) + + def print_stderr(msg): + prefix_print("stderr", msg) + + def print_stdin(msg): + prefix_print("stdin", msg) + + def print_lab(success, cases): + lab_output = {"success": success, "test_cases": cases} + prefix_print("lab_output", json.dumps(lab_output)) + + def c(cl=[]): """Aux procedure, run the given command line and output to stdout.""" """Returns a tuple of (Boolean success, list stdout, returncode).""" @@ -161,11 +178,11 @@ def safe_run(workdir, mode, lab): stderr_line = p.stderr.readline().replace(workdir, '.') if stderr_line != '': - print("stderr:{}".format(stderr_line)) + print_stderr(stderr_line) sys.stderr.flush() if stdout_line != '': - print("stdout:{}".format(stdout_line)) + print_stdout(stdout_line) stdout_list.append(stdout_line) sys.stdout.flush() else: @@ -176,10 +193,10 @@ def safe_run(workdir, mode, lab): sys.stderr.flush() if p.returncode == INTERRUPT_RETURNCODE: - print INTERRUPT_STRING + print_stderr(INTERRUPT_STRING) return True, stdout_list, p.returncode except Exception: - print "ERROR when running {}".format(' '.join(cl)) + print_stderr("ERROR when running {}".format(' '.join(cl))) traceback.print_exc() return False, stdout_list, p.returncode @@ -227,7 +244,7 @@ def safe_run(workdir, mode, lab): if os.path.isfile(cli_txt): cli = "`cat {}`".format(cli_txt); with open(cli_txt, 'r') as f: - print("stdin:{}".format(f.read().replace('\n', ' '))) + print_stdin(f.read().replace('\n', ' ')) else: # otherwise pass no arguments to the main cli = "" @@ -266,7 +283,7 @@ def safe_run(workdir, mode, lab): for index, test in sorted(test_cases.items()): # check that this test case has defined ins and outs if "in" in test.keys() and "out" in test.keys(): - print("stdin:{}".format(test["in"])) + print_stdin(test["in"]) errno, stdout, retcode = run(main, workdir, "`echo {}`".format(test["in"])) test["actual"] = " ".join(stdout).replace('\n', '').replace('\r', '') @@ -277,15 +294,14 @@ def safe_run(workdir, mode, lab): else: test["status"] = "Success" else: - print("Malformed test IO sequence in test case #{}. Please report this issue on https://github.com/AdaCore/learn/issues".format(index)) + print_stderr("Malformed test IO sequence in test case #{}. Please report this issue on https://github.com/AdaCore/learn/issues".format(index)) sys.exit(1) - lab_output = {"success": success, "test_cases": test_cases} - print("lab_output:{}".format(json.dumps(lab_output))) + print_lab(success, test_cases) else: # No lab IO resources defined. This is an error in the lab config - print("No submission criteria found for this lab. Please report this issue on https://github.com/AdaCore/learn/issues") + print_stderr("No submission criteria found for this lab. Please report this issue on https://github.com/AdaCore/learn/issues") else: - print("Build failed...") + print_stderr("Build failed...") elif mode == "prove": doctor_main_gpr(workdir, spark_mode=True) @@ -297,7 +313,7 @@ def safe_run(workdir, mode, lab): doctor_main_gpr(workdir, spark_mode=True) prove(["--report=all"]) else: - print "mode not implemented" + print_stderr("mode not implemented") except Exception: traceback.print_exc() @@ -321,7 +337,7 @@ if __name__ == '__main__': else: lab = None else: - print "Error invoking run" + print_stderr("Error invoking run") sys.exit(1) # This is where the compiler is installed From 8ef432092037fbdd12c0a0ce32501b8c868c8d91 Mon Sep 17 00:00:00 2001 From: Robert Tice Date: Wed, 3 Apr 2019 15:38:25 -0400 Subject: [PATCH 8/9] Adding failure state to labs based on process retcode. --- infrastructure/container_payload/run.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/infrastructure/container_payload/run.py b/infrastructure/container_payload/run.py index e479109..3f96976 100644 --- a/infrastructure/container_payload/run.py +++ b/infrastructure/container_payload/run.py @@ -147,6 +147,7 @@ def doctor_main_gpr(tempd, spark_mode=False): def safe_run(workdir, mode, lab): + def prefix_print(prefix, msg): print("{}:{}".format(prefix, msg)) @@ -156,13 +157,12 @@ def safe_run(workdir, mode, lab): def print_stderr(msg): prefix_print("stderr", msg) - def print_stdin(msg): - prefix_print("stdin", msg) - def print_lab(success, cases): lab_output = {"success": success, "test_cases": cases} prefix_print("lab_output", json.dumps(lab_output)) + def print_console(cmd_list): + prefix_print("console", " ".join(cmd_list).replace(workdir, '.')) def c(cl=[]): """Aux procedure, run the given command line and output to stdout.""" @@ -205,9 +205,10 @@ def safe_run(workdir, mode, lab): """Returns a tuple of (Boolean success, list stdout, returncode).""" line = ["gprbuild", "-q", "-P", "main", "-gnatwa"] line.extend(extra_args) + print_console(line) return c(line) - def run(main, workdir, args): + def run(main, workdir, arg_list): """Runs the application""" """Returns a tuple of (Boolean success, list stdout, returncode).""" @@ -218,7 +219,9 @@ def safe_run(workdir, mode, lab): line = ['sudo', '-u', 'unprivileged', 'timeout', '10s', 'bash', '-c', 'LD_PRELOAD=/preloader.so {} {}'.format( - os.path.join(workdir, main.split('.')[0]), args)] + os.path.join(workdir, main.split('.')[0]), "`echo {}`".format(" ".join(arg_list)))] + print_list = [] + print_console(["./{}".format(main)] + arg_list) return c(line) def prove(extra_args): @@ -227,6 +230,7 @@ def safe_run(workdir, mode, lab): line = ["gnatprove", "-P", "main", "--checks-as-errors", "--level=0", "--no-axiom-guard"] line.extend(extra_args) + print_console(line) return c(line) # This is necessary to get the first line from the container. Otherwise @@ -242,12 +246,11 @@ def safe_run(workdir, mode, lab): # Check to see if cli.txt was sent from the front-end cli_txt = os.path.join(workdir, CLI_FILE) if os.path.isfile(cli_txt): - cli = "`cat {}`".format(cli_txt); with open(cli_txt, 'r') as f: - print_stdin(f.read().replace('\n', ' ')) + cli = f.read().split() else: # otherwise pass no arguments to the main - cli = "" + cli = [] run(main, workdir, cli) else: @@ -283,12 +286,11 @@ def safe_run(workdir, mode, lab): for index, test in sorted(test_cases.items()): # check that this test case has defined ins and outs if "in" in test.keys() and "out" in test.keys(): - print_stdin(test["in"]) - errno, stdout, retcode = run(main, workdir, "`echo {}`".format(test["in"])) + errno, stdout, retcode = run(main, workdir, test["in"].split()) test["actual"] = " ".join(stdout).replace('\n', '').replace('\r', '') - if test["actual"] != test["out"]: + if retcode != 0 or test["actual"] != test["out"]: test["status"] = "Failed" success = False else: From 7f25506a1d4bdcbae84222d2f83a175973d7bbb3 Mon Sep 17 00:00:00 2001 From: Robert Tice Date: Thu, 4 Apr 2019 10:10:08 -0400 Subject: [PATCH 9/9] All messages from the server are json strings now. Also fixing docstrings. --- infrastructure/container_payload/run.py | 62 ++++++++++++++++++------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/infrastructure/container_payload/run.py b/infrastructure/container_payload/run.py index 3f96976..f5374d5 100644 --- a/infrastructure/container_payload/run.py +++ b/infrastructure/container_payload/run.py @@ -148,25 +148,31 @@ def doctor_main_gpr(tempd, spark_mode=False): def safe_run(workdir, mode, lab): - def prefix_print(prefix, msg): - print("{}:{}".format(prefix, msg)) + def json_print(pdict): + print(json.dumps(pdict)) def print_stdout(msg): - prefix_print("stdout", msg) + json_print({"stdout": msg}) def print_stderr(msg): - prefix_print("stderr", msg) + json_print({"stderr": msg}) def print_lab(success, cases): - lab_output = {"success": success, "test_cases": cases} - prefix_print("lab_output", json.dumps(lab_output)) + json_print({"lab_output": {"success": success, "test_cases": cases}}) def print_console(cmd_list): - prefix_print("console", " ".join(cmd_list).replace(workdir, '.')) + json_print({"console": " ".join(cmd_list).replace(workdir, '.')}) def c(cl=[]): - """Aux procedure, run the given command line and output to stdout.""" - """Returns a tuple of (Boolean success, list stdout, returncode).""" + """Aux procedure, run the given command line and output to stdout. + + Parameters: + cl (list): The command list to be sent to popen + + Returns: + tuple: of (Boolean success, list stdout, int returncode). + """ + stdout_list = [] try: debug_print("running: {}".format(cl)) @@ -177,11 +183,11 @@ def safe_run(workdir, mode, lab): stdout_line = p.stdout.readline().replace(workdir, '.') stderr_line = p.stderr.readline().replace(workdir, '.') - if stderr_line != '': + if stderr_line: print_stderr(stderr_line) sys.stderr.flush() - if stdout_line != '': + if stdout_line: print_stdout(stdout_line) stdout_list.append(stdout_line) sys.stdout.flush() @@ -201,16 +207,31 @@ def safe_run(workdir, mode, lab): return False, stdout_list, p.returncode def build(extra_args): - """Builds the application using static args and extra_args.""" - """Returns a tuple of (Boolean success, list stdout, returncode).""" + """Builds command string to build the application and passes that to c() + + Parameters: + extra_args (list): The extra build arguments to be passed to the build + + Returns: + tuple: of (Boolean success, list stdout, returncode). + """ + line = ["gprbuild", "-q", "-P", "main", "-gnatwa"] line.extend(extra_args) print_console(line) return c(line) def run(main, workdir, arg_list): - """Runs the application""" - """Returns a tuple of (Boolean success, list stdout, returncode).""" + """Builds command string to run the application and passes that to c() + + Parameters: + main (string): The name of the main + workdir (string): The path of the working directory + arg_list (list): The arguments to be passed to the main + + Returns: + tuple: of (Boolean success, list stdout, returncode). + """ # We run: # - as user 'unprivileged' that has no write access @@ -225,8 +246,15 @@ def safe_run(workdir, mode, lab): return c(line) def prove(extra_args): - """Proves the application""" - """Returns a tuple of (Boolean success, list stdout, returncode).""" + """Builds command string to prove the application and passes that to c() + + Parameters: + extra_args (list): The extra gnatprove arguments to be passed to the prover + + Returns: + tuple: of (Boolean success, list stdout, returncode). + """ + line = ["gnatprove", "-P", "main", "--checks-as-errors", "--level=0", "--no-axiom-guard"] line.extend(extra_args)