From 09db8e19087e4be3380b754ea76d550445819f8f Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Wed, 29 Jan 2020 15:08:31 +0100 Subject: [PATCH 01/30] test-aware instrumentation changes --- init_verification.py | 159 +++++++++++++++++---- instrument.py | 319 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 425 insertions(+), 53 deletions(-) diff --git a/init_verification.py b/init_verification.py index 15b55d1..3d25377 100644 --- a/init_verification.py +++ b/init_verification.py @@ -17,12 +17,20 @@ from VyPR.monitor_synthesis import formula_tree from VyPR.verdict_reports import VerdictReport + # sys.path.append("VyPR/") VERDICT_SERVER_URL = None VYPR_OUTPUT_VERBOSE = True PROJECT_ROOT = None +## USED IN CASE OF FLASK TESTING +#MAP_COPY_VERDICT = {} + +## The purpose of this flag is to exclude verification send_event call +## with 'test_status' option, if "end" option has not occured. +IS_END_OPT = False + # thank you to the CMS Conditions Browser team for this def to_timestamp(obj): @@ -43,7 +51,7 @@ def vypr_output(string, *args): def send_verdict_report(function_name, time_of_call, end_time_of_call, program_path, verdict_report, - binding_to_line_numbers, http_request_time, property_hash): + binding_to_line_numbers, http_request_time, property_hash, test_result = None, test_name = None): """ Send verdict data for a given function call (function name + time of call). """ @@ -59,7 +67,9 @@ def send_verdict_report(function_name, time_of_call, end_time_of_call, program_p "end_time_of_call": end_time_of_call.isoformat(), "function_name": function_name, "property_hash": property_hash, - "program_path": program_path + "program_path": program_path, + "test_result": test_result, + "test_name" : test_name } vypr_output("CALL DATA") vypr_output(call_data) @@ -113,18 +123,30 @@ def send_verdict_report(function_name, time_of_call, end_time_of_call, program_p vypr_output("Verdicts sent.") + + def consumption_thread_function(verification_obj): # the web service has to be considered as running forever, so the monitoring loop for now should also run forever # this needs to be changed for a clean exit INACTIVE_MONITORING = False + + + + while True: + # import pdb + # pdb.set_trace() + + # take top element from the queue try: top_pair = verification_obj.consumption_queue.get(timeout=1) + ## In case of flask testing except: continue + if top_pair[0] == "end-monitoring": # return from the monitoring function to end the monitoring thread vypr_output("Returning from monitoring thread.") @@ -158,7 +180,10 @@ def consumption_thread_function(verification_obj): instrument_type = top_pair[0] function_name = top_pair[1] - # get the maps we need for this function + + # get the maps we need for this function + + maps = verification_obj.function_to_maps[function_name][property_hash] static_qd_to_point_map = maps.static_qd_to_point_map static_qd_to_monitors = maps.static_qd_to_monitors @@ -176,10 +201,14 @@ def consumption_thread_function(verification_obj): # that are updated at runtime scope_event = top_pair[2] if scope_event == "end": - + global IS_END_OPT + IS_END_OPT = True # before resetting the qd -> monitor map, go through it to find monitors # that reached a verdict, and register those in the verdict report + + vypr_output("Set the function name...") + for static_qd_index in static_qd_to_monitors: for monitor in static_qd_to_monitors[static_qd_index]: # check if the monitor has a collapsing atom - only then do we register a verdict @@ -218,28 +247,44 @@ def consumption_thread_function(verification_obj): elif type(bind_var) is CFGEdge: binding_to_line_numbers[bind_space_index].append(bind_var._instruction.lineno) + print(top_pair) + # send the verdict # we send the function name, the time of the function call, the verdict report object, # the map of bindings to their line numbers and the date/time of the request the identify it (single threaded...) - send_verdict_report( - function_name, - maps.latest_time_of_call, - datetime.datetime.now(), - maps.program_path, - verdict_report, - binding_to_line_numbers, - top_pair[3], - top_pair[4] - ) - - # reset the verdict report - maps.verdict_report.reset() - - # reset the function start time for the next time - maps.latest_time_of_call = None - - # reset the program path - maps.program_path = [] + + + + + + + # We only send verdict data to the server when + + test_aware_status = top_pair[7] + if not test_aware_status in ['normal', 'flask']: + + send_verdict_report( + function_name, + maps.latest_time_of_call, + datetime.datetime.now(), + maps.program_path, + verdict_report, + binding_to_line_numbers, + top_pair[4], + top_pair[5] + ) + + + + + # reset the verdict report + maps.verdict_report.reset() + + # reset the function start time for the next time + maps.latest_time_of_call = None + + # reset the program path + maps.program_path = [] elif scope_event == "start": vypr_output("*" * 50) @@ -251,6 +296,7 @@ def consumption_thread_function(verification_obj): vypr_output("*" * 50) + if instrument_type == "trigger": # we've received a trigger instrument @@ -383,6 +429,45 @@ def consumption_thread_function(verification_obj): inst_point_id=instrumentation_point_db_id, program_path=len(program_path), state_dict=state_dict) + + if instrument_type == "test_status": + global IS_END_OPT + if IS_END_OPT: + + status = top_pair[2] + + if status.failures: + test_result = "Fail" + elif status.errors: + test_result = "Error" + else: + test_result = "Success" + vypr_output("Sending verdict report only in case of testing") + + + send_verdict_report( + function_name, + maps.latest_time_of_call, + datetime.datetime.now(), + maps.program_path, + verdict_report, + binding_to_line_numbers, + top_pair[3], + top_pair[4], + test_result + ) + + # reset the verdict report + maps.verdict_report.reset() + + # reset the function start time for the next time + maps.latest_time_of_call = None + + # reset the program path + maps.program_path = [] + + IS_END_OPT = False + # set the task as done verification_obj.consumption_queue.task_done() @@ -415,6 +500,7 @@ def __init__(self, module_name, function_name, property_hash): (module_name.replace(".", "-"), function_name.replace(":", "-")), "rb") as h: index_to_hash_dump = h.read() + # inst_configuration = read_configuration("vypr.config") # get the specification file name @@ -445,14 +531,28 @@ def read_configuration(file): """ Read in 'file', parse into an object and return. """ - with open(file, "r") as h: - contents = h.read() + content = "" + skip = False + for line in open(file): + li = line.strip() + + # Handle multi-line comments + if li.startswith("/*"): + skip = True + if li.endswith("*/"): + skip = False + continue + + if not (li.startswith("#") or li.startswith("//")) and not skip: + content += line + + return json.loads(content) - return json.loads(contents) class Verification(object): + def __init__(self, flask_object=None): """ Sets up the consumption thread for events from instruments. @@ -556,6 +656,7 @@ def endpoint_resume_monitoring(): vypr_output("VyPR monitoring initialisation finished.") def send_event(self, event_description): + print("trying to send an event..") if not (self.initialisation_failure): vypr_output("adding %s to consumption queue" % str(event_description)) self.consumption_queue.put(event_description) @@ -574,3 +675,9 @@ def resume_monitoring(self): if not (self.initialisation_failure): vypr_output("Sending monitoring resume message.") self.consumption_queue.put(("inactive-monitoring-stop",)) + + def get_test_result_in_flask(self,className, methodName, result): + print("Got the name and the status of the test {} {} {}".format(className, methodName, result)) + + + diff --git a/instrument.py b/instrument.py index 5c817d2..d86cfe2 100644 --- a/instrument.py +++ b/instrument.py @@ -39,6 +39,9 @@ PROJECT_ROOT = "" USE_FLASK = True VERIFICATION_INSTRUCTION = "verification.send_event" +TEST_FRAMEWORK = False + + """def print(*s): global VERBOSE @@ -259,10 +262,23 @@ def read_configuration(file): """ Read in 'file', parse into an object and return. """ - with open(file, "r") as h: - contents = h.read() + content = "" + skip = False + for line in open(file): + li = line.strip() + + # Handle multi-line comments + if li.startswith("/*"): + skip = True + if li.endswith("*/"): + skip = False + continue + + if not (li.startswith("#") or li.startswith("//")) and not skip: + content += line + + return json.loads(content) - return json.loads(contents) def get_instrumentation_points_from_comp_sequence(value_from_binding, moves): @@ -342,6 +358,8 @@ def instrument_point_state(state, name, point, binding_space_indices, ) state_recording_instrument += "%s((%s))" % (VERIFICATION_INSTRUCTION, instrument_tuple) + + record_state_ast = ast.parse(state_recording_instrument).body[0] queue_ast = ast.parse(state_recording_instrument).body[1] @@ -455,8 +473,175 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, point._instruction._parent_body.insert(index_in_block, start_ast) +## TESTING RELATED UPDATES --- 22.1.2020 + + +""" + Adds a setUp() method and creates a verification object if it doesn't exists, + otherwise update the setUp() method with creating a verification object. + + METHOD_CONSTRAINT: + +""" + +def create_test_setup_method(enable_normal_testing, current_step, class_name): + """ + :param enable_normal_testing: Checks whether the testing is "normal" of "flask" based. + :param current_step: Contains the AST for the code + :param class_name: Name of the test class + """ + + + setUp_found = False + + if enable_normal_testing == "normal": + + VERIFICATION_IMPORT = "from VyPR.init_verification import Verification" + VERIFICATION_OBJ ="self.verification = Verification()" + + ## Finding the test class. + current_step = filter( lambda entry: (type(entry) is ast.ClassDef and + entry.name == class_name), current_step)[0] + + test_class_body = current_step.body + + + ## Traversing the body of the class in order to look for setUp method + + for test_function in test_class_body: + + + if not (type(test_function) is ast.FunctionDef): + continue + + if test_function.name is 'setUp': + # We found setUp method, now we need to add verification instructions + setUp_found = True + + + verification_import_inst = ast.parse(VERIFICATION_IMPORT).body[0] + verification_import_obj_assign = ast.parse(VERIFICATION_OBJ).body[0] + + test_function.body.insert(0,verification_import_obj_assign) + test_function.body.insert(0,verification_import_inst) + + # If there is no setUp method, then we need to add setUp method in the class. + if not setUp_found: + setUp_method = "def setUp(self):\n\t" + VERIFICATION_IMPORT + '\n\t' + VERIFICATION_OBJ + method_inst = ast.parse(setUp_method).body[0] + test_class_body.insert(0,method_inst) + + + +""" + Adds a teardown method for storing each test status i.e., whether it failed, succeed or whether there was an error. + +""" + + +def create_teardown_method(ast_code, class_name, formula_hash, function, test_aware_type): + + """ + :param ast_code: Code for the AST + :param class_name: Class name + :param formula_hash: Hash value for the formula + :param function: Fully qualified function name + :parame test_aware_type type of testing can be normal or flask + """ + + + tearDown_found = False + function_index = 0 + + + # In case of flask, getting the class name + + + + if detect_testing_frameworks(ast_code): + + ## Finding the test class. + current_step = filter( lambda entry: (type(entry) is ast.ClassDef and + entry.name == class_name), ast_code.body)[0] + + test_class_body = current_step.body + + + if TEST_AWARE == 'flask': + flask_import = "from app import verification\n\t" + else: + flask_import = "" + + verification_call_code = "%s((\"%s\",\"test_status\", \"%s\",self._resultForDoCleanups, datetime.datetime.now(),\"%s\", \"s\"))" % \ + (VERIFICATION_INSTRUCTION, formula_hash, function,formula_hash + ) + + flask_code = flask_import + verification_call_code + + for test_function in test_class_body: + + + if isinstance(test_function, ast.FunctionDef): + function_index = function_index + 1 # Placed teardown as the last function + + + if not (type(test_function) is ast.FunctionDef): + continue + + if test_function.name is 'tearDown': + # We found setUp method, now we need to add verification instructions + tearDown_found = True + verification_call_inst = ast.parse(verification_call_code).body[0] + test_function.body.insert(0,verification_call_inst) + + if not tearDown_found: + tear_method = "def tearDown(self):\n\t" + flask_code + method_inst = ast.parse(tear_method).body[0] + test_class_body.insert((function_index),method_inst) + + + + + + #for test_function in test_class_body: + + + + +""" + Detects whether the instrumented file has a testing framework. It will then generated verification object and instruction + related to testing. See create_setup_method/create_teardown_method. + + CONSTRAINT: Only works for python unittests, we will have to extend it for different python testing frameworks. +""" + +def detect_testing_frameworks(ast_code): + + """ + :param ast_code: Abstract Syntax Tree for the code. + """ + + if TEST_AWARE != None: + + for node in ast_code.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + if node.names[0].__dict__['name'] is 'unittest': + + return True + else: + return False + else: + print("specify testing to either flask or normal in vypr.config file") + + exit() + + if __name__ == "__main__": + + # import pdb; pdb.set_trace() + + parser = argparse.ArgumentParser(description='Instrumentation for VyPR.') parser.add_argument('--verbose', action='store_true', help='If given, output will be turned on for the instrumentation module.', required=False) @@ -484,10 +669,41 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, if inst_configuration.get("use_flask") else "no" VERIFICATION_INSTRUCTION = inst_configuration.get("verification_instruction") \ if inst_configuration.get("verification_instruction") else "verification.send_event" + TEST_AWARE = inst_configuration.get("testing") + FLASK_TEST_FOLDER=inst_configuration.get("flask_test_folder") \ + if inst_configuration.get("flask_test_folder") else "" + + ##Test related checks in configuration file. + if TEST_AWARE not in ['normal', 'flask', 'None']: + print ("Specify normal or flask for testing. None for normal program analysis") + exit() + + + if TEST_AWARE == 'flask': + # In flask-based testing, it is important to specify the file (absolute path) where test case is present. + if inst_configuration.get("flask_test_folder") == None: + print ("Specify the path for flask test file.") + exit() + else: + # If incorrect format of the path is specified. The path should only start with a directory name and end with .py extension. + import re + m = re.search('^([A-z0-9-_+]+\/)*([A-z0-9]+\.(py))',inst_configuration.get("flask_test_folder")) + if m == None: + print ("Specify the correct path for flask test file.") + exit() + + if TEST_AWARE == 'normal': + VERIFICATION_INSTRUCTION = "self.verification.send_event" + # convert the USE_FLASK flag to boolean USE_FLASK = {"yes": True, "no": False}[USE_FLASK] + + SETUP_ONCE =False + TEARDOWN_ONCE = False + + # reset code to non-instrumented for directory in os.walk("."): for file in directory[2]: @@ -546,6 +762,7 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, # and class navigation later on instrument_function_qualifier = "%s.%s" % (module, function.replace(".", ":")) + index_to_hash = [] qualifier_subsequence = get_qualifier_subsequence(function) @@ -561,16 +778,21 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, current_step = asts.body - # traverse sub structures + # Code for handling testing () + if not SETUP_ONCE and TEST_AWARE == 'normal': + test_class_name = function_name[0] + create_test_setup_method(TEST_AWARE, current_step,test_class_name) + SETUP_ONCE = True + for step in hierarchy: + current_step = filter( lambda entry: (type(entry) is ast.ClassDef and entry.name == step), current_step )[0] - # find the final function definition function_def = filter( lambda entry: (type(entry) is ast.FunctionDef and @@ -624,6 +846,36 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, # update the index -> hash map index_to_hash.append(formula_hash) + if not TEARDOWN_ONCE: + if TEST_AWARE == 'normal': + create_teardown_method(asts,test_class_name,formula_hash,instrument_function_qualifier, TEST_AWARE) + + elif TEST_AWARE == 'flask': + flask_test_file = inst_configuration.get("flask_test_folder") + flask_test_file_without_extension = inst_configuration.get("flask_test_folder").replace('.py','') + + code = "".join(open(flask_test_file, "r").readlines()) + flask_test_file_ast = ast.parse(code) + + for node in flask_test_file_ast.body: + if type(node) == ast.ClassDef: + class_name = node.name + create_teardown_method(flask_test_file_ast, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE) + + + backup_flask_file_name = "%s.py.inst" % flask_test_file_without_extension + flask_instrumented_code = compile(flask_test_file_ast,backup_flask_file_name ,"exec") + + # append an underscore to indicate that it's instrumented - removed for now + flask_instrumented_file_name = "%s%s" % (flask_test_file_without_extension, BYTECODE_EXTENSION) + with open(flask_instrumented_file_name, "wb") as h: + h.write(py_compile.MAGIC) + py_compile.wr_long(h, long(time.time())) + marshal.dump(flask_instrumented_code, h) + + + TEARDOWN_ONCE = True + print("FORMULA HASH") print(formula_hash) @@ -760,9 +1012,9 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, point = element[bind_variable_index] - instrument = "%s((\"%s\", \"trigger\", \"%s\", %i, %i))" % \ + instrument = "%s((\"%s\", \"trigger\", \"%s\", %i, %i, \"%s\"))" % \ (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, m, - bind_variable_index) + bind_variable_index, TEST_AWARE) instrument_ast = ast.parse(instrument).body[0] if type(point) is CFGVertex: @@ -1024,9 +1276,9 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, print( "There was a problem with the verdict server at '%s'. Instrumentation cannot be completed." % VERDICT_SERVER_URL) exit() - instrument_code = "%s((\"%s\", \"path\", \"%s\", %i))" % ( + instrument_code = "%s((\"%s\", \"path\", \"%s\", %i, \"%s\"))" % ( VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - branching_condition_id) + branching_condition_id, TEST_AWARE) instrument_ast = ast.parse(instrument_code).body[0] """instrument_ast.lineno = vertex_information[1]._parent_body[0].lineno instrument_ast.col_offset = vertex_information[1]._parent_body[0].col_offset""" @@ -1049,9 +1301,9 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, print( "There was a problem with the verdict server at '%s'. Instrumentation cannot be completed." % VERDICT_SERVER_URL) exit() - instrument_code = "%s((\"%s\", \"path\", \"%s\", %i))" % ( + instrument_code = "%s((\"%s\", \"path\", \"%s\", %i, \"%s\"))" % ( VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - branching_condition_id) + branching_condition_id, TEST_AWARE) instrument_ast = ast.parse(instrument_code).body[0] vertex_information[1].orelse.insert(0, instrument_ast) print("Branch recording instrument placed") @@ -1089,9 +1341,9 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, print( "There was a problem with the verdict server at '%s'. Instrumentation cannot be completed." % VERDICT_SERVER_URL) exit() - instrument_code = "%s((\"%s\", \"path\", \"%s\", %i))" % ( + instrument_code = "%s((\"%s\", \"path\", \"%s\", %i, \"%s\"))" % ( VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - branching_condition_id) + branching_condition_id, TEST_AWARE) instrument_code_ast = ast.parse(instrument_code).body[0] """instrument_code_ast.lineno = vertex_information[1].lineno+1 instrument_code_ast.col_offset = vertex_information[1].col_offset""" @@ -1115,9 +1367,9 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, print( "There was a problem with the verdict server at '%s'. Instrumentation cannot be completed." % VERDICT_SERVER_URL) exit() - instrument_code_inside_loop = "%s((\"%s\", \"path\", \"%s\", %i))" % ( + instrument_code_inside_loop = "%s((\"%s\", \"path\", \"%s\", %i, \"%s\"))" % ( VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - branching_condition_id) + branching_condition_id, TEST_AWARE) instrument_inside_loop_ast = ast.parse(instrument_code_inside_loop).body[0] """instrument_inside_loop_ast.lineno = vertex_information[1].lineno instrument_inside_loop_ast.col_offset = vertex_information[1].col_offset""" @@ -1133,9 +1385,9 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, print( "There was a problem with the verdict server at '%s'. Instrumentation cannot be completed." % VERDICT_SERVER_URL) exit() - instrument_code_outside_loop = "%s((\"%s\", \"path\", \"%s\", %i))" % ( + instrument_code_outside_loop = "%s((\"%s\", \"path\", \"%s\", %i, \"%s\"))" % ( VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - branching_condition_id) + branching_condition_id, TEST_AWARE) instrument_outside_loop_ast = ast.parse(instrument_code_outside_loop).body[0] """instrument_outside_loop_ast.lineno = vertex_information[3].lineno+1 instrument_outside_loop_ast.col_offset = vertex_information[3].col_offset""" @@ -1160,10 +1412,16 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, # NOTE: only problem with this is that the "end" instrument is inserted before the return, # so a function call in the return statement maybe missed if it's part of verification... thread_id_capture = "import threading; __thread_id = threading.current_thread().ident;" - start_instrument = "%s((\"%s\", \"function\", \"%s\", \"start\", flask.g.request_time, \"%s\", __thread_id))" \ - % ( - VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - formula_hash) + #start_instrument = "%s((\"%s\", \"function\", \"%s\", \"start\", flask.g.request_time, \"%s\", __thread_id))" \ + # % ( + # VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, + # formula_hash) + + start_instrument = "%s((\"%s\", \"function\", \"%s\", \"start\", datetime.datetime.now(), \"%s\", __thread_id, \"%s\" ))" \ + % ( + VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, + formula_hash, TEST_AWARE) + threading_import_ast = ast.parse(thread_id_capture).body[0] thread_id_capture_ast = ast.parse(thread_id_capture).body[1] @@ -1192,9 +1450,12 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, # insert the end instrument before every return statement for end_vertex in scfg.return_statements: - end_instrument = "%s((\"%s\", \"function\", \"%s\", \"end\", flask.g.request_time, \"%s\", __thread_id))" \ - % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - formula_hash) + end_instrument = "%s((\"%s\", \"function\", \"%s\", \"end\", %r, flask.g.request_time, \"%s\", __thread_id, \"%s\"))" \ + % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, detect_testing_frameworks(asts), + formula_hash, TEST_AWARE) + + + end_ast = ast.parse(end_instrument).body[0] """end_ast.lineno = end_vertex._previous_edge._instruction.lineno @@ -1212,9 +1473,13 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, # if the last instruction in the ast is not a return statement, add an end instrument at the end if not (type(function_def.body[-1]) is ast.Return): - end_instrument = "%s((\"%s\", \"function\", \"%s\", \"end\", flask.g.request_time, \"%s\"))" \ - % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - formula_hash) + # end_instrument = "%s((\"%s\", \"function\", \"%s\", \"end\", flask.g.request_time, \"%s\"))" \ + # % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, + # formula_hash) + + end_instrument = "%s((\"%s\", \"function\", \"%s\", \"end\", %r ,datetime.datetime.now(), \"%s\", __thread_id, \"%s\"))" \ + % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier,detect_testing_frameworks(asts), + formula_hash, TEST_AWARE) end_ast = ast.parse(end_instrument).body[0] function_def.body.insert(len(function_def.body), end_ast) From 3be6efcc68d84de843eeeef80be4c54ffb0d48b1 Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Wed, 29 Jan 2020 15:57:50 +0100 Subject: [PATCH 02/30] removed TEST_AWARE variable from tigger and path instrumentation types --- instrument.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/instrument.py b/instrument.py index d86cfe2..28b3a0b 100644 --- a/instrument.py +++ b/instrument.py @@ -603,9 +603,6 @@ def create_teardown_method(ast_code, class_name, formula_hash, function, test_aw - #for test_function in test_class_body: - - """ @@ -1012,9 +1009,9 @@ def detect_testing_frameworks(ast_code): point = element[bind_variable_index] - instrument = "%s((\"%s\", \"trigger\", \"%s\", %i, %i, \"%s\"))" % \ + instrument = "%s((\"%s\", \"trigger\", \"%s\", %i, %i))" % \ (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, m, - bind_variable_index, TEST_AWARE) + bind_variable_index) instrument_ast = ast.parse(instrument).body[0] if type(point) is CFGVertex: @@ -1276,9 +1273,9 @@ def detect_testing_frameworks(ast_code): print( "There was a problem with the verdict server at '%s'. Instrumentation cannot be completed." % VERDICT_SERVER_URL) exit() - instrument_code = "%s((\"%s\", \"path\", \"%s\", %i, \"%s\"))" % ( + instrument_code = "%s((\"%s\", \"path\", \"%s\", %i))" % ( VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - branching_condition_id, TEST_AWARE) + branching_condition_id) instrument_ast = ast.parse(instrument_code).body[0] """instrument_ast.lineno = vertex_information[1]._parent_body[0].lineno instrument_ast.col_offset = vertex_information[1]._parent_body[0].col_offset""" @@ -1301,9 +1298,9 @@ def detect_testing_frameworks(ast_code): print( "There was a problem with the verdict server at '%s'. Instrumentation cannot be completed." % VERDICT_SERVER_URL) exit() - instrument_code = "%s((\"%s\", \"path\", \"%s\", %i, \"%s\"))" % ( + instrument_code = "%s((\"%s\", \"path\", \"%s\", %i))" % ( VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - branching_condition_id, TEST_AWARE) + branching_condition_id) instrument_ast = ast.parse(instrument_code).body[0] vertex_information[1].orelse.insert(0, instrument_ast) print("Branch recording instrument placed") @@ -1341,9 +1338,9 @@ def detect_testing_frameworks(ast_code): print( "There was a problem with the verdict server at '%s'. Instrumentation cannot be completed." % VERDICT_SERVER_URL) exit() - instrument_code = "%s((\"%s\", \"path\", \"%s\", %i, \"%s\"))" % ( + instrument_code = "%s((\"%s\", \"path\", \"%s\", %i))" % ( VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - branching_condition_id, TEST_AWARE) + branching_condition_id) instrument_code_ast = ast.parse(instrument_code).body[0] """instrument_code_ast.lineno = vertex_information[1].lineno+1 instrument_code_ast.col_offset = vertex_information[1].col_offset""" @@ -1367,9 +1364,9 @@ def detect_testing_frameworks(ast_code): print( "There was a problem with the verdict server at '%s'. Instrumentation cannot be completed." % VERDICT_SERVER_URL) exit() - instrument_code_inside_loop = "%s((\"%s\", \"path\", \"%s\", %i, \"%s\"))" % ( + instrument_code_inside_loop = "%s((\"%s\", \"path\", \"%s\", %i))" % ( VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - branching_condition_id, TEST_AWARE) + branching_condition_id) instrument_inside_loop_ast = ast.parse(instrument_code_inside_loop).body[0] """instrument_inside_loop_ast.lineno = vertex_information[1].lineno instrument_inside_loop_ast.col_offset = vertex_information[1].col_offset""" @@ -1385,9 +1382,9 @@ def detect_testing_frameworks(ast_code): print( "There was a problem with the verdict server at '%s'. Instrumentation cannot be completed." % VERDICT_SERVER_URL) exit() - instrument_code_outside_loop = "%s((\"%s\", \"path\", \"%s\", %i, \"%s\"))" % ( + instrument_code_outside_loop = "%s((\"%s\", \"path\", \"%s\", %i))" % ( VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - branching_condition_id, TEST_AWARE) + branching_condition_id) instrument_outside_loop_ast = ast.parse(instrument_code_outside_loop).body[0] """instrument_outside_loop_ast.lineno = vertex_information[3].lineno+1 instrument_outside_loop_ast.col_offset = vertex_information[3].col_offset""" @@ -1417,10 +1414,10 @@ def detect_testing_frameworks(ast_code): # VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, # formula_hash) - start_instrument = "%s((\"%s\", \"function\", \"%s\", \"start\", datetime.datetime.now(), \"%s\", __thread_id, \"%s\" ))" \ + start_instrument = "%s((\"%s\", \"function\", \"%s\", \"start\", datetime.datetime.now(), \"%s\", __thread_id))" \ % ( VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - formula_hash, TEST_AWARE) + formula_hash) threading_import_ast = ast.parse(thread_id_capture).body[0] From 30f398d32a6b405ddef094511b1a7e33d895e739 Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Wed, 29 Jan 2020 16:27:48 +0100 Subject: [PATCH 03/30] script that cleans up python process --- stopService.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100755 stopService.sh diff --git a/stopService.sh b/stopService.sh new file mode 100755 index 0000000..ee71348 --- /dev/null +++ b/stopService.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +## +# Script to cleanup the hanging python process. +# + +# This command does four things +# 1) Get the running process information +# 2) Gets only python process +# 3) Excludes the output which will show grep in it +# 4) List python process id +# 5) Loop through each python process, and send a kill signal + +PID=`ps aux | grep Python | grep -v grep | awk '{print $2}'` + +# Loop for process ids to kill python processes. +for pid in $PID +do + kill -9 $pid +done From 05e23d2e49c6c517141ba732bf33c69e6aacbc08 Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Thu, 30 Jan 2020 10:29:15 +0100 Subject: [PATCH 04/30] handling test data --- init_verification.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/init_verification.py b/init_verification.py index 3d25377..ad8aa1a 100644 --- a/init_verification.py +++ b/init_verification.py @@ -59,8 +59,31 @@ def send_verdict_report(function_name, time_of_call, end_time_of_call, program_p verdicts = verdict_report.get_final_verdict_report() vypr_output("Sending verdicts to server") + + # If test data exists. + if test_result !=None: + vypr_output("SENDING TEST DATA") + vypr_output("TEST_NAME {}".format(test_name)) + vypr_output("TEST_RESULT {}".format(test_result)) + + + test_data = { + "test_name" : test_name, + "test_result" : test_result + } + + + test_id = json.loads(requests.post( + os.path.join(VERDICT_SERVER_URL, "insert_test_data/"), + data=json.dumps(test_data) + ).text) + + + + # first, send function call data - this will also insert program path data + call_data = { "http_request_time": http_request_time.isoformat(), "time_of_call": time_of_call.isoformat(), @@ -68,8 +91,7 @@ def send_verdict_report(function_name, time_of_call, end_time_of_call, program_p "function_name": function_name, "property_hash": property_hash, "program_path": program_path, - "test_result": test_result, - "test_name" : test_name + "test_data_id": test_id } vypr_output("CALL DATA") vypr_output(call_data) From 1d6909a0ac124a4b58d57890d82619d8e667107e Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Wed, 12 Feb 2020 11:18:16 +0100 Subject: [PATCH 05/30] changes --- init_verification.py | 33 +++++++++++------ instrument.py | 84 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 92 insertions(+), 25 deletions(-) diff --git a/init_verification.py b/init_verification.py index ad8aa1a..a7a23cd 100644 --- a/init_verification.py +++ b/init_verification.py @@ -51,7 +51,7 @@ def vypr_output(string, *args): def send_verdict_report(function_name, time_of_call, end_time_of_call, program_path, verdict_report, - binding_to_line_numbers, http_request_time, property_hash, test_result = None, test_name = None): + binding_to_line_numbers, transaction_time, property_hash, test_result = None, test_name = None): """ Send verdict data for a given function call (function name + time of call). """ @@ -60,6 +60,8 @@ def send_verdict_report(function_name, time_of_call, end_time_of_call, program_p vypr_output("Sending verdicts to server") + + # If test data exists. if test_result !=None: vypr_output("SENDING TEST DATA") @@ -85,7 +87,7 @@ def send_verdict_report(function_name, time_of_call, end_time_of_call, program_p call_data = { - "http_request_time": http_request_time.isoformat(), + "transaction_time": transaction_time.isoformat(), "time_of_call": time_of_call.isoformat(), "end_time_of_call": end_time_of_call.isoformat(), "function_name": function_name, @@ -93,6 +95,8 @@ def send_verdict_report(function_name, time_of_call, end_time_of_call, program_p "program_path": program_path, "test_data_id": test_id } + + vypr_output("CALL DATA") vypr_output(call_data) insertion_result = json.loads(requests.post( @@ -151,7 +155,7 @@ def consumption_thread_function(verification_obj): # the web service has to be considered as running forever, so the monitoring loop for now should also run forever # this needs to be changed for a clean exit INACTIVE_MONITORING = False - + global IS_END_OPT @@ -191,6 +195,12 @@ def consumption_thread_function(verification_obj): continue # if inactive monitoring is off (so monitoring is running), process what we consumed + + if top_pair[0] == "test_transaction": + transaction = top_pair[1] + vypr_output("Test suite begins at", transaction) + continue + vypr_output("Consuming:") vypr_output(top_pair) @@ -223,7 +233,7 @@ def consumption_thread_function(verification_obj): # that are updated at runtime scope_event = top_pair[2] if scope_event == "end": - global IS_END_OPT + IS_END_OPT = True # before resetting the qd -> monitor map, go through it to find monitors # that reached a verdict, and register those in the verdict report @@ -288,13 +298,13 @@ def consumption_thread_function(verification_obj): send_verdict_report( function_name, maps.latest_time_of_call, - datetime.datetime.now(), + top_pair[-1], maps.program_path, verdict_report, binding_to_line_numbers, - top_pair[4], - top_pair[5] - ) + top_pair[3], + top_pair[4] + ) @@ -453,7 +463,7 @@ def consumption_thread_function(verification_obj): if instrument_type == "test_status": - global IS_END_OPT + if IS_END_OPT: status = top_pair[2] @@ -467,6 +477,8 @@ def consumption_thread_function(verification_obj): vypr_output("Sending verdict report only in case of testing") + + send_verdict_report( function_name, maps.latest_time_of_call, @@ -474,7 +486,8 @@ def consumption_thread_function(verification_obj): maps.program_path, verdict_report, binding_to_line_numbers, - top_pair[3], + transaction, + # top_pair[3], top_pair[4], test_result ) diff --git a/instrument.py b/instrument.py index 28b3a0b..b70468d 100644 --- a/instrument.py +++ b/instrument.py @@ -500,36 +500,53 @@ def create_test_setup_method(enable_normal_testing, current_step, class_name): VERIFICATION_OBJ ="self.verification = Verification()" ## Finding the test class. - current_step = filter( lambda entry: (type(entry) is ast.ClassDef and + current_step = filter( lambda entry: (type(entry) is ast.ClassDef and entry.name == class_name), current_step)[0] - test_class_body = current_step.body + test_class_body = current_step.body + first_test_method = get_test_case_position(test_class_body, 0) - ## Traversing the body of the class in order to look for setUp method + first_test_method = "'" + first_test_method + "'" - for test_function in test_class_body: + transaction_time_statement = "%s((\"test_transaction\", datetime.datetime.now()))" % \ + (VERIFICATION_INSTRUCTION) + check_condition_for_transaction = "if self._testMethodName ==" +first_test_method + ":" + transaction_time_statement - if not (type(test_function) is ast.FunctionDef): - continue + ## Traversing the body of the class in order to look for setUp method - if test_function.name is 'setUp': - # We found setUp method, now we need to add verification instructions - setUp_found = True + for test_function in test_class_body: + + if not (type(test_function) is ast.FunctionDef): + continue + if test_function.name == 'setUp': + # We found setUp method, now we need to add verification instructions + setUp_found = True + check_condition_for_transaction_assign = ast.parse(check_condition_for_transaction).body[0] + test_function.body.insert(0,check_condition_for_transaction_assign) + + if TEST_AWARE == 'normal': verification_import_inst = ast.parse(VERIFICATION_IMPORT).body[0] verification_import_obj_assign = ast.parse(VERIFICATION_OBJ).body[0] - test_function.body.insert(0,verification_import_obj_assign) test_function.body.insert(0,verification_import_inst) + + + # If there is no setUp method, then we need to add setUp method in the class. - if not setUp_found: - setUp_method = "def setUp(self):\n\t" + VERIFICATION_IMPORT + '\n\t' + VERIFICATION_OBJ - method_inst = ast.parse(setUp_method).body[0] - test_class_body.insert(0,method_inst) + if not setUp_found: + if TEST_AWARE == 'normal': + setUp_method = "def setUp(self):\n\t" + VERIFICATION_IMPORT + '\n\t' + VERIFICATION_OBJ + '\n\t' + check_condition_for_transaction + else: + setUp_method = "def setUp(self):\n\t" + check_condition_for_transaction + + method_inst = ast.parse(setUp_method).body[0] + test_class_body.insert(0,method_inst) + @@ -569,15 +586,35 @@ def create_teardown_method(ast_code, class_name, formula_hash, function, test_aw if TEST_AWARE == 'flask': flask_import = "from app import verification\n\t" + verification_name = "verification" else: flask_import = "" + verification_name = "self.verification" - verification_call_code = "%s((\"%s\",\"test_status\", \"%s\",self._resultForDoCleanups, datetime.datetime.now(),\"%s\", \"s\"))" % \ + verification_call_code = "%s((\"%s\",\"test_status\", \"%s\",self._resultForDoCleanups, datetime.datetime.now(),\"%s\"))" % \ (VERIFICATION_INSTRUCTION, formula_hash, function,formula_hash ) + + + + # ## Find the name of last test method + # function_ast_list = filter( lambda entry: (type(entry) is ast.FunctionDef),test_class_body) + # function_name = map(lambda function: function.name, function_ast_list) + # + # function_name.remove('setUp') + # function_name.remove('tearDown') + # + # last_test_method_name = function_name[-1] + + last_test_method_name = get_test_case_position(test_class_body, -1) + + terminate_monitoring_call = "if self._testMethodName ==" +last_test_method_name + ":" +verification_name + ".end_monitoring()" + verification_call_code = verification_call_code + '\n' + terminate_monitoring_call + flask_code = flask_import + verification_call_code + for test_function in test_class_body: @@ -594,6 +631,9 @@ def create_teardown_method(ast_code, class_name, formula_hash, function, test_aw verification_call_inst = ast.parse(verification_call_code).body[0] test_function.body.insert(0,verification_call_inst) + + ## Terminate monitoring after all tests are analyzed + if not tearDown_found: tear_method = "def tearDown(self):\n\t" + flask_code method_inst = ast.parse(tear_method).body[0] @@ -601,7 +641,21 @@ def create_teardown_method(ast_code, class_name, formula_hash, function, test_aw +""" + Identifies the position of a test case in a test suite +""" + +def get_test_case_position(test_class_body, position): + ## Find the name of last test method + function_ast_list = filter( lambda entry: (type(entry) is ast.FunctionDef),test_class_body) + function_name = map(lambda function: function.name, function_ast_list) + + if 'setUp' in function_name: + function_name.remove('setUp') + if 'tearDown' in function_name: + function_name.remove('tearDown') + return function_name[position] From 192461e6c4758d614ad093490ff449cc29bf53aa Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Wed, 12 Feb 2020 14:32:02 +0100 Subject: [PATCH 06/30] push --- init_verification.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/init_verification.py b/init_verification.py index 1d4c072..1d28043 100644 --- a/init_verification.py +++ b/init_verification.py @@ -330,7 +330,6 @@ def consumption_thread_function(verification_obj): elif type(bind_var) is CFGEdge: binding_to_line_numbers[bind_space_index].append(bind_var._instruction.lineno) -<<<<<<< HEAD print(top_pair) # send the verdict @@ -674,7 +673,7 @@ def __init__(self): self.initialisation_failure = True return - def initialise(self, flask_object): + def initialise(self, flask_object): vypr_output("Initialising VyPR alongside service.") @@ -793,7 +792,7 @@ def endpoint_resume_monitoring(): vypr_output("VyPR monitoring initialisation finished.") - def get_time(self): + def get_time(self): """ Returns either the machine local time, or the NTP time (using the initial NTP time obtained when VyPR started up, so we don't query an NTP server everytime we want to measure time). @@ -812,27 +811,27 @@ def get_time(self): vypr_output("Getting time based on local machine.") return datetime.datetime.utcnow() - def send_event(self, event_description): + def send_event(self, event_description): print("trying to send an event..") if not (self.initialisation_failure): self.consumption_queue.put(event_description) - def end_monitoring(self): + def end_monitoring(self): if not (self.initialisation_failure): vypr_output("Ending VyPR monitoring thread.") self.consumption_queue.put(("end-monitoring",)) - def pause_monitoring(self): + def pause_monitoring(self): if not (self.initialisation_failure): vypr_output("Sending monitoring pause message.") self.consumption_queue.put(("inactive-monitoring-start",)) - def resume_monitoring(self): + def resume_monitoring(self): if not (self.initialisation_failure): vypr_output("Sending monitoring resume message.") self.consumption_queue.put(("inactive-monitoring-stop",)) - def get_test_result_in_flask(self,className, methodName, result): + def get_test_result_in_flask(self,className, methodName, result): print("Got the name and the status of the test {} {} {}".format(className, methodName, result)) From 9256a1c9a65394a528b6eee61da312acea4465f8 Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Fri, 14 Feb 2020 15:51:09 +0100 Subject: [PATCH 07/30] merged to Josh updates --- init_verification.py | 14 +- instrument.py | 1486 ++++++++++++++++++------------------------ 2 files changed, 635 insertions(+), 865 deletions(-) diff --git a/init_verification.py b/init_verification.py index 1d28043..0016d48 100644 --- a/init_verification.py +++ b/init_verification.py @@ -115,7 +115,8 @@ def send_verdict_report(function_name, time_of_call, end_time_of_call, program_p os.path.join(VERDICT_SERVER_URL, "insert_test_data/"), data=json.dumps(test_data) ).text) - + else: + test_id = None @@ -237,7 +238,7 @@ def consumption_thread_function(verification_obj): if top_pair[0] == "test_transaction": transaction = top_pair[1] - vypr_output("Test suite begins at", transaction) + vypr_output("Test suite begins at") continue @@ -344,12 +345,13 @@ def consumption_thread_function(verification_obj): # We only send verdict data to the server when test_aware_status = top_pair[7] + if not test_aware_status in ['normal', 'flask']: send_verdict_report( function_name, maps.latest_time_of_call, - top_pair[-1], + top_pair[-2], maps.program_path, verdict_report, binding_to_line_numbers, @@ -359,7 +361,6 @@ def consumption_thread_function(verification_obj): - # reset the verdict report maps.verdict_report.reset() @@ -516,7 +517,8 @@ def consumption_thread_function(verification_obj): if instrument_type == "test_status": - + + if IS_END_OPT: status = top_pair[2] @@ -528,7 +530,7 @@ def consumption_thread_function(verification_obj): else: test_result = "Success" vypr_output("Sending verdict report only in case of testing") - + print("Sending verdict report only in case of testing") diff --git a/instrument.py b/instrument.py index d56b208..4da512b 100644 --- a/instrument.py +++ b/instrument.py @@ -403,21 +403,21 @@ def instrument_point_state(state, name, point, binding_space_indices, if measure_attribute == "length": state_variable_alias = name.replace(".", "_").replace("(", "__").replace(")", "__") state_recording_instrument = "record_state_%s = len(%s); " % (state_variable_alias, name) - time_attained_instrument = "time_attained_%s = vypr.get_time();" % state_variable_alias + time_attained_instrument = "time_attained_%s = %s.get_time();" % (state_variable_alias,VYPR_OBJ) elif measure_attribute == "type": state_variable_alias = name.replace(".", "_").replace("(", "__").replace(")", "__") state_recording_instrument = "record_state_%s = type(%s).__name__; " % (state_variable_alias, name) - time_attained_instrument = "time_attained_%s = vypr.get_time();" % state_variable_alias + time_attained_instrument = "time_attained_%s = %s.get_time();" % (state_variable_alias,VYPR_OBJ) elif measure_attribute == "time_attained": state_variable_alias = "time_attained_%i" % atom_sub_index - state_recording_instrument = "record_state_%s = vypr.get_time(); " % state_variable_alias + state_recording_instrument = "record_state_%s = %s.get_time(); " % (state_variable_alias,VYPR_OBJ) time_attained_instrument = state_recording_instrument # the only purpose here is to match what is expected in the monitoring algorithm name = "time" else: state_variable_alias = name.replace(".", "_").replace("(", "__").replace(")", "__") state_recording_instrument = "record_state_%s = %s; " % (state_variable_alias, name) - time_attained_instrument = "time_attained_%s = vypr.get_time();" % state_variable_alias + time_attained_instrument = "time_attained_%s = %s.get_time();" % (state_variable_alias,VYPR_OBJ) # note that observed_value is used three times: # 1) to capture the time attained by the state for checking of a property - this is duplicated @@ -536,8 +536,8 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, else: state_dict = "{}" - timer_start_statement = "__timer_s = vypr.get_time()" - timer_end_statement = "__timer_e = vypr.get_time()" + timer_start_statement = "__timer_s =" + VYPR_OBJ + ".get_time()" + timer_end_statement = "__timer_e ="+ VYPR_OBJ + ".get_time()" time_difference_statement = "__duration = __timer_e - __timer_s; " instrument_tuple = ("'{formula_hash}', 'instrument', '{function_qualifier}', {binding_space_index}," + @@ -592,7 +592,7 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, """ -def create_test_setup_method(enable_normal_testing, current_step, class_name): +def create_test_setup_method(current_step, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE): """ :param enable_normal_testing: Checks whether the testing is "normal" of "flask" based. :param current_step: Contains the AST for the code @@ -602,10 +602,13 @@ def create_test_setup_method(enable_normal_testing, current_step, class_name): setUp_found = False - if enable_normal_testing == "normal": + if TEST_AWARE == "normal": + + #VERIFICATION_IMPORT = "from VyPR.init_verification import Verification" + #VERIFICATION_OBJ ="self.verification = Verification()" + VERIFICATION_IMPORT = "from VyPR import Monitor; self.vypr = Monitor()" + VERIFICATION_OBJ = "self.vypr.initialise(None)" - VERIFICATION_IMPORT = "from VyPR.init_verification import Verification" - VERIFICATION_OBJ ="self.verification = Verification()" ## Finding the test class. current_step = filter( lambda entry: (type(entry) is ast.ClassDef and @@ -617,7 +620,7 @@ def create_test_setup_method(enable_normal_testing, current_step, class_name): first_test_method = "'" + first_test_method + "'" - transaction_time_statement = "%s((\"test_transaction\", datetime.datetime.now()))" % \ + transaction_time_statement = "%s((\"test_transaction\", vypr_dt.now() ))" % \ (VERIFICATION_INSTRUCTION) check_condition_for_transaction = "if self._testMethodName ==" +first_test_method + ":" + transaction_time_statement @@ -692,14 +695,8 @@ def create_teardown_method(ast_code, class_name, formula_hash, function, test_aw test_class_body = current_step.body - if TEST_AWARE == 'flask': - flask_import = "from app import verification\n\t" - verification_name = "verification" - else: - flask_import = "" - verification_name = "self.verification" - verification_call_code = "%s((\"%s\",\"test_status\", \"%s\",self._resultForDoCleanups, datetime.datetime.now(),\"%s\"))" % \ + verification_call_code = "%s((\"%s\",\"test_status\", \"%s\",self._resultForDoCleanups, vypr_dt.now(),\"%s\"))" % \ (VERIFICATION_INSTRUCTION, formula_hash, function,formula_hash ) @@ -716,11 +713,12 @@ def create_teardown_method(ast_code, class_name, formula_hash, function, test_aw # last_test_method_name = function_name[-1] last_test_method_name = get_test_case_position(test_class_body, -1) + last_test_method_name = "'" + last_test_method_name + "'" + + terminate_monitoring_call = "if self._testMethodName ==" +last_test_method_name + ":" +VYPR_OBJ + ".end_monitoring()" + verification_call_code = verification_call_code + '\n\t' + terminate_monitoring_call - terminate_monitoring_call = "if self._testMethodName ==" +last_test_method_name + ":" +verification_name + ".end_monitoring()" - verification_call_code = verification_call_code + '\n' + terminate_monitoring_call - flask_code = flask_import + verification_call_code for test_function in test_class_body: @@ -734,7 +732,7 @@ def create_teardown_method(ast_code, class_name, formula_hash, function, test_aw continue if test_function.name is 'tearDown': - # We found setUp method, now we need to add verification instructions + # We found tearDown method, now we need to add verification instructions tearDown_found = True verification_call_inst = ast.parse(verification_call_code).body[0] test_function.body.insert(0,verification_call_inst) @@ -742,8 +740,9 @@ def create_teardown_method(ast_code, class_name, formula_hash, function, test_aw ## Terminate monitoring after all tests are analyzed + if not tearDown_found: - tear_method = "def tearDown(self):\n\t" + flask_code + tear_method = "def tearDown(self):\n\t" + verification_call_code method_inst = ast.parse(tear_method).body[0] test_class_body.insert((function_index),method_inst) @@ -779,7 +778,7 @@ def detect_testing_frameworks(ast_code): """ :param ast_code: Abstract Syntax Tree for the code. """ - + if TEST_AWARE != None: for node in ast_code.body: @@ -975,7 +974,7 @@ def place_function_begin_instruments(function_def, formula_hash, instrument_func # NOTE: only problem with this is that the "end" instrument is inserted before the return, # so a function call in the return statement maybe missed if it's part of verification... thread_id_capture = "import threading; __thread_id = threading.current_thread().ident;" - vypr_start_time_instrument = "vypr_start_time = vypr.get_time();" + vypr_start_time_instrument = "vypr_start_time = %s.get_time();" %VYPR_OBJ start_instrument = \ "%s((\"%s\", \"function\", \"%s\", \"start\", vypr_start_time, \"%s\", __thread_id))" \ % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, formula_hash) @@ -996,13 +995,19 @@ def place_function_begin_instruments(function_def, formula_hash, instrument_func function_def.body.insert(0, vypr_start_time_ast) -def place_function_end_instruments(function_def, scfg, formula_hash, instrument_function_qualifier): +def place_function_end_instruments(function_def, scfg, formula_hash, instrument_function_qualifier, TEST_AWARE): # insert the end instrument before every return statement + + if TEST_AWARE == "normal": + time = "vypr_dt.now()" + else: + time = "flask.g.request_time" + for end_vertex in scfg.return_statements: end_instrument = \ - "%s((\"%s\", \"function\", \"%s\", \"end\", flask.g.request_time, \"%s\", __thread_id, " \ - "vypr.get_time()))" \ - % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, formula_hash) + "%s((\"%s\", \"function\", \"%s\", \"end\", %s, \"%s\", __thread_id, " \ + "%s.get_time(), \"%s\"))" \ + % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, time,formula_hash, VYPR_OBJ,TEST_AWARE) end_ast = ast.parse(end_instrument).body[0] end_ast.lineno = end_vertex._previous_edge._instruction._parent_body[-1].lineno @@ -1017,10 +1022,10 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ # if the last instruction in the ast is not a return statement, add an end instrument at the end if not (type(function_def.body[-1]) is ast.Return): - end_instrument = "%s((\"%s\", \"function\", \"%s\", \"end\", flask.g.request_time, \"%s\", __thread_id, " \ - "vypr.get_time()))" \ - % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - formula_hash) + end_instrument = \ + "%s((\"%s\", \"function\", \"%s\", \"end\", %s, \"%s\", __thread_id, " \ + "%s.get_time(), \"%s\"))" \ + % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, time,formula_hash, VYPR_OBJ,TEST_AWARE) end_ast = ast.parse(end_instrument).body[0] logger.log("Placing end instrument at the end of the function body.") @@ -1088,13 +1093,18 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ print ("Specify the correct path for flask test file.") exit() - if TEST_AWARE == 'normal': - VERIFICATION_INSTRUCTION = "self.verification.send_event" + VYPR_MODULE = inst_configuration.get("vypr_module") \ if inst_configuration.get("vypr_module") else "" VERIFICATION_INSTRUCTION = "vypr.send_event" # VERIFICATION_INSTRUCTION = "print" + VYPR_OBJ = "vypr" + + if TEST_AWARE == 'normal': + VYPR_OBJ = "self.vypr" + VERIFICATION_INSTRUCTION = VYPR_OBJ+".send_event" + machine_id = ("%s-" % inst_configuration.get("machine_id")) if inst_configuration.get("machine_id") else "" @@ -1103,27 +1113,26 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ print("Verdict server is not reachable. Ending instrumentation - nothing has been done.") exit() - # initialise instrumentation logger - logger = InstrumentationLog(LOGS_TO_STDOUT) - # set up the file handle - logger.start_logging() - SETUP_ONCE =False TEARDOWN_ONCE = False + # initialise instrumentation logger + logger = InstrumentationLog(LOGS_TO_STDOUT) + # set up the file handle + logger.start_logging() # reset code to non-instrumented for directory in os.walk("."): - for file in directory[2]: - if not ("venv" in file): - f = os.path.join(directory[0], file) - if ".py.inst" in f: - # rename to .py - os.rename(f, f.replace(".py.inst", ".py")) - # delete bytecode - os.remove(f.replace(".py.inst", BYTECODE_EXTENSION)) - logger.log("Reset file %s to uninstrumented version." % f) + for file in directory[2]: + if not ("venv" in file): + f = os.path.join(directory[0], file) + if ".py.inst" in f: + # rename to .py + os.rename(f, f.replace(".py.inst", ".py")) + # delete bytecode + os.remove(f.replace(".py.inst", BYTECODE_EXTENSION)) + logger.log("Reset file %s to uninstrumented version." % f) logger.log("Importing PyCFTL queries...") # load in verification config file @@ -1144,849 +1153,608 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ for module in verified_modules: - logger.log("Processing module '%s'." % module) - - verified_functions = verification_conf[module].keys() - - file_name = module.replace(".", "/") + ".py" - file_name_without_extension = module.replace(".", "/") - - # extract asts from the code in the file - code = "".join(open(file_name, "r").readlines()) - asts = ast.parse(code) + logger.log("Processing module '%s'." % module) - # add import for init_vypr module - import_code = "from %s import vypr" % VYPR_MODULE - import_ast = ast.parse(import_code).body[0] - import_ast.lineno = asts.body[0].lineno - import_ast.col_offset = asts.body[0].col_offset - asts.body.insert(0, import_ast) + verified_functions = verification_conf[module].keys() - # add vypr datetime import - vypr_datetime_import = "from datetime import datetime as vypr_dt" - datetime_import_ast = ast.parse(vypr_datetime_import).body[0] - datetime_import_ast.lineno = asts.body[0].lineno - datetime_import_ast.col_offset = asts.body[0].col_offset - asts.body.insert(0, datetime_import_ast) + file_name = module.replace(".", "/") + ".py" + file_name_without_extension = module.replace(".", "/") - # if we're using flask, we assume a certain architecture - import_code = "import flask" - import_asts = ast.parse(import_code) - flask_import = import_asts.body[0] - asts.body.insert(0, flask_import) + # extract asts from the code in the file + code = "".join(open(file_name, "r").readlines()) + asts = ast.parse(code) - for function in verified_functions: + # add import for init_vypr module + if not TEST_AWARE in ['normal']: + import_code = "from %s import vypr" % VYPR_MODULE + import_ast = ast.parse(import_code).body[0] + import_ast.lineno = asts.body[0].lineno + import_ast.col_offset = asts.body[0].col_offset + asts.body.insert(0, import_ast) - logger.log("Processing function '%s'." % function) + import_code = "import flask" + import_asts = ast.parse(import_code) + flask_import = import_asts.body[0] + asts.body.insert(0, flask_import) - # we replace . with : in function definitions to make sure we can distinguish between module - # and class navigation later on - instrument_function_qualifier = "%s%s.%s" % (machine_id, module, function.replace(".", ":")) + # add vypr datetime import + vypr_datetime_import = "from datetime import datetime as vypr_dt" + datetime_import_ast = ast.parse(vypr_datetime_import).body[0] + datetime_import_ast.lineno = asts.body[0].lineno + datetime_import_ast.col_offset = asts.body[0].col_offset + asts.body.insert(0, datetime_import_ast) + # if we're using flask, we assume a certain architecture - index_to_hash = [] - qualifier_subsequence = get_qualifier_subsequence(function) - function_name = function.split(".") + for function in verified_functions: - # find the function definition + logger.log("Processing function '%s'." % function) - actual_function_name = function_name[-1] - hierarchy = function_name[:-1] + # we replace . with : in function definitions to make sure we can distinguish between module + # and class navigation later on + instrument_function_qualifier = "%s%s.%s" % (machine_id, module, function.replace(".", ":")) - current_step = asts.body + index_to_hash = [] - # Code for handling testing () - if not SETUP_ONCE and TEST_AWARE == 'normal': - test_class_name = function_name[0] - create_test_setup_method(TEST_AWARE, current_step,test_class_name) - SETUP_ONCE = True + qualifier_subsequence = get_qualifier_subsequence(function) + function_name = function.split(".") + # find the function definition - for step in hierarchy: + actual_function_name = function_name[-1] + hierarchy = function_name[:-1] - current_step = list(filter( - lambda entry: (type(entry) is ast.ClassDef and - entry.name == step), - current_step - ))[0] + current_step = asts.body + # traverse sub structures - function_def = list(filter( - lambda entry: (type(entry) is ast.FunctionDef and - entry.name == actual_function_name), - current_step.body if type(current_step) is ast.ClassDef else current_step - ))[0] + for step in hierarchy: + current_step = list(filter( + lambda entry: (type(entry) is ast.ClassDef and + entry.name == step), + current_step + ))[0] - # get all reference variables - reference_variables = [] - for (formula_index, formula_structure) in enumerate(verification_conf[module][function]): - for var in formula_structure._bind_variables: - if hasattr(var, "_treat_as_ref") and var._treat_as_ref: - reference_variables.append(var._name_changed) + # find the final function definition - # construct the scfg of the code inside the function + function_def = list(filter( + lambda entry: (type(entry) is ast.FunctionDef and + entry.name == actual_function_name), + current_step.body if type(current_step) is ast.ClassDef else current_step + ))[0] - scfg = CFG(reference_variables=reference_variables) - scfg_vertices = scfg.process_block(function_def.body) + # get all reference variables + reference_variables = [] + for (formula_index, formula_structure) in enumerate(verification_conf[module][function]): + for var in formula_structure._bind_variables: + if hasattr(var, "_treat_as_ref") and var._treat_as_ref: + reference_variables.append(var._name_changed) - top_level_block = function_def.body + # construct the scfg of the code inside the function - logger.log("SCFG constructed.") - - # write scfg to file - write_scfg_to_file(scfg, "%s-%s-%s.gv" % - (file_name_without_extension.replace(".", ""), module.replace(".", "-"), - function.replace(".", "-"))) - - # for each property, instrument the function for that property - - for (formula_index, formula_structure) in enumerate(verification_conf[module][function]): - - logger.log("Instrumenting for PyCFTL formula %s" % formula_structure) - - # we should be able to use the same scfg for each stage of instrumentation, - # since when we insert instruments we recompute the position of the instrumented instruction - - atoms = formula_structure._formula_atoms - - formula_hash = hashlib.sha1() - serialised_bind_variables = base64.encodestring(pickle.dumps(formula_structure.bind_variables)) - formula_hash.update(serialised_bind_variables) - serialised_bind_variables = serialised_bind_variables.decode('ascii') - serialised_formula_structure = base64.encodestring( - pickle.dumps(formula_structure.get_formula_instance()) - ) - formula_hash.update(serialised_formula_structure) - serialised_formula_structure = serialised_formula_structure.decode('ascii') - formula_hash = formula_hash.hexdigest() - serialised_atom_list = list( - map(lambda item: base64.encodestring(pickle.dumps(item)).decode('ascii'), atoms) - ) + scfg = CFG(reference_variables=reference_variables) + scfg_vertices = scfg.process_block(function_def.body) - # note that this also means giving an empty list [] will result in path instrumentation - # without property instrumentation - if EXPLANATION: - logger.log("=" * 100) - logger.log("Placing path recording instruments.") - place_path_recording_instruments(scfg, instrument_function_qualifier, formula_hash) + top_level_block = function_def.body - # update the index -> hash map - index_to_hash.append(formula_hash) - - if not TEARDOWN_ONCE: - if TEST_AWARE == 'normal': - create_teardown_method(asts,test_class_name,formula_hash,instrument_function_qualifier, TEST_AWARE) - - elif TEST_AWARE == 'flask': - flask_test_file = inst_configuration.get("flask_test_folder") - flask_test_file_without_extension = inst_configuration.get("flask_test_folder").replace('.py','') - - code = "".join(open(flask_test_file, "r").readlines()) - flask_test_file_ast = ast.parse(code) - - for node in flask_test_file_ast.body: - if type(node) == ast.ClassDef: - class_name = node.name - create_teardown_method(flask_test_file_ast, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE) - - - backup_flask_file_name = "%s.py.inst" % flask_test_file_without_extension - flask_instrumented_code = compile(flask_test_file_ast,backup_flask_file_name ,"exec") - - # append an underscore to indicate that it's instrumented - removed for now - flask_instrumented_file_name = "%s%s" % (flask_test_file_without_extension, BYTECODE_EXTENSION) - with open(flask_instrumented_file_name, "wb") as h: - h.write(py_compile.MAGIC) - py_compile.wr_long(h, long(time.time())) - marshal.dump(flask_instrumented_code, h) - - - TEARDOWN_ONCE = True - - print("FORMULA HASH") - print(formula_hash) - logger.log("with formula hash '%s'" % formula_hash) - - # construct reachability of the SCFG - # and derive the binding space based on the formula - - reachability_map = construct_reachability_map(scfg) - bindings = compute_binding_space(formula_structure, scfg, reachability_map) - print(bindings) - - logger.log("Set of static bindings computed is") - logger.log(str(bindings)) - - # using these bindings, we now need to instrument the code - # and then store the (bind space index, bind var index, atom index) - # so the instrumentation mappings can be recovered at runtime without recomputation - - static_qd_to_point_map = {} - vertices_to_triple_list = {} - - # we attach indices to atoms because we need their index in the set of atoms - # of the relevant formula - initial_property_dict = { - "formula_hash": formula_hash, - "function": instrument_function_qualifier, - "serialised_formula_structure": serialised_formula_structure, - "serialised_bind_variables": serialised_bind_variables, - "serialised_atom_list": list(enumerate(serialised_atom_list)) - } - - # send instrumentation data to the verdict database - try: - logger.log( - "Sending property with hash '%s' for function '%s' in module '%s' to server." % - (formula_hash, function, module)) - response = str(post_to_verdict_server("store_property/", data=json.dumps(initial_property_dict))) - response = json.loads(response) - atom_index_to_db_index = response["atom_index_to_db_index"] - function_id = response["function_id"] - except: - logger.log("Unforeseen exception when sending property to verdict server:") - logger.log(traceback.format_exc()) - logger.log( - "There was a problem with the verdict server at '%s'. Instrumentation cannot be completed." - % VERDICT_SERVER_URL) - exit() - - logger.log("Processing set of static bindings:") - - for (m, element) in enumerate(bindings): - - logger.log("Processing binding %s" % element) - - # send the binding to the verdict server - - line_numbers = [] - for el in element: - if type(el) is CFGVertex: - if el._name_changed != ["loop"]: - line_numbers.append(el._previous_edge._instruction.lineno) - else: - line_numbers.append(el._structure_obj.lineno) - else: - line_numbers.append(el._instruction.lineno) - - binding_dictionary = { - "binding_space_index": m, - "function": function_id, - "binding_statement_lines": line_numbers - } - serialised_binding_dictionary = json.dumps(binding_dictionary) - try: - binding_db_id = int( - post_to_verdict_server("store_binding/", data=serialised_binding_dictionary)) - except: - logger.log("Unforeseen exception when sending binding to verdict server:") - logger.log(traceback.format_exc()) - logger.log( - "There was a problem with the verdict server at '%s'. Instrumentation cannot be completed." - % VERDICT_SERVER_URL) - exit() - - static_qd_to_point_map[m] = {} - - for (atom_index, atom) in enumerate(atoms): - - logger.log("Computing instrumentation points for atom %s." % atom) - - static_qd_to_point_map[m][atom_index] = {} - - if type(atom) in [formula_tree.StateValueEqualToMixed, - formula_tree.TransitionDurationLessThanTransitionDurationMixed, - formula_tree.TransitionDurationLessThanStateValueMixed, - formula_tree.TransitionDurationLessThanStateValueLengthMixed, - formula_tree.TimeBetweenInInterval, - formula_tree.TimeBetweenInOpenInterval]: - - # there may be multiple bind variables - composition_sequences = derive_composition_sequence(atom) - - # get lhs and rhs bind variables - lhs_comp_sequence = composition_sequences["lhs"] - lhs_bind_variable = lhs_comp_sequence[-1] - lhs_bind_variable_index = list(formula_structure._bind_variables).index(lhs_bind_variable) - lhs_starting_point = element[lhs_bind_variable_index] - - rhs_comp_sequence = composition_sequences["rhs"] - rhs_bind_variable = rhs_comp_sequence[-1] - rhs_bind_variable_index = list(formula_structure._bind_variables).index(rhs_bind_variable) - rhs_starting_point = element[rhs_bind_variable_index] - - lhs_moves = list(reversed(lhs_comp_sequence[1:-1])) - rhs_moves = list(reversed(rhs_comp_sequence[1:-1])) - - lhs_instrumentation_points = get_instrumentation_points_from_comp_sequence( - lhs_starting_point, lhs_moves) - rhs_instrumentation_points = get_instrumentation_points_from_comp_sequence( - rhs_starting_point, rhs_moves) - - # 0 and 1 are for lhs and rhs respectively - static_qd_to_point_map[m][atom_index][0] = lhs_instrumentation_points - static_qd_to_point_map[m][atom_index][1] = rhs_instrumentation_points - - else: - - # just one composition sequence, so one set of instrumentation points - - composition_sequence = derive_composition_sequence(atom) - bind_variable = composition_sequence[-1] - variable_index = list(formula_structure._bind_variables).index(bind_variable) - value_from_binding = element[variable_index] - moves = list(reversed(composition_sequence[1:-1])) - instrumentation_points = get_instrumentation_points_from_comp_sequence(value_from_binding, - moves) - - # for simple atoms, there is no lhs and rhs so we just use 0 - static_qd_to_point_map[m][atom_index][0] = instrumentation_points - - logger.log("Finished computing instrumentation points. Result is:") - logger.log(static_qd_to_point_map) - - # now, perform the instrumentation - - # first step is to add triggers - - logger.log("Inserting triggers.") - - for (m, element) in enumerate(bindings): - - for bind_variable_index in range(len(formula_structure.bind_variables.keys())): - - logger.log("Adding trigger for static binding/bind variable %i/%i." % (m, bind_variable_index)) - - point = element[bind_variable_index] - - instrument = "%s((\"%s\", \"trigger\", \"%s\", %i, %i))" % \ - (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, m, - bind_variable_index) - - instrument_ast = ast.parse(instrument).body[0] - if type(point) is CFGVertex: - if point._name_changed == ["loop"]: - # triggers for loop variables must be inserted inside the loop - # so we instantiate a new monitor for every iteration - for edge in point.edges: - if edge._condition == ["enter-loop"]: - instruction = edge._instruction - else: - instruction = point._previous_edge._instruction - else: - instruction = point._instruction - - lineno = instruction.lineno - col_offset = instruction.col_offset - - index_in_block = instruction._parent_body.index(instruction) - - instrument_ast.lineno = instruction._parent_body[0].lineno - - # insert triggers before the things that will be measured - instruction._parent_body.insert(index_in_block, instrument_ast) - - # we then invert the map we constructed from triples to instrumentation points so that we can avoid - # overlap of instruments - - logger.log("Inverting instrumentation structure ready for optimisations.") - - point_to_triples = {} - - for (m, element) in enumerate(bindings): - for atom_index in static_qd_to_point_map[m].keys(): - for sub_index in static_qd_to_point_map[m][atom_index].keys(): - points = static_qd_to_point_map[m][atom_index][sub_index] - for (n, point) in enumerate(points): - - if not (point_to_triples.get(point)): - point_to_triples[point] = {} - - atom_index_in_db = atom_index_to_db_index[atom_index] - # for now, we don't need serialised_condition_sequence so we just use a blank string - if type(point) is CFGVertex: - if point._name_changed == ["loop"]: - # find edge leading into loop body and use the path length for the destination - # state - for edge in point.edges: - if edge._condition == ["enter-loop"]: - reaching_path_length = edge._target_state._path_length - else: - reaching_path_length = point._path_length - else: - reaching_path_length = point._target_state._path_length - - instrumentation_point_dictionary = { - "binding": binding_db_id, - # "serialised_condition_sequence": list( - # map(pickle.dumps, point._previous_edge._condition - # if type(point) is CFGVertex else point._condition) - # ), - "serialised_condition_sequence": "", - "reaching_path_length": reaching_path_length, - "atom": atom_index_to_db_index[atom_index] - } - serialised_dictionary = json.dumps(instrumentation_point_dictionary) - try: - instrumentation_point_db_id = int( - post_to_verdict_server("store_instrumentation_point/", - data=serialised_dictionary)) - except: - logger.log("Unforeseen exception when sending instrumentation point to verdict " - "server:") - logger.log(traceback.format_exc()) - logger.log( - "There was a problem with the verdict server at '%s'. Instrumentation cannot " - "be completed." % VERDICT_SERVER_URL) - exit() - - if not (point_to_triples[point].get(atom_index)): - point_to_triples[point][atom_index] = {} - if not (point_to_triples[point][atom_index].get(sub_index)): - point_to_triples[point][atom_index][sub_index] = [] - - point_to_triples[point][atom_index][sub_index].append([m, instrumentation_point_db_id]) - - logger.log("Placing instruments based on inversion.") - - # we now insert the instruments - - for point in point_to_triples.keys(): - logger.log("Placing instruments for point %s." % point) - for atom_index in point_to_triples[point].keys(): - atom = atoms[atom_index] - for atom_sub_index in point_to_triples[point][atom_index].keys(): - logger.log("Placing single instrument at %s for atom %s at index %i and sub index %i" % ( - point, atom, atom_index, atom_sub_index)) - list_of_lists = list(zip(*point_to_triples[point][atom_index][atom_sub_index])) - - # extract the parameters for this instrumentation point - binding_space_indices = list_of_lists[0] - instrumentation_point_db_ids = list_of_lists[1] - - if type(atom) is formula_tree.TransitionDurationInInterval: - - instrument_point_transition(atom, point, binding_space_indices, atom_index, - atom_sub_index, instrumentation_point_db_ids) - - elif type(atom) in [formula_tree.StateValueInInterval, formula_tree.StateValueEqualTo, - formula_tree.StateValueInOpenInterval]: - - instrument_point_state(atom._state, atom._name, point, binding_space_indices, - atom_index, atom_sub_index, instrumentation_point_db_ids) - - elif type(atom) is formula_tree.StateValueTypeEqualTo: - - instrument_point_state(atom._state, atom._name, point, binding_space_indices, - atom_index, atom_sub_index, instrumentation_point_db_ids, - measure_attribute="type") - - elif type(atom) in [formula_tree.StateValueLengthInInterval]: - """ - Instrumentation for the length of a value given is different - because we have to add len() to the instrument. - """ - - instrument_point_state(atom._state, atom._name, point, binding_space_indices, - atom_index, atom_sub_index, instrumentation_point_db_ids, - measure_attribute="length") - - elif type(atom) in [formula_tree.StateValueEqualToMixed]: - """We're instrumenting multiple states, so we need to perform instrumentation on two - separate points. """ - - # for each side of the atom (LHS and RHS), instrument the necessary points - - logger.log( - "instrumenting for a mixed atom %s with sub atom index %i" % (atom, atom_sub_index) - ) - - if atom_sub_index == 0: - # we're instrumenting for the lhs - logger.log("Placing left-hand-side instrument for SCFG object %s." % atom._lhs) - instrument_point_state(atom._lhs, atom._lhs_name, point, binding_space_indices, - atom_index, atom_sub_index, instrumentation_point_db_ids) - else: - # we're instrumenting for the rhs - logger.log("Placing right-hand-side instrument for SCFG object %s." % atom._rhs) - instrument_point_state(atom._rhs, atom._rhs_name, point, binding_space_indices, - atom_index, atom_sub_index, instrumentation_point_db_ids) - - elif type(atom) is formula_tree.TransitionDurationLessThanTransitionDurationMixed: - """We're instrumenting multiple transitions, so we need to perform instrumentation on - two separate points. """ - - # for each side of the atom (LHS and RHS), instrument the necessary points - - logger.log( - "Instrumenting for a mixed atom %s with sub atom index %i." % (atom, atom_sub_index) - ) - - if atom_sub_index == 0: - # we're instrumenting for the lhs - logger.log("placing lhs instrument for scfg object %s" % atom._lhs) - instrument_point_transition(atom, point, binding_space_indices, - atom_index, atom_sub_index, - instrumentation_point_db_ids) - else: - # we're instrumenting for the rhs - logger.log("placing rhs instrument for scfg object %s" % atom._rhs) - instrument_point_transition(atom, point, binding_space_indices, - atom_index, atom_sub_index, - instrumentation_point_db_ids) - - elif type(atom) is formula_tree.TransitionDurationLessThanStateValueMixed: - """We're instrumenting multiple transitions, so we need to perform instrumentation on - two separate points. """ - - # for each side of the atom (LHS and RHS), instrument the necessary points - - logger.log( - "Instrumenting for a mixed atom %s with sub atom index %i." % (atom, atom_sub_index) - ) - - if atom_sub_index == 0: - # we're instrumenting for the lhs - logger.log("Placing left-hand-side instrument for SCFG object %s." % atom._lhs) - instrument_point_transition(atom, point, binding_space_indices, - atom_index, atom_sub_index, - instrumentation_point_db_ids) - else: - # we're instrumenting for the rhs - logger.log("Placing right-hand-side instrument for SCFG object %s." % atom._rhs) - instrument_point_state(atom._rhs, atom._rhs_name, point, binding_space_indices, - atom_index, atom_sub_index, instrumentation_point_db_ids) - - elif type(atom) is formula_tree.TransitionDurationLessThanStateValueLengthMixed: - """We're instrumenting multiple transitions, so we need to perform instrumentation on - two separate points. """ - - # for each side of the atom (LHS and RHS), instrument the necessary points - - logger.log( - "Instrumenting for a mixed atom %s with sub atom index %i." % (atom, atom_sub_index) - ) - - if atom_sub_index == 0: - # we're instrumenting for the lhs - logger.log("Placing left-hand-side instrument for SCFG object %s." % atom._lhs) - instrument_point_transition(atom, point, binding_space_indices, - atom_index, atom_sub_index, - instrumentation_point_db_ids) - else: - # we're instrumenting for the rhs - logger.log("Placing right-hand-side instrument for SCFG object %s." % atom._rhs) - instrument_point_state(atom._rhs, atom._rhs_name, point, binding_space_indices, - atom_index, atom_sub_index, instrumentation_point_db_ids, - measure_attribute="length") - - elif type(atom) in [formula_tree.TimeBetweenInInterval, - formula_tree.TimeBetweenInOpenInterval]: - """We're instrumenting multiple transitions, so we need to perform instrumentation on - two separate points. """ - - # for each side of the atom (LHS and RHS), instrument the necessary points - - logger.log( - "Instrumenting for a mixed atom %s with sub atom index %i." % (atom, atom_sub_index) - ) - - if atom_sub_index == 0: - # we're instrumenting for the lhs - logger.log("Placing left-hand-side instrument for SCFG object %s." % atom._lhs) - instrument_point_state(atom._lhs, None, point, binding_space_indices, - atom_index, atom_sub_index, instrumentation_point_db_ids, - measure_attribute="time_attained") - else: - # we're instrumenting for the rhs - logger.log("Placing right-hand-side instrument for SCFG object %s." % atom._rhs) - instrument_point_state(atom._rhs, None, point, binding_space_indices, - atom_index, atom_sub_index, instrumentation_point_db_ids, - measure_attribute="time_attained") - - if EXPLANATION: - print("=" * 100) - print("INSTRUMENTING FOR EXPLANATION") - - pprint.pprint(scfg.branch_initial_statements) - - # if explanation was turned on in the configuration file, insert path instruments. - - # insert path recording instruments - these don't depend on the formula being checked so - # this is done independent of binding space computation - for vertex_information in scfg.branch_initial_statements: - print("-" * 100) - if vertex_information[0] in ['conditional', 'try-catch']: - if vertex_information[0] == 'conditional': - print( - "Placing branch recording instrument for conditional with first instruction %s in block" % - vertex_information[1]) - # instrument_code = "print(\"appending path condition %s inside conditional\")" % vertex_information[2] - # send branching condition to verdict server, take the ID from the response and use it in the path recording instruments. - condition_dict = { - "serialised_condition": pickle.dumps(vertex_information[2]) - } - else: - print( - "Placing branch recording instrument for try-catch with first instruction %s in block" % - vertex_information[1]) - # send branching condition to verdict server, take the ID from the response and use it in the path recording instruments. - condition_dict = { - "serialised_condition": vertex_information[2] - } - # if the condition already exists in the database, the verdict server will return the existing ID - try: - branching_condition_id = int(post_to_verdict_server("store_branching_condition/", - data=json.dumps(condition_dict))) - except: - print( - "There was a problem with the verdict server at '%s'. Instrumentation cannot be completed." % VERDICT_SERVER_URL) - exit() - instrument_code = "%s((\"%s\", \"path\", \"%s\", %i))" % ( - VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - branching_condition_id) - instrument_ast = ast.parse(instrument_code).body[0] - """instrument_ast.lineno = vertex_information[1]._parent_body[0].lineno - instrument_ast.col_offset = vertex_information[1]._parent_body[0].col_offset""" - index_in_parent = vertex_information[1]._parent_body.index(vertex_information[1]) - vertex_information[1]._parent_body.insert(index_in_parent, instrument_ast) - print("Branch recording instrument placed") - elif vertex_information[0] == "conditional-no-else": - # no else was present in the conditional, so we add a path recording instrument - # to the else block - print("Placing branch recording instrument for conditional with no else") - # send branching condition to verdict server, take the ID from the response and use it in the path recording instruments. - condition_dict = { - "serialised_condition": pickle.dumps(vertex_information[2]) - } - # if the condition already exists in the database, the verdict server will return the existing ID - try: - branching_condition_id = int(post_to_verdict_server("store_branching_condition/", - data=json.dumps(condition_dict))) - except: - print( - "There was a problem with the verdict server at '%s'. Instrumentation cannot be completed." % VERDICT_SERVER_URL) - exit() - instrument_code = "%s((\"%s\", \"path\", \"%s\", %i))" % ( - VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - branching_condition_id) - instrument_ast = ast.parse(instrument_code).body[0] - vertex_information[1].orelse.insert(0, instrument_ast) - print("Branch recording instrument placed") - elif vertex_information[0] in ['post-conditional', 'post-try-catch']: - if vertex_information[0] == 'post-conditional': - print("Processing post conditional path instrument") - print(vertex_information) - # need this to decide if we've left a conditional, since paths lengths reset after conditionals - print( - "Placing branch recording instrument for end of conditional at %s - %i in parent block - line no %i" % \ - (vertex_information[1], - vertex_information[1]._parent_body.index(vertex_information[1]), - vertex_information[1].lineno)) - - condition_dict = { - "serialised_condition": "conditional exited" - } - else: - print("Processing post try-catch path instrument") - print(vertex_information) - # need this to decide if we've left a conditional, since paths lengths reset after conditionals - print( - "Placing branch recording instrument for end of try-catch at %s - %i in parent block - line no %i" % \ - (vertex_information[1], - vertex_information[1]._parent_body.index(vertex_information[1]), - vertex_information[1].lineno)) - - condition_dict = { - "serialised_condition": "try-catch exited" - } - try: - branching_condition_id = int(post_to_verdict_server("store_branching_condition/", - data=json.dumps(condition_dict))) - except: - print( - "There was a problem with the verdict server at '%s'. Instrumentation cannot be completed." % VERDICT_SERVER_URL) - exit() - instrument_code = "%s((\"%s\", \"path\", \"%s\", %i))" % ( - VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - branching_condition_id) - instrument_code_ast = ast.parse(instrument_code).body[0] - """instrument_code_ast.lineno = vertex_information[1].lineno+1 - instrument_code_ast.col_offset = vertex_information[1].col_offset""" - - index_in_parent = vertex_information[1]._parent_body.index(vertex_information[1]) + 1 - print(vertex_information[1]._parent_body) - print(index_in_parent) - vertex_information[1]._parent_body.insert(index_in_parent, instrument_code_ast) - print(vertex_information[1]._parent_body) - elif vertex_information[0] == 'loop': - print("Placing branch recording instrument for loop with first instruction %s in body" % - vertex_information[1]) - condition_dict = { - "serialised_condition": pickle.dumps(vertex_information[2]) - } - # if the condition already exists in the database, the verdict server will return the existing ID - try: - branching_condition_id = int(post_to_verdict_server("store_branching_condition/", - data=json.dumps(condition_dict))) - except: - print( - "There was a problem with the verdict server at '%s'. Instrumentation cannot be completed." % VERDICT_SERVER_URL) - exit() - instrument_code_inside_loop = "%s((\"%s\", \"path\", \"%s\", %i))" % ( - VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - branching_condition_id) - instrument_inside_loop_ast = ast.parse(instrument_code_inside_loop).body[0] - """instrument_inside_loop_ast.lineno = vertex_information[1].lineno - instrument_inside_loop_ast.col_offset = vertex_information[1].col_offset""" - - condition_dict = { - "serialised_condition": pickle.dumps(vertex_information[4]) - } - # if the condition already exists in the database, the verdict server will return the existing ID - try: - branching_condition_id = int(post_to_verdict_server("store_branching_condition/", - data=json.dumps(condition_dict))) - except: - print( - "There was a problem with the verdict server at '%s'. Instrumentation cannot be completed." % VERDICT_SERVER_URL) - exit() - instrument_code_outside_loop = "%s((\"%s\", \"path\", \"%s\", %i))" % ( - VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - branching_condition_id) - instrument_outside_loop_ast = ast.parse(instrument_code_outside_loop).body[0] - """instrument_outside_loop_ast.lineno = vertex_information[3].lineno+1 - instrument_outside_loop_ast.col_offset = vertex_information[3].col_offset""" - - # insert at beginning of loop body - inside_index_in_parent = vertex_information[1]._parent_body.index(vertex_information[1]) - # insert just after loop body - outside_index_in_parent = vertex_information[3]._parent_body.index( - vertex_information[3]) + 1 - - vertex_information[1]._parent_body.insert(inside_index_in_parent, - instrument_inside_loop_ast) - vertex_information[3]._parent_body.insert(outside_index_in_parent, - instrument_outside_loop_ast) - print("Branch recording instrument for conditional placed") - - print("=" * 100) - - # finally, insert an instrument at the beginning to tell the monitoring thread that a new call of the function has started - # and insert one at the end to signal a return - - # NOTE: only problem with this is that the "end" instrument is inserted before the return, - # so a function call in the return statement maybe missed if it's part of verification... - thread_id_capture = "import threading; __thread_id = threading.current_thread().ident;" - #start_instrument = "%s((\"%s\", \"function\", \"%s\", \"start\", flask.g.request_time, \"%s\", __thread_id))" \ - # % ( - # VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - # formula_hash) - - start_instrument = "%s((\"%s\", \"function\", \"%s\", \"start\", datetime.datetime.now(), \"%s\", __thread_id))" \ - % ( - VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - formula_hash) - - - threading_import_ast = ast.parse(thread_id_capture).body[0] - thread_id_capture_ast = ast.parse(thread_id_capture).body[1] - start_ast = ast.parse(start_instrument).body[0] - - print("inserting scope instruments with line number ", function_def.body[0].lineno) - - print(function_def.body) - - threading_import_ast.lineno = function_def.body[0].lineno - thread_id_capture_ast.lineno = function_def.body[0].lineno - start_ast.lineno = function_def.body[0].lineno - - """threading_import_ast.lineno = function_def.body[0].lineno - threading_import_ast.col_offset = function_def.body[0].col_offset - thread_id_capture_ast.lineno = function_def.body[0].lineno - thread_id_capture_ast.col_offset = function_def.body[0].col_offset - start_ast.lineno = function_def.body[0].lineno - start_ast.col_offset = function_def.body[0].col_offset""" - - function_def.body.insert(0, start_ast) - function_def.body.insert(0, thread_id_capture_ast) - function_def.body.insert(0, threading_import_ast) - - print(function_def.body) - - # insert the end instrument before every return statement - for end_vertex in scfg.return_statements: - end_instrument = "%s((\"%s\", \"function\", \"%s\", \"end\", %r, flask.g.request_time, \"%s\", __thread_id, \"%s\"))" \ - % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, detect_testing_frameworks(asts), - formula_hash, TEST_AWARE) - - - - end_ast = ast.parse(end_instrument).body[0] - - """end_ast.lineno = end_vertex._previous_edge._instruction.lineno - end_ast.col_offset = end_vertex._previous_edge._instruction.col_offset""" - - end_ast.lineno = end_vertex._previous_edge._instruction._parent_body[-1].lineno - - print("inserting end instrument at line %i" % end_ast.lineno) - - insertion_position = len(end_vertex._previous_edge._instruction._parent_body) - 1 - - end_vertex._previous_edge._instruction._parent_body.insert(insertion_position, end_ast) - - print(end_vertex._previous_edge._instruction._parent_body) - - # if the last instruction in the ast is not a return statement, add an end instrument at the end - if not (type(function_def.body[-1]) is ast.Return): - # end_instrument = "%s((\"%s\", \"function\", \"%s\", \"end\", flask.g.request_time, \"%s\"))" \ - # % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, - # formula_hash) - - end_instrument = "%s((\"%s\", \"function\", \"%s\", \"end\", %r ,datetime.datetime.now(), \"%s\", __thread_id, \"%s\"))" \ - % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier,detect_testing_frameworks(asts), - formula_hash, TEST_AWARE) - end_ast = ast.parse(end_instrument).body[0] + logger.log("SCFG constructed.") - function_def.body.insert(len(function_def.body), end_ast) - # finally, insert an instrument at the beginning to tell the monitoring thread that a new call of the - # function has started and insert one at the end to signal a return - # also insert instruments at the end(s) of the function - place_function_end_instruments(function_def, scfg, formula_hash, instrument_function_qualifier) + # write scfg to file + write_scfg_to_file(scfg, "%s-%s-%s.gv" % + (file_name_without_extension.replace(".", ""), module.replace(".", "-"), + function.replace(".", "-"))) - # write the instrumented scfg to a file - instrumented_scfg = CFG() - instrumented_scfg.process_block(top_level_block) - write_scfg_to_file(instrumented_scfg, "%s-%s-%s-instrumented.gv" % - (file_name_without_extension.replace(".", ""), module.replace(".", "-"), - function.replace(".", "-"))) + # for each property, instrument the function for that property - # check for existence of directories for intermediate data and create them if not found - if not (os.path.isdir("binding_spaces")): - os.mkdir("binding_spaces") - if not (os.path.isdir("index_hash")): - os.mkdir("index_hash") + for (formula_index, formula_structure) in enumerate(verification_conf[module][function]): - # pickle binding space - pickled_binding_space = pickle.dumps(bindings) + logger.log("Instrumenting for PyCFTL formula %s" % formula_structure) - # write to a file - binding_space_dump_file = "binding_spaces/module-%s-function-%s-property-%s.dump" % \ - (module.replace(".", "-"), function.replace(".", "-"), formula_hash) - with open(binding_space_dump_file, "wb") as h: - h.write(pickled_binding_space) + # we should be able to use the same scfg for each stage of instrumentation, + # since when we insert instruments we recompute the position of the instrumented instruction + + atoms = formula_structure._formula_atoms + + formula_hash = hashlib.sha1() + serialised_bind_variables = base64.encodestring(pickle.dumps(formula_structure.bind_variables)) + formula_hash.update(serialised_bind_variables) + serialised_bind_variables = serialised_bind_variables.decode('ascii') + serialised_formula_structure = base64.encodestring( + pickle.dumps(formula_structure.get_formula_instance()) + ) + formula_hash.update(serialised_formula_structure) + serialised_formula_structure = serialised_formula_structure.decode('ascii') + formula_hash = formula_hash.hexdigest() + serialised_atom_list = list( + map(lambda item: base64.encodestring(pickle.dumps(item)).decode('ascii'), atoms) + ) + + # note that this also means giving an empty list [] will result in path instrumentation + # without property instrumentation + if EXPLANATION: + logger.log("=" * 100) + logger.log("Placing path recording instruments.") + place_path_recording_instruments(scfg, instrument_function_qualifier, formula_hash) + + # update the index -> hash map + index_to_hash.append(formula_hash) + + logger.log("with formula hash '%s'" % formula_hash) + + # construct reachability of the SCFG + # and derive the binding space based on the formula + + reachability_map = construct_reachability_map(scfg) + bindings = compute_binding_space(formula_structure, scfg, reachability_map) + print(bindings) + + logger.log("Set of static bindings computed is") + logger.log(str(bindings)) + + # using these bindings, we now need to instrument the code + # and then store the (bind space index, bind var index, atom index) + # so the instrumentation mappings can be recovered at runtime without recomputation + + static_qd_to_point_map = {} + vertices_to_triple_list = {} + + # we attach indices to atoms because we need their index in the set of atoms + # of the relevant formula + initial_property_dict = { + "formula_hash": formula_hash, + "function": instrument_function_qualifier, + "serialised_formula_structure": serialised_formula_structure, + "serialised_bind_variables": serialised_bind_variables, + "serialised_atom_list": list(enumerate(serialised_atom_list)) + } + + # send instrumentation data to the verdict database + try: + logger.log( + "Sending property with hash '%s' for function '%s' in module '%s' to server." % + (formula_hash, function, module)) + response = str(post_to_verdict_server("store_property/", data=json.dumps(initial_property_dict))) + response = json.loads(response) + atom_index_to_db_index = response["atom_index_to_db_index"] + function_id = response["function_id"] + except: + logger.log("Unforeseen exception when sending property to verdict server:") + logger.log(traceback.format_exc()) + logger.log( + "There was a problem with the verdict server at '%s'. Instrumentation cannot be completed." + % VERDICT_SERVER_URL) + exit() + + logger.log("Processing set of static bindings:") + + for (m, element) in enumerate(bindings): + + logger.log("Processing binding %s" % element) + + # send the binding to the verdict server + + line_numbers = [] + for el in element: + if type(el) is CFGVertex: + if el._name_changed != ["loop"]: + line_numbers.append(el._previous_edge._instruction.lineno) + else: + line_numbers.append(el._structure_obj.lineno) + else: + line_numbers.append(el._instruction.lineno) + + binding_dictionary = { + "binding_space_index": m, + "function": function_id, + "binding_statement_lines": line_numbers + } + serialised_binding_dictionary = json.dumps(binding_dictionary) + try: + binding_db_id = int( + post_to_verdict_server("store_binding/", data=serialised_binding_dictionary)) + except: + logger.log("Unforeseen exception when sending binding to verdict server:") + logger.log(traceback.format_exc()) + logger.log( + "There was a problem with the verdict server at '%s'. Instrumentation cannot be completed." + % VERDICT_SERVER_URL) + exit() + + static_qd_to_point_map[m] = {} + + for (atom_index, atom) in enumerate(atoms): + + logger.log("Computing instrumentation points for atom %s." % atom) + + static_qd_to_point_map[m][atom_index] = {} + + if type(atom) in [formula_tree.StateValueEqualToMixed, + formula_tree.TransitionDurationLessThanTransitionDurationMixed, + formula_tree.TransitionDurationLessThanStateValueMixed, + formula_tree.TransitionDurationLessThanStateValueLengthMixed, + formula_tree.TimeBetweenInInterval, + formula_tree.TimeBetweenInOpenInterval]: + + # there may be multiple bind variables + composition_sequences = derive_composition_sequence(atom) + + # get lhs and rhs bind variables + lhs_comp_sequence = composition_sequences["lhs"] + lhs_bind_variable = lhs_comp_sequence[-1] + lhs_bind_variable_index = list(formula_structure._bind_variables).index(lhs_bind_variable) + lhs_starting_point = element[lhs_bind_variable_index] + + rhs_comp_sequence = composition_sequences["rhs"] + rhs_bind_variable = rhs_comp_sequence[-1] + rhs_bind_variable_index = list(formula_structure._bind_variables).index(rhs_bind_variable) + rhs_starting_point = element[rhs_bind_variable_index] + + lhs_moves = list(reversed(lhs_comp_sequence[1:-1])) + rhs_moves = list(reversed(rhs_comp_sequence[1:-1])) + + lhs_instrumentation_points = get_instrumentation_points_from_comp_sequence( + lhs_starting_point, lhs_moves) + rhs_instrumentation_points = get_instrumentation_points_from_comp_sequence( + rhs_starting_point, rhs_moves) + + # 0 and 1 are for lhs and rhs respectively + static_qd_to_point_map[m][atom_index][0] = lhs_instrumentation_points + static_qd_to_point_map[m][atom_index][1] = rhs_instrumentation_points + + else: + + # just one composition sequence, so one set of instrumentation points + + composition_sequence = derive_composition_sequence(atom) + bind_variable = composition_sequence[-1] + variable_index = list(formula_structure._bind_variables).index(bind_variable) + value_from_binding = element[variable_index] + moves = list(reversed(composition_sequence[1:-1])) + instrumentation_points = get_instrumentation_points_from_comp_sequence(value_from_binding, + moves) + + # for simple atoms, there is no lhs and rhs so we just use 0 + static_qd_to_point_map[m][atom_index][0] = instrumentation_points + + logger.log("Finished computing instrumentation points. Result is:") + logger.log(static_qd_to_point_map) + + # now, perform the instrumentation + + # first step is to add triggers + + logger.log("Inserting triggers.") + + for (m, element) in enumerate(bindings): + + for bind_variable_index in range(len(formula_structure.bind_variables.keys())): + + logger.log("Adding trigger for static binding/bind variable %i/%i." % (m, bind_variable_index)) + + point = element[bind_variable_index] + + instrument = "%s((\"%s\", \"trigger\", \"%s\", %i, %i))" % \ + (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, m, + bind_variable_index) + + instrument_ast = ast.parse(instrument).body[0] + if type(point) is CFGVertex: + if point._name_changed == ["loop"]: + # triggers for loop variables must be inserted inside the loop + # so we instantiate a new monitor for every iteration + for edge in point.edges: + if edge._condition == ["enter-loop"]: + instruction = edge._instruction + else: + instruction = point._previous_edge._instruction + else: + instruction = point._instruction + + lineno = instruction.lineno + col_offset = instruction.col_offset + + index_in_block = instruction._parent_body.index(instruction) + + instrument_ast.lineno = instruction._parent_body[0].lineno + + # insert triggers before the things that will be measured + instruction._parent_body.insert(index_in_block, instrument_ast) + + # we then invert the map we constructed from triples to instrumentation points so that we can avoid + # overlap of instruments + + logger.log("Inverting instrumentation structure ready for optimisations.") + + point_to_triples = {} + + for (m, element) in enumerate(bindings): + for atom_index in static_qd_to_point_map[m].keys(): + for sub_index in static_qd_to_point_map[m][atom_index].keys(): + points = static_qd_to_point_map[m][atom_index][sub_index] + for (n, point) in enumerate(points): + + if not (point_to_triples.get(point)): + point_to_triples[point] = {} + + atom_index_in_db = atom_index_to_db_index[atom_index] + # for now, we don't need serialised_condition_sequence so we just use a blank string + if type(point) is CFGVertex: + if point._name_changed == ["loop"]: + # find edge leading into loop body and use the path length for the destination + # state + for edge in point.edges: + if edge._condition == ["enter-loop"]: + reaching_path_length = edge._target_state._path_length + else: + reaching_path_length = point._path_length + else: + reaching_path_length = point._target_state._path_length + + instrumentation_point_dictionary = { + "binding": binding_db_id, + # "serialised_condition_sequence": list( + # map(pickle.dumps, point._previous_edge._condition + # if type(point) is CFGVertex else point._condition) + # ), + "serialised_condition_sequence": "", + "reaching_path_length": reaching_path_length, + "atom": atom_index_to_db_index[atom_index] + } + serialised_dictionary = json.dumps(instrumentation_point_dictionary) + try: + instrumentation_point_db_id = int( + post_to_verdict_server("store_instrumentation_point/", + data=serialised_dictionary)) + except: + logger.log("Unforeseen exception when sending instrumentation point to verdict " + "server:") + logger.log(traceback.format_exc()) + logger.log( + "There was a problem with the verdict server at '%s'. Instrumentation cannot " + "be completed." % VERDICT_SERVER_URL) + exit() + + if not (point_to_triples[point].get(atom_index)): + point_to_triples[point][atom_index] = {} + if not (point_to_triples[point][atom_index].get(sub_index)): + point_to_triples[point][atom_index][sub_index] = [] + + point_to_triples[point][atom_index][sub_index].append([m, instrumentation_point_db_id]) + + logger.log("Placing instruments based on inversion.") + + # we now insert the instruments + + for point in point_to_triples.keys(): + logger.log("Placing instruments for point %s." % point) + for atom_index in point_to_triples[point].keys(): + atom = atoms[atom_index] + for atom_sub_index in point_to_triples[point][atom_index].keys(): + logger.log("Placing single instrument at %s for atom %s at index %i and sub index %i" % ( + point, atom, atom_index, atom_sub_index)) + list_of_lists = list(zip(*point_to_triples[point][atom_index][atom_sub_index])) + + # extract the parameters for this instrumentation point + binding_space_indices = list_of_lists[0] + instrumentation_point_db_ids = list_of_lists[1] + + if type(atom) is formula_tree.TransitionDurationInInterval: + + instrument_point_transition(atom, point, binding_space_indices, atom_index, + atom_sub_index, instrumentation_point_db_ids) + + elif type(atom) in [formula_tree.StateValueInInterval, formula_tree.StateValueEqualTo, + formula_tree.StateValueInOpenInterval]: + + instrument_point_state(atom._state, atom._name, point, binding_space_indices, + atom_index, atom_sub_index, instrumentation_point_db_ids) + + elif type(atom) is formula_tree.StateValueTypeEqualTo: + + instrument_point_state(atom._state, atom._name, point, binding_space_indices, + atom_index, atom_sub_index, instrumentation_point_db_ids, + measure_attribute="type") + + elif type(atom) in [formula_tree.StateValueLengthInInterval]: + """ + Instrumentation for the length of a value given is different + because we have to add len() to the instrument. + """ + + instrument_point_state(atom._state, atom._name, point, binding_space_indices, + atom_index, atom_sub_index, instrumentation_point_db_ids, + measure_attribute="length") + + elif type(atom) in [formula_tree.StateValueEqualToMixed]: + """We're instrumenting multiple states, so we need to perform instrumentation on two + separate points. """ + + # for each side of the atom (LHS and RHS), instrument the necessary points + + logger.log( + "instrumenting for a mixed atom %s with sub atom index %i" % (atom, atom_sub_index) + ) + + if atom_sub_index == 0: + # we're instrumenting for the lhs + logger.log("Placing left-hand-side instrument for SCFG object %s." % atom._lhs) + instrument_point_state(atom._lhs, atom._lhs_name, point, binding_space_indices, + atom_index, atom_sub_index, instrumentation_point_db_ids) + else: + # we're instrumenting for the rhs + logger.log("Placing right-hand-side instrument for SCFG object %s." % atom._rhs) + instrument_point_state(atom._rhs, atom._rhs_name, point, binding_space_indices, + atom_index, atom_sub_index, instrumentation_point_db_ids) + + elif type(atom) is formula_tree.TransitionDurationLessThanTransitionDurationMixed: + """We're instrumenting multiple transitions, so we need to perform instrumentation on + two separate points. """ + + # for each side of the atom (LHS and RHS), instrument the necessary points + + logger.log( + "Instrumenting for a mixed atom %s with sub atom index %i." % (atom, atom_sub_index) + ) + + if atom_sub_index == 0: + # we're instrumenting for the lhs + logger.log("placing lhs instrument for scfg object %s" % atom._lhs) + instrument_point_transition(atom, point, binding_space_indices, + atom_index, atom_sub_index, + instrumentation_point_db_ids) + else: + # we're instrumenting for the rhs + logger.log("placing rhs instrument for scfg object %s" % atom._rhs) + instrument_point_transition(atom, point, binding_space_indices, + atom_index, atom_sub_index, + instrumentation_point_db_ids) + + elif type(atom) is formula_tree.TransitionDurationLessThanStateValueMixed: + """We're instrumenting multiple transitions, so we need to perform instrumentation on + two separate points. """ + + # for each side of the atom (LHS and RHS), instrument the necessary points + + logger.log( + "Instrumenting for a mixed atom %s with sub atom index %i." % (atom, atom_sub_index) + ) + + if atom_sub_index == 0: + # we're instrumenting for the lhs + logger.log("Placing left-hand-side instrument for SCFG object %s." % atom._lhs) + instrument_point_transition(atom, point, binding_space_indices, + atom_index, atom_sub_index, + instrumentation_point_db_ids) + else: + # we're instrumenting for the rhs + logger.log("Placing right-hand-side instrument for SCFG object %s." % atom._rhs) + instrument_point_state(atom._rhs, atom._rhs_name, point, binding_space_indices, + atom_index, atom_sub_index, instrumentation_point_db_ids) + + elif type(atom) is formula_tree.TransitionDurationLessThanStateValueLengthMixed: + """We're instrumenting multiple transitions, so we need to perform instrumentation on + two separate points. """ + + # for each side of the atom (LHS and RHS), instrument the necessary points + + logger.log( + "Instrumenting for a mixed atom %s with sub atom index %i." % (atom, atom_sub_index) + ) + + if atom_sub_index == 0: + # we're instrumenting for the lhs + logger.log("Placing left-hand-side instrument for SCFG object %s." % atom._lhs) + instrument_point_transition(atom, point, binding_space_indices, + atom_index, atom_sub_index, + instrumentation_point_db_ids) + else: + # we're instrumenting for the rhs + logger.log("Placing right-hand-side instrument for SCFG object %s." % atom._rhs) + instrument_point_state(atom._rhs, atom._rhs_name, point, binding_space_indices, + atom_index, atom_sub_index, instrumentation_point_db_ids, + measure_attribute="length") + + elif type(atom) in [formula_tree.TimeBetweenInInterval, + formula_tree.TimeBetweenInOpenInterval]: + """We're instrumenting multiple transitions, so we need to perform instrumentation on + two separate points. """ + + # for each side of the atom (LHS and RHS), instrument the necessary points + + logger.log( + "Instrumenting for a mixed atom %s with sub atom index %i." % (atom, atom_sub_index) + ) + + if atom_sub_index == 0: + # we're instrumenting for the lhs + logger.log("Placing left-hand-side instrument for SCFG object %s." % atom._lhs) + instrument_point_state(atom._lhs, None, point, binding_space_indices, + atom_index, atom_sub_index, instrumentation_point_db_ids, + measure_attribute="time_attained") + else: + # we're instrumenting for the rhs + logger.log("Placing right-hand-side instrument for SCFG object %s." % atom._rhs) + instrument_point_state(atom._rhs, None, point, binding_space_indices, + atom_index, atom_sub_index, instrumentation_point_db_ids, + measure_attribute="time_attained") + + # finally, insert an instrument at the beginning to tell the monitoring thread that a new call of the + # function has started and insert one at the end to signal a return + place_function_begin_instruments(function_def, formula_hash, instrument_function_qualifier) + # also insert instruments at the end(s) of the function + place_function_end_instruments(function_def, scfg, formula_hash, instrument_function_qualifier,TEST_AWARE) + + + + + if TEST_AWARE == 'flask': + flask_test_file = inst_configuration.get("flask_test_folder") + flask_test_file_without_extension = inst_configuration.get("flask_test_folder").replace('.py','') + + code = "".join(open(flask_test_file, "r").readlines()) + flask_test_file_ast = ast.parse(code) + + + # add vypr datetime import + vypr_datetime_import = "from datetime import datetime as vypr_dt" + datetime_import_ast = ast.parse(vypr_datetime_import).body[0] + datetime_import_ast.lineno = asts.body[0].lineno + datetime_import_ast.col_offset = asts.body[0].col_offset + flask_test_file_ast.body.insert(0, datetime_import_ast) + + if detect_testing_frameworks(flask_test_file_ast): + + for node in flask_test_file_ast.body: + if type(node) == ast.ClassDef: + class_name = node.name + create_test_setup_method(flask_test_file_ast.body, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE) + create_teardown_method(flask_test_file_ast, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE) + compile_bytecode_and_write(flask_test_file_ast,flask_test_file_without_extension) + + + elif TEST_AWARE == 'normal': + for node in asts.body: + if type(node) == ast.ClassDef: + class_name = node.name + create_test_setup_method(asts.body, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE) + create_teardown_method(asts, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE) + + + + + # write the instrumented scfg to a file + instrumented_scfg = CFG() + instrumented_scfg.process_block(top_level_block) + write_scfg_to_file(instrumented_scfg, "%s-%s-%s-instrumented.gv" % + (file_name_without_extension.replace(".", ""), module.replace(".", "-"), + function.replace(".", "-"))) + + # check for existence of directories for intermediate data and create them if not found + if not (os.path.isdir("binding_spaces")): + os.mkdir("binding_spaces") + if not (os.path.isdir("index_hash")): + os.mkdir("index_hash") - # write the index to hash mapping for properties - pickled_index_hash = pickle.dumps(index_to_hash) - index_to_hash_dump_file = "index_hash/module-%s-function-%s.dump" % \ - (module.replace(".", "-"), function.replace(".", "-")) - with open(index_to_hash_dump_file, "wb") as h: - h.write(pickled_index_hash) + # pickle binding space + pickled_binding_space = pickle.dumps(bindings) + + # write to a file + binding_space_dump_file = "binding_spaces/module-%s-function-%s-property-%s.dump" % \ + (module.replace(".", "-"), function.replace(".", "-"), formula_hash) + with open(binding_space_dump_file, "wb") as h: + h.write(pickled_binding_space) + + # write the index to hash mapping for properties + pickled_index_hash = pickle.dumps(index_to_hash) + index_to_hash_dump_file = "index_hash/module-%s-function-%s.dump" % \ + (module.replace(".", "-"), function.replace(".", "-")) + with open(index_to_hash_dump_file, "wb") as h: + h.write(pickled_index_hash) - compile_bytecode_and_write(asts, file_name_without_extension) + compile_bytecode_and_write(asts, file_name_without_extension) logger.log("Instrumentation complete. If VyPR is imported and activated, monitoring will now work.") # close instrumentation log - logger.end_logging() + logger.end_logging() \ No newline at end of file From 9ac375233fa60bb0fcd3cf761802bf1aaf9bc5bd Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Mon, 23 Mar 2020 14:21:46 +0100 Subject: [PATCH 08/30] added condition for setup class --- init_verification.py | 15 ++++++++------- instrument.py | 13 ++++++++++--- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/init_verification.py b/init_verification.py index 0016d48..5de7992 100644 --- a/init_verification.py +++ b/init_verification.py @@ -49,10 +49,12 @@ def __init__(self, logs_to_stdout): def start_logging(self): # open the log file in append mode - self.handle = open(self.log_file_name, "a") + if not self.logs_to_stdout: + self.handle = open(self.log_file_name, "a") def end_logging(self): - self.handle.close() + if not self.logs_to_stdout: + self.handle.close() def log(self, message): if self.handle: @@ -60,8 +62,8 @@ def log(self, message): self.handle.write("%s\n" % message) # flush the contents of the file to disk - this way we get a log even with an unhandled exception self.handle.flush() - if self.logs_to_stdout: - print(message) + elif self.logs_to_stdout: + print(message) def to_timestamp(obj): @@ -345,7 +347,7 @@ def consumption_thread_function(verification_obj): # We only send verdict data to the server when test_aware_status = top_pair[7] - + vypr_output ("Test aware status %s" %test_aware_status) if not test_aware_status in ['normal', 'flask']: send_verdict_report( @@ -518,7 +520,7 @@ def consumption_thread_function(verification_obj): if instrument_type == "test_status": - + vypr_output("Processing test status instrument..") if IS_END_OPT: status = top_pair[2] @@ -530,7 +532,6 @@ def consumption_thread_function(verification_obj): else: test_result = "Success" vypr_output("Sending verdict report only in case of testing") - print("Sending verdict report only in case of testing") diff --git a/instrument.py b/instrument.py index 4da512b..5d9535f 100644 --- a/instrument.py +++ b/instrument.py @@ -599,7 +599,7 @@ def create_test_setup_method(current_step, class_name, formula_hash,instrument_f :param class_name: Name of the test class """ - + skip_transaction = False setUp_found = False if TEST_AWARE == "normal": @@ -632,12 +632,18 @@ def create_test_setup_method(current_step, class_name, formula_hash,instrument_f if not (type(test_function) is ast.FunctionDef): continue + if test_function.name == 'setUpClass': + setup_transaction = ast.parse(transaction_time_statement).body[0] + test_function.body.insert(0,setup_transaction) + skip_transaction = True + if test_function.name == 'setUp': # We found setUp method, now we need to add verification instructions setUp_found = True - check_condition_for_transaction_assign = ast.parse(check_condition_for_transaction).body[0] - test_function.body.insert(0,check_condition_for_transaction_assign) + if not skip_transaction: + check_condition_for_transaction_assign = ast.parse(check_condition_for_transaction).body[0] + test_function.body.insert(0,check_condition_for_transaction_assign) if TEST_AWARE == 'normal': verification_import_inst = ast.parse(VERIFICATION_IMPORT).body[0] @@ -650,6 +656,7 @@ def create_test_setup_method(current_step, class_name, formula_hash,instrument_f # If there is no setUp method, then we need to add setUp method in the class. if not setUp_found: + if TEST_AWARE == 'normal': setUp_method = "def setUp(self):\n\t" + VERIFICATION_IMPORT + '\n\t' + VERIFICATION_OBJ + '\n\t' + check_condition_for_transaction else: From 010a487968adbf42980efdcb065e0c2280571d62 Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Thu, 26 Mar 2020 22:37:26 +0100 Subject: [PATCH 09/30] updates --- init_verification.py | 19 ++-- instrument.py | 245 +++++++++++++++++++++++++++++-------------- 2 files changed, 179 insertions(+), 85 deletions(-) diff --git a/init_verification.py b/init_verification.py index 0016d48..856b5b5 100644 --- a/init_verification.py +++ b/init_verification.py @@ -49,10 +49,12 @@ def __init__(self, logs_to_stdout): def start_logging(self): # open the log file in append mode - self.handle = open(self.log_file_name, "a") + if not self.logs_to_stdout: + self.handle = open(self.log_file_name, "a") def end_logging(self): - self.handle.close() + if not self.logs_to_stdout: + self.handle.close() def log(self, message): if self.handle: @@ -60,8 +62,8 @@ def log(self, message): self.handle.write("%s\n" % message) # flush the contents of the file to disk - this way we get a log even with an unhandled exception self.handle.flush() - if self.logs_to_stdout: - print(message) + elif self.logs_to_stdout: + print(message) def to_timestamp(obj): @@ -87,13 +89,13 @@ def send_verdict_report(function_name, time_of_call, end_time_of_call, program_p binding_to_line_numbers, transaction_time, property_hash, test_result = None, test_name = None): - + vypr_output("Sending verdicts to server...") """ Send verdict data for a given function call (function name + time of call). """ global VERDICT_SERVER_URL verdicts = verdict_report.get_final_verdict_report() - vypr_output("Sending verdicts to server...") + vypr_output("Retrived the verdict %s" %verdicts) @@ -345,7 +347,7 @@ def consumption_thread_function(verification_obj): # We only send verdict data to the server when test_aware_status = top_pair[7] - + vypr_output ("Test aware status %s" %test_aware_status) if not test_aware_status in ['normal', 'flask']: send_verdict_report( @@ -518,7 +520,7 @@ def consumption_thread_function(verification_obj): if instrument_type == "test_status": - + vypr_output("Processing test status instrument with END_OPT status %s" %IS_END_OPT ) if IS_END_OPT: status = top_pair[2] @@ -530,7 +532,6 @@ def consumption_thread_function(verification_obj): else: test_result = "Success" vypr_output("Sending verdict report only in case of testing") - print("Sending verdict report only in case of testing") diff --git a/instrument.py b/instrument.py index 4da512b..9d8a496 100644 --- a/instrument.py +++ b/instrument.py @@ -592,38 +592,42 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, """ -def create_test_setup_method(current_step, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE): +def create_test_setclass_method(current_step, class_name, formula_hash, instrument_function_qualifier, TEST_AWARE): """ :param enable_normal_testing: Checks whether the testing is "normal" of "flask" based. :param current_step: Contains the AST for the code :param class_name: Name of the test class """ - + skip_transaction = False setUp_found = False if TEST_AWARE == "normal": - #VERIFICATION_IMPORT = "from VyPR.init_verification import Verification" - #VERIFICATION_OBJ ="self.verification = Verification()" - VERIFICATION_IMPORT = "from VyPR import Monitor; self.vypr = Monitor()" - VERIFICATION_OBJ = "self.vypr.initialise(None)" + #VERIFICATION_IMPORT = "from VyPR import Monitor; self.vypr = Monitor()" + #VERIFICATION_OBJ = "self.vypr.initialise(None)" + VERIFICATION_IMPORT = "from VyPR import Monitor" + VERIFICATION_OBJ = "cls.vypr = Monitor(); cls.vypr.initialise(None)" ## Finding the test class. current_step = filter( lambda entry: (type(entry) is ast.ClassDef and entry.name == class_name), current_step)[0] + test_class_body = current_step.body - first_test_method = get_test_case_position(test_class_body, 0) + # Update the instruction because it will be used in setUpClass + - first_test_method = "'" + first_test_method + "'" + if TEST_AWARE == 'normal': + VERIFICATION_INSTRUCTION = "cls.vypr.send_event" + elif TEST_AWARE== 'flask': + VERIFICATION_INSTRUCTION = "vypr.send_event" transaction_time_statement = "%s((\"test_transaction\", vypr_dt.now() ))" % \ (VERIFICATION_INSTRUCTION) - check_condition_for_transaction = "if self._testMethodName ==" +first_test_method + ":" + transaction_time_statement ## Traversing the body of the class in order to look for setUp method @@ -632,12 +636,14 @@ def create_test_setup_method(current_step, class_name, formula_hash,instrument_f if not (type(test_function) is ast.FunctionDef): continue - if test_function.name == 'setUp': - # We found setUp method, now we need to add verification instructions + if test_function.name == 'setUpClass': + setUp_found = True - check_condition_for_transaction_assign = ast.parse(check_condition_for_transaction).body[0] - test_function.body.insert(0,check_condition_for_transaction_assign) + setup_transaction = ast.parse(transaction_time_statement).body[0] + test_function.body.insert(0,setup_transaction) + + if TEST_AWARE == 'normal': verification_import_inst = ast.parse(VERIFICATION_IMPORT).body[0] @@ -648,12 +654,13 @@ def create_test_setup_method(current_step, class_name, formula_hash,instrument_f - # If there is no setUp method, then we need to add setUp method in the class. + # If there is no setUpClass method, then we need to add setUp method in the class. if not setUp_found: + if TEST_AWARE == 'normal': - setUp_method = "def setUp(self):\n\t" + VERIFICATION_IMPORT + '\n\t' + VERIFICATION_OBJ + '\n\t' + check_condition_for_transaction + setUp_method = "@classmethod\ndef setUpClass(cls):\n\t" + VERIFICATION_IMPORT + '\n\t' + VERIFICATION_OBJ + '\n\t' + transaction_time_statement else: - setUp_method = "def setUp(self):\n\t" + check_condition_for_transaction + setUp_method = "@classmethod\ndef setUpClass(cls):\n\t" + transaction_time_statement method_inst = ast.parse(setUp_method).body[0] test_class_body.insert(0,method_inst) @@ -667,7 +674,7 @@ def create_test_setup_method(current_step, class_name, formula_hash,instrument_f """ -def create_teardown_method(ast_code, class_name, formula_hash, function, test_aware_type): +def create_teardownclass_method(ast_code, class_name, formula_hash, function, test_aware_type): """ :param ast_code: Code for the AST @@ -695,43 +702,22 @@ def create_teardown_method(ast_code, class_name, formula_hash, function, test_aw test_class_body = current_step.body + # Updating insttuction for teardownclass method + if test_aware_type == 'flask': + VYPR_OBJ='vypr' + else: + VYPR_OBJ='cls.vypr' - verification_call_code = "%s((\"%s\",\"test_status\", \"%s\",self._resultForDoCleanups, vypr_dt.now(),\"%s\"))" % \ - (VERIFICATION_INSTRUCTION, formula_hash, function,formula_hash - ) - - - - - # ## Find the name of last test method - # function_ast_list = filter( lambda entry: (type(entry) is ast.FunctionDef),test_class_body) - # function_name = map(lambda function: function.name, function_ast_list) - # - # function_name.remove('setUp') - # function_name.remove('tearDown') - # - # last_test_method_name = function_name[-1] - - last_test_method_name = get_test_case_position(test_class_body, -1) - last_test_method_name = "'" + last_test_method_name + "'" - - terminate_monitoring_call = "if self._testMethodName ==" +last_test_method_name + ":" +VYPR_OBJ + ".end_monitoring()" - verification_call_code = verification_call_code + '\n\t' + terminate_monitoring_call - + verification_call_code = VYPR_OBJ + ".end_monitoring()" for test_function in test_class_body: - - if isinstance(test_function, ast.FunctionDef): - function_index = function_index + 1 # Placed teardown as the last function - - if not (type(test_function) is ast.FunctionDef): continue - if test_function.name is 'tearDown': + if test_function.name is 'tearDownClass': # We found tearDown method, now we need to add verification instructions tearDown_found = True verification_call_inst = ast.parse(verification_call_code).body[0] @@ -742,27 +728,10 @@ def create_teardown_method(ast_code, class_name, formula_hash, function, test_aw if not tearDown_found: - tear_method = "def tearDown(self):\n\t" + verification_call_code + tear_method = "@classmethod\ndef tearDownClass(cls):\n\t" + verification_call_code method_inst = ast.parse(tear_method).body[0] - test_class_body.insert((function_index),method_inst) - - - -""" - Identifies the position of a test case in a test suite -""" + test_class_body.insert(len(test_class_body),method_inst) -def get_test_case_position(test_class_body, position): - ## Find the name of last test method - function_ast_list = filter( lambda entry: (type(entry) is ast.FunctionDef),test_class_body) - function_name = map(lambda function: function.name, function_ast_list) - - if 'setUp' in function_name: - function_name.remove('setUp') - if 'tearDown' in function_name: - function_name.remove('tearDown') - - return function_name[position] @@ -792,6 +761,83 @@ def detect_testing_frameworks(ast_code): print("specify testing to either flask or normal in vypr.config file") exit() + + + + +""" + +This method is used for flask based testing. +We have to insert a send_event of type 'end' in the method which calls the flask app method. +TODO: Regex would be a better way to compare the end-point + +""" +def get_method_name(end_point, file_path ): + + """ + + :param end_point: + :param file_path: + :return: name of the test method based on the end-point + + """ + + if os.path.exists(file_path): + None + elif os.path.exists(file_path+'.inst'): + file_path = file_path+'.inst' + + else: + print("File for getting method name does not exist..") + exit() + + file = open(file_path, 'r') + for line in file: + if 'def' in line: + def_method_name = line + + if end_point in line : + without_def = def_method_name.split( )[1] + position_before_arg = without_def.find('(') + method_name = without_def[0:position_before_arg] + return method_name + + +def instrument_test_method(flask_test_file_ast, class_name, test_method_name_to_instrument, formula_hash,instrument_function_qualifier, TEST_AWARE): + + + print (instrument_function_qualifier) + + ## Finding the test class. + current_step = filter( lambda entry: (type(entry) is ast.ClassDef and + entry.name == class_name), flask_test_file_ast.body)[0] + + test_class_body = current_step.body + + + # Updating instruction for flask test + if TEST_AWARE == 'normal': + VERIFICATION_INSTRUCTION="self.vypr.send_event" + else: + VERIFICATION_INSTRUCTION="vypr.send_event" + + verification_call_code = "%s((\"%s\",\"test_status\", \"%s\",self._resultForDoCleanups, vypr_dt.now(),\"%s\"))" % \ + (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, formula_hash + ) + + + for test_function in test_class_body: + + + if not (type(test_function) is ast.FunctionDef): + continue + + + if test_function.name == test_method_name_to_instrument: + verification_call_inst = ast.parse(verification_call_code).body[0] + test_function.body.insert(len(test_function.body),verification_call_inst) + + def place_path_recording_instruments(scfg, instrument_function_qualifier, formula_hash): # insert path recording instruments - these don't depend on the formula being checked so # this is done independent of binding space computation @@ -1028,10 +1074,22 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, time,formula_hash, VYPR_OBJ,TEST_AWARE) end_ast = ast.parse(end_instrument).body[0] + + # In case of test, there is no return instruction therefore we add the test_status instruction after the end instruction + logger.log("Placing end instrument at the end of the function body.") function_def.body.insert(len(function_def.body), end_ast) + if TEST_AWARE == 'normal': + test_status_instrument = \ + "%s((\"%s\",\"test_status\", \"%s\",self._resultForDoCleanups, vypr_dt.now(),\"%s\"))" % \ + (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, formula_hash + ) + test_status_ast = ast.parse(test_status_instrument).body[0] + logger.log("Placing test_status instrument at the end of the function body.") + function_def.body.insert(len(function_def.body), test_status_ast) + if __name__ == "__main__": @@ -1114,8 +1172,7 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ exit() - SETUP_ONCE =False - TEARDOWN_ONCE = False + SETUP_ONCE = False # initialise instrumentation logger logger = InstrumentationLog(LOGS_TO_STDOUT) @@ -1153,6 +1210,9 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ for module in verified_modules: + # Initialize to False for a new file. + SETUP_ONCE = False + logger.log("Processing module '%s'." % module) verified_functions = verification_conf[module].keys() @@ -1164,6 +1224,23 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ code = "".join(open(file_name, "r").readlines()) asts = ast.parse(code) + + if TEST_AWARE == 'flask': + + + + flask_test_file = inst_configuration.get("flask_test_folder") + + flask_test_file_for_inst = flask_test_file + + flask_test_file_without_extension = inst_configuration.get("flask_test_folder").replace('.py','') + + + code = "".join(open(flask_test_file, "r").readlines()) + flask_test_file_ast = ast.parse(code) + + + # add import for init_vypr module if not TEST_AWARE in ['normal']: import_code = "from %s import vypr" % VYPR_MODULE @@ -1191,6 +1268,7 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ logger.log("Processing function '%s'." % function) + # we replace . with : in function definitions to make sure we can distinguish between module # and class navigation later on instrument_function_qualifier = "%s%s.%s" % (machine_id, module, function.replace(".", ":")) @@ -1689,11 +1767,12 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ if TEST_AWARE == 'flask': - flask_test_file = inst_configuration.get("flask_test_folder") - flask_test_file_without_extension = inst_configuration.get("flask_test_folder").replace('.py','') - - code = "".join(open(flask_test_file, "r").readlines()) - flask_test_file_ast = ast.parse(code) + # flask_test_file = inst_configuration.get("flask_test_folder") + # print (flask_test_file) + # flask_test_file_without_extension = inst_configuration.get("flask_test_folder").replace('.py','') + # + # code = "".join(open(flask_test_file, "r").readlines()) + # flask_test_file_ast = ast.parse(code) # add vypr datetime import @@ -1708,18 +1787,29 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ for node in flask_test_file_ast.body: if type(node) == ast.ClassDef: class_name = node.name - create_test_setup_method(flask_test_file_ast.body, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE) - create_teardown_method(flask_test_file_ast, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE) - compile_bytecode_and_write(flask_test_file_ast,flask_test_file_without_extension) + + + if not SETUP_ONCE: + create_test_setclass_method(flask_test_file_ast.body, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE) + + create_teardownclass_method(flask_test_file_ast, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE) + SETUP_ONCE = True + # We are adding a send_event instruction with 'test_status' at the end of the method having the necessary end-point + + test_method_name_to_instrument = get_method_name(function, flask_test_file_for_inst) + instrument_test_method(flask_test_file_ast, class_name, test_method_name_to_instrument, formula_hash,instrument_function_qualifier, TEST_AWARE) + elif TEST_AWARE == 'normal': for node in asts.body: if type(node) == ast.ClassDef: class_name = node.name - create_test_setup_method(asts.body, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE) - create_teardown_method(asts, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE) + if not SETUP_ONCE: + create_test_setclass_method(asts.body, class_name, formula_hash, instrument_function_qualifier, TEST_AWARE) + create_teardownclass_method(asts, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE) + SETUP_ONCE=True @@ -1754,6 +1844,9 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ compile_bytecode_and_write(asts, file_name_without_extension) + if TEST_AWARE == 'flask': + compile_bytecode_and_write(flask_test_file_ast,flask_test_file_without_extension) + logger.log("Instrumentation complete. If VyPR is imported and activated, monitoring will now work.") # close instrumentation log From b025c9a049f4d89ac7845d48493fc4cebe0a3d18 Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Mon, 30 Mar 2020 10:50:39 +0200 Subject: [PATCH 10/30] removed anti-patterns --- init_verification.py | 29 ++-- instrument.py | 374 +++++++++++++++++++++++-------------------- 2 files changed, 218 insertions(+), 185 deletions(-) diff --git a/init_verification.py b/init_verification.py index 856b5b5..b1467be 100644 --- a/init_verification.py +++ b/init_verification.py @@ -32,6 +32,10 @@ IS_END_OPT = False + + + + class MonitoringLog(object): """ Class to handle monitoring logging. @@ -66,6 +70,9 @@ def log(self, message): print(message) + + + def to_timestamp(obj): if type(obj) is datetime.datetime: return obj.isoformat() @@ -127,6 +134,7 @@ def send_verdict_report(function_name, time_of_call, end_time_of_call, program_p vypr_output("Function end time was %s" % end_time_of_call) + call_data = { "transaction_time": transaction_time.isoformat(), "time_of_call": time_of_call.isoformat(), @@ -200,17 +208,17 @@ def consumption_thread_function(verification_obj): continue_monitoring = True while continue_monitoring: - - # import pdb # pdb.set_trace() - # take top element from the queue try: top_pair = verification_obj.consumption_queue.get(timeout=1) ## In case of flask testing except: + # Changing flag to false here because in normal testing, end-monitoring does not change to False. + # If exception is raised we just terminate the monitoring + continue @@ -346,9 +354,12 @@ def consumption_thread_function(verification_obj): # We only send verdict data to the server when - test_aware_status = top_pair[7] - vypr_output ("Test aware status %s" %test_aware_status) - if not test_aware_status in ['normal', 'flask']: + is_test = top_pair[7] + vypr_output ("Test aware status %s" %is_test) + vypr_output ("Type %s" %type(is_test)) + + # Not flask-testing nor normal-testing + if not is_test: send_verdict_report( function_name, @@ -362,7 +373,6 @@ def consumption_thread_function(verification_obj): ) - # reset the verdict report maps.verdict_report.reset() @@ -534,7 +544,6 @@ def consumption_thread_function(verification_obj): vypr_output("Sending verdict report only in case of testing") - send_verdict_report( function_name, maps.latest_time_of_call, @@ -558,6 +567,7 @@ def consumption_thread_function(verification_obj): maps.program_path = [] IS_END_OPT = False + # Finish the loop # set the task as done verification_obj.consumption_queue.task_done() @@ -567,7 +577,7 @@ def consumption_thread_function(verification_obj): vypr_output("=" * 100) # if we reach this point, the monitoring thread is ending - vypr_logger.end_logging() + #vypr_logger.end_logging() class PropertyMapGroup(object): @@ -821,6 +831,7 @@ def send_event(self, event_description): def end_monitoring(self): if not (self.initialisation_failure): + print ("End monitoring signal") vypr_output("Ending VyPR monitoring thread.") self.consumption_queue.put(("end-monitoring",)) diff --git a/instrument.py b/instrument.py index 9d8a496..dcff3e4 100644 --- a/instrument.py +++ b/instrument.py @@ -588,48 +588,40 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, Adds a setUp() method and creates a verification object if it doesn't exists, otherwise update the setUp() method with creating a verification object. - METHOD_CONSTRAINT: + TODO: Too many checks for distinguishing between flask and non-flask testing. """ -def create_test_setclass_method(current_step, class_name, formula_hash, instrument_function_qualifier, TEST_AWARE): +def create_test_setclass_method(current_step, class_name, flask): """ :param enable_normal_testing: Checks whether the testing is "normal" of "flask" based. :param current_step: Contains the AST for the code :param class_name: Name of the test class """ - skip_transaction = False setUp_found = False - if TEST_AWARE == "normal": - - #VERIFICATION_IMPORT = "from VyPR import Monitor; self.vypr = Monitor()" - #VERIFICATION_OBJ = "self.vypr.initialise(None)" + # Setting instuctions for setUp methods. + if flask: + VERIFICATION_INSTRUCTION = "vypr.send_event" + else: + VERIFICATION_INSTRUCTION = "cls.vypr.send_event" VERIFICATION_IMPORT = "from VyPR import Monitor" VERIFICATION_OBJ = "cls.vypr = Monitor(); cls.vypr.initialise(None)" - ## Finding the test class. + ## Finding the test class. current_step = filter( lambda entry: (type(entry) is ast.ClassDef and entry.name == class_name), current_step)[0] test_class_body = current_step.body - # Update the instruction because it will be used in setUpClass - - - if TEST_AWARE == 'normal': - VERIFICATION_INSTRUCTION = "cls.vypr.send_event" - elif TEST_AWARE== 'flask': - VERIFICATION_INSTRUCTION = "vypr.send_event" - transaction_time_statement = "%s((\"test_transaction\", vypr_dt.now() ))" % \ (VERIFICATION_INSTRUCTION) - ## Traversing the body of the class in order to look for setUp method + ## Traversing the body of the class in order to look for setUpClass method for test_function in test_class_body: @@ -643,25 +635,25 @@ def create_test_setclass_method(current_step, class_name, formula_hash, instrume setup_transaction = ast.parse(transaction_time_statement).body[0] test_function.body.insert(0,setup_transaction) - - - if TEST_AWARE == 'normal': + if not flask: verification_import_inst = ast.parse(VERIFICATION_IMPORT).body[0] verification_import_obj_assign = ast.parse(VERIFICATION_OBJ).body[0] test_function.body.insert(0,verification_import_obj_assign) test_function.body.insert(0,verification_import_inst) - - # If there is no setUpClass method, then we need to add setUp method in the class. if not setUp_found: - if TEST_AWARE == 'normal': - setUp_method = "@classmethod\ndef setUpClass(cls):\n\t" + VERIFICATION_IMPORT + '\n\t' + VERIFICATION_OBJ + '\n\t' + transaction_time_statement - else: + if flask: + setUp_method = "@classmethod\ndef setUpClass(cls):\n\t" + transaction_time_statement + else: + + setUp_method = "@classmethod\ndef setUpClass(cls):\n\t" + VERIFICATION_IMPORT + '\n\t' + VERIFICATION_OBJ + '\n\t' + transaction_time_statement + + method_inst = ast.parse(setUp_method).body[0] test_class_body.insert(0,method_inst) @@ -674,7 +666,7 @@ def create_test_setclass_method(current_step, class_name, formula_hash, instrume """ -def create_teardownclass_method(ast_code, class_name, formula_hash, function, test_aware_type): +def create_teardownclass_method(ast_code, class_name, flask): """ :param ast_code: Code for the AST @@ -686,53 +678,44 @@ def create_teardownclass_method(ast_code, class_name, formula_hash, function, te tearDown_found = False - function_index = 0 - # In case of flask, getting the class name - - - if detect_testing_frameworks(ast_code): - - ## Finding the test class. - current_step = filter( lambda entry: (type(entry) is ast.ClassDef and + ## Finding the test class. + current_step = filter( lambda entry: (type(entry) is ast.ClassDef and entry.name == class_name), ast_code.body)[0] - test_class_body = current_step.body - + test_class_body = current_step.body - # Updating insttuction for teardownclass method - if test_aware_type == 'flask': - VYPR_OBJ='vypr' - else: - VYPR_OBJ='cls.vypr' + # Updating insttuction for teardownclass method + if flask: + VYPR_OBJ='vypr' + else: + VYPR_OBJ='cls.vypr' - verification_call_code = VYPR_OBJ + ".end_monitoring()" + verification_call_code = VYPR_OBJ + ".end_monitoring()" - for test_function in test_class_body: + for test_function in test_class_body: - if not (type(test_function) is ast.FunctionDef): - continue + if not (type(test_function) is ast.FunctionDef): + continue - if test_function.name is 'tearDownClass': - # We found tearDown method, now we need to add verification instructions - tearDown_found = True - verification_call_inst = ast.parse(verification_call_code).body[0] - test_function.body.insert(0,verification_call_inst) + if test_function.name is 'tearDownClass': + # We found tearDown method, now we need to add verification instructions + tearDown_found = True + verification_call_inst = ast.parse(verification_call_code).body[0] + test_function.body.insert(0,verification_call_inst) ## Terminate monitoring after all tests are analyzed - if not tearDown_found: - tear_method = "@classmethod\ndef tearDownClass(cls):\n\t" + verification_call_code - method_inst = ast.parse(tear_method).body[0] - test_class_body.insert(len(test_class_body),method_inst) - - + if not tearDown_found: + tear_method = "@classmethod\ndef tearDownClass(cls):\n\t" + verification_call_code + method_inst = ast.parse(tear_method).body[0] + test_class_body.insert(len(test_class_body),method_inst) """ @@ -748,20 +731,12 @@ def detect_testing_frameworks(ast_code): :param ast_code: Abstract Syntax Tree for the code. """ - if TEST_AWARE != None: - - for node in ast_code.body: - if isinstance(node, (ast.Import, ast.ImportFrom)): - if node.names[0].__dict__['name'] is 'unittest': - + for node in ast_code.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + if node.names[0].__dict__['name'] is 'unittest': return True else: return False - else: - print("specify testing to either flask or normal in vypr.config file") - - exit() - @@ -803,10 +778,11 @@ def get_method_name(end_point, file_path ): return method_name -def instrument_test_method(flask_test_file_ast, class_name, test_method_name_to_instrument, formula_hash,instrument_function_qualifier, TEST_AWARE): +def instrument_test_method(flask_test_file_ast, class_name, test_method_name_to_instrument, formula_hash,instrument_function_qualifier, flask_status_dict): + is_testing = flask_status_dict['normal_test'] + flask = flask_status_dict['flask_status'] - print (instrument_function_qualifier) ## Finding the test class. current_step = filter( lambda entry: (type(entry) is ast.ClassDef and @@ -816,11 +792,16 @@ def instrument_test_method(flask_test_file_ast, class_name, test_method_name_to_ # Updating instruction for flask test - if TEST_AWARE == 'normal': - VERIFICATION_INSTRUCTION="self.vypr.send_event" - else: + if flask: + VERIFICATION_INSTRUCTION="vypr.send_event" + elif not flask and is_testing: + + VERIFICATION_INSTRUCTION="self.vypr.send_event" + + + verification_call_code = "%s((\"%s\",\"test_status\", \"%s\",self._resultForDoCleanups, vypr_dt.now(),\"%s\"))" % \ (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, formula_hash ) @@ -1041,19 +1022,33 @@ def place_function_begin_instruments(function_def, formula_hash, instrument_func function_def.body.insert(0, vypr_start_time_ast) -def place_function_end_instruments(function_def, scfg, formula_hash, instrument_function_qualifier, TEST_AWARE): +def place_function_end_instruments(function_def, scfg, formula_hash, instrument_function_qualifier, flask_status_dict): # insert the end instrument before every return statement - if TEST_AWARE == "normal": - time = "vypr_dt.now()" - else: + # Use time accordingly, if its flask or normal testing + + flask = flask_status_dict['flask_status'] + is_testing = flask_status_dict['normal_test'] + + if flask: + time = "flask.g.request_time" + else: + + time = "vypr_dt.now()" + + + + # This condition is used send data to verdict server. We send either directly on 'end' instrument type or 'test_aware' type + test_aware = True if flask or is_testing else False + + for end_vertex in scfg.return_statements: end_instrument = \ "%s((\"%s\", \"function\", \"%s\", \"end\", %s, \"%s\", __thread_id, " \ - "%s.get_time(), \"%s\"))" \ - % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, time,formula_hash, VYPR_OBJ,TEST_AWARE) + "%s.get_time(),%r))" \ + % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, time,formula_hash, VYPR_OBJ, test_aware) end_ast = ast.parse(end_instrument).body[0] end_ast.lineno = end_vertex._previous_edge._instruction._parent_body[-1].lineno @@ -1070,8 +1065,8 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ if not (type(function_def.body[-1]) is ast.Return): end_instrument = \ "%s((\"%s\", \"function\", \"%s\", \"end\", %s, \"%s\", __thread_id, " \ - "%s.get_time(), \"%s\"))" \ - % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, time,formula_hash, VYPR_OBJ,TEST_AWARE) + "%s.get_time(), %r))" \ + % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, time,formula_hash, VYPR_OBJ,test_aware) end_ast = ast.parse(end_instrument).body[0] @@ -1081,7 +1076,7 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ function_def.body.insert(len(function_def.body), end_ast) - if TEST_AWARE == 'normal': + if flask or is_testing: test_status_instrument = \ "%s((\"%s\",\"test_status\", \"%s\",self._resultForDoCleanups, vypr_dt.now(),\"%s\"))" % \ (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, formula_hash @@ -1091,6 +1086,63 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ function_def.body.insert(len(function_def.body), test_status_ast) + +def detect_flask_test_case(function , ast): + + test = detect_testing_frameworks(ast) + + # This flag is use to help decide the instrumentation about which instructions to insert in setupclass and teardownclass methods in test class + is_flask=False + for directory in os.walk("."): + for file in directory[2]: + f = os.path.join(directory[0], file) + if str(f).endswith('.py'): + # rename to .py + with open (f) as file: + contents = file.read() + search_end_point = 'self.client.post(\'/' + function + if search_end_point in contents: + is_flask = True + # Returns the the flask status and test file for instrumenting + return {'flask_status' : is_flask, 'test_file': f, 'normal_test': test} + + # This would be the case for normal testing + return {'flask_status' : is_flask, 'test_file': None, 'normal_test': test} + + + +def add_vypr_datetime_import(asts): + + vypr_datetime_import = "from datetime import datetime as vypr_dt" + datetime_import_ast = ast.parse(vypr_datetime_import).body[0] + datetime_import_ast.lineno = asts.body[0].lineno + datetime_import_ast.col_offset = asts.body[0].col_offset + asts.body.insert(0, datetime_import_ast) + + +def add_setup_instruction(asts, flask_status_dict): + + # Get test and flask status + + is_testing = flask_status_dict['normal_test'] + + flask_status = flask_status_dict['flask_status'] + + if flask_status or is_testing: + + for node in asts.body: + if type(node) == ast.ClassDef: + class_name = node.name + + create_test_setclass_method(asts.body, class_name, flask_status) + + create_teardownclass_method(asts, class_name, flask_status) + + return class_name + + # In case of non-testing + return None + if __name__ == "__main__": @@ -1128,29 +1180,6 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ if inst_configuration.get("use_flask") else "no" VERIFICATION_INSTRUCTION = inst_configuration.get("verification_instruction") \ if inst_configuration.get("verification_instruction") else "verification.send_event" - TEST_AWARE = inst_configuration.get("testing") - FLASK_TEST_FOLDER=inst_configuration.get("flask_test_folder") \ - if inst_configuration.get("flask_test_folder") else "" - - ##Test related checks in configuration file. - if TEST_AWARE not in ['normal', 'flask', 'None']: - print ("Specify normal or flask for testing. None for normal program analysis") - exit() - - - if TEST_AWARE == 'flask': - # In flask-based testing, it is important to specify the file (absolute path) where test case is present. - if inst_configuration.get("flask_test_folder") == None: - print ("Specify the path for flask test file.") - exit() - else: - # If incorrect format of the path is specified. The path should only start with a directory name and end with .py extension. - import re - m = re.search('^([A-z0-9-_+]+\/)*([A-z0-9]+\.(py))',inst_configuration.get("flask_test_folder")) - if m == None: - print ("Specify the correct path for flask test file.") - exit() - VYPR_MODULE = inst_configuration.get("vypr_module") \ @@ -1159,11 +1188,6 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ # VERIFICATION_INSTRUCTION = "print" VYPR_OBJ = "vypr" - if TEST_AWARE == 'normal': - VYPR_OBJ = "self.vypr" - VERIFICATION_INSTRUCTION = VYPR_OBJ+".send_event" - - machine_id = ("%s-" % inst_configuration.get("machine_id")) if inst_configuration.get("machine_id") else "" # first, check that the verdict server is reachable @@ -1172,6 +1196,7 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ exit() + flask_test_file_for_inst = None SETUP_ONCE = False # initialise instrumentation logger @@ -1213,6 +1238,13 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ # Initialize to False for a new file. SETUP_ONCE = False + # Reset the instructions + VERIFICATION_INSTRUCTION = "vypr.send_event" + + # VERIFICATION_INSTRUCTION = "print" + VYPR_OBJ = "vypr" + + logger.log("Processing module '%s'." % module) verified_functions = verification_conf[module].keys() @@ -1220,46 +1252,58 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ file_name = module.replace(".", "/") + ".py" file_name_without_extension = module.replace(".", "/") + print (file_name) + # extract asts from the code in the file code = "".join(open(file_name, "r").readlines()) asts = ast.parse(code) + # If testing is enabled, we detect where the corresponding test case resides so that we can instrument the test file accordingly. + flask_status_dict = detect_flask_test_case(verified_functions[0], asts) - if TEST_AWARE == 'flask': + # Detecting whether the file is a test case. Therefore, we change the verification instruction accordingly + if flask_status_dict['normal_test']: + VYPR_OBJ = "self.vypr" + VERIFICATION_INSTRUCTION = VYPR_OBJ+".send_event" - flask_test_file = inst_configuration.get("flask_test_folder") + # If we find a corresponding test file in case of flask testing, we will instrument it. + if flask_status_dict['test_file'] != None: - flask_test_file_for_inst = flask_test_file + flask_test_file = flask_status_dict['test_file'] - flask_test_file_without_extension = inst_configuration.get("flask_test_folder").replace('.py','') + flask_test_file_for_inst = flask_test_file + flask_test_file_without_extension = flask_test_file.replace('.py','') - code = "".join(open(flask_test_file, "r").readlines()) - flask_test_file_ast = ast.parse(code) + code = "".join(open(flask_test_file, "r").readlines()) + flask_test_file_ast = ast.parse(code) + # We first add imports to the corresponding test file + add_vypr_datetime_import(flask_test_file_ast) + # add import for init_vypr module - if not TEST_AWARE in ['normal']: - import_code = "from %s import vypr" % VYPR_MODULE - import_ast = ast.parse(import_code).body[0] - import_ast.lineno = asts.body[0].lineno - import_ast.col_offset = asts.body[0].col_offset - asts.body.insert(0, import_ast) - - import_code = "import flask" - import_asts = ast.parse(import_code) - flask_import = import_asts.body[0] - asts.body.insert(0, flask_import) + # For normal testing we don't need to add this to the test class file + + if not flask_status_dict['normal_test']: + import_code = "from %s import vypr" % VYPR_MODULE + import_ast = ast.parse(import_code).body[0] + import_ast.lineno = asts.body[0].lineno + import_ast.col_offset = asts.body[0].col_offset + asts.body.insert(0, import_ast) + + import_code = "import flask" + import_asts = ast.parse(import_code) + flask_import = import_asts.body[0] + asts.body.insert(0, flask_import) + # add vypr datetime import - vypr_datetime_import = "from datetime import datetime as vypr_dt" - datetime_import_ast = ast.parse(vypr_datetime_import).body[0] - datetime_import_ast.lineno = asts.body[0].lineno - datetime_import_ast.col_offset = asts.body[0].col_offset - asts.body.insert(0, datetime_import_ast) + add_vypr_datetime_import(asts) + # if we're using flask, we assume a certain architecture @@ -1761,55 +1805,32 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ # function has started and insert one at the end to signal a return place_function_begin_instruments(function_def, formula_hash, instrument_function_qualifier) # also insert instruments at the end(s) of the function - place_function_end_instruments(function_def, scfg, formula_hash, instrument_function_qualifier,TEST_AWARE) - - - - - if TEST_AWARE == 'flask': - # flask_test_file = inst_configuration.get("flask_test_folder") - # print (flask_test_file) - # flask_test_file_without_extension = inst_configuration.get("flask_test_folder").replace('.py','') - # - # code = "".join(open(flask_test_file, "r").readlines()) - # flask_test_file_ast = ast.parse(code) - + place_function_end_instruments(function_def, scfg, formula_hash, instrument_function_qualifier, flask_status_dict) - # add vypr datetime import - vypr_datetime_import = "from datetime import datetime as vypr_dt" - datetime_import_ast = ast.parse(vypr_datetime_import).body[0] - datetime_import_ast.lineno = asts.body[0].lineno - datetime_import_ast.col_offset = asts.body[0].col_offset - flask_test_file_ast.body.insert(0, datetime_import_ast) - - if detect_testing_frameworks(flask_test_file_ast): - - for node in flask_test_file_ast.body: - if type(node) == ast.ClassDef: - class_name = node.name - - - if not SETUP_ONCE: - create_test_setclass_method(flask_test_file_ast.body, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE) - - create_teardownclass_method(flask_test_file_ast, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE) - SETUP_ONCE = True - # We are adding a send_event instruction with 'test_status' at the end of the method having the necessary end-point - - test_method_name_to_instrument = get_method_name(function, flask_test_file_for_inst) - instrument_test_method(flask_test_file_ast, class_name, test_method_name_to_instrument, formula_hash,instrument_function_qualifier, TEST_AWARE) + # Adding setupClass and teardownClass for testing + if not SETUP_ONCE: + if flask_status_dict['flask_status']: + # In case of Flask we only instrument test case. + ast_to_modify = flask_test_file_ast + class_name = add_setup_instruction(ast_to_modify, flask_status_dict) + else: + # In case of normal testing, we instrument the normal program + ast_to_modify = asts + add_setup_instruction(ast_to_modify, flask_status_dict) + # We only add setup instructions once. + SETUP_ONCE = True - elif TEST_AWARE == 'normal': - for node in asts.body: - if type(node) == ast.ClassDef: - class_name = node.name + # We add instruction in flask test file. + if flask_test_file_for_inst: + test_method_name_to_instrument = get_method_name(function, flask_test_file_for_inst) - if not SETUP_ONCE: - create_test_setclass_method(asts.body, class_name, formula_hash, instrument_function_qualifier, TEST_AWARE) - create_teardownclass_method(asts, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE) - SETUP_ONCE=True + instrument_test_method(flask_test_file_ast, + class_name, + test_method_name_to_instrument, + formula_hash,instrument_function_qualifier, + flask_status_dict) @@ -1844,7 +1865,8 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ compile_bytecode_and_write(asts, file_name_without_extension) - if TEST_AWARE == 'flask': + # Generate a bytecode file only in case of flask-based testing + if flask_status_dict['flask_status']: compile_bytecode_and_write(flask_test_file_ast,flask_test_file_without_extension) logger.log("Instrumentation complete. If VyPR is imported and activated, monitoring will now work.") From a1c542ece536ab2e6b9d1a541016685ab83b1c5e Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Thu, 2 Apr 2020 23:02:27 +0200 Subject: [PATCH 11/30] updates based on Joshs new approach to path --- init_verification.py | 46 ++-- instrument.py | 484 ++++++++++++++++++++++++++----------------- 2 files changed, 326 insertions(+), 204 deletions(-) diff --git a/init_verification.py b/init_verification.py index 5de7992..2a607cf 100644 --- a/init_verification.py +++ b/init_verification.py @@ -32,6 +32,10 @@ IS_END_OPT = False + + + + class MonitoringLog(object): """ Class to handle monitoring logging. @@ -66,6 +70,9 @@ def log(self, message): print(message) + + + def to_timestamp(obj): if type(obj) is datetime.datetime: return obj.isoformat() @@ -89,13 +96,13 @@ def send_verdict_report(function_name, time_of_call, end_time_of_call, program_p binding_to_line_numbers, transaction_time, property_hash, test_result = None, test_name = None): - + vypr_output("Sending verdicts to server...") """ Send verdict data for a given function call (function name + time of call). """ global VERDICT_SERVER_URL verdicts = verdict_report.get_final_verdict_report() - vypr_output("Sending verdicts to server...") + vypr_output("Retrived the verdict %s" %verdicts) @@ -127,6 +134,7 @@ def send_verdict_report(function_name, time_of_call, end_time_of_call, program_p vypr_output("Function end time was %s" % end_time_of_call) + call_data = { "transaction_time": transaction_time.isoformat(), "time_of_call": time_of_call.isoformat(), @@ -200,17 +208,17 @@ def consumption_thread_function(verification_obj): continue_monitoring = True while continue_monitoring: - - # import pdb # pdb.set_trace() - # take top element from the queue try: top_pair = verification_obj.consumption_queue.get(timeout=1) ## In case of flask testing except: + # Changing flag to false here because in normal testing, end-monitoring does not change to False. + # If exception is raised we just terminate the monitoring + continue @@ -346,9 +354,12 @@ def consumption_thread_function(verification_obj): # We only send verdict data to the server when - test_aware_status = top_pair[7] - vypr_output ("Test aware status %s" %test_aware_status) - if not test_aware_status in ['normal', 'flask']: + is_test = top_pair[7] + vypr_output ("Test aware status %s" %is_test) + vypr_output ("Type %s" %type(is_test)) + + # Not flask-testing nor normal-testing + if not is_test: send_verdict_report( function_name, @@ -362,7 +373,6 @@ def consumption_thread_function(verification_obj): ) - # reset the verdict report maps.verdict_report.reset() @@ -457,6 +467,8 @@ def consumption_thread_function(verification_obj): new_monitor.atom_to_state_dict[atom_index][sub_index] = \ monitor.atom_to_state_dict[atom_index][sub_index] + vypr_output(" New monitor construction finished.") + elif len(monitor._monitor_instantiation_time) == bind_variable_index: vypr_output(" Updating existing monitor timestamp sequence") # extend the monitor's timestamp sequence @@ -520,7 +532,7 @@ def consumption_thread_function(verification_obj): if instrument_type == "test_status": - vypr_output("Processing test status instrument..") + vypr_output("Processing test status instrument with END_OPT status %s" %IS_END_OPT ) if IS_END_OPT: status = top_pair[2] @@ -534,7 +546,6 @@ def consumption_thread_function(verification_obj): vypr_output("Sending verdict report only in case of testing") - send_verdict_report( function_name, maps.latest_time_of_call, @@ -558,6 +569,7 @@ def consumption_thread_function(verification_obj): maps.program_path = [] IS_END_OPT = False + # Finish the loop # set the task as done verification_obj.consumption_queue.task_done() @@ -567,7 +579,7 @@ def consumption_thread_function(verification_obj): vypr_output("=" * 100) # if we reach this point, the monitoring thread is ending - vypr_logger.end_logging() + #vypr_logger.end_logging() class PropertyMapGroup(object): @@ -717,9 +729,11 @@ def initialise(self, flask_object): if flask_object: def prepare_vypr(): import datetime + from app import vypr # this function runs inside a request, so flask.g exists # we store just the request time - flask.g.request_time = datetime.datetime.now() + #flask.g.request_time = datetime.datetime.now() + flask.g.request_time = vypr.get_time() flask_object.before_request(prepare_vypr) @@ -795,7 +809,7 @@ def endpoint_resume_monitoring(): vypr_output("VyPR monitoring initialisation finished.") - def get_time(self): + def get_time(self, callee=""): """ Returns either the machine local time, or the NTP time (using the initial NTP time obtained when VyPR started up, so we don't query an NTP server everytime we want to measure time). @@ -811,16 +825,16 @@ def get_time(self): current_ntp_time = self.ntp_start_time + difference return current_ntp_time else: - vypr_output("Getting time based on local machine.") + vypr_output("Getting time based on local machine - %s" % callee) return datetime.datetime.utcnow() def send_event(self, event_description): - print("trying to send an event..") if not (self.initialisation_failure): self.consumption_queue.put(event_description) def end_monitoring(self): if not (self.initialisation_failure): + print ("End monitoring signal") vypr_output("Ending VyPR monitoring thread.") self.consumption_queue.put(("end-monitoring",)) diff --git a/instrument.py b/instrument.py index 5d9535f..3ce1033 100644 --- a/instrument.py +++ b/instrument.py @@ -418,7 +418,7 @@ def instrument_point_state(state, name, point, binding_space_indices, state_variable_alias = name.replace(".", "_").replace("(", "__").replace(")", "__") state_recording_instrument = "record_state_%s = %s; " % (state_variable_alias, name) time_attained_instrument = "time_attained_%s = %s.get_time();" % (state_variable_alias,VYPR_OBJ) - + time_attained_variable = "time_attained_%s" % state_variable_alias # note that observed_value is used three times: # 1) to capture the time attained by the state for checking of a property - this is duplicated # because we have the start and end time of the state, which is the same because states are instantaneous. @@ -435,7 +435,7 @@ def instrument_point_state(state, name, point, binding_space_indices, atom_sub_index=atom_sub_index, instrumentation_point_db_id=instrumentation_point_db_ids, atom_program_variable=name, - time_attained = ("time_attained_%s" % state_variable_alias), + time_attained = time_attained_variable, observed_value=("record_state_%s" % state_variable_alias) ) state_recording_instrument += "%s((%s))" % (VERIFICATION_INSTRUCTION, instrument_tuple) @@ -588,44 +588,40 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, Adds a setUp() method and creates a verification object if it doesn't exists, otherwise update the setUp() method with creating a verification object. - METHOD_CONSTRAINT: + TODO: Too many checks for distinguishing between flask and non-flask testing. """ -def create_test_setup_method(current_step, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE): +def create_test_setclass_method(current_step, class_name, flask): """ :param enable_normal_testing: Checks whether the testing is "normal" of "flask" based. :param current_step: Contains the AST for the code :param class_name: Name of the test class """ - skip_transaction = False setUp_found = False - if TEST_AWARE == "normal": - - #VERIFICATION_IMPORT = "from VyPR.init_verification import Verification" - #VERIFICATION_OBJ ="self.verification = Verification()" - VERIFICATION_IMPORT = "from VyPR import Monitor; self.vypr = Monitor()" - VERIFICATION_OBJ = "self.vypr.initialise(None)" + # Setting instuctions for setUp methods. + if flask: + VERIFICATION_INSTRUCTION = "vypr.send_event" + else: + VERIFICATION_INSTRUCTION = "cls.vypr.send_event" + VERIFICATION_IMPORT = "from VyPR import Monitor" + VERIFICATION_OBJ = "cls.vypr = Monitor(); cls.vypr.initialise(None)" - ## Finding the test class. + ## Finding the test class. current_step = filter( lambda entry: (type(entry) is ast.ClassDef and entry.name == class_name), current_step)[0] - test_class_body = current_step.body - - first_test_method = get_test_case_position(test_class_body, 0) - first_test_method = "'" + first_test_method + "'" + test_class_body = current_step.body transaction_time_statement = "%s((\"test_transaction\", vypr_dt.now() ))" % \ (VERIFICATION_INSTRUCTION) - check_condition_for_transaction = "if self._testMethodName ==" +first_test_method + ":" + transaction_time_statement - ## Traversing the body of the class in order to look for setUp method + ## Traversing the body of the class in order to look for setUpClass method for test_function in test_class_body: @@ -633,34 +629,30 @@ def create_test_setup_method(current_step, class_name, formula_hash,instrument_f continue if test_function.name == 'setUpClass': - setup_transaction = ast.parse(transaction_time_statement).body[0] - test_function.body.insert(0,setup_transaction) - skip_transaction = True - if test_function.name == 'setUp': - # We found setUp method, now we need to add verification instructions setUp_found = True - if not skip_transaction: - check_condition_for_transaction_assign = ast.parse(check_condition_for_transaction).body[0] - test_function.body.insert(0,check_condition_for_transaction_assign) + setup_transaction = ast.parse(transaction_time_statement).body[0] + test_function.body.insert(0,setup_transaction) - if TEST_AWARE == 'normal': + if not flask: verification_import_inst = ast.parse(VERIFICATION_IMPORT).body[0] verification_import_obj_assign = ast.parse(VERIFICATION_OBJ).body[0] test_function.body.insert(0,verification_import_obj_assign) test_function.body.insert(0,verification_import_inst) + # If there is no setUpClass method, then we need to add setUp method in the class. + if not setUp_found: + if flask: - # If there is no setUp method, then we need to add setUp method in the class. - if not setUp_found: + setUp_method = "@classmethod\ndef setUpClass(cls):\n\t" + transaction_time_statement - if TEST_AWARE == 'normal': - setUp_method = "def setUp(self):\n\t" + VERIFICATION_IMPORT + '\n\t' + VERIFICATION_OBJ + '\n\t' + check_condition_for_transaction else: - setUp_method = "def setUp(self):\n\t" + check_condition_for_transaction + + setUp_method = "@classmethod\ndef setUpClass(cls):\n\t" + VERIFICATION_IMPORT + '\n\t' + VERIFICATION_OBJ + '\n\t' + transaction_time_statement + method_inst = ast.parse(setUp_method).body[0] test_class_body.insert(0,method_inst) @@ -674,7 +666,7 @@ def create_test_setup_method(current_step, class_name, formula_hash,instrument_f """ -def create_teardown_method(ast_code, class_name, formula_hash, function, test_aware_type): +def create_teardownclass_method(ast_code, class_name, flask): """ :param ast_code: Code for the AST @@ -686,119 +678,147 @@ def create_teardown_method(ast_code, class_name, formula_hash, function, test_aw tearDown_found = False - function_index = 0 - # In case of flask, getting the class name + ## Finding the test class. + current_step = filter( lambda entry: (type(entry) is ast.ClassDef and + entry.name == class_name), ast_code.body)[0] + test_class_body = current_step.body - if detect_testing_frameworks(ast_code): + # Updating insttuction for teardownclass method + if flask: + VYPR_OBJ='vypr' + else: + VYPR_OBJ='cls.vypr' - ## Finding the test class. - current_step = filter( lambda entry: (type(entry) is ast.ClassDef and - entry.name == class_name), ast_code.body)[0] + verification_call_code = VYPR_OBJ + ".end_monitoring()" - test_class_body = current_step.body + for test_function in test_class_body: + + if not (type(test_function) is ast.FunctionDef): + continue + + if test_function.name is 'tearDownClass': + # We found tearDown method, now we need to add verification instructions + tearDown_found = True + verification_call_inst = ast.parse(verification_call_code).body[0] + test_function.body.insert(0,verification_call_inst) + + + ## Terminate monitoring after all tests are analyzed + + + if not tearDown_found: + tear_method = "@classmethod\ndef tearDownClass(cls):\n\t" + verification_call_code + method_inst = ast.parse(tear_method).body[0] + test_class_body.insert(len(test_class_body),method_inst) + + +""" + Detects whether the instrumented file has a testing framework. It will then generated verification object and instruction + related to testing. See create_setup_method/create_teardown_method. + + CONSTRAINT: Only works for python unittests, we will have to extend it for different python testing frameworks. +""" + +def detect_testing_frameworks(ast_code): + + """ + :param ast_code: Abstract Syntax Tree for the code. + """ - verification_call_code = "%s((\"%s\",\"test_status\", \"%s\",self._resultForDoCleanups, vypr_dt.now(),\"%s\"))" % \ - (VERIFICATION_INSTRUCTION, formula_hash, function,formula_hash - ) + for node in ast_code.body: + if isinstance(node, (ast.Import, ast.ImportFrom)): + if node.names[0].__dict__['name'] is 'unittest': + return True + else: + return False +""" - # ## Find the name of last test method - # function_ast_list = filter( lambda entry: (type(entry) is ast.FunctionDef),test_class_body) - # function_name = map(lambda function: function.name, function_ast_list) - # - # function_name.remove('setUp') - # function_name.remove('tearDown') - # - # last_test_method_name = function_name[-1] +This method is used for flask based testing. +We have to insert a send_event of type 'end' in the method which calls the flask app method. +TODO: Regex would be a better way to compare the end-point - last_test_method_name = get_test_case_position(test_class_body, -1) - last_test_method_name = "'" + last_test_method_name + "'" +""" +def get_method_name(end_point, file_path ): - terminate_monitoring_call = "if self._testMethodName ==" +last_test_method_name + ":" +VYPR_OBJ + ".end_monitoring()" - verification_call_code = verification_call_code + '\n\t' + terminate_monitoring_call + """ + :param end_point: + :param file_path: + :return: name of the test method based on the end-point + """ + if os.path.exists(file_path): + None + elif os.path.exists(file_path+'.inst'): + file_path = file_path+'.inst' - for test_function in test_class_body: + else: + print("File for getting method name does not exist..") + exit() + file = open(file_path, 'r') + for line in file: + if 'def' in line: + def_method_name = line - if isinstance(test_function, ast.FunctionDef): - function_index = function_index + 1 # Placed teardown as the last function + if end_point in line : + without_def = def_method_name.split( )[1] + position_before_arg = without_def.find('(') + method_name = without_def[0:position_before_arg] + return method_name - if not (type(test_function) is ast.FunctionDef): - continue +def instrument_test_method(flask_test_file_ast, class_name, test_method_name_to_instrument, formula_hash,instrument_function_qualifier, flask_status_dict): - if test_function.name is 'tearDown': - # We found tearDown method, now we need to add verification instructions - tearDown_found = True - verification_call_inst = ast.parse(verification_call_code).body[0] - test_function.body.insert(0,verification_call_inst) + is_testing = flask_status_dict['normal_test'] + flask = flask_status_dict['flask_status'] - ## Terminate monitoring after all tests are analyzed + ## Finding the test class. + current_step = filter( lambda entry: (type(entry) is ast.ClassDef and + entry.name == class_name), flask_test_file_ast.body)[0] + test_class_body = current_step.body - if not tearDown_found: - tear_method = "def tearDown(self):\n\t" + verification_call_code - method_inst = ast.parse(tear_method).body[0] - test_class_body.insert((function_index),method_inst) + # Updating instruction for flask test + if flask: + VERIFICATION_INSTRUCTION="vypr.send_event" -""" - Identifies the position of a test case in a test suite -""" + elif not flask and is_testing: -def get_test_case_position(test_class_body, position): - ## Find the name of last test method - function_ast_list = filter( lambda entry: (type(entry) is ast.FunctionDef),test_class_body) - function_name = map(lambda function: function.name, function_ast_list) + VERIFICATION_INSTRUCTION="self.vypr.send_event" - if 'setUp' in function_name: - function_name.remove('setUp') - if 'tearDown' in function_name: - function_name.remove('tearDown') - return function_name[position] + verification_call_code = "%s((\"%s\",\"test_status\", \"%s\",self._resultForDoCleanups, vypr_dt.now(),\"%s\"))" % \ + (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, formula_hash + ) -""" - Detects whether the instrumented file has a testing framework. It will then generated verification object and instruction - related to testing. See create_setup_method/create_teardown_method. - - CONSTRAINT: Only works for python unittests, we will have to extend it for different python testing frameworks. -""" + for test_function in test_class_body: -def detect_testing_frameworks(ast_code): - """ - :param ast_code: Abstract Syntax Tree for the code. - """ + if not (type(test_function) is ast.FunctionDef): + continue - if TEST_AWARE != None: - for node in ast_code.body: - if isinstance(node, (ast.Import, ast.ImportFrom)): - if node.names[0].__dict__['name'] is 'unittest': + if test_function.name == test_method_name_to_instrument: + verification_call_inst = ast.parse(verification_call_code).body[0] + test_function.body.insert(len(test_function.body),verification_call_inst) - return True - else: - return False - else: - print("specify testing to either flask or normal in vypr.config file") - exit() def place_path_recording_instruments(scfg, instrument_function_qualifier, formula_hash): # insert path recording instruments - these don't depend on the formula being checked so # this is done independent of binding space computation @@ -1002,19 +1022,33 @@ def place_function_begin_instruments(function_def, formula_hash, instrument_func function_def.body.insert(0, vypr_start_time_ast) -def place_function_end_instruments(function_def, scfg, formula_hash, instrument_function_qualifier, TEST_AWARE): +def place_function_end_instruments(function_def, scfg, formula_hash, instrument_function_qualifier, flask_status_dict): # insert the end instrument before every return statement - if TEST_AWARE == "normal": - time = "vypr_dt.now()" - else: + # Use time accordingly, if its flask or normal testing + + flask = flask_status_dict['flask_status'] + is_testing = flask_status_dict['normal_test'] + + if flask: + time = "flask.g.request_time" + else: + + time = "vypr_dt.now()" + + + + # This condition is used send data to verdict server. We send either directly on 'end' instrument type or 'test_aware' type + test_aware = True if flask or is_testing else False + + for end_vertex in scfg.return_statements: end_instrument = \ "%s((\"%s\", \"function\", \"%s\", \"end\", %s, \"%s\", __thread_id, " \ - "%s.get_time(), \"%s\"))" \ - % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, time,formula_hash, VYPR_OBJ,TEST_AWARE) + "%s.get_time(),%r))" \ + % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, time,formula_hash, VYPR_OBJ, test_aware) end_ast = ast.parse(end_instrument).body[0] end_ast.lineno = end_vertex._previous_edge._instruction._parent_body[-1].lineno @@ -1031,14 +1065,83 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ if not (type(function_def.body[-1]) is ast.Return): end_instrument = \ "%s((\"%s\", \"function\", \"%s\", \"end\", %s, \"%s\", __thread_id, " \ - "%s.get_time(), \"%s\"))" \ - % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, time,formula_hash, VYPR_OBJ,TEST_AWARE) + "%s.get_time(), %r))" \ + % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, time,formula_hash, VYPR_OBJ,test_aware) end_ast = ast.parse(end_instrument).body[0] + + # In case of test, there is no return instruction therefore we add the test_status instruction after the end instruction + logger.log("Placing end instrument at the end of the function body.") function_def.body.insert(len(function_def.body), end_ast) + if flask or is_testing: + test_status_instrument = \ + "%s((\"%s\",\"test_status\", \"%s\",self._resultForDoCleanups, vypr_dt.now(),\"%s\"))" % \ + (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, formula_hash + ) + test_status_ast = ast.parse(test_status_instrument).body[0] + logger.log("Placing test_status instrument at the end of the function body.") + function_def.body.insert(len(function_def.body), test_status_ast) + + + +def detect_flask_test_case(function , ast): + + test = detect_testing_frameworks(ast) + + # This flag is use to help decide the instrumentation about which instructions to insert in setupclass and teardownclass methods in test class + is_flask=False + for directory in os.walk("."): + for file in directory[2]: + f = os.path.join(directory[0], file) + if str(f).endswith('.py'): + # rename to .py + with open (f) as file: + contents = file.read() + search_end_point = 'self.client.post(\'/' + function + if search_end_point in contents: + is_flask = True + # Returns the the flask status and test file for instrumenting + return {'flask_status' : is_flask, 'test_file': f, 'normal_test': test} + + # This would be the case for normal testing + return {'flask_status' : is_flask, 'test_file': None, 'normal_test': test} + + + +def add_vypr_datetime_import(asts): + + vypr_datetime_import = "from datetime import datetime as vypr_dt" + datetime_import_ast = ast.parse(vypr_datetime_import).body[0] + datetime_import_ast.lineno = asts.body[0].lineno + datetime_import_ast.col_offset = asts.body[0].col_offset + asts.body.insert(0, datetime_import_ast) + + +def add_setup_instruction(asts, flask_status_dict): + + # Get test and flask status + + is_testing = flask_status_dict['normal_test'] + + flask_status = flask_status_dict['flask_status'] + + if flask_status or is_testing: + + for node in asts.body: + if type(node) == ast.ClassDef: + class_name = node.name + + create_test_setclass_method(asts.body, class_name, flask_status) + + create_teardownclass_method(asts, class_name, flask_status) + + return class_name + + # In case of non-testing + return None if __name__ == "__main__": @@ -1077,29 +1180,6 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ if inst_configuration.get("use_flask") else "no" VERIFICATION_INSTRUCTION = inst_configuration.get("verification_instruction") \ if inst_configuration.get("verification_instruction") else "verification.send_event" - TEST_AWARE = inst_configuration.get("testing") - FLASK_TEST_FOLDER=inst_configuration.get("flask_test_folder") \ - if inst_configuration.get("flask_test_folder") else "" - - ##Test related checks in configuration file. - if TEST_AWARE not in ['normal', 'flask', 'None']: - print ("Specify normal or flask for testing. None for normal program analysis") - exit() - - - if TEST_AWARE == 'flask': - # In flask-based testing, it is important to specify the file (absolute path) where test case is present. - if inst_configuration.get("flask_test_folder") == None: - print ("Specify the path for flask test file.") - exit() - else: - # If incorrect format of the path is specified. The path should only start with a directory name and end with .py extension. - import re - m = re.search('^([A-z0-9-_+]+\/)*([A-z0-9]+\.(py))',inst_configuration.get("flask_test_folder")) - if m == None: - print ("Specify the correct path for flask test file.") - exit() - VYPR_MODULE = inst_configuration.get("vypr_module") \ @@ -1108,11 +1188,6 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ # VERIFICATION_INSTRUCTION = "print" VYPR_OBJ = "vypr" - if TEST_AWARE == 'normal': - VYPR_OBJ = "self.vypr" - VERIFICATION_INSTRUCTION = VYPR_OBJ+".send_event" - - machine_id = ("%s-" % inst_configuration.get("machine_id")) if inst_configuration.get("machine_id") else "" # first, check that the verdict server is reachable @@ -1121,8 +1196,8 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ exit() - SETUP_ONCE =False - TEARDOWN_ONCE = False + flask_test_file_for_inst = None + SETUP_ONCE = False # initialise instrumentation logger logger = InstrumentationLog(LOGS_TO_STDOUT) @@ -1160,6 +1235,16 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ for module in verified_modules: + # Initialize to False for a new file. + SETUP_ONCE = False + + # Reset the instructions + VERIFICATION_INSTRUCTION = "vypr.send_event" + + # VERIFICATION_INSTRUCTION = "print" + VYPR_OBJ = "vypr" + + logger.log("Processing module '%s'." % module) verified_functions = verification_conf[module].keys() @@ -1167,29 +1252,58 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ file_name = module.replace(".", "/") + ".py" file_name_without_extension = module.replace(".", "/") + print (file_name) + # extract asts from the code in the file code = "".join(open(file_name, "r").readlines()) asts = ast.parse(code) + # If testing is enabled, we detect where the corresponding test case resides so that we can instrument the test file accordingly. + flask_status_dict = detect_flask_test_case(verified_functions[0], asts) + + + # Detecting whether the file is a test case. Therefore, we change the verification instruction accordingly + if flask_status_dict['normal_test']: + VYPR_OBJ = "self.vypr" + VERIFICATION_INSTRUCTION = VYPR_OBJ+".send_event" + + + # If we find a corresponding test file in case of flask testing, we will instrument it. + if flask_status_dict['test_file'] != None: + + flask_test_file = flask_status_dict['test_file'] + + flask_test_file_for_inst = flask_test_file + + flask_test_file_without_extension = flask_test_file.replace('.py','') + + code = "".join(open(flask_test_file, "r").readlines()) + + flask_test_file_ast = ast.parse(code) + + + # We first add imports to the corresponding test file + add_vypr_datetime_import(flask_test_file_ast) + # add import for init_vypr module - if not TEST_AWARE in ['normal']: - import_code = "from %s import vypr" % VYPR_MODULE - import_ast = ast.parse(import_code).body[0] - import_ast.lineno = asts.body[0].lineno - import_ast.col_offset = asts.body[0].col_offset - asts.body.insert(0, import_ast) - - import_code = "import flask" - import_asts = ast.parse(import_code) - flask_import = import_asts.body[0] - asts.body.insert(0, flask_import) + # For normal testing we don't need to add this to the test class file + + if not flask_status_dict['normal_test']: + import_code = "from %s import vypr" % VYPR_MODULE + import_ast = ast.parse(import_code).body[0] + import_ast.lineno = asts.body[0].lineno + import_ast.col_offset = asts.body[0].col_offset + asts.body.insert(0, import_ast) + + import_code = "import flask" + import_asts = ast.parse(import_code) + flask_import = import_asts.body[0] + asts.body.insert(0, flask_import) + # add vypr datetime import - vypr_datetime_import = "from datetime import datetime as vypr_dt" - datetime_import_ast = ast.parse(vypr_datetime_import).body[0] - datetime_import_ast.lineno = asts.body[0].lineno - datetime_import_ast.col_offset = asts.body[0].col_offset - asts.body.insert(0, datetime_import_ast) + add_vypr_datetime_import(asts) + # if we're using flask, we assume a certain architecture @@ -1198,6 +1312,7 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ logger.log("Processing function '%s'." % function) + # we replace . with : in function definitions to make sure we can distinguish between module # and class navigation later on instrument_function_qualifier = "%s%s.%s" % (machine_id, module, function.replace(".", ":")) @@ -1690,43 +1805,32 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ # function has started and insert one at the end to signal a return place_function_begin_instruments(function_def, formula_hash, instrument_function_qualifier) # also insert instruments at the end(s) of the function - place_function_end_instruments(function_def, scfg, formula_hash, instrument_function_qualifier,TEST_AWARE) + place_function_end_instruments(function_def, scfg, formula_hash, instrument_function_qualifier, flask_status_dict) + # Adding setupClass and teardownClass for testing + if not SETUP_ONCE: + if flask_status_dict['flask_status']: + # In case of Flask we only instrument test case. + ast_to_modify = flask_test_file_ast + class_name = add_setup_instruction(ast_to_modify, flask_status_dict) + else: + # In case of normal testing, we instrument the normal program + ast_to_modify = asts + add_setup_instruction(ast_to_modify, flask_status_dict) + # We only add setup instructions once. + SETUP_ONCE = True - if TEST_AWARE == 'flask': - flask_test_file = inst_configuration.get("flask_test_folder") - flask_test_file_without_extension = inst_configuration.get("flask_test_folder").replace('.py','') - - code = "".join(open(flask_test_file, "r").readlines()) - flask_test_file_ast = ast.parse(code) - - - # add vypr datetime import - vypr_datetime_import = "from datetime import datetime as vypr_dt" - datetime_import_ast = ast.parse(vypr_datetime_import).body[0] - datetime_import_ast.lineno = asts.body[0].lineno - datetime_import_ast.col_offset = asts.body[0].col_offset - flask_test_file_ast.body.insert(0, datetime_import_ast) - - if detect_testing_frameworks(flask_test_file_ast): - - for node in flask_test_file_ast.body: - if type(node) == ast.ClassDef: - class_name = node.name - create_test_setup_method(flask_test_file_ast.body, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE) - create_teardown_method(flask_test_file_ast, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE) - compile_bytecode_and_write(flask_test_file_ast,flask_test_file_without_extension) - - - elif TEST_AWARE == 'normal': - for node in asts.body: - if type(node) == ast.ClassDef: - class_name = node.name - create_test_setup_method(asts.body, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE) - create_teardown_method(asts, class_name, formula_hash,instrument_function_qualifier, TEST_AWARE) + # We add instruction in flask test file. + if flask_test_file_for_inst: + test_method_name_to_instrument = get_method_name(function, flask_test_file_for_inst) + instrument_test_method(flask_test_file_ast, + class_name, + test_method_name_to_instrument, + formula_hash,instrument_function_qualifier, + flask_status_dict) @@ -1761,6 +1865,10 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ compile_bytecode_and_write(asts, file_name_without_extension) + # Generate a bytecode file only in case of flask-based testing + if flask_status_dict['flask_status']: + compile_bytecode_and_write(flask_test_file_ast,flask_test_file_without_extension) + logger.log("Instrumentation complete. If VyPR is imported and activated, monitoring will now work.") # close instrumentation log From b1030b43c1b81a0636a582cc075953d4e12ef5fd Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Thu, 2 Apr 2020 23:27:31 +0200 Subject: [PATCH 12/30] updates to VyPR --- QueryBuilding/formula_building.py | 48 +++++++++++++++++---- monitor_synthesis/formula_tree.py | 72 ++++++++++++++++++++++++++----- 2 files changed, 101 insertions(+), 19 deletions(-) diff --git a/QueryBuilding/formula_building.py b/QueryBuilding/formula_building.py index 4fd28b0..2f5e16d 100644 --- a/QueryBuilding/formula_building.py +++ b/QueryBuilding/formula_building.py @@ -278,6 +278,8 @@ class SourceStaticState(StaticState): def __init__(self, outgoing_transition): self._outgoing_transition = outgoing_transition + self._arithmetic_stack = [] + self._arithmetic_build = False def __repr__(self): return "(%s).input()" % self._outgoing_transition @@ -294,6 +296,8 @@ class DestinationStaticState(StaticState): def __init__(self, incoming_transition): self._incoming_transition = incoming_transition + self._arithmetic_stack = [] + self._arithmetic_build = False def __repr__(self): return "(%s).result()" % self._incoming_transition @@ -360,22 +364,26 @@ def __mul__(self, value): add an object to the arithmetic stack so it can be applied later when values are checked. """ - if self._state._arithmetic_build: + base_variable = composition_sequence_from_value([self._state], self._state)[-1] + if base_variable._arithmetic_build: self._state._arithmetic_stack.append(formula_tree.ArithmeticMultiply(value)) return self def __add__(self, value): - if self._state._arithmetic_build: + base_variable = composition_sequence_from_value([self._state], self._state)[-1] + if base_variable._arithmetic_build: self._state._arithmetic_stack.append(formula_tree.ArithmeticAdd(value)) return self def __sub__(self, value): - if self._state._arithmetic_build: + base_variable = composition_sequence_from_value([self._state], self._state)[-1] + if base_variable._arithmetic_build: self._state._arithmetic_stack.append(formula_tree.ArithmeticSubtract(value)) return self def __truediv__(self, value): - if self._state._arithmetic_build: + base_variable = composition_sequence_from_value([self._state], self._state)[-1] + if base_variable._arithmetic_build: self._state._arithmetic_stack.append(formula_tree.ArithmeticTrueDivide(value)) return self @@ -399,6 +407,25 @@ def _in(self, interval): interval ) + """ + Overload comparison operators. + """ + + def __lt__(self, value): + """ + Generates an atom. + """ + # TODO: extend for multiple types + if type(value) is StateValueLength: + # the RHS of the comparison requires observation of another state or transition + # so we use a different class to deal with this + return formula_tree.StateValueLengthLessThanStateValueLengthMixed( + self._state, + self._name, + value._state, + value._name + ) + """ Arithmetic overloading is useful for mixed atoms when observed quantities are being compared to each other. @@ -410,22 +437,26 @@ def __mul__(self, value): add an object to the arithmetic stack so it can be applied later when values are checked. """ - if self._state._arithmetic_build: + base_variable = composition_sequence_from_value([self._state], self._state)[-1] + if base_variable._arithmetic_build: self._state._arithmetic_stack.append(formula_tree.ArithmeticMultiply(value)) return self def __add__(self, value): - if self._state._arithmetic_build: + base_variable = composition_sequence_from_value([self._state], self._state)[-1] + if base_variable._arithmetic_build: self._state._arithmetic_stack.append(formula_tree.ArithmeticAdd(value)) return self def __sub__(self, value): - if self._state._arithmetic_build: + base_variable = composition_sequence_from_value([self._state], self._state)[-1] + if base_variable._arithmetic_build: self._state._arithmetic_stack.append(formula_tree.ArithmeticSubtract(value)) return self def __truediv__(self, value): - if self._state._arithmetic_build: + base_variable = composition_sequence_from_value([self._state], self._state)[-1] + if base_variable._arithmetic_build: self._state._arithmetic_stack.append(formula_tree.ArithmeticTrueDivide(value)) return self @@ -642,6 +673,7 @@ def derive_composition_sequence(atom): current_operator = atom if type(atom) in [formula_tree.StateValueEqualToMixed, + formula_tree.StateValueLengthLessThanStateValueLengthMixed, formula_tree.TransitionDurationLessThanTransitionDurationMixed, formula_tree.TransitionDurationLessThanStateValueMixed, formula_tree.TransitionDurationLessThanStateValueLengthMixed, diff --git a/monitor_synthesis/formula_tree.py b/monitor_synthesis/formula_tree.py index 2dd4d2f..689d427 100644 --- a/monitor_synthesis/formula_tree.py +++ b/monitor_synthesis/formula_tree.py @@ -120,7 +120,7 @@ def __init__(self, state, name, interval): self.verdict = None def __repr__(self): - return "(%s)(%s) in %s" % (self._state, self._name, self._interval) + return "(%s)(%s) in %s" % (self._state, self._name, self.d_interval) def __eq__(self, other_atom): if type(other_atom) is StateValueInInterval: @@ -203,7 +203,9 @@ def __repr__(self): def __eq__(self, other_atom): if type(other_atom) is StateValueEqualToMixed: return (self._lhs == other_atom._lhs - and self._lhs_name == self._rhs_name) + and self._lhs_name == other_atom._lhs_name + and self._rhs == other_atom._rhs + and self._rhs_name == other_atom._rhs_name) else: return False @@ -216,16 +218,61 @@ def check(self, cummulative_state): return None else: lhs_with_arithmetic = apply_arithmetic_stack( - self._lhs.arithmetic_stack, - cummulative_state[0][0] + self._lhs._arithmetic_stack, + cummulative_state[0][0][self._lhs_name] ) rhs_with_arithmetic = apply_arithmetic_stack( - self._rhs.arithmetic_stack, - cummulative_state[1][0] + self._rhs._arithmetic_stack, + cummulative_state[1][0][self._rhs_name] ) return lhs_with_arithmetic == rhs_with_arithmetic +class StateValueLengthLessThanStateValueLengthMixed(Atom): + """ + This class models the atom (s1(x).length() < s2(y).length()). + """ + + def __init__(self, lhs, lhs_name, rhs, rhs_name): + self._lhs = lhs + self._rhs = rhs + self._lhs_name = lhs_name + self._rhs_name = rhs_name + self.verdict = None + + def __repr__(self): + return "(%s)(%s).length() < (%s)(%s).length()" % (self._lhs, self._lhs_name, self._rhs, self._rhs_name) + + def __eq__(self, other_atom): + if type(other_atom) is StateValueLengthLessThanStateValueLengthMixed: + return (self._lhs == other_atom._lhs + and self._lhs_name == other_atom._lhs_name + and self._rhs == other_atom._rhs + and self._rhs_name == other_atom._rhs_name) + else: + return False + + def check(self, cummulative_state): + """ + If either the RHS or LHS are None, we don't try to reach a truth value. + But if they are both not equal to None, we check for equality. + """ + if cummulative_state.get(0) is None or cummulative_state.get(1) is None: + return None + else: + lhs_with_arithmetic = apply_arithmetic_stack( + self._lhs._arithmetic_stack, + cummulative_state[0][0][self._lhs_name] + ) + rhs_with_arithmetic = apply_arithmetic_stack( + self._rhs._arithmetic_stack, + cummulative_state[1][0][self._rhs_name] + ) + print(lhs_with_arithmetic, rhs_with_arithmetic) + print(lhs_with_arithmetic < rhs_with_arithmetic) + return lhs_with_arithmetic < rhs_with_arithmetic + + class StateValueLengthInInterval(Atom): """ This class models the atom (len(s(x)) in I). @@ -797,10 +844,10 @@ def construct_atom_formula_occurrence_map(self, formula): self.construct_atom_formula_occurrence_map(formula.operands[n]) elif formula_is_atom(formula): if not(formula in self.sub_formulas): - self.sub_formulas.append(formula) - formula_index_in_sub_formulas = len(self.sub_formulas)-1 + self.sub_formulas.append(formula) + formula_index_in_sub_formulas = len(self.sub_formulas)-1 else: - formula_index_in_sub_formulas = self.sub_formulas.index(formula) + formula_index_in_sub_formulas = self.sub_formulas.index(formula) if formula in self.atom_to_occurrence_map.keys(): self.atom_to_occurrence_map[formula_index_in_sub_formulas].append(formula) @@ -817,6 +864,7 @@ def check_atom_truth_value(self, atom, value): an indication of whether the observation is for the lhs or rhs """ check_value = atom.check(value) + print("resulting truth value", check_value) if check_value == True: result = self.check(self._formula, atom) elif check_value == False: @@ -841,7 +889,7 @@ def process_atom_and_value(self, atom, observation_time, observation_end_time, v self.atom_to_state_dict[atom_index] = {} if not (self.atom_to_observation[atom_index].get(atom_sub_index)): - self.atom_to_observation[atom_index][atom_sub_index] =\ + self.atom_to_observation[atom_index][atom_sub_index] = \ (value, inst_point_id, observation_time, observation_end_time) # self.atom_to_program_path[atom_index][atom_sub_index] = [v for v in program_path] # we deal with integer indices now, so no need to copy a list @@ -1063,8 +1111,10 @@ def check(self, formula, symbol, level=0): self._formula.verdict = True return True elif formula_is_derived_from_atom(formula): + print("simple formula") if formula == symbol: if level == 0: + print("reached true verdict") self._formula.verdict = True return True else: @@ -1074,4 +1124,4 @@ def check(self, formula, symbol, level=0): def new_monitor(formula, optimised=False): - return Checker(formula, optimised) + return Checker(formula, optimised) \ No newline at end of file From 955ac80beed627b22f8566b5a9d792ad4687ffa7 Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Thu, 2 Apr 2020 23:41:30 +0200 Subject: [PATCH 13/30] updated requirement file --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 4d95609..fbbb9ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ graphviz +requests From facb1d9b858568a22a502dbd041278c844c1a3f6 Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Sun, 5 Apr 2020 15:32:45 +0200 Subject: [PATCH 14/30] fixed minor issue --- init_verification.py | 3 ++- instrument.py | 13 +++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/init_verification.py b/init_verification.py index 3412a25..3f2fbd9 100644 --- a/init_verification.py +++ b/init_verification.py @@ -283,6 +283,7 @@ def consumption_thread_function(verification_obj): # for now, we can just process "end" - we reset the contents of the maps # that are updated at runtime scope_event = top_pair[2] + vypr_output("SCOPE_EVENT %s" %scope_event) if scope_event == "end": @@ -733,7 +734,7 @@ def prepare_vypr(): flask.g.request_time = vypr.get_time() flask_object.before_request(prepare_vypr) - + vypr_output("Completed the time") # add VyPR end points - we may use this for statistics collection on the server # add the safe exist end point @flask_object.route("/vypr/stop-monitoring/") diff --git a/instrument.py b/instrument.py index 3e0b1e8..2ca5f17 100644 --- a/instrument.py +++ b/instrument.py @@ -1087,7 +1087,11 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ -def detect_flask_test_case(function , ast): +def detect_flask_test_case(function , ast, test_flag): + + # In case of no testing at all + if not test_flag: + return {'flask_status' : False, 'test_file': None, 'normal_test': False} test = detect_testing_frameworks(ast) @@ -1180,6 +1184,11 @@ def add_setup_instruction(asts, flask_status_dict): if inst_configuration.get("use_flask") else "no" VERIFICATION_INSTRUCTION = inst_configuration.get("verification_instruction") \ if inst_configuration.get("verification_instruction") else "verification.send_event" + TEST_FRAMEWORK = inst_configuration.get("testing") \ + if inst_configuration.get("testing") else False + + if TEST_FRAMEWORK in ['yes']: + TEST_FRAMEWORK = True VYPR_MODULE = inst_configuration.get("vypr_module") \ @@ -1259,7 +1268,7 @@ def add_setup_instruction(asts, flask_status_dict): asts = ast.parse(code) # If testing is enabled, we detect where the corresponding test case resides so that we can instrument the test file accordingly. - flask_status_dict = detect_flask_test_case(verified_functions[0], asts) + flask_status_dict = detect_flask_test_case(verified_functions[0], asts, TEST_FRAMEWORK) # Detecting whether the file is a test case. Therefore, we change the verification instruction accordingly From ba25e7298dd896027ae904fe8475df7ae62fa310 Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Fri, 24 Apr 2020 08:05:11 +0200 Subject: [PATCH 15/30] added flask --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index fbbb9ae..d0547ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ graphviz +flask requests From 29b391132abc276a124be263a87e0d7e622912a3 Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Thu, 14 May 2020 16:37:27 +0200 Subject: [PATCH 16/30] updates based on verification coverage issue --- init_verification.py | 199 ++++++++------ instrument.py | 630 ++++++++++++++++++------------------------- 2 files changed, 374 insertions(+), 455 deletions(-) diff --git a/init_verification.py b/init_verification.py index 3f2fbd9..8b045d1 100644 --- a/init_verification.py +++ b/init_verification.py @@ -24,12 +24,13 @@ VYPR_OUTPUT_VERBOSE = True PROJECT_ROOT = None + ## USED IN CASE OF FLASK TESTING #MAP_COPY_VERDICT = {} ## The purpose of this flag is to exclude verification send_event call ## with 'test_status' option, if "end" option has not occured. -IS_END_OPT = False +TEST_DIR = '' @@ -94,7 +95,7 @@ def vypr_output(string): def send_verdict_report(function_name, time_of_call, end_time_of_call, program_path, verdict_report, - binding_to_line_numbers, transaction_time, property_hash, test_result = None, test_name = None): + binding_to_line_numbers, transaction_time, property_hash): vypr_output("Sending verdicts to server...") """ @@ -106,29 +107,6 @@ def send_verdict_report(function_name, time_of_call, end_time_of_call, program_p - - # If test data exists. - if test_result !=None: - vypr_output("SENDING TEST DATA") - vypr_output("TEST_NAME {}".format(test_name)) - vypr_output("TEST_RESULT {}".format(test_result)) - - - test_data = { - "test_name" : test_name, - "test_result" : test_result - } - - - test_id = json.loads(requests.post( - os.path.join(VERDICT_SERVER_URL, "insert_test_data/"), - data=json.dumps(test_data) - ).text) - else: - test_id = None - - - # first, send function call data - this will also insert program path data vypr_output("Function start time was %s" % time_of_call) vypr_output("Function end time was %s" % end_time_of_call) @@ -141,8 +119,7 @@ def send_verdict_report(function_name, time_of_call, end_time_of_call, program_p "end_time_of_call": end_time_of_call.isoformat(), "function_name": function_name, "property_hash": property_hash, - "program_path": program_path, - "test_data_id": test_id + "program_path": program_path } @@ -203,9 +180,14 @@ def consumption_thread_function(verification_obj): # this needs to be changed for a clean exit INACTIVE_MONITORING = False - global IS_END_OPT + #list_test_cases = total_test_cases() + #list_test_cases = ['test_index', 'test__python_version', 'test_upload_session', 'test_check_hashes', 'test_store_payload', + # 'test_upload_metadata', 'test_close_upload_session'] + + list_test_cases = ['test_orm_objects_to_dicts', 'test_dicts_to_orm_objects' ] + transaction = -1 continue_monitoring = True while continue_monitoring: # import pdb @@ -218,7 +200,7 @@ def consumption_thread_function(verification_obj): except: # Changing flag to false here because in normal testing, end-monitoring does not change to False. # If exception is raised we just terminate the monitoring - + print("Getting stuck here! because queue is empty") continue @@ -248,7 +230,6 @@ def consumption_thread_function(verification_obj): if top_pair[0] == "test_transaction": transaction = top_pair[1] - vypr_output("Test suite begins at") continue @@ -287,7 +268,6 @@ def consumption_thread_function(verification_obj): if scope_event == "end": - IS_END_OPT = True vypr_output("*" * 50) @@ -342,46 +322,41 @@ def consumption_thread_function(verification_obj): elif type(bind_var) is CFGEdge: binding_to_line_numbers[bind_space_index].append(bind_var._instruction.lineno) - print(top_pair) # send the verdict # we send the function name, the time of the function call, the verdict report object, # the map of bindings to their line numbers and the date/time of the request the identify it (single threaded...) + if transaction != -1: + transaction_time = transaction + else: + transaction_time = top_pair[3] - - - - # We only send verdict data to the server when - - is_test = top_pair[7] - vypr_output ("Test aware status %s" %is_test) - vypr_output ("Type %s" %type(is_test)) - - # Not flask-testing nor normal-testing - if not is_test: - - send_verdict_report( + send_verdict_report( function_name, maps.latest_time_of_call, - top_pair[-2], + top_pair[-1], maps.program_path, verdict_report, binding_to_line_numbers, - top_pair[3], + transaction_time, + #top_pair[3], top_pair[4] - ) + ) - # reset the verdict report - maps.verdict_report.reset() + # reset transaction value + transaction = -1 - # reset the function start time for the next time - maps.latest_time_of_call = None + # reset the verdict report + maps.verdict_report.reset() - # reset the program path - maps.program_path = [] + # reset the function start time for the next time + maps.latest_time_of_call = None + + # reset the program path + maps.program_path = [] elif scope_event == "start": vypr_output("Function '%s' has started." % function_name) @@ -531,51 +506,60 @@ def consumption_thread_function(verification_obj): if instrument_type == "test_status": - vypr_output("Processing test status instrument with END_OPT status %s" %IS_END_OPT ) - if IS_END_OPT: - status = top_pair[2] - if status.failures: + # verified_function = top_pair[1] + status = top_pair[2] + start_test_time = top_pair[3] + end_test_time = top_pair[4] + test_name = top_pair[6] + + print (list_test_cases) + # We are trying to empty all the test cases in order to terminate the monitoring + if test_name in list_test_cases: + list_test_cases.remove(test_name) + + if len(list_test_cases) == 0: + continue_monitoring = False + + if status.failures: test_result = "Fail" - elif status.errors: + elif status.errors: test_result = "Error" - else: + else: test_result = "Success" - vypr_output("Sending verdict report only in case of testing") - send_verdict_report( - function_name, - maps.latest_time_of_call, - datetime.datetime.now(), - maps.program_path, - verdict_report, - binding_to_line_numbers, - transaction, - # top_pair[3], - top_pair[4], - test_result - ) + # If test data exists. - # reset the verdict report - maps.verdict_report.reset() - # reset the function start time for the next time - maps.latest_time_of_call = None + test_data = { + "test_name" : test_name, + "test_result" : test_result, + "start_time" : start_test_time.isoformat(), + "end_time" : end_test_time.isoformat() + + } - # reset the program path - maps.program_path = [] - IS_END_OPT = False - # Finish the loop + json.loads(requests.post( + os.path.join(VERDICT_SERVER_URL, "insert_test_data/"), + data=json.dumps(test_data) + ).text) + + # To terminate + #if top_pair[5] >= TOTAL_TEST_RUN: + # continue_monitoring = False + + # + # set the task as done - verification_obj.consumption_queue.task_done() + verification_obj.consumption_queue.task_done() - vypr_output("Consumption finished.") + vypr_output("Consumption finished.") - vypr_output("=" * 100) + vypr_output("=" * 100) # if we reach this point, the monitoring thread is ending #vypr_logger.end_logging() @@ -633,6 +617,7 @@ def __init__(self, module_name, function_name, property_hash): self.program_path = [] + def read_configuration(file): """ Read in 'file', parse into an object and return. @@ -655,6 +640,36 @@ def read_configuration(file): return json.loads(content) +def total_test_cases(): + + from os.path import dirname, abspath + import re + + global TEST_DIR + + ROOT_DIR = dirname(dirname(abspath(__file__))) + + test_cases = [] + + path = ROOT_DIR+'/' + TEST_DIR + + total_tests = [] + + #for r, d, f in os.walk(os.environ('PATH')): + for r, d, f in os.walk(path): + + for file in f: + + if file.startswith("test_") and (file.endswith('.py') or file.endswith('.py.inst')): + + readfile = open(os.path.join(r, file), "r") + + for line in readfile: + if re.search('(def)\s(test.*)', line): + test_cases.append( line[5:line.index('(')]) + return test_cases + + class Verification(object): @@ -672,12 +687,13 @@ def __init__(self): # read configuration file inst_configuration = read_configuration("vypr.config") - global VERDICT_SERVER_URL, VYPR_OUTPUT_VERBOSE, PROJECT_ROOT + global VERDICT_SERVER_URL, VYPR_OUTPUT_VERBOSE, PROJECT_ROOT, TOTAL_TEST_RUN VERDICT_SERVER_URL = inst_configuration.get("verdict_server_url") if inst_configuration.get( "verdict_server_url") else "http://localhost:9001/" VYPR_OUTPUT_VERBOSE = inst_configuration.get("verbose") if inst_configuration.get("verbose") else True PROJECT_ROOT = inst_configuration.get("project_root") if inst_configuration.get("project_root") else "" + # try to connect to the verdict server before we set anything up try: attempt = requests.get(VERDICT_SERVER_URL) @@ -699,6 +715,19 @@ def initialise(self, flask_object): VYPR_OUTPUT_VERBOSE = inst_configuration.get("verbose") if inst_configuration.get("verbose") else True PROJECT_ROOT = inst_configuration.get("project_root") if inst_configuration.get("project_root") else "" + TEST_FRAMEWORK = inst_configuration.get("test") \ + if inst_configuration.get("test") else "" + + # If testing is set then we should specify the test module + if TEST_FRAMEWORK in ['yes']: + TEST_DIR = inst_configuration.get("test_module") \ + if inst_configuration.get("test_module") else '' + + if TEST_DIR == '': + print ('Specify test module. Ending instrumentation - nothing has been done') + exit() + + self.machine_id = ("%s-" % inst_configuration.get("machine_id")) if inst_configuration.get("machine_id") else "" # check if there's an NTP server given that we should use for time diff --git a/instrument.py b/instrument.py index 2ca5f17..0001425 100644 --- a/instrument.py +++ b/instrument.py @@ -17,10 +17,14 @@ import py_compile import time +from os.path import dirname, abspath + # for now, we remove the final occurrence of VyPR from the first path to look in for modules rindex = sys.path[0].rfind("/VyPR") sys.path[0] = sys.path[0][:rindex] + sys.path[0][rindex + len("/VyPR"):] +# Getting the root directory of the project + # get the formula building functions before we evaluate the configuration code from VyPR.QueryBuilding import * from VyPR.SCFG.construction import * @@ -581,243 +585,6 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, point._instruction._parent_body.insert(index_in_block, start_ast) -## TESTING RELATED UPDATES --- 22.1.2020 - - -""" - Adds a setUp() method and creates a verification object if it doesn't exists, - otherwise update the setUp() method with creating a verification object. - - TODO: Too many checks for distinguishing between flask and non-flask testing. - -""" - -def create_test_setclass_method(current_step, class_name, flask): - """ - :param enable_normal_testing: Checks whether the testing is "normal" of "flask" based. - :param current_step: Contains the AST for the code - :param class_name: Name of the test class - """ - - setUp_found = False - - # Setting instuctions for setUp methods. - if flask: - VERIFICATION_INSTRUCTION = "vypr.send_event" - else: - VERIFICATION_INSTRUCTION = "cls.vypr.send_event" - VERIFICATION_IMPORT = "from VyPR import Monitor" - VERIFICATION_OBJ = "cls.vypr = Monitor(); cls.vypr.initialise(None)" - - - ## Finding the test class. - current_step = filter( lambda entry: (type(entry) is ast.ClassDef and - entry.name == class_name), current_step)[0] - - - test_class_body = current_step.body - - transaction_time_statement = "%s((\"test_transaction\", vypr_dt.now() ))" % \ - (VERIFICATION_INSTRUCTION) - - - ## Traversing the body of the class in order to look for setUpClass method - - for test_function in test_class_body: - - if not (type(test_function) is ast.FunctionDef): - continue - - if test_function.name == 'setUpClass': - - setUp_found = True - - setup_transaction = ast.parse(transaction_time_statement).body[0] - test_function.body.insert(0,setup_transaction) - - if not flask: - verification_import_inst = ast.parse(VERIFICATION_IMPORT).body[0] - verification_import_obj_assign = ast.parse(VERIFICATION_OBJ).body[0] - test_function.body.insert(0,verification_import_obj_assign) - test_function.body.insert(0,verification_import_inst) - - - # If there is no setUpClass method, then we need to add setUp method in the class. - if not setUp_found: - - if flask: - - setUp_method = "@classmethod\ndef setUpClass(cls):\n\t" + transaction_time_statement - - else: - - setUp_method = "@classmethod\ndef setUpClass(cls):\n\t" + VERIFICATION_IMPORT + '\n\t' + VERIFICATION_OBJ + '\n\t' + transaction_time_statement - - - method_inst = ast.parse(setUp_method).body[0] - test_class_body.insert(0,method_inst) - - - - -""" - Adds a teardown method for storing each test status i.e., whether it failed, succeed or whether there was an error. - -""" - - -def create_teardownclass_method(ast_code, class_name, flask): - - """ - :param ast_code: Code for the AST - :param class_name: Class name - :param formula_hash: Hash value for the formula - :param function: Fully qualified function name - :parame test_aware_type type of testing can be normal or flask - """ - - - tearDown_found = False - - - - ## Finding the test class. - current_step = filter( lambda entry: (type(entry) is ast.ClassDef and - entry.name == class_name), ast_code.body)[0] - - test_class_body = current_step.body - - # Updating insttuction for teardownclass method - if flask: - VYPR_OBJ='vypr' - else: - VYPR_OBJ='cls.vypr' - - verification_call_code = VYPR_OBJ + ".end_monitoring()" - - - - for test_function in test_class_body: - - if not (type(test_function) is ast.FunctionDef): - continue - - if test_function.name is 'tearDownClass': - # We found tearDown method, now we need to add verification instructions - tearDown_found = True - verification_call_inst = ast.parse(verification_call_code).body[0] - test_function.body.insert(0,verification_call_inst) - - - ## Terminate monitoring after all tests are analyzed - - - if not tearDown_found: - tear_method = "@classmethod\ndef tearDownClass(cls):\n\t" + verification_call_code - method_inst = ast.parse(tear_method).body[0] - test_class_body.insert(len(test_class_body),method_inst) - - -""" - Detects whether the instrumented file has a testing framework. It will then generated verification object and instruction - related to testing. See create_setup_method/create_teardown_method. - - CONSTRAINT: Only works for python unittests, we will have to extend it for different python testing frameworks. -""" - -def detect_testing_frameworks(ast_code): - - """ - :param ast_code: Abstract Syntax Tree for the code. - """ - - for node in ast_code.body: - if isinstance(node, (ast.Import, ast.ImportFrom)): - if node.names[0].__dict__['name'] is 'unittest': - return True - else: - return False - - - -""" - -This method is used for flask based testing. -We have to insert a send_event of type 'end' in the method which calls the flask app method. -TODO: Regex would be a better way to compare the end-point - -""" -def get_method_name(end_point, file_path ): - - """ - - :param end_point: - :param file_path: - :return: name of the test method based on the end-point - - """ - - if os.path.exists(file_path): - None - elif os.path.exists(file_path+'.inst'): - file_path = file_path+'.inst' - - else: - print("File for getting method name does not exist..") - exit() - - file = open(file_path, 'r') - for line in file: - if 'def' in line: - def_method_name = line - - if end_point in line : - without_def = def_method_name.split( )[1] - position_before_arg = without_def.find('(') - method_name = without_def[0:position_before_arg] - return method_name - - -def instrument_test_method(flask_test_file_ast, class_name, test_method_name_to_instrument, formula_hash,instrument_function_qualifier, flask_status_dict): - - is_testing = flask_status_dict['normal_test'] - flask = flask_status_dict['flask_status'] - - - ## Finding the test class. - current_step = filter( lambda entry: (type(entry) is ast.ClassDef and - entry.name == class_name), flask_test_file_ast.body)[0] - - test_class_body = current_step.body - - - # Updating instruction for flask test - if flask: - - VERIFICATION_INSTRUCTION="vypr.send_event" - - elif not flask and is_testing: - - VERIFICATION_INSTRUCTION="self.vypr.send_event" - - - - verification_call_code = "%s((\"%s\",\"test_status\", \"%s\",self._resultForDoCleanups, vypr_dt.now(),\"%s\"))" % \ - (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, formula_hash - ) - - - for test_function in test_class_body: - - - if not (type(test_function) is ast.FunctionDef): - continue - - - if test_function.name == test_method_name_to_instrument: - verification_call_inst = ast.parse(verification_call_code).body[0] - test_function.body.insert(len(test_function.body),verification_call_inst) - def place_path_recording_instruments(scfg, instrument_function_qualifier, formula_hash): # insert path recording instruments - these don't depend on the formula being checked so @@ -1002,53 +769,48 @@ def place_function_begin_instruments(function_def, formula_hash, instrument_func # so a function call in the return statement maybe missed if it's part of verification... thread_id_capture = "import threading; __thread_id = threading.current_thread().ident;" vypr_start_time_instrument = "vypr_start_time = %s.get_time();" %VYPR_OBJ + verification_test_start_code = "start_test_time = vypr_dt.now()" + + + start_instrument = \ "%s((\"%s\", \"function\", \"%s\", \"start\", vypr_start_time, \"%s\", __thread_id))" \ % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, formula_hash) + verfication_test_start_time_ast = ast.parse(verification_test_start_code).body[0] threading_import_ast = ast.parse(thread_id_capture).body[0] thread_id_capture_ast = ast.parse(thread_id_capture).body[1] vypr_start_time_ast = ast.parse(vypr_start_time_instrument).body[0] start_ast = ast.parse(start_instrument).body[0] + verfication_test_start_time_ast.lineno = function_def.body[0].lineno threading_import_ast.lineno = function_def.body[0].lineno thread_id_capture_ast.lineno = function_def.body[0].lineno vypr_start_time_ast.lineno = function_def.body[0].lineno start_ast.lineno = function_def.body[0].lineno + function_def.body.insert(0,verfication_test_start_time_ast) function_def.body.insert(0, start_ast) function_def.body.insert(0, thread_id_capture_ast) function_def.body.insert(0, threading_import_ast) function_def.body.insert(0, vypr_start_time_ast) -def place_function_end_instruments(function_def, scfg, formula_hash, instrument_function_qualifier, flask_status_dict): +def place_function_end_instruments(function_def, scfg, formula_hash, instrument_function_qualifier, is_flask): # insert the end instrument before every return statement # Use time accordingly, if its flask or normal testing - - flask = flask_status_dict['flask_status'] - is_testing = flask_status_dict['normal_test'] - - if flask: - + if is_flask: time = "flask.g.request_time" - else: - - time = "vypr_dt.now()" - - - - # This condition is used send data to verdict server. We send either directly on 'end' instrument type or 'test_aware' type - test_aware = True if flask or is_testing else False + time = "vypr.get_time()" for end_vertex in scfg.return_statements: end_instrument = \ "%s((\"%s\", \"function\", \"%s\", \"end\", %s, \"%s\", __thread_id, " \ - "%s.get_time(),%r))" \ - % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, time,formula_hash, VYPR_OBJ, test_aware) + "%s.get_time('end-instrument')))" \ + % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, time,formula_hash, VYPR_OBJ) end_ast = ast.parse(end_instrument).body[0] end_ast.lineno = end_vertex._previous_edge._instruction._parent_body[-1].lineno @@ -1065,8 +827,8 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ if not (type(function_def.body[-1]) is ast.Return): end_instrument = \ "%s((\"%s\", \"function\", \"%s\", \"end\", %s, \"%s\", __thread_id, " \ - "%s.get_time(), %r))" \ - % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, time,formula_hash, VYPR_OBJ,test_aware) + "%s.get_time('end-instrument')))" \ + % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, time,formula_hash, VYPR_OBJ) end_ast = ast.parse(end_instrument).body[0] @@ -1076,80 +838,218 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ function_def.body.insert(len(function_def.body), end_ast) - if flask or is_testing: - test_status_instrument = \ - "%s((\"%s\",\"test_status\", \"%s\",self._resultForDoCleanups, vypr_dt.now(),\"%s\"))" % \ - (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, formula_hash - ) - test_status_ast = ast.parse(test_status_instrument).body[0] + + + + +def instrument_modules(module_list, formula_hash, instrument_function_qualifier): + + for module in module_list: + + + if module.endswith('.py'): + ext = '.py' + elif module.endswith('.inst'): + ext = '.py.inst' + + file_name = module + + file_name_without_extension = module.replace(ext, "") + + + # extract asts from the code in the file + code = "".join(open(file_name, "r").readlines()) + + asts = ast.parse(code) + + for node in asts.body: + if type(node) == ast.ClassDef: + class_name = node.name + + create_test_setclass_method(asts.body, class_name) + + instrument_test_cases(asts, class_name, formula_hash, instrument_function_qualifier) + + + # compile bytecode + compile_bytecode_and_write(asts,file_name_without_extension) + + + +def instrument_test_cases(test_ast, class_name, formula_hash, instrument_function_qualifier): + + ## Adding imports first to the test cases + + import_code = "from %s import vypr" % VYPR_MODULE + vypr_add_import = 'from test import vypr' +# datetime_import = 'from datetime import datetime as vypr_dt' + + import_vypr_add_ast = ast.parse(vypr_add_import).body[0] + import_vypr_add_ast.lineno = test_ast.body[0].lineno + import_vypr_add_ast.col_offset = test_ast.body[0].col_offset + + +# datetime_import_ast = ast.parse(datetime_import).body[0] +# datetime_import_ast.lineno = test_ast.body[0].lineno +# datetime_import_ast.col_offset = test_ast.body[0].col_offset + + test_ast.body.insert(0, import_vypr_add_ast) + test_ast.body.insert(0, datetime_import_ast) + + + + start_test_time_statement = "test_start_time = vypr.get_time()" + end_test_time_statement = \ + "%s((\"%s\",\"test_status\", \"%s\",self._resultForDoCleanups, test_start_time, vypr.get_time(), \"%s\", self._testMethodName, ))" % \ + (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, formula_hash + ) + + + ## Finding the test class. + current_step = filter( lambda entry: (type(entry) is ast.ClassDef and + entry.name == class_name), test_ast.body)[0] + + + test_class_body = current_step.body + + + + ## Traversing the body of the class in order to look for setUpClass method + + for test_function in test_class_body: + + + + + # We only want to instrument test cases + if not (type(test_function) is ast.FunctionDef) or test_function.name in ['setUpClass', 'setUp', 'tearDown']: + continue + + + setup_transaction = ast.parse(start_test_time_statement).body[0] + test_function.body.insert(0,setup_transaction) + + + test_status_ast = ast.parse(end_test_time_statement).body[0] logger.log("Placing test_status instrument at the end of the function body.") - function_def.body.insert(len(function_def.body), test_status_ast) + test_function.body.insert(len(function_def.body), test_status_ast) -def detect_flask_test_case(function , ast, test_flag): - # In case of no testing at all - if not test_flag: - return {'flask_status' : False, 'test_file': None, 'normal_test': False} - test = detect_testing_frameworks(ast) +def get_test_modules(is_test_module , module = None): - # This flag is use to help decide the instrumentation about which instructions to insert in setupclass and teardownclass methods in test class - is_flask=False - for directory in os.walk("."): - for file in directory[2]: - f = os.path.join(directory[0], file) - if str(f).endswith('.py'): - # rename to .py - with open (f) as file: - contents = file.read() - search_end_point = 'self.client.post(\'/' + function - if search_end_point in contents: - is_flask = True - # Returns the the flask status and test file for instrumenting - return {'flask_status' : is_flask, 'test_file': f, 'normal_test': test} - # This would be the case for normal testing - return {'flask_status' : is_flask, 'test_file': None, 'normal_test': test} + from os.path import dirname, abspath + ROOT_DIR = dirname(dirname(abspath(__file__))) + module_path = [] -def add_vypr_datetime_import(asts): + path = ROOT_DIR+'/' + TEST_DIR - vypr_datetime_import = "from datetime import datetime as vypr_dt" - datetime_import_ast = ast.parse(vypr_datetime_import).body[0] - datetime_import_ast.lineno = asts.body[0].lineno - datetime_import_ast.col_offset = asts.body[0].col_offset - asts.body.insert(0, datetime_import_ast) -def add_setup_instruction(asts, flask_status_dict): + #for r, d, f in os.walk(os.environ('PATH')): + for r, d, f in os.walk(path): - # Get test and flask status + for file in f: - is_testing = flask_status_dict['normal_test'] + if file.startswith("test_") and (file.endswith('.py') or file.endswith('.py.inst')): - flask_status = flask_status_dict['flask_status'] + if file == module[module.rindex('.')+1:]+'.py' and is_test_module: + continue - if flask_status or is_testing: + module_path.append(os.path.join(r, file)) - for node in asts.body: - if type(node) == ast.ClassDef: - class_name = node.name - create_test_setclass_method(asts.body, class_name, flask_status) + return module_path - create_teardownclass_method(asts, class_name, flask_status) - return class_name +def create_test_setclass_method(current_step, class_name): + """ + :param enable_normal_testing: Checks whether the testing is "normal" of "flask" based. + :param current_step: Contains the AST for the code + :param class_name: Name of the test class + """ + + setUp_found = False + + ## Finding the test class. + current_step = filter( lambda entry: (type(entry) is ast.ClassDef and + entry.name == class_name), current_step)[0] + + + test_class_body = current_step.body + + transaction_time_statement = "%s((\"test_transaction\", vypr_dt.now() ))" % \ + (VERIFICATION_INSTRUCTION) + + + ## Traversing the body of the class in order to look for setUpClass method + + for test_function in test_class_body: + + if not (type(test_function) is ast.FunctionDef): + continue + + if test_function.name == 'setUpClass': + + setUp_found = True + + setup_transaction = ast.parse(transaction_time_statement).body[0] + test_function.body.insert(0,setup_transaction) + + # If there is no setUpClass method, then we need to add setUp method in the class. + if not setUp_found: + + setUp_method = "@classmethod\ndef setUpClass(cls):\n\t" + transaction_time_statement + + method_inst = ast.parse(setUp_method).body[0] + test_class_body.insert(0,method_inst) + + + +def if_file_is_test(name): + + test_path = dirname(dirname(abspath(__file__))) + '/' + TEST_DIR + + + for root, dirs, files in os.walk(test_path): + + test_file = name[name.rindex('/')+1:] + + if test_file in files: + return True + + return False + + +def if_file_is_flask(name): + + file = open(name, 'r') + + data = file.readlines() + + for line in data: + + stripped_line = line.strip() + + + + if 'from flask' in stripped_line: + return True + + + return False + - # In case of non-testing - return None if __name__ == "__main__": + # import pdb; pdb.set_trace() @@ -1184,19 +1084,30 @@ def add_setup_instruction(asts, flask_status_dict): if inst_configuration.get("use_flask") else "no" VERIFICATION_INSTRUCTION = inst_configuration.get("verification_instruction") \ if inst_configuration.get("verification_instruction") else "verification.send_event" - TEST_FRAMEWORK = inst_configuration.get("testing") \ - if inst_configuration.get("testing") else False + TEST_FRAMEWORK = inst_configuration.get("test") \ + if inst_configuration.get("test") else "" - if TEST_FRAMEWORK in ['yes']: - TEST_FRAMEWORK = True VYPR_MODULE = inst_configuration.get("vypr_module") \ if inst_configuration.get("vypr_module") else "" + VERIFICATION_INSTRUCTION = "vypr.send_event" - # VERIFICATION_INSTRUCTION = "print" + VYPR_OBJ = "vypr" + # If testing is set then we should specify the test module + if TEST_FRAMEWORK in ['yes']: + TEST_DIR = inst_configuration.get("test_module") \ + if inst_configuration.get("test_module") else '' + + if TEST_DIR != '': + VYPR_MODULE = TEST_DIR + else: + print ('Specify test module. Ending instrumentation - nothing has been done') + exit() + + machine_id = ("%s-" % inst_configuration.get("machine_id")) if inst_configuration.get("machine_id") else "" # first, check that the verdict server is reachable @@ -1204,10 +1115,9 @@ def add_setup_instruction(asts, flask_status_dict): print("Verdict server is not reachable. Ending instrumentation - nothing has been done.") exit() - - flask_test_file_for_inst = None SETUP_ONCE = False + # initialise instrumentation logger logger = InstrumentationLog(LOGS_TO_STDOUT) # set up the file handle @@ -1244,8 +1154,12 @@ def add_setup_instruction(asts, flask_status_dict): for module in verified_modules: - # Initialize to False for a new file. - SETUP_ONCE = False + # If we are writing specification over tests + is_test_module = False + + # Check if the file is flask or not to select the appropriate timestamp for flask + is_flask = False + # Reset the instructions VERIFICATION_INSTRUCTION = "vypr.send_event" @@ -1261,48 +1175,39 @@ def add_setup_instruction(asts, flask_status_dict): file_name = module.replace(".", "/") + ".py" file_name_without_extension = module.replace(".", "/") - print (file_name) - # extract asts from the code in the file - code = "".join(open(file_name, "r").readlines()) - asts = ast.parse(code) + # Instrument test cases with 'test_status' instrumentation type - # If testing is enabled, we detect where the corresponding test case resides so that we can instrument the test file accordingly. - flask_status_dict = detect_flask_test_case(verified_functions[0], asts, TEST_FRAMEWORK) + if if_file_is_test(file_name): + is_test_module = True - # Detecting whether the file is a test case. Therefore, we change the verification instruction accordingly - if flask_status_dict['normal_test']: - VYPR_OBJ = "self.vypr" - VERIFICATION_INSTRUCTION = VYPR_OBJ+".send_event" + if if_file_is_flask(file_name): + is_flask = True - # If we find a corresponding test file in case of flask testing, we will instrument it. - if flask_status_dict['test_file'] != None: - - flask_test_file = flask_status_dict['test_file'] - - flask_test_file_for_inst = flask_test_file - - flask_test_file_without_extension = flask_test_file.replace('.py','') - - code = "".join(open(flask_test_file, "r").readlines()) - - flask_test_file_ast = ast.parse(code) - + # extract asts from the code in the file + code = "".join(open(file_name, "r").readlines()) + asts = ast.parse(code) - # We first add imports to the corresponding test file - add_vypr_datetime_import(flask_test_file_ast) # add import for init_vypr module # For normal testing we don't need to add this to the test class file - if not flask_status_dict['normal_test']: - import_code = "from %s import vypr" % VYPR_MODULE - import_ast = ast.parse(import_code).body[0] - import_ast.lineno = asts.body[0].lineno - import_ast.col_offset = asts.body[0].col_offset - asts.body.insert(0, import_ast) + # add vypr datetime import + vypr_datetime_import = "from datetime import datetime as vypr_dt" + datetime_import_ast = ast.parse(vypr_datetime_import).body[0] + datetime_import_ast.lineno = asts.body[0].lineno + datetime_import_ast.col_offset = asts.body[0].col_offset + asts.body.insert(0, datetime_import_ast) + + + import_code = "from %s import vypr" % VYPR_MODULE + #import_code = "from test import vypr" + import_ast = ast.parse(import_code).body[0] + import_ast.lineno = asts.body[0].lineno + import_ast.col_offset = asts.body[0].col_offset + asts.body.insert(0, import_ast) import_code = "import flask" import_asts = ast.parse(import_code) @@ -1310,9 +1215,6 @@ def add_setup_instruction(asts, flask_status_dict): asts.body.insert(0, flask_import) - # add vypr datetime import - add_vypr_datetime_import(asts) - # if we're using flask, we assume a certain architecture @@ -1322,6 +1224,7 @@ def add_setup_instruction(asts, flask_status_dict): logger.log("Processing function '%s'." % function) + # we replace . with : in function definitions to make sure we can distinguish between module # and class navigation later on instrument_function_qualifier = "%s%s.%s" % (machine_id, module, function.replace(".", ":")) @@ -1815,32 +1718,19 @@ def add_setup_instruction(asts, flask_status_dict): # function has started and insert one at the end to signal a return place_function_begin_instruments(function_def, formula_hash, instrument_function_qualifier) # also insert instruments at the end(s) of the function - place_function_end_instruments(function_def, scfg, formula_hash, instrument_function_qualifier, flask_status_dict) - - # Adding setupClass and teardownClass for testing + place_function_end_instruments(function_def, scfg, formula_hash, instrument_function_qualifier, is_flask) if not SETUP_ONCE: - if flask_status_dict['flask_status']: - # In case of Flask we only instrument test case. - ast_to_modify = flask_test_file_ast - class_name = add_setup_instruction(ast_to_modify, flask_status_dict) - else: - # In case of normal testing, we instrument the normal program - ast_to_modify = asts - add_setup_instruction(ast_to_modify, flask_status_dict) - - # We only add setup instructions once. + test_modules = get_test_modules(is_test_module, module) + instrument_modules(test_modules, formula_hash, instrument_function_qualifier) SETUP_ONCE = True - # We add instruction in flask test file. - if flask_test_file_for_inst: - test_method_name_to_instrument = get_method_name(function, flask_test_file_for_inst) + if is_test_module: + for node in asts.body: + if type(node) == ast.ClassDef: + class_name = node.name - instrument_test_method(flask_test_file_ast, - class_name, - test_method_name_to_instrument, - formula_hash,instrument_function_qualifier, - flask_status_dict) + instrument_test_cases(asts, class_name, formula_hash, instrument_function_qualifier) @@ -1876,10 +1766,10 @@ def add_setup_instruction(asts, flask_status_dict): compile_bytecode_and_write(asts, file_name_without_extension) # Generate a bytecode file only in case of flask-based testing - if flask_status_dict['flask_status']: - compile_bytecode_and_write(flask_test_file_ast,flask_test_file_without_extension) + # if flask_status_dict['flask_status']: + # compile_bytecode_and_write(flask_test_file_ast,flask_test_file_without_extension) logger.log("Instrumentation complete. If VyPR is imported and activated, monitoring will now work.") # close instrumentation log - logger.end_logging() \ No newline at end of file + logger.end_logging() From 6ae81de001015c2c7a9f300c815fbb00a45d9099 Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Fri, 15 May 2020 00:33:58 +0200 Subject: [PATCH 17/30] added --- init_verification.py | 4 +++- instrument.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/init_verification.py b/init_verification.py index 8b045d1..259b677 100644 --- a/init_verification.py +++ b/init_verification.py @@ -185,7 +185,9 @@ def consumption_thread_function(verification_obj): #list_test_cases = ['test_index', 'test__python_version', 'test_upload_session', 'test_check_hashes', 'test_store_payload', # 'test_upload_metadata', 'test_close_upload_session'] - list_test_cases = ['test_orm_objects_to_dicts', 'test_dicts_to_orm_objects' ] + #list_test_cases = ['test_orm_objects_to_dicts', 'test_dicts_to_orm_objects' ] + list_test_cases = ['test_insert_iovs', 'test_construct_tags'] + transaction = -1 continue_monitoring = True diff --git a/instrument.py b/instrument.py index 0001425..18b0370 100644 --- a/instrument.py +++ b/instrument.py @@ -983,7 +983,7 @@ def create_test_setclass_method(current_step, class_name): test_class_body = current_step.body - transaction_time_statement = "%s((\"test_transaction\", vypr_dt.now() ))" % \ + transaction_time_statement = "%s((\"test_transaction\", vypr.get_time() ))" % \ (VERIFICATION_INSTRUCTION) From f8aaabc1016e7335ea3e0d9a21f770727afcecca Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Fri, 15 May 2020 02:22:42 +0200 Subject: [PATCH 18/30] handling test cases for terminating the monitoring thread --- init_verification.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/init_verification.py b/init_verification.py index 259b677..e6788c1 100644 --- a/init_verification.py +++ b/init_verification.py @@ -181,12 +181,12 @@ def consumption_thread_function(verification_obj): INACTIVE_MONITORING = False - #list_test_cases = total_test_cases() + list_test_cases = total_test_cases() #list_test_cases = ['test_index', 'test__python_version', 'test_upload_session', 'test_check_hashes', 'test_store_payload', # 'test_upload_metadata', 'test_close_upload_session'] #list_test_cases = ['test_orm_objects_to_dicts', 'test_dicts_to_orm_objects' ] - list_test_cases = ['test_insert_iovs', 'test_construct_tags'] + #list_test_cases = ['test_insert_iovs', 'test_construct_tags'] transaction = -1 From 1c649d79f300436dca188a8c2126baa5e6608848 Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Fri, 15 May 2020 16:21:59 +0200 Subject: [PATCH 19/30] fixed the transaction issue for analysis library --- init_verification.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/init_verification.py b/init_verification.py index e6788c1..39ce461 100644 --- a/init_verification.py +++ b/init_verification.py @@ -23,7 +23,7 @@ VERDICT_SERVER_URL = None VYPR_OUTPUT_VERBOSE = True PROJECT_ROOT = None - +TEST_FRAMEWORK = 'no' ## USED IN CASE OF FLASK TESTING #MAP_COPY_VERDICT = {} @@ -329,11 +329,11 @@ def consumption_thread_function(verification_obj): # we send the function name, the time of the function call, the verdict report object, # the map of bindings to their line numbers and the date/time of the request the identify it (single threaded...) - - if transaction != -1: - transaction_time = transaction + if 'yes' in TEST_FRAMEWORK : + transaction_time = transaction else: - transaction_time = top_pair[3] + transaction_time = top_pair[3] + send_verdict_report( function_name, @@ -348,9 +348,6 @@ def consumption_thread_function(verification_obj): ) - # reset transaction value - transaction = -1 - # reset the verdict report maps.verdict_report.reset() @@ -516,7 +513,6 @@ def consumption_thread_function(verification_obj): end_test_time = top_pair[4] test_name = top_pair[6] - print (list_test_cases) # We are trying to empty all the test cases in order to terminate the monitoring if test_name in list_test_cases: list_test_cases.remove(test_name) @@ -711,7 +707,7 @@ def initialise(self, flask_object): # read configuration file inst_configuration = read_configuration("vypr.config") - global VERDICT_SERVER_URL, VYPR_OUTPUT_VERBOSE, PROJECT_ROOT + global VERDICT_SERVER_URL, VYPR_OUTPUT_VERBOSE, PROJECT_ROOT, TEST_FRAMEWORK VERDICT_SERVER_URL = inst_configuration.get("verdict_server_url") if inst_configuration.get( "verdict_server_url") else "http://localhost:9001/" VYPR_OUTPUT_VERBOSE = inst_configuration.get("verbose") if inst_configuration.get("verbose") else True @@ -858,7 +854,7 @@ def get_time(self, callee=""): return datetime.datetime.utcnow() def send_event(self, event_description): - print("trying to send an event..") + print( "Looking at the events --->>> id : %s , event: %s"%(id(self),event_description)) if not (self.initialisation_failure): self.consumption_queue.put(event_description) From ec55c7d4c45bc480fcbed7d9f9f9a73fa7f12194 Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Sat, 16 May 2020 15:18:42 +0200 Subject: [PATCH 20/30] removed the problem of monitor termination based on unit tests --- init_verification.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/init_verification.py b/init_verification.py index 39ce461..50053b5 100644 --- a/init_verification.py +++ b/init_verification.py @@ -181,7 +181,8 @@ def consumption_thread_function(verification_obj): INACTIVE_MONITORING = False - list_test_cases = total_test_cases() + + #list_test_cases = total_test_cases() #list_test_cases = ['test_index', 'test__python_version', 'test_upload_session', 'test_check_hashes', 'test_store_payload', # 'test_upload_metadata', 'test_close_upload_session'] @@ -329,6 +330,7 @@ def consumption_thread_function(verification_obj): # we send the function name, the time of the function call, the verdict report object, # the map of bindings to their line numbers and the date/time of the request the identify it (single threaded...) + if 'yes' in TEST_FRAMEWORK : transaction_time = transaction else: @@ -513,12 +515,12 @@ def consumption_thread_function(verification_obj): end_test_time = top_pair[4] test_name = top_pair[6] - # We are trying to empty all the test cases in order to terminate the monitoring - if test_name in list_test_cases: - list_test_cases.remove(test_name) - - if len(list_test_cases) == 0: - continue_monitoring = False + # # We are trying to empty all the test cases in order to terminate the monitoring + # if test_name in list_test_cases: + # list_test_cases.remove(test_name) + # + # if len(list_test_cases) == 0: + # continue_monitoring = False if status.failures: test_result = "Fail" @@ -539,7 +541,6 @@ def consumption_thread_function(verification_obj): } - json.loads(requests.post( os.path.join(VERDICT_SERVER_URL, "insert_test_data/"), data=json.dumps(test_data) @@ -854,7 +855,6 @@ def get_time(self, callee=""): return datetime.datetime.utcnow() def send_event(self, event_description): - print( "Looking at the events --->>> id : %s , event: %s"%(id(self),event_description)) if not (self.initialisation_failure): self.consumption_queue.put(event_description) From 148c8224a5b98fa67d3c0bd974fe04aacdf5b3ec Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Sat, 16 May 2020 21:33:39 +0200 Subject: [PATCH 21/30] removed test analysis problems --- init_verification.py | 1 + instrument.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/init_verification.py b/init_verification.py index 50053b5..7ca4b88 100644 --- a/init_verification.py +++ b/init_verification.py @@ -233,6 +233,7 @@ def consumption_thread_function(verification_obj): if top_pair[0] == "test_transaction": transaction = top_pair[1] + vypr_output("Test Transaction >> %s" %str(top_pair)) continue diff --git a/instrument.py b/instrument.py index 18b0370..737d126 100644 --- a/instrument.py +++ b/instrument.py @@ -881,7 +881,7 @@ def instrument_test_cases(test_ast, class_name, formula_hash, instrument_functio ## Adding imports first to the test cases import_code = "from %s import vypr" % VYPR_MODULE - vypr_add_import = 'from test import vypr' + vypr_add_import = 'from %s import vypr' % VYPR_MODULE # datetime_import = 'from datetime import datetime as vypr_dt' import_vypr_add_ast = ast.parse(vypr_add_import).body[0] From 73a05f6399d8fabe6bc82079e04da65c08a3559d Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Mon, 18 May 2020 00:40:12 +0200 Subject: [PATCH 22/30] updated the instrumentation for fixing the time between property violation issue --- instrument.py | 143 +++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 135 insertions(+), 8 deletions(-) diff --git a/instrument.py b/instrument.py index 737d126..43b480f 100644 --- a/instrument.py +++ b/instrument.py @@ -393,7 +393,6 @@ def get_instrumentation_points_from_comp_sequence(value_from_binding, moves): return instrumentation_points - def instrument_point_state(state, name, point, binding_space_indices, atom_index, atom_sub_index, instrumentation_point_db_ids, measure_attribute=None): @@ -407,21 +406,29 @@ def instrument_point_state(state, name, point, binding_space_indices, if measure_attribute == "length": state_variable_alias = name.replace(".", "_").replace("(", "__").replace(")", "__") state_recording_instrument = "record_state_%s = len(%s); " % (state_variable_alias, name) - time_attained_instrument = "time_attained_%s = %s.get_time();" % (state_variable_alias,VYPR_OBJ) + time_attained_instrument = "time_attained_%s = %s.get_time('point instrument');" % \ + (state_variable_alias, VYPR_OBJ) + time_attained_variable = "time_attained_%s" % state_variable_alias elif measure_attribute == "type": state_variable_alias = name.replace(".", "_").replace("(", "__").replace(")", "__") state_recording_instrument = "record_state_%s = type(%s).__name__; " % (state_variable_alias, name) - time_attained_instrument = "time_attained_%s = %s.get_time();" % (state_variable_alias,VYPR_OBJ) + time_attained_instrument = "time_attained_%s = %s.get_time('point instrument');" % \ + (state_variable_alias, VYPR_OBJ) + time_attained_variable = "time_attained_%s" % state_variable_alias elif measure_attribute == "time_attained": state_variable_alias = "time_attained_%i" % atom_sub_index - state_recording_instrument = "record_state_%s = %s.get_time(); " % (state_variable_alias,VYPR_OBJ) + state_recording_instrument = "record_state_%s = %s.get_time('point instrument'); " % \ + (state_variable_alias, VYPR_OBJ) time_attained_instrument = state_recording_instrument + time_attained_variable = "record_state_%s" % state_variable_alias # the only purpose here is to match what is expected in the monitoring algorithm name = "time" else: state_variable_alias = name.replace(".", "_").replace("(", "__").replace(")", "__") state_recording_instrument = "record_state_%s = %s; " % (state_variable_alias, name) - time_attained_instrument = "time_attained_%s = %s.get_time();" % (state_variable_alias,VYPR_OBJ) + time_attained_instrument = "time_attained_%s = %s.get_time('point instrument');" % \ + (state_variable_alias, VYPR_OBJ) + time_attained_variable = "time_attained_%s" % state_variable_alias # note that observed_value is used three times: # 1) to capture the time attained by the state for checking of a property - this is duplicated @@ -439,13 +446,11 @@ def instrument_point_state(state, name, point, binding_space_indices, atom_sub_index=atom_sub_index, instrumentation_point_db_id=instrumentation_point_db_ids, atom_program_variable=name, - time_attained = ("time_attained_%s" % state_variable_alias), + time_attained=time_attained_variable, observed_value=("record_state_%s" % state_variable_alias) ) state_recording_instrument += "%s((%s))" % (VERIFICATION_INSTRUCTION, instrument_tuple) - - time_attained_ast = ast.parse(time_attained_instrument).body[0] record_state_ast = ast.parse(state_recording_instrument).body[0] queue_ast = ast.parse(state_recording_instrument).body[1] @@ -516,6 +521,128 @@ def instrument_point_state(state, name, point, binding_space_indices, parent_block.insert(index_in_block + 1, record_state_ast) parent_block.insert(index_in_block + 1, time_attained_ast) +# def instrument_point_state(state, name, point, binding_space_indices, +# atom_index, atom_sub_index, instrumentation_point_db_ids, +# measure_attribute=None): +# """ +# state is the PyCFTL object, and point is the part of the SCFG found by traversal. +# """ +# global VERIFICATION_INSTRUCTION +# +# logger.log("Instrumenting point %s" % point) +# +# if measure_attribute == "length": +# state_variable_alias = name.replace(".", "_").replace("(", "__").replace(")", "__") +# state_recording_instrument = "record_state_%s = len(%s); " % (state_variable_alias, name) +# time_attained_instrument = "time_attained_%s = %s.get_time();" % (state_variable_alias,VYPR_OBJ) +# elif measure_attribute == "type": +# state_variable_alias = name.replace(".", "_").replace("(", "__").replace(")", "__") +# state_recording_instrument = "record_state_%s = type(%s).__name__; " % (state_variable_alias, name) +# time_attained_instrument = "time_attained_%s = %s.get_time();" % (state_variable_alias,VYPR_OBJ) +# elif measure_attribute == "time_attained": +# state_variable_alias = "time_attained_%i" % atom_sub_index +# state_recording_instrument = "record_state_%s = %s.get_time(); " % (state_variable_alias,VYPR_OBJ) +# time_attained_instrument = state_recording_instrument +# # the only purpose here is to match what is expected in the monitoring algorithm +# name = "time" +# else: +# state_variable_alias = name.replace(".", "_").replace("(", "__").replace(")", "__") +# state_recording_instrument = "record_state_%s = %s; " % (state_variable_alias, name) +# time_attained_instrument = "time_attained_%s = %s.get_time();" % (state_variable_alias,VYPR_OBJ) +# +# # note that observed_value is used three times: +# # 1) to capture the time attained by the state for checking of a property - this is duplicated +# # because we have the start and end time of the state, which is the same because states are instantaneous. +# # 3) to capture the time at which an observation was received - it makes sense that these times would +# # be the same. +# instrument_tuple = ("'{formula_hash}', 'instrument', '{function_qualifier}', {binding_space_index}, " +# "{atom_index}, {atom_sub_index}, {instrumentation_point_db_id}, {time_attained}, " +# "{time_attained}, {{ '{atom_program_variable}' : {observed_value} }}, __thread_id") \ +# .format( +# formula_hash=formula_hash, +# function_qualifier=instrument_function_qualifier, +# binding_space_index=binding_space_indices, +# atom_index=atom_index, +# atom_sub_index=atom_sub_index, +# instrumentation_point_db_id=instrumentation_point_db_ids, +# atom_program_variable=name, +# time_attained = ("time_attained_%s" % state_variable_alias), +# observed_value=("record_state_%s" % state_variable_alias) +# ) +# state_recording_instrument += "%s((%s))" % (VERIFICATION_INSTRUCTION, instrument_tuple) +# +# +# +# time_attained_ast = ast.parse(time_attained_instrument).body[0] +# record_state_ast = ast.parse(state_recording_instrument).body[0] +# queue_ast = ast.parse(state_recording_instrument).body[1] +# +# if type(state) is SourceStaticState or type(state) is DestinationStaticState: +# # if the state we're measuring a property of is derived from a source/destination operator, +# # then the instrumentation point we're given is an SCFG edge which contains +# # an instruction for us to place a state recording instrument before +# +# logger.log("Adding state recording instrument for source or target.") +# +# parent_block = point._instruction._parent_body +# +# record_state_ast.lineno = point._instruction.lineno +# record_state_ast.col_offset = point._instruction.col_offset +# queue_ast.lineno = point._instruction.lineno +# queue_ast.col_offset = point._instruction.col_offset +# +# index_in_block = parent_block.index(point._instruction) +# +# if type(state) is SourceStaticState: +# # for source state recording, we record the state, but only insert its value after +# # this is so triggers can be inserted before normal instruments without introducing +# # a special case for trigger insertion +# parent_block.insert(index_in_block, queue_ast) +# parent_block.insert(index_in_block, record_state_ast) +# parent_block.insert(index_in_block, time_attained_ast) +# elif type(state) is DestinationStaticState: +# parent_block.insert(index_in_block + 1, queue_ast) +# parent_block.insert(index_in_block + 1, record_state_ast) +# parent_block.insert(index_in_block + 1, time_attained_ast) +# +# else: +# +# if point._name_changed == ["loop"]: +# # we're instrumenting the change of a loop variable +# logger.log("Performing instrumentation for loop variable.") +# # determine the edge leading into the loop body +# for edge in point.edges: +# if edge._condition == ["enter-loop"]: +# # place an instrument before the instruction on this edge +# parent_block = edge._instruction._parent_body +# record_state_ast.lineno = edge._instruction.lineno +# record_state_ast.col_offset = edge._instruction.col_offset +# queue_ast.lineno = edge._instruction.lineno +# queue_ast.col_offset = edge._instruction.col_offset +# index_in_block = parent_block.index(edge._instruction) +# # insert instruments in reverse order +# parent_block.insert(index_in_block + 1, queue_ast) +# parent_block.insert(index_in_block + 1, record_state_ast) +# parent_block.insert(index_in_block + 1, time_attained_ast) +# else: +# # we're instrumenting a normal vertex where there is an explicit instruction +# logger.log("Not source or destination state - performing normal instrumentation.") +# incident_edge = point._previous_edge +# parent_block = incident_edge._instruction._parent_body +# +# record_state_ast.lineno = incident_edge._instruction.lineno +# record_state_ast.col_offset = incident_edge._instruction.col_offset +# queue_ast.lineno = incident_edge._instruction.lineno +# queue_ast.col_offset = incident_edge._instruction.col_offset +# +# index_in_block = parent_block.index(incident_edge._instruction) +# +# # insert instruments in reverse order +# +# parent_block.insert(index_in_block + 1, queue_ast) +# parent_block.insert(index_in_block + 1, record_state_ast) +# parent_block.insert(index_in_block + 1, time_attained_ast) + def instrument_point_transition(atom, point, binding_space_indices, atom_index, atom_sub_index, instrumentation_point_db_ids): From 3cd672312ce1a680dc1ce4b82b4278bc398ccc8e Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Tue, 19 May 2020 00:04:48 +0200 Subject: [PATCH 23/30] modified instrumentation for taking only timestimp with vypr.get_time() --- instrument.py | 141 ++++---------------------------------------------- 1 file changed, 9 insertions(+), 132 deletions(-) diff --git a/instrument.py b/instrument.py index 43b480f..beadc16 100644 --- a/instrument.py +++ b/instrument.py @@ -521,128 +521,6 @@ def instrument_point_state(state, name, point, binding_space_indices, parent_block.insert(index_in_block + 1, record_state_ast) parent_block.insert(index_in_block + 1, time_attained_ast) -# def instrument_point_state(state, name, point, binding_space_indices, -# atom_index, atom_sub_index, instrumentation_point_db_ids, -# measure_attribute=None): -# """ -# state is the PyCFTL object, and point is the part of the SCFG found by traversal. -# """ -# global VERIFICATION_INSTRUCTION -# -# logger.log("Instrumenting point %s" % point) -# -# if measure_attribute == "length": -# state_variable_alias = name.replace(".", "_").replace("(", "__").replace(")", "__") -# state_recording_instrument = "record_state_%s = len(%s); " % (state_variable_alias, name) -# time_attained_instrument = "time_attained_%s = %s.get_time();" % (state_variable_alias,VYPR_OBJ) -# elif measure_attribute == "type": -# state_variable_alias = name.replace(".", "_").replace("(", "__").replace(")", "__") -# state_recording_instrument = "record_state_%s = type(%s).__name__; " % (state_variable_alias, name) -# time_attained_instrument = "time_attained_%s = %s.get_time();" % (state_variable_alias,VYPR_OBJ) -# elif measure_attribute == "time_attained": -# state_variable_alias = "time_attained_%i" % atom_sub_index -# state_recording_instrument = "record_state_%s = %s.get_time(); " % (state_variable_alias,VYPR_OBJ) -# time_attained_instrument = state_recording_instrument -# # the only purpose here is to match what is expected in the monitoring algorithm -# name = "time" -# else: -# state_variable_alias = name.replace(".", "_").replace("(", "__").replace(")", "__") -# state_recording_instrument = "record_state_%s = %s; " % (state_variable_alias, name) -# time_attained_instrument = "time_attained_%s = %s.get_time();" % (state_variable_alias,VYPR_OBJ) -# -# # note that observed_value is used three times: -# # 1) to capture the time attained by the state for checking of a property - this is duplicated -# # because we have the start and end time of the state, which is the same because states are instantaneous. -# # 3) to capture the time at which an observation was received - it makes sense that these times would -# # be the same. -# instrument_tuple = ("'{formula_hash}', 'instrument', '{function_qualifier}', {binding_space_index}, " -# "{atom_index}, {atom_sub_index}, {instrumentation_point_db_id}, {time_attained}, " -# "{time_attained}, {{ '{atom_program_variable}' : {observed_value} }}, __thread_id") \ -# .format( -# formula_hash=formula_hash, -# function_qualifier=instrument_function_qualifier, -# binding_space_index=binding_space_indices, -# atom_index=atom_index, -# atom_sub_index=atom_sub_index, -# instrumentation_point_db_id=instrumentation_point_db_ids, -# atom_program_variable=name, -# time_attained = ("time_attained_%s" % state_variable_alias), -# observed_value=("record_state_%s" % state_variable_alias) -# ) -# state_recording_instrument += "%s((%s))" % (VERIFICATION_INSTRUCTION, instrument_tuple) -# -# -# -# time_attained_ast = ast.parse(time_attained_instrument).body[0] -# record_state_ast = ast.parse(state_recording_instrument).body[0] -# queue_ast = ast.parse(state_recording_instrument).body[1] -# -# if type(state) is SourceStaticState or type(state) is DestinationStaticState: -# # if the state we're measuring a property of is derived from a source/destination operator, -# # then the instrumentation point we're given is an SCFG edge which contains -# # an instruction for us to place a state recording instrument before -# -# logger.log("Adding state recording instrument for source or target.") -# -# parent_block = point._instruction._parent_body -# -# record_state_ast.lineno = point._instruction.lineno -# record_state_ast.col_offset = point._instruction.col_offset -# queue_ast.lineno = point._instruction.lineno -# queue_ast.col_offset = point._instruction.col_offset -# -# index_in_block = parent_block.index(point._instruction) -# -# if type(state) is SourceStaticState: -# # for source state recording, we record the state, but only insert its value after -# # this is so triggers can be inserted before normal instruments without introducing -# # a special case for trigger insertion -# parent_block.insert(index_in_block, queue_ast) -# parent_block.insert(index_in_block, record_state_ast) -# parent_block.insert(index_in_block, time_attained_ast) -# elif type(state) is DestinationStaticState: -# parent_block.insert(index_in_block + 1, queue_ast) -# parent_block.insert(index_in_block + 1, record_state_ast) -# parent_block.insert(index_in_block + 1, time_attained_ast) -# -# else: -# -# if point._name_changed == ["loop"]: -# # we're instrumenting the change of a loop variable -# logger.log("Performing instrumentation for loop variable.") -# # determine the edge leading into the loop body -# for edge in point.edges: -# if edge._condition == ["enter-loop"]: -# # place an instrument before the instruction on this edge -# parent_block = edge._instruction._parent_body -# record_state_ast.lineno = edge._instruction.lineno -# record_state_ast.col_offset = edge._instruction.col_offset -# queue_ast.lineno = edge._instruction.lineno -# queue_ast.col_offset = edge._instruction.col_offset -# index_in_block = parent_block.index(edge._instruction) -# # insert instruments in reverse order -# parent_block.insert(index_in_block + 1, queue_ast) -# parent_block.insert(index_in_block + 1, record_state_ast) -# parent_block.insert(index_in_block + 1, time_attained_ast) -# else: -# # we're instrumenting a normal vertex where there is an explicit instruction -# logger.log("Not source or destination state - performing normal instrumentation.") -# incident_edge = point._previous_edge -# parent_block = incident_edge._instruction._parent_body -# -# record_state_ast.lineno = incident_edge._instruction.lineno -# record_state_ast.col_offset = incident_edge._instruction.col_offset -# queue_ast.lineno = incident_edge._instruction.lineno -# queue_ast.col_offset = incident_edge._instruction.col_offset -# -# index_in_block = parent_block.index(incident_edge._instruction) -# -# # insert instruments in reverse order -# -# parent_block.insert(index_in_block + 1, queue_ast) -# parent_block.insert(index_in_block + 1, record_state_ast) -# parent_block.insert(index_in_block + 1, time_attained_ast) - def instrument_point_transition(atom, point, binding_space_indices, atom_index, atom_sub_index, instrumentation_point_db_ids): @@ -926,18 +804,14 @@ def place_function_begin_instruments(function_def, formula_hash, instrument_func def place_function_end_instruments(function_def, scfg, formula_hash, instrument_function_qualifier, is_flask): # insert the end instrument before every return statement - # Use time accordingly, if its flask or normal testing - if is_flask: - time = "flask.g.request_time" - else: - time = "vypr.get_time()" - for end_vertex in scfg.return_statements: + end_instrument = \ - "%s((\"%s\", \"function\", \"%s\", \"end\", %s, \"%s\", __thread_id, " \ + "%s((\"%s\", \"function\", \"%s\", \"end\", vypr.get_time(), \"%s\", __thread_id, " \ "%s.get_time('end-instrument')))" \ - % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, time,formula_hash, VYPR_OBJ) + % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, formula_hash, VYPR_OBJ) + end_ast = ast.parse(end_instrument).body[0] end_ast.lineno = end_vertex._previous_edge._instruction._parent_body[-1].lineno @@ -953,9 +827,11 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ # if the last instruction in the ast is not a return statement, add an end instrument at the end if not (type(function_def.body[-1]) is ast.Return): end_instrument = \ - "%s((\"%s\", \"function\", \"%s\", \"end\", %s, \"%s\", __thread_id, " \ + "%s((\"%s\", \"function\", \"%s\", \"end\", vypr.get_time(), \"%s\", __thread_id, " \ "%s.get_time('end-instrument')))" \ - % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, time,formula_hash, VYPR_OBJ) + % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier,formula_hash, VYPR_OBJ) + + end_ast = ast.parse(end_instrument).body[0] @@ -1343,6 +1219,7 @@ def if_file_is_flask(name): + # if we're using flask, we assume a certain architecture From 5f41f0b538d670089f4d3092aa0fbd98fe47d4ba Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Wed, 27 May 2020 15:23:18 +0200 Subject: [PATCH 24/30] updated --- instrument.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/instrument.py b/instrument.py index beadc16..27f685d 100644 --- a/instrument.py +++ b/instrument.py @@ -1116,7 +1116,7 @@ def if_file_is_flask(name): # first, check that the verdict server is reachable if not (is_verdict_server_reachable()): print("Verdict server is not reachable. Ending instrumentation - nothing has been done.") - exit() + SETUP_ONCE = False @@ -1138,6 +1138,8 @@ def if_file_is_flask(name): os.remove(f.replace(".py.inst", BYTECODE_EXTENSION)) logger.log("Reset file %s to uninstrumented version." % f) + + logger.log("Importing PyCFTL queries...") # load in verification config file # to do this, we read in the existing one, write a temporary one with imports added and import that one From be48984938c1fb671a73d51925fa27529f101439 Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Wed, 27 May 2020 22:33:57 +0200 Subject: [PATCH 25/30] removed comments --- monitor_synthesis/formula_tree.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/monitor_synthesis/formula_tree.py b/monitor_synthesis/formula_tree.py index 689d427..a145356 100644 --- a/monitor_synthesis/formula_tree.py +++ b/monitor_synthesis/formula_tree.py @@ -268,8 +268,8 @@ def check(self, cummulative_state): self._rhs._arithmetic_stack, cummulative_state[1][0][self._rhs_name] ) - print(lhs_with_arithmetic, rhs_with_arithmetic) - print(lhs_with_arithmetic < rhs_with_arithmetic) + # print(lhs_with_arithmetic, rhs_with_arithmetic) + # print(lhs_with_arithmetic < rhs_with_arithmetic) return lhs_with_arithmetic < rhs_with_arithmetic @@ -864,7 +864,7 @@ def check_atom_truth_value(self, atom, value): an indication of whether the observation is for the lhs or rhs """ check_value = atom.check(value) - print("resulting truth value", check_value) + #print("resulting truth value", check_value) if check_value == True: result = self.check(self._formula, atom) elif check_value == False: @@ -1111,10 +1111,10 @@ def check(self, formula, symbol, level=0): self._formula.verdict = True return True elif formula_is_derived_from_atom(formula): - print("simple formula") + #print("simple formula") if formula == symbol: if level == 0: - print("reached true verdict") + #print("reached true verdict") self._formula.verdict = True return True else: From ca4bcb8fbddba560da34437b690cc4b1b7f8f44c Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Wed, 27 May 2020 22:54:30 +0200 Subject: [PATCH 26/30] removed comments in monitoring thread --- init_verification.py | 1 - 1 file changed, 1 deletion(-) diff --git a/init_verification.py b/init_verification.py index 7ca4b88..c1df18d 100644 --- a/init_verification.py +++ b/init_verification.py @@ -203,7 +203,6 @@ def consumption_thread_function(verification_obj): except: # Changing flag to false here because in normal testing, end-monitoring does not change to False. # If exception is raised we just terminate the monitoring - print("Getting stuck here! because queue is empty") continue From b50d0d4328f77047aa08560ff05d2e6d231cd453 Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Mon, 15 Jun 2020 11:22:39 +0200 Subject: [PATCH 27/30] fixed path problem --- SCFG/construction.py | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) mode change 100755 => 100644 SCFG/construction.py diff --git a/SCFG/construction.py b/SCFG/construction.py old mode 100755 new mode 100644 index 8a4f547..3d611e0 --- a/SCFG/construction.py +++ b/SCFG/construction.py @@ -156,6 +156,20 @@ def get_attr_name_string(obj, omit_subscripts=False): return attr_string +def get_first_instruction_from_block(block): + """ + Given a list of AST objects, get the first one that isn't a comment. + """ + first_non_comment_index = 0 + for (n, obj) in enumerate(block): + if type(obj) is ast.Expr and hasattr(obj, "value") and hasattr(obj.value, "s"): + # this statement is a comment so we skipe it + continue + else: + first_non_comment_index = n + return first_non_comment_index + + class CFGVertex(object): """ This class represents a vertex in a control flow graph. @@ -477,9 +491,11 @@ def process_block(self, block, starting_vertices=None, condition=[], closest_loo ) # add to the list of final vertices that need to be connected to the post-conditional vertex final_conditional_vertices += final_vertices + # get first non comment instruction index in block + first_non_comment_index = get_first_instruction_from_block(current_conditional[0].body) # add the branching statement self.branch_initial_statements.append( - ["conditional", current_conditional[0].body[0], branch_number] + ["conditional", current_conditional[0].body[first_non_comment_index], branch_number] ) branch_number += 1 @@ -501,9 +517,11 @@ def process_block(self, block, starting_vertices=None, condition=[], closest_loo ) # add to the list of final vertices that need to be connected to the post-conditional vertex final_conditional_vertices += final_vertices + # get first non comment instruction index in block + first_non_comment_index = get_first_instruction_from_block(current_conditional[0].body) # add the branching statement self.branch_initial_statements.append( - ["conditional", current_conditional[0].body[0], branch_number] + ["conditional", current_conditional[0].body[first_non_comment_index], branch_number] ) branch_number += 1 @@ -605,12 +623,20 @@ def process_block(self, block, starting_vertices=None, condition=[], closest_loo current_vertices = [empty_conditional_vertex] blocks = [] - self.branch_initial_statements.append(["try-catch", entry.body[0], "try-catch-main"]) + # get first non comment instruction index in block + first_non_comment_index = get_first_instruction_from_block(entry.body) + self.branch_initial_statements.append( + ["try-catch", entry.body[first_non_comment_index], "try-catch-main"] + ) # print("except handling blocks are:") for except_handler in entry.handlers: - self.branch_initial_statements.append(["try-catch", except_handler.body[0], "try-catch-handler"]) + # get first non comment instruction index in block + first_non_comment_index = get_first_instruction_from_block(except_handler.body) + self.branch_initial_statements.append( + ["try-catch", except_handler.body[first_non_comment_index], "try-catch-handler"] + ) # print(except_handler.body) blocks.append(except_handler.body) @@ -714,9 +740,13 @@ def process_block(self, block, starting_vertices=None, condition=[], closest_loo empty_post_loop_vertex ) + # get first non comment instruction index in block + first_non_comment_index = get_first_instruction_from_block(entry.body) # for a for loop, we add a path recording instrument at the beginning of the loop body # and after the loop body - self.branch_initial_statements.append(["loop", entry.body[0], "enter-loop", entry, "end-loop"]) + self.branch_initial_statements.append( + ["loop", entry.body[first_non_comment_index], "enter-loop", entry, "end-loop"] + ) # add 2 edges from the final_vertex - one going back to the pre-loop vertex # with the positive condition, and one going to the post loop vertex. From 188bafe6324213298a517a8b15c19ee15a0c1224 Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Mon, 22 Jun 2020 14:20:27 +0200 Subject: [PATCH 28/30] updated SCFG --- SCFG/construction.py | 192 +++++++++++++++++++++++++------------------ instrument.py | 43 ++++++---- 2 files changed, 140 insertions(+), 95 deletions(-) diff --git a/SCFG/construction.py b/SCFG/construction.py index 3d611e0..3993437 100644 --- a/SCFG/construction.py +++ b/SCFG/construction.py @@ -455,13 +455,15 @@ def process_block(self, block, starting_vertices=None, condition=[], closest_loo elif ast_is_if(entry): + print("--processing if-statement at line %i--" % entry.lineno) + entry._parent_body = block path_length += 1 # if this conditional isn't the last element in its block, we need to place a post-conditional # path recording instrument after it if entry != entry._parent_body[-1]: - self.branch_initial_statements.append(["post-conditional", entry]) + self.branch_initial_statements.append(["post-conditional", entry, entry.lineno]) # insert intermediate control flow vertex at the beginning of the block empty_conditional_vertex = CFGVertex(structure_obj=entry) @@ -480,94 +482,56 @@ def process_block(self, block, starting_vertices=None, condition=[], closest_loo current_conditional = [entry] final_else_is_present = False final_conditional_vertices = [] - branch_number = 0 - # process the main body, and then iterate downwards + # get first non comment instruction index in block + first_non_comment_index = get_first_instruction_from_block(current_conditional[0].body) + + # add the branching statement for the conditional body + self.branch_initial_statements.append( + ["conditional-body", current_conditional[0], 0, current_conditional[0].lineno] + ) + + # recurse on the main body final_vertices = self.process_block( current_conditional[0].body, current_vertices, [current_conditional[0].test], closest_loop ) + # add to the list of final vertices that need to be connected to the post-conditional vertex final_conditional_vertices += final_vertices - # get first non comment instruction index in block - first_non_comment_index = get_first_instruction_from_block(current_conditional[0].body) - # add the branching statement + + # recurse on the else block, if there's anything there + if len(current_conditional[0].orelse) > 0: + print("else block found for line %i" % entry.lineno) + # recurse on else-block + final_vertices = self.process_block( + current_conditional[0].orelse, + current_vertices, + [current_conditional[0].test], + closest_loop + ) + # add final vertices to list of vertices that need to be linked to the end of the current if-block + final_conditional_vertices += final_vertices + # remember that we've seen a final else-block + final_else_is_present = True + else: + print("no else block found for line %i" % entry.lineno) + + # add the branching statement for the else-block self.branch_initial_statements.append( - ["conditional", current_conditional[0].body[first_non_comment_index], branch_number] + ["conditional-else", current_conditional[0], 1, current_conditional[0].lineno] ) - branch_number += 1 - - # we now repeat the same, but iterating through the conditional structure - while type(current_conditional[0]) is ast.If: - current_conditional = current_conditional[0].orelse - if len(current_conditional) == 1: - - # there is just another conditional block, so process it as if it were a branch - if type(current_conditional[0]) is ast.If: - # pairs.append( - # (current_condition_set + [current_conditional[0].test], current_conditional[0].body)) - # current_condition_set.append(formula_tree.lnot(current_conditional[0].test)) - final_vertices = self.process_block( - current_conditional[0].body, - current_vertices, - [current_conditional[0].test], - closest_loop - ) - # add to the list of final vertices that need to be connected to the post-conditional vertex - final_conditional_vertices += final_vertices - # get first non comment instruction index in block - first_non_comment_index = get_first_instruction_from_block(current_conditional[0].body) - # add the branching statement - self.branch_initial_statements.append( - ["conditional", current_conditional[0].body[first_non_comment_index], branch_number] - ) - branch_number += 1 - else: - # the else block contains an instruction that isn't a conditional - # pairs.append((current_condition_set, current_conditional)) - final_vertices = self.process_block( - current_conditional, - current_vertices, - ["else"], - closest_loop - ) - # we reached an else block - final_else_is_present = True - # add to the list of final vertices that need to be connected to the post-conditional vertex - final_conditional_vertices += final_vertices - # add the branching statement - self.branch_initial_statements.append( - ["conditional", current_conditional[0], branch_number] - ) - branch_number += 1 - - elif len(current_conditional) > 1: - # there are multiple blocks inside the orelse, so we can't treat this like another branch - final_vertices = self.process_block( - current_conditional, - current_vertices, - ["else"], - closest_loop - ) - final_conditional_vertices += final_vertices - self.branch_initial_statements.append( - ["conditional", current_conditional[0], branch_number] - ) - # we reached an else block - final_else_is_present = True - else: - # nowhere else to go in the traversal - break - - # we include the vertex before the conditional, only if there was no else - if not (final_else_is_present): - # we add a branching statement - the branch number is just the number of pairs we found - self.branch_initial_statements.append(["conditional-no-else", entry, branch_number]) + # if we didn't find an else-block, we need an edge going around the if-block + if not final_else_is_present: + # all vertices present before the final conditional also need to be linked up + # since there was no else-block current_vertices = final_conditional_vertices + current_vertices else: + # the vertices that now need to be linked up by the next block processed + # are those constructed for the if-block current_vertices = final_conditional_vertices # filter out vertices that were returns or raises @@ -579,8 +543,7 @@ def process_block(self, block, starting_vertices=None, condition=[], closest_loo )) # add an empty "control flow" vertex after the conditional - # to avoid transition duplication along the edges leaving - # the conditional + # to avoid duplication of edges leaving the conditional if len(current_vertices) > 0: empty_vertex = CFGVertex() empty_vertex._name_changed = ['post-conditional'] @@ -603,6 +566,73 @@ def process_block(self, block, starting_vertices=None, condition=[], closest_loo # reset path length for instructions after conditional path_length = 0 + # branch_number += 1 + # + # # we now repeat the same, but iterating through the conditional structure + # while type(current_conditional[0]) is ast.If: + # current_conditional = current_conditional[0].orelse + # if len(current_conditional) == 1: + # + # # there is just another conditional block, so process it as if it were a branch + # if type(current_conditional[0]) is ast.If: + # print("processing else block with other if block") + # # pairs.append( + # # (current_condition_set + [current_conditional[0].test], current_conditional[0].body)) + # # current_condition_set.append(formula_tree.lnot(current_conditional[0].test)) + # final_vertices = self.process_block( + # current_conditional[0].body, + # current_vertices, + # [current_conditional[0].test], + # closest_loop + # ) + # # add to the list of final vertices that need to be connected to the post-conditional vertex + # final_conditional_vertices += final_vertices + # # get first non comment instruction index in block + # first_non_comment_index = get_first_instruction_from_block(current_conditional[0].body) + # # add the branching statement + # self.branch_initial_statements.append( + # ["conditional", current_conditional[0].body[first_non_comment_index], branch_number] + # ) + # branch_number += 1 + # + # else: + # print("processing else block with non-if block") + # # the else block contains an instruction that isn't a conditional + # # pairs.append((current_condition_set, current_conditional)) + # final_vertices = self.process_block( + # current_conditional, + # current_vertices, + # ["else"], + # closest_loop + # ) + # # we reached an else block + # final_else_is_present = True + # # add to the list of final vertices that need to be connected to the post-conditional vertex + # final_conditional_vertices += final_vertices + # # add the branching statement + # self.branch_initial_statements.append( + # ["conditional", current_conditional[0], branch_number] + # ) + # branch_number += 1 + # + # elif len(current_conditional) > 1: + # # there are multiple blocks inside the orelse, so we can't treat this like another branch + # final_vertices = self.process_block( + # current_conditional, + # current_vertices, + # ["else"], + # closest_loop + # ) + # final_conditional_vertices += final_vertices + # self.branch_initial_statements.append( + # ["conditional", current_conditional[0], branch_number] + # ) + # # we reached an else block + # final_else_is_present = True + # else: + # # nowhere else to go in the traversal + # break + elif ast_is_try(entry): entry._parent_body = block path_length += 1 @@ -624,18 +654,18 @@ def process_block(self, block, starting_vertices=None, condition=[], closest_loo blocks = [] # get first non comment instruction index in block - first_non_comment_index = get_first_instruction_from_block(entry.body) + #first_non_comment_index = get_first_instruction_from_block(entry.body) self.branch_initial_statements.append( - ["try-catch", entry.body[first_non_comment_index], "try-catch-main"] + ["try-catch", entry, "try-catch-main"] ) # print("except handling blocks are:") for except_handler in entry.handlers: # get first non comment instruction index in block - first_non_comment_index = get_first_instruction_from_block(except_handler.body) + #first_non_comment_index = get_first_instruction_from_block(except_handler.body) self.branch_initial_statements.append( - ["try-catch", except_handler.body[first_non_comment_index], "try-catch-handler"] + ["try-catch", except_handler, "try-catch-handler"] ) # print(except_handler.body) blocks.append(except_handler.body) diff --git a/instrument.py b/instrument.py index 27f685d..21671be 100644 --- a/instrument.py +++ b/instrument.py @@ -596,8 +596,12 @@ def place_path_recording_instruments(scfg, instrument_function_qualifier, formul # this is done independent of binding space computation for vertex_information in scfg.branch_initial_statements: logger.log("-" * 100) - if vertex_information[0] in ['conditional', 'try-catch']: - if vertex_information[0] == 'conditional': + if vertex_information[0] in ['conditional-body', 'try-catch']: + + print("placing main clause instrument for conditional at line %i" % vertex_information[1].lineno) + + if vertex_information[0] == 'conditional-body': + logger.log( "Placing branch recording instrument for conditional with first instruction %s in " "block" % @@ -618,6 +622,7 @@ def place_path_recording_instruments(scfg, instrument_function_qualifier, formul condition_dict = { "serialised_condition": vertex_information[2] } + # if the condition already exists in the database, the verdict server will return the # existing ID try: @@ -633,13 +638,15 @@ def place_path_recording_instruments(scfg, instrument_function_qualifier, formul VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, branching_condition_id) instrument_ast = ast.parse(instrument_code).body[0] - index_in_parent = vertex_information[1]._parent_body.index(vertex_information[1]) - vertex_information[1]._parent_body.insert(index_in_parent, instrument_ast) + #index_in_parent = vertex_information[1].body.index(vertex_information[1]) + vertex_information[1].body.insert(0, instrument_ast) logger.log("Branch recording instrument placed") - elif vertex_information[0] == "conditional-no-else": - # no else was present in the conditional, so we add a path recording instrument - # to the else block - logger.log("Placing branch recording instrument for conditional with no else") + + elif vertex_information[0] == "conditional-else": + + # an else block was found, so we add an instrument at the beginning + logger.log("Placing branch recording instrument for conditional with else") + # send branching condition to verdict server, take the ID from the response and use it in # the path recording instruments. condition_dict = { @@ -659,10 +666,13 @@ def place_path_recording_instruments(scfg, instrument_function_qualifier, formul instrument_code = "%s((\"%s\", \"path\", \"%s\", %i))" % ( VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, branching_condition_id) + instrument_ast = ast.parse(instrument_code).body[0] vertex_information[1].orelse.insert(0, instrument_ast) logger.log("Branch recording instrument placed") + elif vertex_information[0] in ['post-conditional', 'post-try-catch']: + if vertex_information[0] == 'post-conditional': logger.log("Processing post conditional path instrument") logger.log(vertex_information) @@ -712,7 +722,9 @@ def place_path_recording_instruments(scfg, instrument_function_qualifier, formul logger.log(index_in_parent) vertex_information[1]._parent_body.insert(index_in_parent, instrument_code_ast) logger.log(vertex_information[1]._parent_body) + elif vertex_information[0] == 'loop': + logger.log( "Placing branch recording instrument for loop with first instruction %s in body" % vertex_information[1]) @@ -1276,6 +1288,9 @@ def if_file_is_flask(name): scfg = CFG(reference_variables=reference_variables) scfg_vertices = scfg.process_block(function_def.body) + import pprint + pprint.pprint(scfg.branch_initial_statements) + top_level_block = function_def.body logger.log("SCFG constructed.") @@ -1740,12 +1755,12 @@ def if_file_is_flask(name): - # write the instrumented scfg to a file - instrumented_scfg = CFG() - instrumented_scfg.process_block(top_level_block) - write_scfg_to_file(instrumented_scfg, "%s-%s-%s-instrumented.gv" % - (file_name_without_extension.replace(".", ""), module.replace(".", "-"), - function.replace(".", "-"))) + # # write the instrumented scfg to a file + # instrumented_scfg = CFG() + # instrumented_scfg.process_block(top_level_block) + # write_scfg_to_file(instrumented_scfg, "%s-%s-%s-instrumented.gv" % + # (file_name_without_extension.replace(".", ""), module.replace(".", "-"), + # function.replace(".", "-"))) # check for existence of directories for intermediate data and create them if not found if not (os.path.isdir("binding_spaces")): From b0516a077edd263d2b1569043382f136b9bf54f6 Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Mon, 17 Aug 2020 11:46:19 +0200 Subject: [PATCH 29/30] sync local changes --- QueryBuilding/formula_building.py | 433 +++++++++++++---- SCFG/construction.py | 315 +++++++------ init_verification.py | 760 +++++++++++++++--------------- instrument.py | 318 +++++++++---- monitor_synthesis/formula_tree.py | 527 +++++++++++++++------ 5 files changed, 1500 insertions(+), 853 deletions(-) diff --git a/QueryBuilding/formula_building.py b/QueryBuilding/formula_building.py index 2f5e16d..c45f997 100644 --- a/QueryBuilding/formula_building.py +++ b/QueryBuilding/formula_building.py @@ -12,16 +12,90 @@ StateValueInInterval - models the atom (s(x) in I) for some state s, a name x and an interval (list) I. TransitionDurationInInterval - models the atom (d(delta t) in I) for some transition delta t and an interval (list) I. -Atoms are generated once states or transitions have been described by calling - """ import inspect # be careful with versions here... from collections import OrderedDict +import ast from VyPR.monitor_synthesis import formula_tree +""" +Function selection. +""" + + +class Module(object): + """ + Used to point to a module/function in specifications. + """ + + def __init__(self, module_name): + self._module_name = module_name + + def Function(self, function_name): + self._function_name = function_name + return self + + +class Functions(object): + """ + Given as keys in the initial specification config file dictionary by the developer. + """ + + def __init__(self, **parameter_dictionary): + """ + Parameters given here, with their values, act as criteria to select a set of functions in a project. + Multiple parameter values will be used in conjunction. + """ + # set the parameters given as object parameters that can be used in the search performed by instrumentation + self.criteria_dict = parameter_dictionary + + def is_satisfied_by(self, function_ast, scfg): + """ + Given a Function AST, decide whether the criteria defined by this instance are satisfied. + :param function_ast: + :param scfg: + :return: True or False + """ + for criterion_name in self.criteria_dict: + if criterion_name == "containing_call_of": + edges = scfg.edges + call_found = False + for edge in edges: + if edge._operates_on and self.criteria_dict[criterion_name] in edge._operates_on: + call_found = True + # if no call was found, this criterion has not been met, + # so return False because the conjunction is broken + if not call_found: + return False + elif criterion_name == "containing_change_of": + vertices = scfg.vertices + change_found = False + for vertex in vertices: + if self.criteria_dict[criterion_name] in vertex._name_changed: + change_found = True + elif hasattr(vertex, "_structure_obj"): + if type(vertex._structure_obj) is ast.For: + if type(vertex._structure_obj.target) is ast.Tuple: + if (self.criteria_dict[criterion_name] in + map(lambda item: item.id, vertex._structure_obj.target)): + change_found = True + else: + if self.criteria_dict[criterion_name] == vertex._structure_obj.target.id: + change_found = True + # if no change was found, this criterion has not been met, + # so return False because the conjunction is broken + if not (change_found): + return False + # we found no evidence against satisfaction, so return True + return True + + def __repr__(self): + return "<%s>" % str(self.criteria_dict) + + """ General structure-building classes and methods. """ @@ -95,7 +169,7 @@ def __repr__(self): if self._formula is None: return "Forall(%s)" % self.bind_variables else: - return "Forall(%s).Formula(%s)" % \ + return "Forall(%s).Check(%s)" % \ (self.bind_variables, self.get_formula_instance()) def Forall(self, **bind_variable): @@ -226,6 +300,54 @@ def requires_state_or_transition(obj): return type(obj) in [StateValue, StateValueLength] +class PossiblyNumeric(object): + """ + Models a quantity that one can perform arithmetic on. + Should be inherited by any class that wants a user to be able to perform arithmetic with it. + """ + + def __mul__(self, value): + """ + Given a constant (we assume this for now), + add an object to the arithmetic stack so it can be applied + later when values are checked. + """ + if type(value) in [int, float]: + base_variable = composition_sequence_from_value([self._state], self._state)[-1] + if base_variable._arithmetic_build: + self._state._arithmetic_stack.append(formula_tree.ArithmeticMultiply(value)) + return self + else: + raise Exception("Value used to multiply quantity %s must be of type int or float." % self) + + def __add__(self, value): + if type(value) in [int, float]: + base_variable = composition_sequence_from_value([self._state], self._state)[-1] + if base_variable._arithmetic_build: + self._state._arithmetic_stack.append(formula_tree.ArithmeticAdd(value)) + return self + else: + raise Exception("Value added to quantity %s must be of type int or float." % self) + + def __sub__(self, value): + if type(value) in [int, float]: + base_variable = composition_sequence_from_value([self._state], self._state)[-1] + if base_variable._arithmetic_build: + self._state._arithmetic_stack.append(formula_tree.ArithmeticSubtract(value)) + return self + else: + raise Exception("Value subtracted from quantity %s must be of type int or float." % self) + + def __truediv__(self, value): + if type(value) in [int, float] and value != 0: + base_variable = composition_sequence_from_value([self._state], self._state)[-1] + if base_variable._arithmetic_build: + self._state._arithmetic_stack.append(formula_tree.ArithmeticTrueDivide(value)) + return self + else: + raise Exception("Value used to divide quantity %s must be non-zero and of type int or float." % self) + + class StaticState(object): """ Models a state attained by the monitored program at runtime. @@ -244,7 +366,11 @@ def __init__(self, bind_variable_name, name_changed, instruction_coordinates, us self._arithmetic_build = False def __call__(self, name): - return StateValue(self, name) + if type(name) is str: + return StateValue(self, name) + else: + raise Exception("Value given to state '%s' must be of type str, not %s." % (self, name.__class__.__name__)) + def next_call(self, function, record=None): """ @@ -252,15 +378,19 @@ def next_call(self, function, record=None): points if there is nesting. It is a list of variable values to record at the start of the next call to function. """ - return NextStaticTransition(self, function, record=record) + if type(function) is str: + return NextStaticTransition(self, function, record=record) + else: + raise Exception("Value given to (%s).next_call must be of type str, not %s." % + (self, function.__class__.__name__)) def __repr__(self): if self._required_binding: - return "%s = StaticState(changes=%s, uses=%s)" % \ + return "%s = changes('%s', uses=%s)" % \ (self._bind_variable_name, self._name_changed, self._required_binding) else: - return "%s = StaticState(changes=%s)" % \ + return "%s = changes('%s')" % \ (self._bind_variable_name, self._name_changed) def __eq__(self, other): @@ -307,7 +437,7 @@ def __eq__(self, other): self._incoming_transition == other._incoming_transition) -class StateValue(object): +class StateValue(PossiblyNumeric): """ Models the value to which some state maps a program variable. """ @@ -320,11 +450,63 @@ def _in(self, interval): """ Generates an atom. """ - return formula_tree.StateValueInInterval( - self._state, - self._name, - interval - ) + if type(interval) is list and len(interval) == 2: + return formula_tree.StateValueInInterval( + self._state, + self._name, + interval + ) + elif type(interval) is tuple and len(interval) == 2: + return formula_tree.StateValueInOpenInterval( + self._state, + self._name, + interval + ) + else: + raise Exception("Value of type %s given to _in operator with %s('%s') is not supported." % + (interval.__class__.__name__, self._state, self._name)) + + def __lt__(self, value): + """ + Generates an atom. + """ + if type(value) is int: + return formula_tree.StateValueInOpenInterval( + self._state, + self._name, + [0, value] + ) + elif type(value) is StateValue: + return formula_tree.StateValueLessThanEqualStateValueMixed( + self._state, + self._name, + value._state, + value._name + ) + else: + raise Exception("Value of type %s given to < comparison with %s('%s') is not supported." % + (value.__class__.__name__, self._state, self._name)) + + def __le__(self, value): + """ + Generates an atom. + """ + if type(value) is int: + return formula_tree.StateValueInInterval( + self._state, + self._name, + [0, value] + ) + elif type(value) is StateValue: + return formula_tree.StateValueLessThanStateValueMixed( + self._state, + self._name, + value._state, + value._name + ) + else: + raise Exception("Value of type %s given to < comparison with %s('%s') is not supported." % + (value.__class__.__name__, self._state, self._name)) def equals(self, value): """ @@ -353,42 +535,8 @@ def length(self): def type(self): return StateValueType(self._state, self._name) - """ - Arithmetic overloading is useful for mixed atoms - when observed quantities are being compared to each other. - """ - - def __mul__(self, value): - """ - Given a constant (we assume this for now), - add an object to the arithmetic stack so it can be applied - later when values are checked. - """ - base_variable = composition_sequence_from_value([self._state], self._state)[-1] - if base_variable._arithmetic_build: - self._state._arithmetic_stack.append(formula_tree.ArithmeticMultiply(value)) - return self - - def __add__(self, value): - base_variable = composition_sequence_from_value([self._state], self._state)[-1] - if base_variable._arithmetic_build: - self._state._arithmetic_stack.append(formula_tree.ArithmeticAdd(value)) - return self - - def __sub__(self, value): - base_variable = composition_sequence_from_value([self._state], self._state)[-1] - if base_variable._arithmetic_build: - self._state._arithmetic_stack.append(formula_tree.ArithmeticSubtract(value)) - return self - - def __truediv__(self, value): - base_variable = composition_sequence_from_value([self._state], self._state)[-1] - if base_variable._arithmetic_build: - self._state._arithmetic_stack.append(formula_tree.ArithmeticTrueDivide(value)) - return self - -class StateValueLength(object): +class StateValueLength(PossiblyNumeric): """ Models the length being measured of a value given by a state. """ @@ -401,11 +549,21 @@ def _in(self, interval): """ Generates an atom. """ - return formula_tree.StateValueLengthInInterval( - self._state, - self._name, - interval - ) + if type(interval) is list and len(interval) == 2: + return formula_tree.StateValueLengthInInterval( + self._state, + self._name, + interval + ) + elif type(interval) is tuple and len(interval) == 2: + return formula_tree.StateValueLengthInOpenInterval( + self._state, + self._name, + interval + ) + else: + raise Exception("Value of type %s given to _in comparison with %s('%s').length() is not supported." % + (interval.__class__.__name__, self._state, self._name)) """ Overload comparison operators. @@ -415,50 +573,43 @@ def __lt__(self, value): """ Generates an atom. """ - # TODO: extend for multiple types - if type(value) is StateValueLength: - # the RHS of the comparison requires observation of another state or transition - # so we use a different class to deal with this + if type(value) is int: + return formula_tree.StateValueLengthInOpenInterval( + self._state, + self._name, + [0, value] + ) + elif type(value) is StateValueLength: return formula_tree.StateValueLengthLessThanStateValueLengthMixed( self._state, self._name, value._state, value._name ) + else: + raise Exception("Value of type %s given to _in comparison with %s('%s').length() is not supported." % + (value.__class__.__name__, self._state, self._name)) - """ - Arithmetic overloading is useful for mixed atoms - when observed quantities are being compared to each other. - """ - - def __mul__(self, value): + def __le__(self, value): """ - Given a constant (we assume this for now), - add an object to the arithmetic stack so it can be applied - later when values are checked. + Generates an atom. """ - base_variable = composition_sequence_from_value([self._state], self._state)[-1] - if base_variable._arithmetic_build: - self._state._arithmetic_stack.append(formula_tree.ArithmeticMultiply(value)) - return self - - def __add__(self, value): - base_variable = composition_sequence_from_value([self._state], self._state)[-1] - if base_variable._arithmetic_build: - self._state._arithmetic_stack.append(formula_tree.ArithmeticAdd(value)) - return self - - def __sub__(self, value): - base_variable = composition_sequence_from_value([self._state], self._state)[-1] - if base_variable._arithmetic_build: - self._state._arithmetic_stack.append(formula_tree.ArithmeticSubtract(value)) - return self - - def __truediv__(self, value): - base_variable = composition_sequence_from_value([self._state], self._state)[-1] - if base_variable._arithmetic_build: - self._state._arithmetic_stack.append(formula_tree.ArithmeticTrueDivide(value)) - return self + if type(value) is int: + return formula_tree.StateValueLengthInInterval( + self._state, + self._name, + [0, value] + ) + elif type(value) is StateValueLength: + return formula_tree.StateValueLengthLessThanEqualStateValueLengthMixed( + self._state, + self._name, + value._state, + value._name + ) + else: + raise Exception("Value of type %s given to _in comparison with %s('%s').length() is not supported." % + (value.__class__.__name__, self._state, self._name)) class StateValueType(object): @@ -501,7 +652,11 @@ def next_call(self, function, record=None): record will only matter for the final instrumentation points if there is nesting. It is a list of variable values to record at the start of the next call to function. """ - return NextStaticTransition(self, function, record=record) + if type(function) is str: + return NextStaticTransition(self, function, record=record) + else: + raise Exception("Value given to state '%s' must be of type str, not %s." % + (self, function.__class__.__name__)) def input(self): return SourceStaticState(self) @@ -512,19 +667,19 @@ def result(self): def __repr__(self): if self._required_binding: if self._record: - return "%s = StaticTransition(operates_on=%s, uses=%s, record=%s)" % \ + return "%s = calls('%s', uses=%s, record=%s)" % \ (self._bind_variable_name, self._operates_on, self._required_binding, str(self._record)) else: - return "%s = StaticTransition(operates_on=%s, uses=%s)" % \ + return "%s = calls('%s', uses=%s)" % \ (self._bind_variable_name, self._operates_on, self._required_binding) else: if self._record: - return "%s = StaticTransition(operates_on=%s, record=%s)" % \ + return "%s = calls('%s', record=%s)" % \ (self._bind_variable_name, self._operates_on, str(self._record)) else: - return "%s = StaticTransition(operates_on=%s)" % \ + return "%s = calls('%s')" % \ (self._bind_variable_name, self._operates_on) def __eq__(self, other): @@ -557,7 +712,7 @@ def __eq__(self, other): self._operates_on == other._operates_on) -class Duration(object): +class Duration(PossiblyNumeric): """ Models the duration of a transition. """ @@ -580,7 +735,8 @@ def _in(self, interval): interval ) else: - raise Exception("Duration predicate wasn't defined properly.") + raise Exception("Value of type %s given to _in comparison on %s is not supported." % + (interval.__class__.__name__, self._transition)) def __lt__(self, value): """ @@ -604,6 +760,44 @@ def __lt__(self, value): self._transition, value._transition, ) + elif type(value) in [int, float]: + return formula_tree.TransitionDurationInOpenInterval( + self._transition, + [0, value] + ) + else: + raise Exception("Value of type %s given to < comparison on %s is not supported." % + (value.__class__.__name__, self._transition)) + + def __le__(self, value): + """ + Generates an atom. + """ + if type(value) is StateValue: + return formula_tree.TransitionDurationLessThanEqualStateValueMixed( + self._transition, + value._state, + value._name + ) + elif type(value) is StateValueLength: + return formula_tree.TransitionDurationLessThanEqualStateValueLengthMixed( + self._transition, + value._state, + value._name + ) + elif type(value) is Duration: + return formula_tree.TransitionDurationLessThanEqualTransitionDurationMixed( + self._transition, + value._transition, + ) + elif type(value) in [int, float]: + return formula_tree.TransitionDurationInInterval( + self._transition, + [0, value] + ) + else: + raise Exception("Value of type %s given to <= comparison on %s is not supported." % + (value.__class__.__name__, self._transition)) """ @@ -641,9 +835,42 @@ def _in(self, interval): interval ) else: - raise Exception("TimeBetween predicate wasn't defined properly.") + raise Exception("Value of type %s given to _in comparison on %s is not supported." % + (interval.__class__.__name__, self)) + + def __lt__(self, value): + """ + Generates an atom. + """ + if type(value) in [int, float]: + return formula_tree.TimeBetweenInOpenInterval( + self._lhs, + self._rhs, + [0, value] + ) + else: + raise Exception("Value of type %s given to < comparison on %s is not supported." % + (value.__class__.__name__, self)) + + def __le__(self, value): + """ + Generates an atom. + """ + if type(value) in [int, float]: + return formula_tree.TimeBetweenInInterval( + self._lhs, + self._rhs, + [0, value] + ) + else: + raise Exception("Value of type %s given to <= comparison on %s is not supported." % + (value.__class__.__name__, self)) +""" +Utility functions to extract information from formulas. +""" + def composition_sequence_from_value(sequence, current_operator): while not (type(current_operator) in [StaticState, StaticTransition]): sequence.append(current_operator) @@ -672,13 +899,7 @@ def derive_composition_sequence(atom): else: current_operator = atom - if type(atom) in [formula_tree.StateValueEqualToMixed, - formula_tree.StateValueLengthLessThanStateValueLengthMixed, - formula_tree.TransitionDurationLessThanTransitionDurationMixed, - formula_tree.TransitionDurationLessThanStateValueMixed, - formula_tree.TransitionDurationLessThanStateValueLengthMixed, - formula_tree.TimeBetweenInInterval, - formula_tree.TimeBetweenInOpenInterval]: + if formula_tree.is_mixed_atom(atom): # atom is mixed - two composition sequences @@ -699,12 +920,18 @@ def derive_composition_sequence(atom): if type(current_operator) == formula_tree.TransitionDurationInInterval: current_operator = current_operator._transition + if type(current_operator) == formula_tree.TransitionDurationInOpenInterval: + current_operator = current_operator._transition elif type(current_operator) == formula_tree.StateValueEqualTo: current_operator = current_operator._state elif type(current_operator) == formula_tree.StateValueLengthInInterval: current_operator = current_operator._state + elif type(current_operator) == formula_tree.StateValueLengthInOpenInterval: + current_operator = current_operator._state elif type(current_operator) == formula_tree.StateValueInInterval: current_operator = current_operator._state + elif type(current_operator) == formula_tree.StateValueInOpenInterval: + current_operator = current_operator._state elif type(current_operator) == formula_tree.StateValueTypeEqualTo: current_operator = current_operator._state diff --git a/SCFG/construction.py b/SCFG/construction.py index 3993437..31f6622 100644 --- a/SCFG/construction.py +++ b/SCFG/construction.py @@ -129,10 +129,19 @@ def get_reversed_string_list(obj, omit_subscripts=False): elif type(obj.slice.value) is ast.Name: return ["[%s]" % obj.slice.value.id] + get_reversed_string_list(obj.value, omit_subscripts=omit_subscripts) + elif type(obj.slice.value) is ast.Subscript: + return ["[...]"] + get_reversed_string_list(obj.value, omit_subscripts=omit_subscripts) + elif type(obj.slice.value) is ast.Call: + if type(obj.slice.value.func) is ast.Attribute: + return [get_attr_name_string(obj.slice.value.func)] + else: + return [obj.slice.value.func.id] elif type(obj) is ast.Call: return get_function_name_strings(obj) elif type(obj) is ast.Str: return [obj.s] + else: + return [str(obj)] def get_attr_name_string(obj, omit_subscripts=False): @@ -156,20 +165,6 @@ def get_attr_name_string(obj, omit_subscripts=False): return attr_string -def get_first_instruction_from_block(block): - """ - Given a list of AST objects, get the first one that isn't a comment. - """ - first_non_comment_index = 0 - for (n, obj) in enumerate(block): - if type(obj) is ast.Expr and hasattr(obj, "value") and hasattr(obj.value, "s"): - # this statement is a comment so we skipe it - continue - else: - first_non_comment_index = n - return first_non_comment_index - - class CFGVertex(object): """ This class represents a vertex in a control flow graph. @@ -211,7 +206,13 @@ def __init__(self, entry=None, path_length=None, structure_obj=None, reference_v # nothing else could be changed self._name_changed = [] elif type(entry) is ast.Raise: - self._name_changed = [entry.type.func.id] + if type(entry.type) is ast.Call: + if type(entry.type.func) is ast.Attribute: + self._name_changed = [get_attr_name_string(entry.type.func)] + else: + self._name_changed = [entry.type.func.id] + else: + self._name_changed = [] elif type(entry) is ast.Pass: self._name_changed = ["pass"] elif type(entry) is ast.Continue: @@ -260,7 +261,14 @@ def __init__(self, condition, instruction=None): elif type(self._instruction) is ast.Return and type(self._instruction.value) is ast.Call: self._operates_on = get_function_name_strings(self._instruction.value) elif type(self._instruction) is ast.Raise: - self._operates_on = [self._instruction.type.func.id] + if type(self._instruction.type) is ast.Call: + if type(self._instruction.type.func) is ast.Attribute: + self._operates_on = [get_attr_name_string(self._instruction.type.func)] + else: + self._operates_on = [self._instruction.type.func.id] + else: + self._operates_on = [] + elif type(self._instruction) is ast.Pass: self._operates_on = ["pass"] else: @@ -416,9 +424,9 @@ def process_block(self, block, starting_vertices=None, condition=[], closest_loo for vertex in current_vertices: vertex.add_outgoing_edge(loop_ending_edge) - # direct all new edges to this new vertex + """# direct all new edges to this new vertex for edge in new_edges: - edge.set_target_state(new_vertex) + edge.set_target_state(new_vertex)""" # set the current_vertices to empty so no constructs can make an edge # from the preceding statement @@ -455,15 +463,13 @@ def process_block(self, block, starting_vertices=None, condition=[], closest_loo elif ast_is_if(entry): - print("--processing if-statement at line %i--" % entry.lineno) - entry._parent_body = block path_length += 1 # if this conditional isn't the last element in its block, we need to place a post-conditional # path recording instrument after it if entry != entry._parent_body[-1]: - self.branch_initial_statements.append(["post-conditional", entry, entry.lineno]) + self.branch_initial_statements.append(["post-conditional", entry]) # insert intermediate control flow vertex at the beginning of the block empty_conditional_vertex = CFGVertex(structure_obj=entry) @@ -482,56 +488,90 @@ def process_block(self, block, starting_vertices=None, condition=[], closest_loo current_conditional = [entry] final_else_is_present = False final_conditional_vertices = [] + branch_number = 0 - # get first non comment instruction index in block - first_non_comment_index = get_first_instruction_from_block(current_conditional[0].body) - - # add the branching statement for the conditional body - self.branch_initial_statements.append( - ["conditional-body", current_conditional[0], 0, current_conditional[0].lineno] - ) - - # recurse on the main body + # process the main body, and then iterate downwards final_vertices = self.process_block( current_conditional[0].body, current_vertices, [current_conditional[0].test], closest_loop ) - # add to the list of final vertices that need to be connected to the post-conditional vertex final_conditional_vertices += final_vertices - - # recurse on the else block, if there's anything there - if len(current_conditional[0].orelse) > 0: - print("else block found for line %i" % entry.lineno) - # recurse on else-block - final_vertices = self.process_block( - current_conditional[0].orelse, - current_vertices, - [current_conditional[0].test], - closest_loop - ) - # add final vertices to list of vertices that need to be linked to the end of the current if-block - final_conditional_vertices += final_vertices - # remember that we've seen a final else-block - final_else_is_present = True - else: - print("no else block found for line %i" % entry.lineno) - - # add the branching statement for the else-block + # add the branching statement self.branch_initial_statements.append( - ["conditional-else", current_conditional[0], 1, current_conditional[0].lineno] + ["conditional", current_conditional[0].body[0], branch_number] ) + branch_number += 1 + + # we now repeat the same, but iterating through the conditional structure + while type(current_conditional[0]) is ast.If: + current_conditional = current_conditional[0].orelse + if len(current_conditional) == 1: + + # there is just another conditional block, so process it as if it were a branch + if type(current_conditional[0]) is ast.If: + # pairs.append( + # (current_condition_set + [current_conditional[0].test], current_conditional[0].body)) + # current_condition_set.append(formula_tree.lnot(current_conditional[0].test)) + final_vertices = self.process_block( + current_conditional[0].body, + current_vertices, + [current_conditional[0].test], + closest_loop + ) + # add to the list of final vertices that need to be connected to the post-conditional vertex + final_conditional_vertices += final_vertices + # add the branching statement + self.branch_initial_statements.append( + ["conditional", current_conditional[0].body[0], branch_number] + ) + branch_number += 1 + + else: + # the else block contains an instruction that isn't a conditional + # pairs.append((current_condition_set, current_conditional)) + final_vertices = self.process_block( + current_conditional, + current_vertices, + ["else"], + closest_loop + ) + # we reached an else block + final_else_is_present = True + # add to the list of final vertices that need to be connected to the post-conditional vertex + final_conditional_vertices += final_vertices + # add the branching statement + self.branch_initial_statements.append( + ["conditional", current_conditional[0], branch_number] + ) + branch_number += 1 + + elif len(current_conditional) > 1: + # there are multiple blocks inside the orelse, so we can't treat this like another branch + final_vertices = self.process_block( + current_conditional, + current_vertices, + ["else"], + closest_loop + ) + final_conditional_vertices += final_vertices + self.branch_initial_statements.append( + ["conditional", current_conditional[0], branch_number] + ) + # we reached an else block + final_else_is_present = True + else: + # nowhere else to go in the traversal + break - # if we didn't find an else-block, we need an edge going around the if-block - if not final_else_is_present: - # all vertices present before the final conditional also need to be linked up - # since there was no else-block + # we include the vertex before the conditional, only if there was no else + if not (final_else_is_present): + # we add a branching statement - the branch number is just the number of pairs we found + self.branch_initial_statements.append(["conditional-no-else", entry, branch_number]) current_vertices = final_conditional_vertices + current_vertices else: - # the vertices that now need to be linked up by the next block processed - # are those constructed for the if-block current_vertices = final_conditional_vertices # filter out vertices that were returns or raises @@ -543,7 +583,8 @@ def process_block(self, block, starting_vertices=None, condition=[], closest_loo )) # add an empty "control flow" vertex after the conditional - # to avoid duplication of edges leaving the conditional + # to avoid transition duplication along the edges leaving + # the conditional if len(current_vertices) > 0: empty_vertex = CFGVertex() empty_vertex._name_changed = ['post-conditional'] @@ -566,73 +607,6 @@ def process_block(self, block, starting_vertices=None, condition=[], closest_loo # reset path length for instructions after conditional path_length = 0 - # branch_number += 1 - # - # # we now repeat the same, but iterating through the conditional structure - # while type(current_conditional[0]) is ast.If: - # current_conditional = current_conditional[0].orelse - # if len(current_conditional) == 1: - # - # # there is just another conditional block, so process it as if it were a branch - # if type(current_conditional[0]) is ast.If: - # print("processing else block with other if block") - # # pairs.append( - # # (current_condition_set + [current_conditional[0].test], current_conditional[0].body)) - # # current_condition_set.append(formula_tree.lnot(current_conditional[0].test)) - # final_vertices = self.process_block( - # current_conditional[0].body, - # current_vertices, - # [current_conditional[0].test], - # closest_loop - # ) - # # add to the list of final vertices that need to be connected to the post-conditional vertex - # final_conditional_vertices += final_vertices - # # get first non comment instruction index in block - # first_non_comment_index = get_first_instruction_from_block(current_conditional[0].body) - # # add the branching statement - # self.branch_initial_statements.append( - # ["conditional", current_conditional[0].body[first_non_comment_index], branch_number] - # ) - # branch_number += 1 - # - # else: - # print("processing else block with non-if block") - # # the else block contains an instruction that isn't a conditional - # # pairs.append((current_condition_set, current_conditional)) - # final_vertices = self.process_block( - # current_conditional, - # current_vertices, - # ["else"], - # closest_loop - # ) - # # we reached an else block - # final_else_is_present = True - # # add to the list of final vertices that need to be connected to the post-conditional vertex - # final_conditional_vertices += final_vertices - # # add the branching statement - # self.branch_initial_statements.append( - # ["conditional", current_conditional[0], branch_number] - # ) - # branch_number += 1 - # - # elif len(current_conditional) > 1: - # # there are multiple blocks inside the orelse, so we can't treat this like another branch - # final_vertices = self.process_block( - # current_conditional, - # current_vertices, - # ["else"], - # closest_loop - # ) - # final_conditional_vertices += final_vertices - # self.branch_initial_statements.append( - # ["conditional", current_conditional[0], branch_number] - # ) - # # we reached an else block - # final_else_is_present = True - # else: - # # nowhere else to go in the traversal - # break - elif ast_is_try(entry): entry._parent_body = block path_length += 1 @@ -653,20 +627,12 @@ def process_block(self, block, starting_vertices=None, condition=[], closest_loo current_vertices = [empty_conditional_vertex] blocks = [] - # get first non comment instruction index in block - #first_non_comment_index = get_first_instruction_from_block(entry.body) - self.branch_initial_statements.append( - ["try-catch", entry, "try-catch-main"] - ) + self.branch_initial_statements.append(["try-catch", entry.body[0], "try-catch-main"]) # print("except handling blocks are:") for except_handler in entry.handlers: - # get first non comment instruction index in block - #first_non_comment_index = get_first_instruction_from_block(except_handler.body) - self.branch_initial_statements.append( - ["try-catch", except_handler, "try-catch-handler"] - ) + self.branch_initial_statements.append(["try-catch", except_handler.body[0], "try-catch-handler"]) # print(except_handler.body) blocks.append(except_handler.body) @@ -770,13 +736,9 @@ def process_block(self, block, starting_vertices=None, condition=[], closest_loo empty_post_loop_vertex ) - # get first non comment instruction index in block - first_non_comment_index = get_first_instruction_from_block(entry.body) # for a for loop, we add a path recording instrument at the beginning of the loop body # and after the loop body - self.branch_initial_statements.append( - ["loop", entry.body[first_non_comment_index], "enter-loop", entry, "end-loop"] - ) + self.branch_initial_statements.append(["loop", entry.body[0], "enter-loop", entry, "end-loop"]) # add 2 edges from the final_vertex - one going back to the pre-loop vertex # with the positive condition, and one going to the post loop vertex. @@ -817,7 +779,7 @@ def process_block(self, block, starting_vertices=None, condition=[], closest_loo path_length = 0 elif ast_is_while(entry): - # needs work - but while loops haven't been a thing we've needed to handle so far + """# needs work - but while loops haven't been a thing we've needed to handle so far # need to add code to deal with branching vertices path_length += 1 @@ -833,7 +795,76 @@ def process_block(self, block, starting_vertices=None, condition=[], closest_loo final_vertex.add_outgoing_edge(new_positive_edge) new_positive_edge.set_target_state(base_vertex) - current_vertices = final_vertices + current_vertices = final_vertices""" + + entry._parent_body = block + path_length += 1 + + empty_pre_loop_vertex = CFGVertex(structure_obj=entry) + empty_pre_loop_vertex._name_changed = ['while'] + empty_post_loop_vertex = CFGVertex() + empty_post_loop_vertex._name_changed = ['post-while'] + self.vertices.append(empty_pre_loop_vertex) + self.vertices.append(empty_post_loop_vertex) + + # link current_vertices to the pre-loop vertex + for vertex in current_vertices: + new_edge = CFGEdge(entry.test, "while") + self.edges.append(new_edge) + vertex.add_outgoing_edge(new_edge) + new_edge.set_target_state(empty_pre_loop_vertex) + + current_vertices = [empty_pre_loop_vertex] + + # process loop body + final_vertices = self.process_block( + entry.body, + current_vertices, + ['enter-while'], + empty_post_loop_vertex + ) + + # for a for loop, we add a path recording instrument at the beginning of the loop body + # and after the loop body + self.branch_initial_statements.append(["while", entry.body[0], "enter-while", entry, "end-while"]) + + # add 2 edges from the final_vertex - one going back to the pre-loop vertex + # with the positive condition, and one going to the post loop vertex. + + for final_vertex in final_vertices: + # there will probably only ever be one final vertex, but we register a branching vertex + # self.branching_vertices.append(final_vertex) + for base_vertex in current_vertices: + new_positive_edge = CFGEdge('while-jump', 'while-jump') + self.edges.append(new_positive_edge) + final_vertex.add_outgoing_edge(new_positive_edge) + new_positive_edge.set_target_state(base_vertex) + + new_post_edge = CFGEdge("post-while", "post-while") + self.edges.append(new_post_edge) + final_vertex.add_outgoing_edge(new_post_edge) + new_post_edge.set_target_state(empty_post_loop_vertex) + + # process all of the continue vertices on the stack + for continue_vertex in self.continue_vertex_stack: + new_edge = CFGEdge("post-while", "post-while") + self.edges.append(new_edge) + continue_vertex.add_outgoing_edge(new_edge) + new_edge.set_target_state(empty_pre_loop_vertex) + self.continue_vertex_stack.remove(continue_vertex) + + skip_edge = CFGEdge(formula_tree.lnot(entry.test), "while-skip") + empty_pre_loop_vertex.add_outgoing_edge(skip_edge) + # skip_edge.set_target_state(final_vertices[0]) + skip_edge.set_target_state(empty_post_loop_vertex) + + current_vertices = [empty_post_loop_vertex] + # current_vertices = final_vertices + + condition.append("skip-while-loop") + + # reset path length for instructions after loop + path_length = 0 return current_vertices diff --git a/init_verification.py b/init_verification.py index c1df18d..11fbadd 100644 --- a/init_verification.py +++ b/init_verification.py @@ -8,6 +8,8 @@ import threading import flask import traceback +import requests +import base64 from Queue import Queue import requests @@ -16,27 +18,14 @@ from VyPR.monitor_synthesis import formula_tree from VyPR.verdict_reports import VerdictReport - - - - VERDICT_SERVER_URL = None -VYPR_OUTPUT_VERBOSE = True +VYPR_OUTPUT_VERBOSE = False PROJECT_ROOT = None TEST_FRAMEWORK = 'no' - -## USED IN CASE OF FLASK TESTING -#MAP_COPY_VERDICT = {} - -## The purpose of this flag is to exclude verification send_event call -## with 'test_status' option, if "end" option has not occured. TEST_DIR = '' - - - class MonitoringLog(object): """ Class to handle monitoring logging. @@ -54,12 +43,10 @@ def __init__(self, logs_to_stdout): def start_logging(self): # open the log file in append mode - if not self.logs_to_stdout: - self.handle = open(self.log_file_name, "a") + self.handle = open(self.log_file_name, "a") def end_logging(self): - if not self.logs_to_stdout: - self.handle.close() + self.handle.close() def log(self, message): if self.handle: @@ -67,10 +54,8 @@ def log(self, message): self.handle.write("%s\n" % message) # flush the contents of the file to disk - this way we get a log even with an unhandled exception self.handle.flush() - elif self.logs_to_stdout: - print(message) - - + if self.logs_to_stdout: + print(message) @@ -93,41 +78,42 @@ def vypr_output(string): vypr_logger.log(string) -def send_verdict_report(function_name, time_of_call, end_time_of_call, program_path, verdict_report, - - binding_to_line_numbers, transaction_time, property_hash): - - vypr_output("Sending verdicts to server...") +def send_function_call_data(function_name, time_of_call, end_time_of_call, program_path, transaction_time): """ - Send verdict data for a given function call (function name + time of call). + Send a function call to the verdict server. """ global VERDICT_SERVER_URL - verdicts = verdict_report.get_final_verdict_report() - vypr_output("Retrived the verdict %s" %verdicts) - - + vypr_output("Sending function call data to server...") # first, send function call data - this will also insert program path data vypr_output("Function start time was %s" % time_of_call) vypr_output("Function end time was %s" % end_time_of_call) - - call_data = { "transaction_time": transaction_time.isoformat(), "time_of_call": time_of_call.isoformat(), "end_time_of_call": end_time_of_call.isoformat(), "function_name": function_name, - "property_hash": property_hash, "program_path": program_path } - - insertion_result = json.loads(requests.post( os.path.join(VERDICT_SERVER_URL, "insert_function_call_data/"), data=json.dumps(call_data) ).text) + vypr_output("Function call data sent.") + + return insertion_result + + +def send_verdict_report(verdict_report, property_hash, function_id, function_call_id): + """ + Send verdict data for a given function call. + """ + global VERDICT_SERVER_URL + verdicts = verdict_report.get_final_verdict_report() + vypr_output("Sending verdicts to server...") + # second, send verdict data - all data in one request # for this, we first build the structure # that we'll send over HTTP @@ -148,15 +134,15 @@ def send_verdict_report(function_name, time_of_call, end_time_of_call, program_p verdict[4], verdict[5], verdict[6] - ], - "line_numbers": json.dumps(binding_to_line_numbers[bind_space_index]), + ] } verdict_dict_list.append(verdict_dict) request_body_dict = { - "function_call_id": insertion_result["function_call_id"], - "function_id": insertion_result["function_id"], - "verdicts": verdict_dict_list + "function_call_id": function_call_id, + "function_id": function_id, + "verdicts": verdict_dict_list, + "property_hash": property_hash } # send request @@ -179,22 +165,9 @@ def consumption_thread_function(verification_obj): # the web service has to be considered as running forever, so the monitoring loop for now should also run forever # this needs to be changed for a clean exit INACTIVE_MONITORING = False - - - - #list_test_cases = total_test_cases() - #list_test_cases = ['test_index', 'test__python_version', 'test_upload_session', 'test_check_hashes', 'test_store_payload', - # 'test_upload_metadata', 'test_close_upload_session'] - - #list_test_cases = ['test_orm_objects_to_dicts', 'test_dicts_to_orm_objects' ] - #list_test_cases = ['test_insert_iovs', 'test_construct_tags'] - - transaction = -1 continue_monitoring = True while continue_monitoring: - # import pdb - # pdb.set_trace() # take top element from the queue try: @@ -236,273 +209,307 @@ def consumption_thread_function(verification_obj): continue - - vypr_output("Consuming: %s" % str(top_pair)) - property_hash = top_pair[0] - - # remove the property hash and just deal with the rest of the data - top_pair = top_pair[1:] - - instrument_type = top_pair[0] - function_name = top_pair[1] - - - # get the maps we need for this function - + first_element = top_pair[0] - maps = verification_obj.function_to_maps[function_name][property_hash] - static_qd_to_monitors = maps.static_qd_to_monitors - formula_structure = maps.formula_structure - bindings = maps.binding_space - program_path = maps.program_path + if first_element == "function": - verdict_report = maps.verdict_report + # we have a function start/end instrument - atoms = formula_structure._formula_atoms - - if instrument_type == "function": - # we've received a point telling us that a function has started or ended - # for now, we can just process "end" - we reset the contents of the maps - # that are updated at runtime - scope_event = top_pair[2] - vypr_output("SCOPE_EVENT %s" %scope_event) + property_hash_list = top_pair[1] + function_name = top_pair[2] + scope_event = top_pair[3] if scope_event == "end": + # first, send the function call data independently of any property + # the latest time of call and program path are the same everywhere + latest_time_of_call = verification_obj.function_to_maps[function_name][ + verification_obj.function_to_maps[function_name].keys()[0] + ].latest_time_of_call + program_path = verification_obj.function_to_maps[function_name][ + verification_obj.function_to_maps[function_name].keys()[0] + ].program_path - vypr_output("*" * 50) - - vypr_output("Function '%s' started at time %s has ended at %s." - % (function_name, str(maps.latest_time_of_call), str(top_pair[-1]))) - - - # before resetting the qd -> monitor map, go through it to find monitors - # that reached a verdict, and register those in the verdict report - - - vypr_output("Set the function name...") - - for static_qd_index in static_qd_to_monitors: - for monitor in static_qd_to_monitors[static_qd_index]: - # check if the monitor has a collapsing atom - only then do we register a verdict - if monitor.collapsing_atom: - verdict_report.add_verdict( - static_qd_index, - monitor._formula.verdict, - monitor.atom_to_observation, - monitor.atom_to_program_path, - atoms.index(monitor.collapsing_atom), - monitor.collapsing_atom_sub_index, - monitor.atom_to_state_dict - ) - - # everything not here is static data that we need, and should be left - maps.static_qd_to_monitors = {} - - # generate the verdict report - - report_map = verdict_report.get_final_verdict_report() - - binding_to_line_numbers = {} - - for bind_space_index in report_map.keys(): - - binding = bindings[bind_space_index] - - binding_to_line_numbers[bind_space_index] = [] - - # for each element of the binding, print the appropriate representation - for bind_var in binding: - - if type(bind_var) is CFGVertex: - if bind_var._name_changed == ["loop"]: - binding_to_line_numbers[bind_space_index].append(bind_var._structure_obj.lineno) - else: - binding_to_line_numbers[bind_space_index].append( - bind_var._previous_edge._instruction.lineno) - elif type(bind_var) is CFGEdge: - binding_to_line_numbers[bind_space_index].append(bind_var._instruction.lineno) - + if 'yes' in TEST_FRAMEWORK : + transaction_time = transaction + else: + transaction_time = top_pair[3] + + insertion_data = send_function_call_data( + function_name, + latest_time_of_call, + top_pair[-1], + program_path, + transaction_time + ) - # send the verdict - # we send the function name, the time of the function call, the verdict report object, - # the map of bindings to their line numbers and the date/time of the request the identify it (single threaded...) + # now handle the verdict data we have for each property + for property_hash in property_hash_list: + + maps = verification_obj.function_to_maps[function_name][property_hash] + static_qd_to_monitors = maps.static_qd_to_monitors + verdict_report = maps.verdict_report + + vypr_output("*" * 50) + + # before resetting the qd -> monitor map, go through it to find monitors + # that reached a verdict, and register those in the verdict report + + for static_qd_index in static_qd_to_monitors: + for monitor in static_qd_to_monitors[static_qd_index]: + # check if the monitor has a collapsing atom - only then do we register a verdict + if monitor.collapsing_atom_index is not None: + verdict_report.add_verdict( + static_qd_index, + monitor._formula.verdict, + monitor.atom_to_observation, + monitor.atom_to_program_path, + monitor.collapsing_atom_index, + monitor.collapsing_atom_sub_index, + monitor.atom_to_state_dict + ) + + # reset the monitors + maps.static_qd_to_monitors = {} + + # send the verdict data + send_verdict_report( + verdict_report, + property_hash, + insertion_data["function_id"], + insertion_data["function_call_id"] + ) + # reset the verdict report + maps.verdict_report.reset() - if 'yes' in TEST_FRAMEWORK : - transaction_time = transaction - else: - transaction_time = top_pair[3] + # reset the function start time for the next time + maps.latest_time_of_call = None + elif scope_event == "start": + vypr_output("Function '%s' has started." % function_name) - send_verdict_report( - function_name, - maps.latest_time_of_call, - top_pair[-1], - maps.program_path, - verdict_report, - binding_to_line_numbers, - transaction_time, - #top_pair[3], - top_pair[4] - ) + for property_hash in property_hash_list: + # reset anything that might have been left over from the previous call, + # especially if an unhandled exception caused the function to end without + # vypr instruments sending an end signal + maps = verification_obj.function_to_maps[function_name][property_hash] + maps.static_qd_to_monitors = {} + maps.verdict_report.reset() - # reset the verdict report - maps.verdict_report.reset() + # remember when the function call started + maps.latest_time_of_call = top_pair[4] - # reset the function start time for the next time - maps.latest_time_of_call = None + vypr_output("Set start time to %s" % maps.latest_time_of_call) # reset the program path maps.program_path = [] - elif scope_event == "start": - vypr_output("Function '%s' has started." % function_name) + vypr_output("*" * 50) + else: - # remember when the function call started - maps.latest_time_of_call = top_pair[3] + # we have another kind of instrument that is specific to a property - vypr_output("Set start time to %s" % maps.latest_time_of_call) + property_hash = top_pair[0] - vypr_output("*" * 50) + # remove the property hash and just deal with the rest of the data + top_pair = top_pair[1:] + instrument_type = top_pair[0] + function_name = top_pair[1] + + # get the maps we need for this function + maps = verification_obj.function_to_maps[function_name][property_hash] + static_qd_to_monitors = maps.static_qd_to_monitors + formula_structure = maps.formula_structure + program_path = maps.program_path + + verdict_report = maps.verdict_report + atoms = formula_structure._formula_atoms - if instrument_type == "trigger": - # we've received a trigger instrument + if instrument_type == "trigger": + # we've received a trigger instrument - vypr_output("Processing trigger - dealing with monitor instantiation") + vypr_output("Processing trigger - dealing with monitor instantiation") - static_qd_index = top_pair[2] - bind_variable_index = top_pair[3] + static_qd_index = top_pair[2] + bind_variable_index = top_pair[3] - vypr_output("Trigger is for bind variable %i" % bind_variable_index) - if bind_variable_index == 0: - vypr_output("Instantiating new, clean monitor") - # we've encountered a trigger for the first bind variable, so we simply instantiate a new monitor - new_monitor = formula_tree.new_monitor(formula_structure.get_formula_instance()) + vypr_output("Trigger is for bind variable %i" % bind_variable_index) + if bind_variable_index == 0: + vypr_output("Instantiating new, clean monitor") + # we've encountered a trigger for the first bind variable, so we simply instantiate a new monitor + new_monitor = formula_tree.new_monitor(formula_structure.get_formula_instance()) + try: + static_qd_to_monitors[static_qd_index].append(new_monitor) + except: + static_qd_to_monitors[static_qd_index] = [new_monitor] + else: + vypr_output("Processing existing monitors") + # we look at the monitors' timestamps, and decide whether to generate a new monitor + # and copy over existing information, or update timestamps of existing monitors + new_monitors = [] + subsequences_processed = [] + for monitor in static_qd_to_monitors[static_qd_index]: + # check if the monitor's timestamp sequence includes a timestamp for this bind variable + vypr_output( + " Processing monitor with timestamp sequence %s" % str(monitor._monitor_instantiation_time)) + if len(monitor._monitor_instantiation_time) == bind_variable_index + 1: + if monitor._monitor_instantiation_time[:bind_variable_index] in subsequences_processed: + # the same subsequence might have been copied and extended multiple times + # we only care about one + continue + else: + subsequences_processed.append(monitor._monitor_instantiation_time[:bind_variable_index]) + # construct new monitor + vypr_output(" Instantiating new monitor with modified timestamp sequence") + # instantiate a new monitor using the timestamp subsequence excluding the current bind + # variable and copy over all observation, path and state information + + old_instantiation_time = list(monitor._monitor_instantiation_time) + updated_instantiation_time = tuple( + old_instantiation_time[:bind_variable_index] + [datetime.datetime.now()]) + new_monitor = formula_tree.new_monitor(formula_structure.get_formula_instance()) + new_monitors.append(new_monitor) + + # copy timestamp sequence, observation, path and state information + new_monitor._monitor_instantiation_time = updated_instantiation_time + + # iterate through the observations stored by the previous monitor + # for bind variables before the current one and use them to update the new monitor + for atom_index in monitor._state._state: + + atom = atoms[atom_index] + + if not (type(atom) is formula_tree.LogicalNot): + + # the copy we do for the information related to the atom + # depends on whether the atom is mixed or not + + if formula_tree.is_mixed_atom(atom): + + # for mixed atoms, the return value here is a list + base_variables = get_base_variable(atom) + # determine the base variables which are before the current bind variable + relevant_base_variables = filter( + lambda var : ( + formula_structure._bind_variables.index(var) < bind_variable_index + ), + base_variables + ) + # determine the base variables' sub-indices in the current atom + relevant_sub_indices = map( + lambda var : base_variables.index(var), + relevant_base_variables + ) + + # copy over relevant information for the sub indices + # whose base variables had positions less than the current variable index + # relevant_sub_indices can contain at most 0 and 1. + for sub_index in relevant_sub_indices: + # set up keys in new monitor state if they aren't already there + if not(new_monitor.atom_to_observation.get(atom_index)): + new_monitor.atom_to_observation[atom_index] = {} + new_monitor.atom_to_program_path[atom_index] = {} + new_monitor.atom_to_state_dict[atom_index] = {} + + # copy over observation, program path and state information + new_monitor.atom_to_observation[atom_index][sub_index] = \ + monitor.atom_to_observation[atom_index][sub_index] + new_monitor.atom_to_program_path[atom_index][sub_index] = \ + monitor.atom_to_program_path[atom_index][sub_index] + new_monitor.atom_to_state_dict[atom_index][sub_index] = \ + monitor.atom_to_state_dict[atom_index][sub_index] + + # update the state of the monitor + new_monitor.check_atom_truth_value(atom, atom_index, atom_sub_index) + else: + + # the atom is not mixed, so copying over information is simpler + + if (formula_structure._bind_variables.index( + get_base_variable(atom)) < bind_variable_index + and not (monitor._state._state[atom] is None)): + + # decide how to update the new monitor based on the existing monitor's truth + # value for it + if monitor._state._state[atom_index] == True: + new_monitor.check_optimised(atom) + elif monitor._state._state[atom_index] == False: + new_monitor.check_optimised(formula_tree.lnot(atom)) + + # copy over observation, program path and state information + new_monitor.atom_to_observation[atom_index][0] = \ + monitor.atom_to_observation[atom_index][0] + new_monitor.atom_to_program_path[atom_index][0] = \ + monitor.atom_to_program_path[atom_index][0] + new_monitor.atom_to_state_dict[atom_index][0] = \ + monitor.atom_to_state_dict[atom_index][0] + + vypr_output(" New monitor construction finished.") + + elif len(monitor._monitor_instantiation_time) == bind_variable_index: + vypr_output(" Updating existing monitor timestamp sequence") + # extend the monitor's timestamp sequence + tmp_sequence = list(monitor._monitor_instantiation_time) + tmp_sequence.append(datetime.datetime.now()) + monitor._monitor_instantiation_time = tuple(tmp_sequence) + + # add the new monitors + static_qd_to_monitors[static_qd_index] += new_monitors + + if instrument_type == "path": + # we've received a path recording instrument + # append the branching condition to the program path encountered so far for this function. + program_path.append(top_pair[2]) + + if instrument_type == "instrument": + + static_qd_indices = top_pair[2] + atom_index = top_pair[3] + atom_sub_index = top_pair[4] + instrumentation_point_db_ids = top_pair[5] + observation_time = top_pair[6] + observation_end_time = top_pair[7] + observed_value = top_pair[8] + thread_id = top_pair[9] try: - static_qd_to_monitors[static_qd_index].append(new_monitor) + state_dict = top_pair[10] except: - static_qd_to_monitors[static_qd_index] = [new_monitor] - else: - vypr_output("Processing existing monitors") - # we look at the monitors' timestamps, and decide whether to generate a new monitor - # and copy over existing information, or update timestamps of existing monitors - new_monitors = [] - subsequences_processed = [] - for monitor in static_qd_to_monitors[static_qd_index]: - # check if the monitor's timestamp sequence includes a timestamp for this bind variable - vypr_output( - " Processing monitor with timestamp sequence %s" % str(monitor._monitor_instantiation_time)) - if len(monitor._monitor_instantiation_time) == bind_variable_index + 1: - if monitor._monitor_instantiation_time[:bind_variable_index] in subsequences_processed: - # the same subsequence might have been copied and extended multiple times - # we only care about one - continue - else: - subsequences_processed.append(monitor._monitor_instantiation_time[:bind_variable_index]) - # construct new monitor - vypr_output(" Instantiating new monitor with modified timestamp sequence") - # instantiate a new monitor using the timestamp subsequence excluding the current bind - # variable and copy over all observation, path and state information - - old_instantiation_time = list(monitor._monitor_instantiation_time) - updated_instantiation_time = tuple( - old_instantiation_time[:bind_variable_index] + [datetime.datetime.now()]) - new_monitor = formula_tree.new_monitor(formula_structure.get_formula_instance()) - new_monitors.append(new_monitor) - - # copy timestamp sequence, observation, path and state information - new_monitor._monitor_instantiation_time = updated_instantiation_time - - # iterate through the observations stored by the previous monitor - # for bind variables before the current one and use them to update the new monitor - for atom in monitor._state._state: - if not (type(atom) is formula_tree.LogicalNot): - if (formula_structure._bind_variables.index( - get_base_variable(atom)) < bind_variable_index - and not (monitor._state._state[atom] is None)): - if monitor._state._state[atom] == True: - new_monitor.check_optimised(atom) - elif monitor._state._state[atom] == False: - new_monitor.check_optimised(formula_tree.lnot(atom)) - - atom_index = atoms.index(atom) - - for sub_index in monitor.atom_to_observation[atom_index].keys(): - new_monitor.atom_to_observation[atom_index][sub_index] = \ - monitor.atom_to_observation[atom_index][sub_index] - - for sub_index in monitor.atom_to_program_path[atom_index].keys(): - new_monitor.atom_to_program_path[atom_index][sub_index] = \ - monitor.atom_to_program_path[atom_index][sub_index] - - for sub_index in monitor.atom_to_state_dict[atom_index].keys(): - new_monitor.atom_to_state_dict[atom_index][sub_index] = \ - monitor.atom_to_state_dict[atom_index][sub_index] - - elif len(monitor._monitor_instantiation_time) == bind_variable_index: - vypr_output(" Updating existing monitor timestamp sequence") - # extend the monitor's timestamp sequence - tmp_sequence = list(monitor._monitor_instantiation_time) - tmp_sequence.append(datetime.datetime.now()) - monitor._monitor_instantiation_time = tuple(tmp_sequence) - - # add the new monitors - static_qd_to_monitors[static_qd_index] += new_monitors - - if instrument_type == "path": - # we've received a path recording instrument - # append the branching condition to the program path encountered so far for this function. - program_path.append(top_pair[2]) - - if instrument_type == "instrument": - - static_qd_indices = top_pair[2] - atom_index = top_pair[3] - atom_sub_index = top_pair[4] - instrumentation_point_db_ids = top_pair[5] - observation_time = top_pair[6] - observation_end_time = top_pair[7] - observed_value = top_pair[8] - thread_id = top_pair[9] - try: - state_dict = top_pair[10] - except: - # instrument isn't from a transition measurement - state_dict = None + # instrument isn't from a transition measurement + state_dict = None - vypr_output("Consuming data from an instrument in thread %i" % thread_id) + vypr_output("Consuming data from an instrument in thread %i" % thread_id) - lists = zip(static_qd_indices, instrumentation_point_db_ids) + lists = zip(static_qd_indices, instrumentation_point_db_ids) - for values in lists: + for values in lists: - static_qd_index = values[0] - instrumentation_point_db_id = values[1] + static_qd_index = values[0] + instrumentation_point_db_id = values[1] - vypr_output("Binding space index : %i" % static_qd_index) - vypr_output("Atom index : %i" % atom_index) - vypr_output("Atom sub index : %i" % atom_sub_index) - vypr_output("Instrumentation point db id : %i" % instrumentation_point_db_id) - vypr_output("Observation time : %s" % str(observation_time)) - vypr_output("Observation end time : %s" % str(observation_end_time)) - vypr_output("Observed value : %s" % observed_value) - vypr_output("State dictionary : %s" % str(state_dict)) + vypr_output("Binding space index : %i" % static_qd_index) + vypr_output("Atom index : %i" % atom_index) + vypr_output("Atom sub index : %i" % atom_sub_index) + vypr_output("Instrumentation point db id : %i" % instrumentation_point_db_id) + vypr_output("Observation time : %s" % str(observation_time)) + vypr_output("Observation end time : %s" % str(observation_end_time)) + vypr_output("Observed value : %s" % observed_value) + vypr_output("State dictionary : %s" % str(state_dict)) - instrumentation_atom = atoms[atom_index] + instrumentation_atom = atoms[atom_index] - # update all monitors associated with static_qd_index - if static_qd_to_monitors.get(static_qd_index): - for (n, monitor) in enumerate(static_qd_to_monitors[static_qd_index]): - # checking for previous observation of the atom is done by the monitor's internal logic - monitor.process_atom_and_value(instrumentation_atom, observation_time, observation_end_time, - observed_value, atom_index, atom_sub_index, - inst_point_id=instrumentation_point_db_id, - program_path=len(program_path), state_dict=state_dict) + # update all monitors associated with static_qd_index + if static_qd_to_monitors.get(static_qd_index): + for (n, monitor) in enumerate(static_qd_to_monitors[static_qd_index]): + # checking for previous observation of the atom is done by the monitor's internal logic + monitor.process_atom_and_value(instrumentation_atom, observation_time, observation_end_time, + observed_value, atom_index, atom_sub_index, + inst_point_id=instrumentation_point_db_id, + program_path=len(program_path), state_dict=state_dict) if instrument_type == "test_status": @@ -578,39 +585,27 @@ def __init__(self, module_name, function_name, property_hash): (module_name.replace(".", "-"), function_name.replace(":", "-"), property_hash), "rb") as h: binding_space_dump = h.read() - # read in index hash map - with open(os.path.join(PROJECT_ROOT, "index_hash/module-%s-function-%s.dump") % \ - (module_name.replace(".", "-"), function_name.replace(":", "-")), "rb") as h: - index_to_hash_dump = h.read() - - # - inst_configuration = read_configuration("vypr.config") - - # get the specification file name - verification_conf_file = inst_configuration.get("specification_file") \ - if inst_configuration.get("specification_file") else "verification_conf.py" - - # reconstruct formula structure - # there's probably a better way to do this - # exec("".join(open(verification_conf_file, "r").readlines())) - try: from VyPR_queries_with_imports import verification_conf except ImportError: print("Query file generated by instrumentation couldn't be found. Run VyPR instrumentation first.") exit() - index_to_hash = pickle.loads(index_to_hash_dump) - property_index = index_to_hash.index(property_hash) + # to get the property structure, + property_data = json.loads( + requests.get( + os.path.join(VERDICT_SERVER_URL, "get_property_from_hash/%s/" % property_hash) + ).text + ) + property_index = property_data["index_in_specification_file"] vypr_output("Queries imported.") - # might just change the syntax in the verification conf file at some point to use : instead of . + # store all the data we have self.formula_structure = verification_conf[module_name][function_name.replace(":", ".")][property_index] self.binding_space = pickle.loads(binding_space_dump) self.static_qd_to_monitors = {} self.static_bindings_to_monitor_states = {} - self.static_bindings_to_trigger_points = {} self.verdict_report = VerdictReport() self.latest_time_of_call = None self.program_path = [] @@ -686,12 +681,24 @@ def __init__(self): # read configuration file inst_configuration = read_configuration("vypr.config") - global VERDICT_SERVER_URL, VYPR_OUTPUT_VERBOSE, PROJECT_ROOT, TOTAL_TEST_RUN + global VERDICT_SERVER_URL, VYPR_OUTPUT_VERBOSE, PROJECT_ROOT, VYPR_MODULE, TOTAL_TEST_RUN VERDICT_SERVER_URL = inst_configuration.get("verdict_server_url") if inst_configuration.get( "verdict_server_url") else "http://localhost:9001/" VYPR_OUTPUT_VERBOSE = inst_configuration.get("verbose") if inst_configuration.get("verbose") else True PROJECT_ROOT = inst_configuration.get("project_root") if inst_configuration.get("project_root") else "" + VYPR_MODULE = inst_configuration.get("vypr_module") if inst_configuration.get("vypr_module") else "" + TEST_FRAMEWORK = inst_configuration.get("test") \ + if inst_configuration.get("test") else "" + + # If testing is set then we should specify the test module + if TEST_FRAMEWORK in ['yes']: + TEST_DIR = inst_configuration.get("test_module") \ + if inst_configuration.get("test_module") else '' + + if TEST_DIR == '': + print ('Specify test module. Ending instrumentation - nothing has been done') + exit() # try to connect to the verdict server before we set anything up try: @@ -702,32 +709,8 @@ def __init__(self): self.initialisation_failure = True return - def initialise(self, flask_object): - - vypr_output("Initialising VyPR alongside service.") - - # read configuration file - inst_configuration = read_configuration("vypr.config") - global VERDICT_SERVER_URL, VYPR_OUTPUT_VERBOSE, PROJECT_ROOT, TEST_FRAMEWORK - VERDICT_SERVER_URL = inst_configuration.get("verdict_server_url") if inst_configuration.get( - "verdict_server_url") else "http://localhost:9001/" - VYPR_OUTPUT_VERBOSE = inst_configuration.get("verbose") if inst_configuration.get("verbose") else True - PROJECT_ROOT = inst_configuration.get("project_root") if inst_configuration.get("project_root") else "" - - TEST_FRAMEWORK = inst_configuration.get("test") \ - if inst_configuration.get("test") else "" - - # If testing is set then we should specify the test module - if TEST_FRAMEWORK in ['yes']: - TEST_DIR = inst_configuration.get("test_module") \ - if inst_configuration.get("test_module") else '' - - if TEST_DIR == '': - print ('Specify test module. Ending instrumentation - nothing has been done') - exit() - - - self.machine_id = ("%s-" % inst_configuration.get("machine_id")) if inst_configuration.get("machine_id") else "" + self.machine_id = ("%s-" % inst_configuration.get("machine_id")) if inst_configuration.get( + "machine_id") else "" # check if there's an NTP server given that we should use for time self.ntp_server = inst_configuration.get("ntp_server") @@ -753,46 +736,14 @@ def initialise(self, flask_object): print("Couldn't set time based on NTP server '%s'." % self.ntp_server) exit() - if flask_object: - def prepare_vypr(): - import datetime - from app import vypr - # this function runs inside a request, so flask.g exists - # we store just the request time - flask.g.request_time = vypr.get_time() - - flask_object.before_request(prepare_vypr) - vypr_output("Completed the time") - # add VyPR end points - we may use this for statistics collection on the server - # add the safe exist end point - @flask_object.route("/vypr/stop-monitoring/") - def endpoint_stop_monitoring(): - from app import vypr - # send end-monitoring message - vypr.end_monitoring() - # wait for the thread to end - vypr.consumption_thread.join() - return "VyPR monitoring thread exited. The server must be restarted to turn monitoring back on.\n" - - @flask_object.route("/vypr/pause-monitoring/") - def endpoint_pause_monitoring(): - from app import vypr - vypr.pause_monitoring() - return "VyPR monitoring paused - thread is still running.\n" - - @flask_object.route("/vypr/resume-monitoring/") - def endpoint_resume_monitoring(): - from app import vypr - vypr.resume_monitoring() - return "VyPR monitoring resumed.\n" - # set up the maps that the monitoring algorithm that the consumption thread runs # we need the list of functions that we have instrumentation data from, so read the instrumentation maps # directory dump_files = filter(lambda filename: ".dump" in filename, os.listdir(os.path.join(PROJECT_ROOT, "binding_spaces"))) - functions_and_properties = map(lambda function_dump_file: function_dump_file.replace(".dump", ""), dump_files) + functions_and_properties = map(lambda function_dump_file: function_dump_file.replace(".dump", ""), + dump_files) tokens = map(lambda string: string.split("-"), functions_and_properties) self.function_to_maps = {} @@ -822,19 +773,92 @@ def endpoint_resume_monitoring(): vypr_output(self.function_to_maps) - vypr_output("Setting up monitoring thread.") + def initialise(self, flask_object=None): + + vypr_output("Initialising VyPR alongside service.") + + if flask_object: + if VYPR_MODULE != "": + # if a VyPR module is given, this means VyPR will be running between requests + + def prepare_vypr(): + import datetime + from app import vypr + # this function runs inside a request, so flask.g exists + # we store just the request time + flask.g.request_time = vypr.get_time() + + if flask_object != None: + flask_object.before_request(prepare_vypr) + + # add VyPR end points - we may use this for statistics collection on the server + # add the safe exist end point + @flask_object.route("/vypr/stop-monitoring/") + def endpoint_stop_monitoring(): + from app import vypr + # send end-monitoring message + vypr.end_monitoring() + # wait for the thread to end + vypr.consumption_thread.join() + return "VyPR monitoring thread exited. The server must be restarted to turn monitoring back on.\n" + + @flask_object.route("/vypr/pause-monitoring/") + def endpoint_pause_monitoring(): + from app import vypr + vypr.pause_monitoring() + return "VyPR monitoring paused - thread is still running.\n" + + @flask_object.route("/vypr/resume-monitoring/") + def endpoint_resume_monitoring(): + from app import vypr + vypr.resume_monitoring() + return "VyPR monitoring resumed.\n" + + + # setup consumption queue and store it globally across requests + + else: + # if no VyPR module is given, this means VyPR will have to run per request + + def start_vypr(): + # setup consumption queue and store it within the request context + from flask import g + self.consumption_queue = Queue() + # setup consumption thread + self.consumption_thread = threading.Thread( + target=consumption_thread_function, + args=[self] + ) + # store vypr object in request context + g.vypr = self + self.consumption_thread.start() + g.request_time = g.vypr.get_time() + + if flask_object != None: + flask_object.before_request(start_vypr) + + # set up tear down function + + def stop_vypr(e): + # send kill message to consumption thread + # for now, we don't wait for the thread to end + from flask import g + g.vypr.end_monitoring() + + if flask_object != None: + flask_object.teardown_request(stop_vypr) - # setup consumption queue and store it globally across requests self.consumption_queue = Queue() # setup consumption thread self.consumption_thread = threading.Thread( target=consumption_thread_function, args=[self] - ) + ) self.consumption_thread.start() vypr_output("VyPR monitoring initialisation finished.") + def get_time(self, callee=""): """ Returns either the machine local time, or the NTP time (using the initial NTP time diff --git a/instrument.py b/instrument.py index 21671be..8c7784c 100644 --- a/instrument.py +++ b/instrument.py @@ -23,8 +23,6 @@ rindex = sys.path[0].rfind("/VyPR") sys.path[0] = sys.path[0][:rindex] + sys.path[0][rindex + len("/VyPR"):] -# Getting the root directory of the project - # get the formula building functions before we evaluate the configuration code from VyPR.QueryBuilding import * from VyPR.SCFG.construction import * @@ -37,9 +35,148 @@ vypr_module = "." VERIFICATION_INSTRUCTION = "verification.send_event" TEST_FRAMEWORK = False +LOGS_TO_STDOUT = False + +""" +Specification compilation. +""" -LOGS_TO_STDOUT = False +def get_function_asts_in_module(module_ast): + """ + Given a Module AST object, traverse it and find all the functions. + :param module_ast: + :return: List of function names. + """ + all_function_names = [] + # map elements of ast to pairs - left hand side is module path, right hand side is ast object + stack = list(map(lambda item : ("", item), module_ast.body)) + while len(stack) > 0: + top = stack.pop() + module_path = top[0] + ast_obj = top[1] + if type(ast_obj) is ast.FunctionDef: + all_function_names.append(("%s%s" % (top[0], top[1].name), top[1])) + elif hasattr(ast_obj, "body"): + if type(ast_obj) is ast.If: + stack += map( + lambda item : (module_path, item), + ast_obj.body + ) + elif type(ast_obj) is ast.ClassDef: + stack += map( + lambda item: ("%s%s%s:" % + (module_path, "." if (module_path != "" and module_path[-1] != ":") else "", + ast_obj.name), item), + ast_obj.body + ) + return all_function_names + +def compile_queries(specification_file): + """ + Given a specification file, complete the syntax and add imports, then inspect the objects + used as keys to build the final dictionary structure. + :param specification_file: + :return: fully-expanded dictionary structure + """ + logger.log("Compiling queries...") + + # load in verification config file + # to do this, we read in the existing one, write a temporary one with imports added and import that one + # this is to allow users to write specifications without caring about importing anything from QueryBuilding + # when we read in the original specification code, we add verification_conf = {} to turn it into valid specification + # syntax + specification_code = "verification_conf = %s" % open(specification_file, "r").read() + # replace empty lists with a fake property + fake_property = "[Forall(q = changes('fake_vypr_var')).Check(lambda q : q('fake_vypr_var').equals(True))]" + specification_code = specification_code.replace("[]", fake_property) + with_imports = "from VyPR.QueryBuilding import *\n%s" % specification_code + with open("VyPR_queries_with_imports.py", "w") as h: + h.write(with_imports) + + # we now import the specification file + try: + from VyPR_queries_with_imports import verification_conf + except AttributeError: + print("Cannot continue with instrumentation - " + "looks like you tried to call a non-existent method on an object!\n") + print(traceback.format_exc()) + exit() + except: + # check for errors in the specification + print(traceback.format_exc()) + exit() + + + # this hierarchy will be populated as functions are found in the project that satisfy requirements + compiled_hierarchy = {} + + # finally, we process the keys of the verification_conf dictionary + # these keys are objects representing searches we should perform to generate the final expanded specification + # with fully-qualified module names + # we have this initial step to enable developers to write more sophisticated selection criteria the functions + # to which they want to apply a query + + # we go through each function in the project we're instrumenting, check if there's a key in the initial + # configuration file whose criteria are fulfilled by the function and, if so, add to the hierarchy + for (root, directories, files) in os.walk("."): + for file in files: + # read in the file, traverse its AST structure to find all the functions and then determine + # whether it satisfies a function selector + # only consider Python files + filename = os.path.join(root, file) + module_name = "%s.%s" % (root[1:].replace("/", ""), file.replace(".py", "")) + if (filename[-3:] == ".py" + and "VyPR" not in filename + and "venv" not in filename): + # traverse the AST structure + code = open(filename, "r").read() + module_asts = ast.parse(code) + function_asts = get_function_asts_in_module(module_asts) + # process each function + for (function_name, function_ast) in function_asts: + # construct the SCFG + scfg = CFG() + scfg.process_block(function_ast.body) + # process each function selector + for function_selector in verification_conf: + if type(function_selector) is Functions: + if function_selector.is_satisfied_by(function_ast, scfg): + # add to the compiled hierarchy + if not(compiled_hierarchy.get(module_name)): + compiled_hierarchy[module_name] = {} + if not(compiled_hierarchy[module_name].get(function_name)): + compiled_hierarchy[module_name][function_name] = verification_conf[function_selector] + else: + compiled_hierarchy[module_name][function_name] += verification_conf[function_selector] + elif type(function_selector) is Module: + if (module_name == function_selector._module_name + and function_name == function_selector._function_name): + # add to the final hierarchy + if not (compiled_hierarchy.get(module_name)): + compiled_hierarchy[module_name] = {} + if not (compiled_hierarchy[module_name].get(function_name)): + compiled_hierarchy[module_name][function_name] = verification_conf[function_selector] + else: + compiled_hierarchy[module_name][function_name] += verification_conf[function_selector] + + # now merge the specifications written for specific functions with the compiled specifications + for top_level in verification_conf: + if type(top_level) is str: + if compiled_hierarchy.get(top_level): + for bottom_level in verification_conf[top_level]: + # if top_level was a string, for now bottom_level will be as well + # check whether the compiled part of the specification has already assigned a property here + if compiled_hierarchy.get(top_level) and compiled_hierarchy[top_level].get(bottom_level): + compiled_hierarchy[top_level][bottom_level] += verification_conf[top_level][bottom_level] + else: + compiled_hierarchy[top_level][bottom_level] = verification_conf[top_level][bottom_level] + else: + compiled_hierarchy[top_level] = {} + for bottom_level in verification_conf[top_level]: + compiled_hierarchy[top_level][bottom_level] = verification_conf[top_level][bottom_level] + + return compiled_hierarchy class InstrumentationLog(object): @@ -166,7 +303,6 @@ def compute_binding_space(quantifier_sequence, scfg, reachability_map, current_b vertex._structure_obj.target.id == variable_changed): # the variable we're looking for was found as a simple loop variable qd.append(vertex) - print("adding loop vertex to static binding") elif (type(vertex._structure_obj.target) is ast.Tuple and variable_changed in list(map(lambda item: item.id, vertex._structure_obj.target))): # the loop variable we're looking for was found inside a tuple @@ -406,18 +542,18 @@ def instrument_point_state(state, name, point, binding_space_indices, if measure_attribute == "length": state_variable_alias = name.replace(".", "_").replace("(", "__").replace(")", "__") state_recording_instrument = "record_state_%s = len(%s); " % (state_variable_alias, name) - time_attained_instrument = "time_attained_%s = %s.get_time('point instrument');" % \ + time_attained_instrument = "time_attained_%s = %s.get_time('point instrument');" %\ (state_variable_alias, VYPR_OBJ) time_attained_variable = "time_attained_%s" % state_variable_alias elif measure_attribute == "type": state_variable_alias = name.replace(".", "_").replace("(", "__").replace(")", "__") state_recording_instrument = "record_state_%s = type(%s).__name__; " % (state_variable_alias, name) - time_attained_instrument = "time_attained_%s = %s.get_time('point instrument');" % \ + time_attained_instrument = "time_attained_%s = %s.get_time('point instrument');" %\ (state_variable_alias, VYPR_OBJ) time_attained_variable = "time_attained_%s" % state_variable_alias elif measure_attribute == "time_attained": state_variable_alias = "time_attained_%i" % atom_sub_index - state_recording_instrument = "record_state_%s = %s.get_time('point instrument'); " % \ + state_recording_instrument = "record_state_%s = %s.get_time('point instrument'); " %\ (state_variable_alias, VYPR_OBJ) time_attained_instrument = state_recording_instrument time_attained_variable = "record_state_%s" % state_variable_alias @@ -426,7 +562,7 @@ def instrument_point_state(state, name, point, binding_space_indices, else: state_variable_alias = name.replace(".", "_").replace("(", "__").replace(")", "__") state_recording_instrument = "record_state_%s = %s; " % (state_variable_alias, name) - time_attained_instrument = "time_attained_%s = %s.get_time('point instrument');" % \ + time_attained_instrument = "time_attained_%s = %s.get_time('point instrument');" %\ (state_variable_alias, VYPR_OBJ) time_attained_variable = "time_attained_%s" % state_variable_alias @@ -545,8 +681,8 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, else: state_dict = "{}" - timer_start_statement = "__timer_s =" + VYPR_OBJ + ".get_time()" - timer_end_statement = "__timer_e ="+ VYPR_OBJ + ".get_time()" + timer_start_statement = "__timer_s = %s.get_time('transition instrument')" % VYPR_OBJ + timer_end_statement = "__timer_e = %s.get_time('transition instrument')" % VYPR_OBJ time_difference_statement = "__duration = __timer_e - __timer_s; " instrument_tuple = ("'{formula_hash}', 'instrument', '{function_qualifier}', {binding_space_index}," + @@ -590,18 +726,13 @@ def instrument_point_transition(atom, point, binding_space_indices, atom_index, point._instruction._parent_body.insert(index_in_block, start_ast) - def place_path_recording_instruments(scfg, instrument_function_qualifier, formula_hash): # insert path recording instruments - these don't depend on the formula being checked so # this is done independent of binding space computation for vertex_information in scfg.branch_initial_statements: logger.log("-" * 100) - if vertex_information[0] in ['conditional-body', 'try-catch']: - - print("placing main clause instrument for conditional at line %i" % vertex_information[1].lineno) - - if vertex_information[0] == 'conditional-body': - + if vertex_information[0] in ['conditional', 'try-catch']: + if vertex_information[0] == 'conditional': logger.log( "Placing branch recording instrument for conditional with first instruction %s in " "block" % @@ -622,7 +753,6 @@ def place_path_recording_instruments(scfg, instrument_function_qualifier, formul condition_dict = { "serialised_condition": vertex_information[2] } - # if the condition already exists in the database, the verdict server will return the # existing ID try: @@ -638,15 +768,13 @@ def place_path_recording_instruments(scfg, instrument_function_qualifier, formul VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, branching_condition_id) instrument_ast = ast.parse(instrument_code).body[0] - #index_in_parent = vertex_information[1].body.index(vertex_information[1]) - vertex_information[1].body.insert(0, instrument_ast) + index_in_parent = vertex_information[1]._parent_body.index(vertex_information[1]) + vertex_information[1]._parent_body.insert(index_in_parent, instrument_ast) logger.log("Branch recording instrument placed") - - elif vertex_information[0] == "conditional-else": - - # an else block was found, so we add an instrument at the beginning - logger.log("Placing branch recording instrument for conditional with else") - + elif vertex_information[0] == "conditional-no-else": + # no else was present in the conditional, so we add a path recording instrument + # to the else block + logger.log("Placing branch recording instrument for conditional with no else") # send branching condition to verdict server, take the ID from the response and use it in # the path recording instruments. condition_dict = { @@ -666,13 +794,10 @@ def place_path_recording_instruments(scfg, instrument_function_qualifier, formul instrument_code = "%s((\"%s\", \"path\", \"%s\", %i))" % ( VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, branching_condition_id) - instrument_ast = ast.parse(instrument_code).body[0] vertex_information[1].orelse.insert(0, instrument_ast) logger.log("Branch recording instrument placed") - elif vertex_information[0] in ['post-conditional', 'post-try-catch']: - if vertex_information[0] == 'post-conditional': logger.log("Processing post conditional path instrument") logger.log(vertex_information) @@ -722,9 +847,7 @@ def place_path_recording_instruments(scfg, instrument_function_qualifier, formul logger.log(index_in_parent) vertex_information[1]._parent_body.insert(index_in_parent, instrument_code_ast) logger.log(vertex_information[1]._parent_body) - elif vertex_information[0] == 'loop': - logger.log( "Placing branch recording instrument for loop with first instruction %s in body" % vertex_information[1]) @@ -781,15 +904,14 @@ def place_path_recording_instruments(scfg, instrument_function_qualifier, formul logger.log("=" * 100) -def place_function_begin_instruments(function_def, formula_hash, instrument_function_qualifier): +def place_function_begin_instruments(function_def, formula_hashes, instrument_function_qualifier): # NOTE: only problem with this is that the "end" instrument is inserted before the return, # so a function call in the return statement maybe missed if it's part of verification... thread_id_capture = "import threading; __thread_id = threading.current_thread().ident;" - vypr_start_time_instrument = "vypr_start_time = %s.get_time();" %VYPR_OBJ + vypr_start_time_instrument = "vypr_start_time = %s.get_time('begin instrument');" % VYPR_OBJ verification_test_start_code = "start_test_time = vypr_dt.now()" - - - + formula_hashes = ",".join(map(lambda hash : "'%s'" % hash, formula_hashes)) + formula_hashes_as_list = "[%s]" % formula_hashes start_instrument = \ "%s((\"%s\", \"function\", \"%s\", \"start\", vypr_start_time, \"%s\", __thread_id))" \ % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, formula_hash) @@ -813,16 +935,17 @@ def place_function_begin_instruments(function_def, formula_hash, instrument_func function_def.body.insert(0, vypr_start_time_ast) -def place_function_end_instruments(function_def, scfg, formula_hash, instrument_function_qualifier, is_flask): +def place_function_end_instruments(function_def, scfg, formula_hashes, instrument_function_qualifier, is_flask): # insert the end instrument before every return statement - + formula_hashes = ",".join(map(lambda hash : "'%s'" % hash, formula_hashes)) + formula_hashes_as_list = "[%s]" % formula_hashes for end_vertex in scfg.return_statements: end_instrument = \ "%s((\"%s\", \"function\", \"%s\", \"end\", vypr.get_time(), \"%s\", __thread_id, " \ "%s.get_time('end-instrument')))" \ - % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, formula_hash, VYPR_OBJ) + % (VERIFICATION_INSTRUCTION, formula_hashes_as_list, instrument_function_qualifier, formula_hash, VYPR_OBJ) end_ast = ast.parse(end_instrument).body[0] @@ -841,7 +964,7 @@ def place_function_end_instruments(function_def, scfg, formula_hash, instrument_ end_instrument = \ "%s((\"%s\", \"function\", \"%s\", \"end\", vypr.get_time(), \"%s\", __thread_id, " \ "%s.get_time('end-instrument')))" \ - % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier,formula_hash, VYPR_OBJ) + % (VERIFICATION_INSTRUCTION, formula_hash_as_list, instrument_function_qualifier,formula_hash, VYPR_OBJ) end_ast = ast.parse(end_instrument).body[0] @@ -1107,9 +1230,15 @@ def if_file_is_flask(name): VYPR_MODULE = inst_configuration.get("vypr_module") \ if inst_configuration.get("vypr_module") else "" + # if VYPR_MODULE is empty, we assume that the central object + # will be accessible in the request context + if VYPR_MODULE == "": + VYPR_OBJECT = "g.vypr" + else: + VYPR_OBJECT = "vypr" + VERIFICATION_INSTRUCTION = "vypr.send_event" - VYPR_OBJ = "vypr" # If testing is set then we should specify the test module if TEST_FRAMEWORK in ['yes']: @@ -1128,7 +1257,7 @@ def if_file_is_flask(name): # first, check that the verdict server is reachable if not (is_verdict_server_reachable()): print("Verdict server is not reachable. Ending instrumentation - nothing has been done.") - + exit() SETUP_ONCE = False @@ -1150,20 +1279,23 @@ def if_file_is_flask(name): os.remove(f.replace(".py.inst", BYTECODE_EXTENSION)) logger.log("Reset file %s to uninstrumented version." % f) + for directory in os.walk("."): + if directory == "binding_spaces": + for file in directory[2]: + try: + os.remove(file) + except: + print("Could not remove file '%s' from dump directories. Stopping instrumentation." % file) + exit() + + logger.log("Importing PyCFTL queries...") # load in verification config file # to do this, we read in the existing one, write a temporary one with imports added and import that one # this is to allow users to write specifications without caring about importing anything from QueryBuilding - specification_code = open("VyPR_queries.py", "r").read() - # replace empty lists with a fake property - fake_property = "[Forall(q = changes('fake_vypr_var')).Check(lambda q : q('fake_vypr_var').equals(True))]" - specification_code = specification_code.replace("[]", fake_property) - with_imports = "from VyPR.QueryBuilding import *\n%s" % specification_code - with open("VyPR_queries_with_imports.py", "w") as h: - h.write(with_imports) - from VyPR_queries_with_imports import verification_conf + verification_conf = compile_queries("VyPR_queries.py") verified_modules = verification_conf.keys() @@ -1302,6 +1434,8 @@ def if_file_is_flask(name): # for each property, instrument the function for that property + property_hashes = [] + for (formula_index, formula_structure) in enumerate(verification_conf[module][function]): logger.log("Instrumenting for PyCFTL formula %s" % formula_structure) @@ -1325,6 +1459,8 @@ def if_file_is_flask(name): map(lambda item: base64.encodestring(pickle.dumps(item)).decode('ascii'), atoms) ) + property_hashes.append(formula_hash) + # note that this also means giving an empty list [] will result in path instrumentation # without property instrumentation if EXPLANATION: @@ -1342,7 +1478,6 @@ def if_file_is_flask(name): reachability_map = construct_reachability_map(scfg) bindings = compute_binding_space(formula_structure, scfg, reachability_map) - print(bindings) logger.log("Set of static bindings computed is") logger.log(str(bindings)) @@ -1361,7 +1496,8 @@ def if_file_is_flask(name): "function": instrument_function_qualifier, "serialised_formula_structure": serialised_formula_structure, "serialised_bind_variables": serialised_bind_variables, - "serialised_atom_list": list(enumerate(serialised_atom_list)) + "serialised_atom_list": list(enumerate(serialised_atom_list)), + "formula_index": formula_index } # send instrumentation data to the verdict database @@ -1402,7 +1538,8 @@ def if_file_is_flask(name): binding_dictionary = { "binding_space_index": m, "function": function_id, - "binding_statement_lines": line_numbers + "binding_statement_lines": line_numbers, + "property_hash": formula_hash } serialised_binding_dictionary = json.dumps(binding_dictionary) try: @@ -1424,13 +1561,7 @@ def if_file_is_flask(name): static_qd_to_point_map[m][atom_index] = {} - if type(atom) in [formula_tree.StateValueEqualToMixed, - formula_tree.StateValueLengthLessThanStateValueLengthMixed, - formula_tree.TransitionDurationLessThanTransitionDurationMixed, - formula_tree.TransitionDurationLessThanStateValueMixed, - formula_tree.TransitionDurationLessThanStateValueLengthMixed, - formula_tree.TimeBetweenInInterval, - formula_tree.TimeBetweenInOpenInterval]: + if formula_tree.is_mixed_atom(atom): # there may be multiple bind variables composition_sequences = derive_composition_sequence(atom) @@ -1595,24 +1726,27 @@ def if_file_is_flask(name): binding_space_indices = list_of_lists[0] instrumentation_point_db_ids = list_of_lists[1] - if type(atom) is formula_tree.TransitionDurationInInterval: + if type(atom) in [formula_tree.TransitionDurationInInterval, + formula_tree.TransitionDurationInOpenInterval]: instrument_point_transition(atom, point, binding_space_indices, atom_index, - atom_sub_index, instrumentation_point_db_ids) + atom_sub_index, instrumentation_point_db_ids) - elif type(atom) in [formula_tree.StateValueInInterval, formula_tree.StateValueEqualTo, - formula_tree.StateValueInOpenInterval]: + elif type(atom) in [formula_tree.StateValueInInterval, + formula_tree.StateValueEqualTo, + formula_tree.StateValueInOpenInterval]: instrument_point_state(atom._state, atom._name, point, binding_space_indices, - atom_index, atom_sub_index, instrumentation_point_db_ids) + atom_index, atom_sub_index, instrumentation_point_db_ids) elif type(atom) is formula_tree.StateValueTypeEqualTo: instrument_point_state(atom._state, atom._name, point, binding_space_indices, - atom_index, atom_sub_index, instrumentation_point_db_ids, - measure_attribute="type") + atom_index, atom_sub_index, instrumentation_point_db_ids, + measure_attribute="type") - elif type(atom) in [formula_tree.StateValueLengthInInterval]: + elif type(atom) in [formula_tree.StateValueLengthInInterval, + formula_tree.StateValueLengthInOpenInterval]: """ Instrumentation for the length of a value given is different because we have to add len() to the instrument. @@ -1643,32 +1777,34 @@ def if_file_is_flask(name): instrument_point_state(atom._rhs, atom._rhs_name, point, binding_space_indices, atom_index, atom_sub_index, instrumentation_point_db_ids) - elif type(atom) is formula_tree.TransitionDurationLessThanTransitionDurationMixed: - """We're instrumenting multiple transitions, so we need to perform instrumentation on - two separate points. """ + elif type(atom) in [formula_tree.StateValueLengthLessThanStateValueLengthMixed, + formula_tree.StateValueLengthLessThanEqualStateValueLengthMixed]: + """We're instrumenting multiple states, so we need to perform instrumentation on two + separate points. """ # for each side of the atom (LHS and RHS), instrument the necessary points logger.log( - "Instrumenting for a mixed atom %s with sub atom index %i." % (atom, atom_sub_index) + "instrumenting for a mixed atom %s with sub atom index %i" % (atom, atom_sub_index) ) if atom_sub_index == 0: # we're instrumenting for the lhs - logger.log("placing lhs instrument for scfg object %s" % atom._lhs) - instrument_point_transition(atom, point, binding_space_indices, - atom_index, atom_sub_index, - instrumentation_point_db_ids) + logger.log("Placing left-hand-side instrument for SCFG object %s." % atom._lhs) + instrument_point_state(atom._lhs, atom._lhs_name, point, binding_space_indices, + atom_index, atom_sub_index, instrumentation_point_db_ids, + measure_attribute="length") else: # we're instrumenting for the rhs - logger.log("placing rhs instrument for scfg object %s" % atom._rhs) - instrument_point_transition(atom, point, binding_space_indices, - atom_index, atom_sub_index, - instrumentation_point_db_ids) + logger.log("Placing right-hand-side instrument for SCFG object %s." % atom._rhs) + instrument_point_state(atom._rhs, atom._rhs_name, point, binding_space_indices, + atom_index, atom_sub_index, instrumentation_point_db_ids, + measure_attribute="length") - elif type(atom) is formula_tree.TransitionDurationLessThanStateValueMixed: + elif type(atom) in [formula_tree.TransitionDurationLessThanTransitionDurationMixed, + formula_tree.TransitionDurationLessThanEqualTransitionDurationMixed]: """We're instrumenting multiple transitions, so we need to perform instrumentation on - two separate points. """ + two separate points. """ # for each side of the atom (LHS and RHS), instrument the necessary points @@ -1688,7 +1824,8 @@ def if_file_is_flask(name): instrument_point_state(atom._rhs, atom._rhs_name, point, binding_space_indices, atom_index, atom_sub_index, instrumentation_point_db_ids) - elif type(atom) is formula_tree.TransitionDurationLessThanStateValueLengthMixed: + elif type(atom) in [formula_tree.TransitionDurationLessThanStateValueLengthMixed, + formula_tree.TransitionDurationLessThanEqualStateValueLengthMixed]: """We're instrumenting multiple transitions, so we need to perform instrumentation on two separate points. """ @@ -1755,18 +1892,17 @@ def if_file_is_flask(name): - # # write the instrumented scfg to a file - # instrumented_scfg = CFG() - # instrumented_scfg.process_block(top_level_block) - # write_scfg_to_file(instrumented_scfg, "%s-%s-%s-instrumented.gv" % - # (file_name_without_extension.replace(".", ""), module.replace(".", "-"), - # function.replace(".", "-"))) + # write the instrumented scfg to a file + instrumented_scfg = CFG() + instrumented_scfg.process_block(top_level_block) + write_scfg_to_file(instrumented_scfg, "%s-%s-%s-instrumented.gv" % + (file_name_without_extension.replace(".", ""), module.replace(".", "-"), + function.replace(".", "-"))) # check for existence of directories for intermediate data and create them if not found if not (os.path.isdir("binding_spaces")): os.mkdir("binding_spaces") - if not (os.path.isdir("index_hash")): - os.mkdir("index_hash") + # pickle binding space pickled_binding_space = pickle.dumps(bindings) diff --git a/monitor_synthesis/formula_tree.py b/monitor_synthesis/formula_tree.py index a145356..88bc7fe 100644 --- a/monitor_synthesis/formula_tree.py +++ b/monitor_synthesis/formula_tree.py @@ -123,8 +123,9 @@ def __repr__(self): return "(%s)(%s) in %s" % (self._state, self._name, self.d_interval) def __eq__(self, other_atom): - if type(other_atom) is StateValueInInterval: - return self._state == other_atom._state and self._name == other_atom._name and self._interval == other_atom._interval + if type(other_atom) is StateValueInOpenInterval: + return (self._state == other_atom._state and self._name == other_atom._name + and self._interval == other_atom._interval) else: return False @@ -151,7 +152,7 @@ def __repr__(self): def __eq__(self, other_atom): if type(other_atom) is StateValueEqualTo: - return (self._state == other_atom._state and self._name == other_atom._name \ + return (self._state == other_atom._state and self._name == other_atom._name and self._value == other_atom._value) else: return False @@ -176,7 +177,7 @@ def __repr__(self): def __eq__(self, other_atom): if type(other_atom) is StateValueTypeEqualTo: - return (self._state == other_atom._state and self._name == other_atom._name \ + return (self._state == other_atom._state and self._name == other_atom._name and self._value == other_atom._value) else: return False @@ -227,6 +228,90 @@ def check(self, cummulative_state): ) return lhs_with_arithmetic == rhs_with_arithmetic +class StateValueLessThanStateValueMixed(Atom): + """ + This class models the atom (s1(x) < s2(y)). + """ + + def __init__(self, lhs, lhs_name, rhs, rhs_name): + self._lhs = lhs + self._rhs = rhs + self._lhs_name = lhs_name + self._rhs_name = rhs_name + self.verdict = None + + def __repr__(self): + return "(%s)(%s) < (%s)(%s)" % (self._lhs, self._lhs_name, self._rhs, self._rhs_name) + + def __eq__(self, other_atom): + if type(other_atom) is StateValueLessThanStateValueMixed: + return (self._lhs == other_atom._lhs + and self._lhs_name == other_atom._lhs_name + and self._rhs == other_atom._rhs + and self._rhs_name == other_atom._rhs_name) + else: + return False + + def check(self, cummulative_state): + """ + If either the RHS or LHS are None, we don't try to reach a truth value. + But if they are both not equal to None, we check for equality. + """ + if cummulative_state.get(0) is None or cummulative_state.get(1) is None: + return None + else: + lhs_with_arithmetic = apply_arithmetic_stack( + self._lhs._arithmetic_stack, + cummulative_state[0][0][self._lhs_name] + ) + rhs_with_arithmetic = apply_arithmetic_stack( + self._rhs._arithmetic_stack, + cummulative_state[1][0][self._rhs_name] + ) + return lhs_with_arithmetic < rhs_with_arithmetic + +class StateValueLessThanEqualStateValueMixed(Atom): + """ + This class models the atom (s1(x) <= s2(y)). + """ + + def __init__(self, lhs, lhs_name, rhs, rhs_name): + self._lhs = lhs + self._rhs = rhs + self._lhs_name = lhs_name + self._rhs_name = rhs_name + self.verdict = None + + def __repr__(self): + return "(%s)(%s) <= (%s)(%s)" % (self._lhs, self._lhs_name, self._rhs, self._rhs_name) + + def __eq__(self, other_atom): + if type(other_atom) is StateValueLessThanEqualStateValueMixed: + return (self._lhs == other_atom._lhs + and self._lhs_name == other_atom._lhs_name + and self._rhs == other_atom._rhs + and self._rhs_name == other_atom._rhs_name) + else: + return False + + def check(self, cummulative_state): + """ + If either the RHS or LHS are None, we don't try to reach a truth value. + But if they are both not equal to None, we check for equality. + """ + if cummulative_state.get(0) is None or cummulative_state.get(1) is None: + return None + else: + lhs_with_arithmetic = apply_arithmetic_stack( + self._lhs._arithmetic_stack, + cummulative_state[0][0][self._lhs_name] + ) + rhs_with_arithmetic = apply_arithmetic_stack( + self._rhs._arithmetic_stack, + cummulative_state[1][0][self._rhs_name] + ) + return lhs_with_arithmetic <= rhs_with_arithmetic + class StateValueLengthLessThanStateValueLengthMixed(Atom): """ @@ -268,11 +353,52 @@ def check(self, cummulative_state): self._rhs._arithmetic_stack, cummulative_state[1][0][self._rhs_name] ) - # print(lhs_with_arithmetic, rhs_with_arithmetic) - # print(lhs_with_arithmetic < rhs_with_arithmetic) return lhs_with_arithmetic < rhs_with_arithmetic +class StateValueLengthLessThanEqualStateValueLengthMixed(Atom): + """ + This class models the atom (s1(x).length() < s2(y).length()). + """ + + def __init__(self, lhs, lhs_name, rhs, rhs_name): + self._lhs = lhs + self._rhs = rhs + self._lhs_name = lhs_name + self._rhs_name = rhs_name + self.verdict = None + + def __repr__(self): + return "(%s)(%s).length() <= (%s)(%s).length()" % (self._lhs, self._lhs_name, self._rhs, self._rhs_name) + + def __eq__(self, other_atom): + if type(other_atom) is StateValueLengthLessThanEqualStateValueLengthMixed: + return (self._lhs == other_atom._lhs + and self._lhs_name == other_atom._lhs_name + and self._rhs == other_atom._rhs + and self._rhs_name == other_atom._rhs_name) + else: + return False + + def check(self, cummulative_state): + """ + If either the RHS or LHS are None, we don't try to reach a truth value. + But if they are both not equal to None, we check for equality. + """ + if cummulative_state.get(0) is None or cummulative_state.get(1) is None: + return None + else: + lhs_with_arithmetic = apply_arithmetic_stack( + self._lhs._arithmetic_stack, + cummulative_state[0][0][self._lhs_name] + ) + rhs_with_arithmetic = apply_arithmetic_stack( + self._rhs._arithmetic_stack, + cummulative_state[1][0][self._rhs_name] + ) + return lhs_with_arithmetic <= rhs_with_arithmetic + + class StateValueLengthInInterval(Atom): """ This class models the atom (len(s(x)) in I). @@ -300,6 +426,33 @@ def check(self, value): """ return self._interval[0] <= value[0][0][self._name] <= self._interval[1] +class StateValueLengthInOpenInterval(Atom): + """ + This class models the atom (len(s(x)) in I) for open I. + """ + + def __init__(self, state, name, interval): + self._state = state + self._name = name + self._interval = interval + self.verdict = None + + def __repr__(self): + return "(%s(%s)).length() in %s" % (self._state, self._name, self._interval) + + def __eq__(self, other_atom): + if type(other_atom) is StateValueLengthInOpenInterval: + return (self._state == other_atom._state and self._name == other_atom._name + and self._interval == other_atom._interval) + else: + return False + + def check(self, value): + """ + Mandatory check method used by formula trees to compute truth values. + """ + return self._interval[0] < value[0][0][self._name] < self._interval[1] + class TransitionDurationInInterval(Atom): """ @@ -316,7 +469,7 @@ def __repr__(self): def __eq__(self, other_atom): if type(other_atom) is TransitionDurationInInterval: - return (self._transition == other_atom._transition and self._interval == other_atom._interval) + return self._transition == other_atom._transition and self._interval == other_atom._interval else: return False @@ -324,6 +477,29 @@ def check(self, value): return self._interval[0] <= value[0][0].total_seconds() <= self._interval[1] +class TransitionDurationInOpenInterval(Atom): + """ + This class models the atom (d(delta t) in I). + """ + + def __init__(self, transition, interval): + self._transition = transition + self._interval = interval + self.verdict = None + + def __repr__(self): + return "d(%s) in %s" % (self._transition, self._interval) + + def __eq__(self, other_atom): + if type(other_atom) is TransitionDurationInOpenInterval: + return self._transition == other_atom._transition and self._interval == other_atom._interval + else: + return False + + def check(self, value): + return self._interval[0] < value[0][0].total_seconds() < self._interval[1] + + class TransitionDurationLessThanTransitionDurationMixed(Atom): """ This class models the atom (duration(t1) < duration(t2)) @@ -340,6 +516,37 @@ def __repr__(self): def __eq__(self, other_atom): if type(other_atom) is TransitionDurationLessThanTransitionDurationMixed: + return self._lhs == other_atom._lhs and self._rhs == other_atom._rhs + else: + return False + + def check(self, cummulative_state): + """ + If either the RHS or LHS are None, we don't try to reach a truth value. + But if they are both not equal to None, we check for equality. + """ + if cummulative_state.get(0) is None or cummulative_state.get(1) is None: + return None + else: + return cummulative_state[0][0] < cummulative_state[1][0] + + +class TransitionDurationLessThanEqualTransitionDurationMixed(Atom): + """ + This class models the atom (duration(t1) < duration(t2)) + for v the duration of another transition. + """ + + def __init__(self, lhs_transition, rhs_transition): + self._lhs = lhs_transition + self._rhs = rhs_transition + self.verdict = None + + def __repr__(self): + return "d(%s) <= d(%s)" % (self._lhs, self._rhs) + + def __eq__(self, other_atom): + if type(other_atom) is TransitionDurationLessThanEqualTransitionDurationMixed: return (self._lhs == other_atom._lhs and self._rhs == other_atom._rhs) else: @@ -353,7 +560,7 @@ def check(self, cummulative_state): if cummulative_state.get(0) is None or cummulative_state.get(1) is None: return None else: - return cummulative_state[0][0] < cummulative_state[1][0] + return cummulative_state[0][0] <= cummulative_state[1][0] class TransitionDurationLessThanStateValueMixed(Atom): @@ -394,6 +601,44 @@ def check(self, cummulative_state): return cummulative_state[0][0].total_seconds() < rhs_with_arithmetic +class TransitionDurationLessThanEqualStateValueMixed(Atom): + """ + This class models the atom (duration(t) <= v) + for v a value given by a state. + """ + + def __init__(self, transition, state, name): + self._lhs = transition + self._rhs = state + self._rhs_name = name + self.verdict = None + + def __repr__(self): + return "d(%s) <= (%s)(%s)" % (self._lhs, self._rhs, self._rhs_name) + + def __eq__(self, other_atom): + if type(other_atom) is TransitionDurationLessThanEqualStateValueMixed: + return (self._lhs == other_atom._lhs and + self._rhs == other_atom._rhs and + self._rhs_name == other_atom._rhs_name) + else: + return False + + def check(self, cummulative_state): + """ + If either the RHS or LHS are None, we don't try to reach a truth value. + But if they are both not equal to None, we check for equality. + """ + if cummulative_state.get(0) is None or cummulative_state.get(1) is None: + return None + else: + rhs_with_arithmetic = apply_arithmetic_stack( + self._rhs._arithmetic_stack, + cummulative_state[1][0][self._rhs_name] + ) + return cummulative_state[0][0].total_seconds() <= rhs_with_arithmetic + + class TransitionDurationLessThanStateValueLengthMixed(Atom): """ This class models the atom (duration(t) < v.length()) @@ -432,6 +677,44 @@ def check(self, cummulative_state): return cummulative_state[0][0].total_seconds() < rhs_with_arithmetic +class TransitionDurationLessThanEqualStateValueLengthMixed(Atom): + """ + This class models the atom (duration(t) < v.length()) + for v a value given by a state. + """ + + def __init__(self, transition, state, name): + self._lhs = transition + self._rhs = state + self._rhs_name = name + self.verdict = None + + def __repr__(self): + return "d(%s) <= (%s)(%s).length()" % (self._lhs, self._rhs, self._rhs_name) + + def __eq__(self, other_atom): + if type(other_atom) is TransitionDurationLessThanEqualStateValueLengthMixed: + return (self._lhs == other_atom._lhs and + self._rhs == other_atom._rhs and + self._rhs_name == other_atom._rhs_name) + else: + return False + + def check(self, cummulative_state): + """ + If either the RHS or LHS are None, we don't try to reach a truth value. + But if they are both not equal to None, we check for equality. + """ + if cummulative_state.get(0) is None or cummulative_state.get(1) is None: + return None + else: + rhs_with_arithmetic = apply_arithmetic_stack( + self._rhs._arithmetic_stack, + cummulative_state[1][0][self._rhs_name] + ) + return cummulative_state[0][0].total_seconds() <= rhs_with_arithmetic + + class TimeBetweenInInterval(Atom): """ This class models the atom (timeBetween(q1, q2)._in([n, m])) @@ -501,6 +784,29 @@ def check(self, cummulative_state): result = self._interval[0] < duration < self._interval[1] return result +def is_mixed_atom(atom): + """ + Given an atom, check if its class is in the list of mixed atom classes. + :param atom: + :return: True or False. + """ + mixed_atom_class_list = [ + StateValueEqualToMixed, + StateValueLessThanStateValueMixed, + StateValueLessThanEqualStateValueMixed, + StateValueLengthLessThanStateValueLengthMixed, + StateValueLengthLessThanEqualStateValueLengthMixed, + TransitionDurationLessThanTransitionDurationMixed, + TransitionDurationLessThanEqualTransitionDurationMixed, + TransitionDurationLessThanStateValueMixed, + TransitionDurationLessThanEqualStateValueMixed, + TransitionDurationLessThanStateValueLengthMixed, + TransitionDurationLessThanEqualStateValueLengthMixed, + TimeBetweenInOpenInterval, + TimeBetweenInInterval + ] + return type(atom) in mixed_atom_class_list + """ Classes for propositional logical connectives. @@ -782,7 +1088,7 @@ def __init__(self, formula, optimised=False): self.atom_to_observation = {} self.atom_to_program_path = {} self.atom_to_state_dict = {} - self.collapsing_atom = None + self.collapsing_atom_index = None self.collapsing_atom_sub_index = None self.sub_formulas = [] # we use a tuple to record the instantiation time for each encountered bind variable @@ -858,21 +1164,32 @@ def __repr__(self): return "Monitor for formula %s:\n timestamps: %s\n state: %s\n verdict: %s" % ( self._original_formula, str(self._monitor_instantiation_time), str(self._formula), self._formula.verdict) - def check_atom_truth_value(self, atom, value): + def check_atom_truth_value(self, atom, atom_index, atom_sub_index): """ - Given an atom, an observation and, if the atom is mixed, + Given an atom (with index and sub-index), an observation and, if the atom is mixed, an indication of whether the observation is for the lhs or rhs """ - check_value = atom.check(value) - #print("resulting truth value", check_value) + # take the initial verdict so we can check the difference after update + initial_verdict = self._formula.verdict + # check the value of the atom given the value observed + check_value = atom.check(self.atom_to_observation[atom_index]) + # update the monitor accordingly based on the truth value given by the check if check_value == True: result = self.check(self._formula, atom) elif check_value == False: result = self.check(self._formula, lnot(atom)) elif check_value == None: - # mixed atoms can still be unconclusive if only part of them has been given an observation + # mixed atoms can still be inconclusive if only part of them has been given an observation # in this case, the atom maintains state so no changes are required to the formula tree result = None + # record the new truth value for comparison + final_verdict = self._formula.verdict + # if the verdict has changed, then the atom/sub-atom indices we just used for the update + # are the collapsing ones + if initial_verdict != final_verdict: + # for each monitor, this branch can only ever be traversed once + self.collapsing_atom_index = atom_index + self.collapsing_atom_sub_index = atom_sub_index return result def process_atom_and_value(self, atom, observation_time, observation_end_time, value, atom_index, atom_sub_index, @@ -889,116 +1206,19 @@ def process_atom_and_value(self, atom, observation_time, observation_end_time, v self.atom_to_state_dict[atom_index] = {} if not (self.atom_to_observation[atom_index].get(atom_sub_index)): - self.atom_to_observation[atom_index][atom_sub_index] = \ + self.atom_to_observation[atom_index][atom_sub_index] =\ (value, inst_point_id, observation_time, observation_end_time) - # self.atom_to_program_path[atom_index][atom_sub_index] = [v for v in program_path] - # we deal with integer indices now, so no need to copy a list self.atom_to_program_path[atom_index][atom_sub_index] = program_path self.atom_to_state_dict[atom_index][atom_sub_index] = state_dict else: # the observation has already been processed - no need to do anything return - initial_verdict = self._formula.verdict - - result = self.check_atom_truth_value(atom, self.atom_to_observation[atom_index]) - - final_verdict = self._formula.verdict - - if initial_verdict != final_verdict: - # for each monitor, this branch can only ever be traversed once - self.collapsing_atom = atom - self.collapsing_atom_sub_index = atom_sub_index + # check the truth value of the relevant atom based on the state that we've built up so far + result = self.check_atom_truth_value(atom, atom_index, atom_sub_index) return result - def check_optimised(self, symbol, force_monitor_update=False): - """ - Given a symbol, find the formula occurrences that contain this symbol. - For each of the occurrences, replace the atom with the appropriate value (T or F). - Then loop up through the parents while each successive parent can be collapsed to a truth value. - """ - - if not (force_monitor_update) and not (self._formula.verdict is None): - return self._formula.verdict - - if symbol in self.observed_atoms or lnot(symbol) in self.observed_atoms: - return - else: - self.observed_atoms.append(symbol) - - # NOTE: BE AWARE THAT THE ALPHABET USED TO INITIALLY POPULATE _STATE DOES NOT INCLUDE NEGATIVES - # OF EVERY ATOM - - # update state for the monitoring algorithm to use - self._state.set_state(symbol) - - # temporary fix for Python 3 - the long term solution needs to be more robust - index_of_symbol_in_sub_formulas = self.sub_formulas.index(symbol) - if index_of_symbol_in_sub_formulas in self.atom_to_occurrence_map.keys(): - positives = self.atom_to_occurrence_map.get(index_of_symbol_in_sub_formulas) - else: - positives = [] - - negatives = [] - - all_occurences = positives + negatives - - for occurrence in all_occurences: - # find the position of the atom in the subformula - index_in_formula = 0 - # if the formula to which this atom belongs is an atom, - # this can only happen when a formula consists of only an atom - if formula_is_atom(occurrence): - if formula_is_derived_from_atom(symbol): - if formula_is_derived_from_atom(occurrence): - self._formula.verdict = True - return True - else: - self._formula.verdict = False - return False - else: - if formula_is_derived_from_atom(occurrence): - self._formula.verdict = False - return False - else: - self._formula.verdict = True - return True - else: - for n in range(len(occurrence.operands)): - if occurrence.operands[n] in [symbol, lnot(symbol)]: - index_in_formula = n - - # replace the atom we've observed accordingly - if formula_is_derived_from_atom(symbol): - if formula_is_derived_from_atom(occurrence.operands[index_in_formula]): - occurrence.operands[index_in_formula] = 'T' - else: - occurrence.operands[index_in_formula] = 'F' - else: - if formula_is_derived_from_atom(occurrence.operands[index_in_formula]): - occurrence.operands[index_in_formula] = 'F' - else: - occurrence.operands[index_in_formula] = 'T' - - # iterate up through the tree, collapsing sub-formulas to truth values as far as we can - current_formula = occurrence - current_collapsed_value = collapsed_formula(current_formula) - # iterate while the current formula is collapsible to a truth value - while not (current_collapsed_value is None): - if not (current_formula.parent_formula is None): - current_formula.parent_formula.operands[ - current_formula.index_in_parent] = current_collapsed_value - current_formula = current_formula.parent_formula - current_collapsed_value = collapsed_formula(current_formula) - else: - # we have collapsed the root to a truth value - truth_value_to_boolean = {'T': True, 'F': False, '?': None} - self._formula.verdict = truth_value_to_boolean[current_collapsed_value] - return self._formula.verdict - - return None - def check(self, formula, symbol, level=0): """ Given a formula and a symbol that is true, @@ -1021,86 +1241,96 @@ def check(self, formula, symbol, level=0): sub_verdict = None - indent = " " * level - + # we go through the possible forms of the formula if type(formula) is LogicalAnd or type(formula) is LogicalOr: - # first check if the disjunction or conjunction can be immediately - # collapsed to a truth value - if type(formula) is LogicalAnd: - if 'F' in formula.operands: - if level == 0: - self._formula.verdict = False - return False - elif type(formula) is LogicalOr: - if 'T' in formula.operands: - if level == 0: - self._formula.verdict = True - return True - - if len(set(formula.operands)) == 1: - if formula.operands[0] == 'T': - if level == 0: - self._formula.verdict = True - return True - elif formula.operands[0] == 'F': - if level == 0: - self._formula.verdict = False - return False # if not, iterate through the operands for n in range(len(formula.operands)): if not (formula.operands[n] in ['T', 'F']): + + # recursive base case - we have an atom if formula_is_atom(formula.operands[n]): + + # deal with negation + if ((formula_is_derived_from_atom(formula.operands[n]) and formula_is_derived_from_atom( symbol) and formula.operands[n] == symbol) or (type(formula.operands[n]) is LogicalNot and type(symbol) is LogicalNot and formula.operands[n] == symbol)): formula.operands[n] = 'T' if type(formula) is LogicalOr: - formula = 'T' + # we have at least one true subformula, so we can return true if level == 0: self._formula.verdict = True return True elif type(formula) is LogicalAnd: formula.true_clauses += 1 if formula.true_clauses == len(formula.operands): - formula = 'T' + # all subformulas are true, so we can return true if level == 0: self._formula.verdict = True return True + elif ((formula_is_derived_from_atom(formula.operands[n]) and type(symbol) is LogicalNot and formula.operands[n] == symbol.operand) or (type(formula.operands[n]) is LogicalNot and formula.operands[n].operand == symbol)): formula.operands[n] = 'F' if type(formula) is LogicalAnd: - formula = 'F' + # at least one subformula is false, so return false if level == 0: self._formula.verdict = False return False + elif type(formula) is LogicalOr: + if len(set(formula.operands)) == 1: + # for disjunction, we only care about false + if formula.operands[0] == 'F': + if level == 0: + self._formula.verdict = False + return False + else: + + # recursive on the subformula + sub_verdict = self.check(formula.operands[n], symbol, level + 1) + + # in the cases here, we don't care about None, since that means no truth value + # has been reached in the subformula + if sub_verdict: + formula.operands[n] = 'T' if type(formula) is LogicalOr: - formula = 'T' + # we have at least one true subformula, so we can return true if level == 0: self._formula.verdict = True return True elif type(formula) is LogicalAnd: + # if all subformulas are true, we can return true formula.true_clauses += 1 if formula.true_clauses == len(formula.operands): - formula = 'T' if level == 0: self._formula.verdict = True return True - elif sub_verdict == False: # explicitly not including None + + elif sub_verdict is False: # explicitly not including None + formula.operands[n] = 'F' - if type(formula) is LogicalAnd: - formula = 'F' + if type(formula) is LogicalOr: + # check for all false + if len(set(formula.operands)) == 1: + if formula.operands[0] == 'F': + if level == 0: + self._formula.verdict = False + return False + elif type(formula) is LogicalAnd: if level == 0: self._formula.verdict = False return False + + # we couldn't make any decisions based on subformulas, so return sub_verdict return sub_verdict + elif type(formula) is LogicalNot: if formula_is_derived_from_atom(formula.operand) and formula.operand == symbol: if level == 0: @@ -1110,11 +1340,10 @@ def check(self, formula, symbol, level=0): if level == 0: self._formula.verdict = True return True + elif formula_is_derived_from_atom(formula): - #print("simple formula") if formula == symbol: if level == 0: - #print("reached true verdict") self._formula.verdict = True return True else: @@ -1124,4 +1353,4 @@ def check(self, formula, symbol, level=0): def new_monitor(formula, optimised=False): - return Checker(formula, optimised) \ No newline at end of file + return Checker(formula, optimised) From 8ece76db5c547b0beacd98cb2f3d10ff1c4b57ed Mon Sep 17 00:00:00 2001 From: Omar Javed Date: Mon, 17 Aug 2020 15:41:10 +0200 Subject: [PATCH 30/30] sync final --- init_verification.py | 65 ++++++++++++++++++++------------------------ instrument.py | 28 +++++++++++-------- 2 files changed, 47 insertions(+), 46 deletions(-) diff --git a/init_verification.py b/init_verification.py index 11fbadd..37a1bd5 100644 --- a/init_verification.py +++ b/init_verification.py @@ -512,52 +512,47 @@ def consumption_thread_function(verification_obj): program_path=len(program_path), state_dict=state_dict) - if instrument_type == "test_status": + if instrument_type == "test_status": - # verified_function = top_pair[1] - status = top_pair[2] - start_test_time = top_pair[3] - end_test_time = top_pair[4] - test_name = top_pair[6] + # verified_function = top_pair[1] + status = top_pair[2] + start_test_time = top_pair[3] + end_test_time = top_pair[4] + test_name = top_pair[6] - # # We are trying to empty all the test cases in order to terminate the monitoring - # if test_name in list_test_cases: - # list_test_cases.remove(test_name) - # - # if len(list_test_cases) == 0: - # continue_monitoring = False - - if status.failures: - test_result = "Fail" - elif status.errors: - test_result = "Error" - else: - test_result = "Success" + # # We are trying to empty all the test cases in order to terminate the monitoring + # if test_name in list_test_cases: + # list_test_cases.remove(test_name) + # + # if len(list_test_cases) == 0: + # continue_monitoring = False + if status.failures: + test_result = "Fail" + elif status.errors: + test_result = "Error" + else: + test_result = "Success" - # If test data exists. + # If test data exists. - test_data = { - "test_name" : test_name, - "test_result" : test_result, - "start_time" : start_test_time.isoformat(), - "end_time" : end_test_time.isoformat() - } + test_data = { + "test_name" : test_name, + "test_result" : test_result, + "start_time" : start_test_time.isoformat(), + "end_time" : end_test_time.isoformat() - json.loads(requests.post( - os.path.join(VERDICT_SERVER_URL, "insert_test_data/"), - data=json.dumps(test_data) - ).text) + } - # To terminate - #if top_pair[5] >= TOTAL_TEST_RUN: - # continue_monitoring = False + json.loads(requests.post( + os.path.join(VERDICT_SERVER_URL, "insert_test_data/"), + data=json.dumps(test_data) + ).text) - # # set the task as done @@ -681,7 +676,7 @@ def __init__(self): # read configuration file inst_configuration = read_configuration("vypr.config") - global VERDICT_SERVER_URL, VYPR_OUTPUT_VERBOSE, PROJECT_ROOT, VYPR_MODULE, TOTAL_TEST_RUN + global VERDICT_SERVER_URL, VYPR_OUTPUT_VERBOSE, PROJECT_ROOT, VYPR_MODULE, TOTAL_TEST_RUN, TEST_FRAMEWORK VERDICT_SERVER_URL = inst_configuration.get("verdict_server_url") if inst_configuration.get( "verdict_server_url") else "http://localhost:9001/" VYPR_OUTPUT_VERBOSE = inst_configuration.get("verbose") if inst_configuration.get("verbose") else True diff --git a/instrument.py b/instrument.py index 8c7784c..04dcead 100644 --- a/instrument.py +++ b/instrument.py @@ -913,8 +913,8 @@ def place_function_begin_instruments(function_def, formula_hashes, instrument_fu formula_hashes = ",".join(map(lambda hash : "'%s'" % hash, formula_hashes)) formula_hashes_as_list = "[%s]" % formula_hashes start_instrument = \ - "%s((\"%s\", \"function\", \"%s\", \"start\", vypr_start_time, \"%s\", __thread_id))" \ - % (VERIFICATION_INSTRUCTION, formula_hash, instrument_function_qualifier, formula_hash) + "%s((\"function\", %s, \"%s\", \"start\", vypr_start_time, \"%s\", __thread_id))" \ + % (VERIFICATION_INSTRUCTION, formula_hashes_as_list, instrument_function_qualifier, formula_hash) verfication_test_start_time_ast = ast.parse(verification_test_start_code).body[0] threading_import_ast = ast.parse(thread_id_capture).body[0] @@ -943,9 +943,15 @@ def place_function_end_instruments(function_def, scfg, formula_hashes, instrumen for end_vertex in scfg.return_statements: end_instrument = \ - "%s((\"%s\", \"function\", \"%s\", \"end\", vypr.get_time(), \"%s\", __thread_id, " \ - "%s.get_time('end-instrument')))" \ - % (VERIFICATION_INSTRUCTION, formula_hashes_as_list, instrument_function_qualifier, formula_hash, VYPR_OBJ) + "%s((\"function\", %s, \"%s\", \"end\", flask.g.request_time, \"%s\", __thread_id, " \ + "%s.get_time('end instrument')))" \ + % ( + VERIFICATION_INSTRUCTION, + formula_hashes_as_list, + instrument_function_qualifier, + formula_hash, + VYPR_OBJ + ) end_ast = ast.parse(end_instrument).body[0] @@ -961,10 +967,10 @@ def place_function_end_instruments(function_def, scfg, formula_hashes, instrumen # if the last instruction in the ast is not a return statement, add an end instrument at the end if not (type(function_def.body[-1]) is ast.Return): - end_instrument = \ - "%s((\"%s\", \"function\", \"%s\", \"end\", vypr.get_time(), \"%s\", __thread_id, " \ - "%s.get_time('end-instrument')))" \ - % (VERIFICATION_INSTRUCTION, formula_hash_as_list, instrument_function_qualifier,formula_hash, VYPR_OBJ) + end_instrument = "%s((\"function\", %s, \"%s\", \"end\", flask.g.request_time, \"%s\", __thread_id, " \ + "%s.get_time('end instrument')))" \ + % (VERIFICATION_INSTRUCTION, formula_hashes_as_list, instrument_function_qualifier, + formula_hash, VYPR_OBJ) end_ast = ast.parse(end_instrument).body[0] @@ -1874,9 +1880,9 @@ def if_file_is_flask(name): # finally, insert an instrument at the beginning to tell the monitoring thread that a new call of the # function has started and insert one at the end to signal a return - place_function_begin_instruments(function_def, formula_hash, instrument_function_qualifier) + place_function_begin_instruments(function_def, property_hashes, instrument_function_qualifier) # also insert instruments at the end(s) of the function - place_function_end_instruments(function_def, scfg, formula_hash, instrument_function_qualifier, is_flask) + place_function_end_instruments(function_def, scfg, property_hashes, instrument_function_qualifier, is_flask) if not SETUP_ONCE: test_modules = get_test_modules(is_test_module, module)