From 7423ddceda37943171a1e0c00659910840cdb5d8 Mon Sep 17 00:00:00 2001 From: "Christopher S. Meiklejohn" Date: Sat, 5 Nov 2022 01:40:17 -0400 Subject: [PATCH 01/14] Add Dockerfile. --- Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..e21bd05 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.9 +WORKDIR /app +COPY . . +RUN make install +CMD ["/usr/local/bin/filibuster", "--server-only"] +EXPOSE 5005 From f832136da3af23b78ebc53f85d7db56cef5a7567 Mon Sep 17 00:00:00 2001 From: "Christopher S. Meiklejohn" Date: Sat, 5 Nov 2022 01:40:55 -0400 Subject: [PATCH 02/14] Handle possible environment variables from Docker container. --- filibuster_cli.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/filibuster_cli.py b/filibuster_cli.py index eb23367..d2d9b6e 100644 --- a/filibuster_cli.py +++ b/filibuster_cli.py @@ -91,6 +91,12 @@ def test(server_only, else: test_to_execute = functional_test + if os.environ.get('SHOULD_SUPPRESS_COMBINATIONS', ''): + should_suppress_combinations = True + + if os.environ.get('DISABLE_DYNAMIC_REDUCTION', ''): + disable_dynamic_reduction = True + start_filibuster_server_and_run_test(test_to_execute, abs_analysis_file, counterexample_file, From df6b2c38b1dd4105c77a2628f6fafd5823de35c3 Mon Sep 17 00:00:00 2001 From: "Christopher S. Meiklejohn" Date: Sat, 5 Nov 2022 01:41:05 -0400 Subject: [PATCH 03/14] Remove documentation requirements. --- requirements.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9ebd083..cb6168a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ click>=7.1.2 -Flask==1.0.0 requests -sphinx -sphinx-rtd-theme -myst-parser \ No newline at end of file +Flask==2.0.0 +pytest +requests +Jinja2 \ No newline at end of file From c71aab98259731ecb36d9b63186636040b00177e Mon Sep 17 00:00:00 2001 From: "Christopher S. Meiklejohn" Date: Sat, 5 Nov 2022 01:41:20 -0400 Subject: [PATCH 04/14] Remove throwing debug statement. --- filibuster/lifecycle/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filibuster/lifecycle/__init__.py b/filibuster/lifecycle/__init__.py index b37b625..785ca58 100644 --- a/filibuster/lifecycle/__init__.py +++ b/filibuster/lifecycle/__init__.py @@ -12,7 +12,7 @@ def num_services_running(services): num_running = len(services) for service in services: if not service_running(service): - debug("! service " + service + " not yet running!") + # debug("! service " + service + " not yet running!") num_running -= 1 return num_running From 03f56cd3a18e7134fe8a2ce6c1b88369b22f1eb1 Mon Sep 17 00:00:00 2001 From: "Christopher S. Meiklejohn" Date: Sat, 5 Nov 2022 01:41:44 -0400 Subject: [PATCH 05/14] Set code conditional on server mode. --- filibuster/server/__init__.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/filibuster/server/__init__.py b/filibuster/server/__init__.py index ee2b0f8..9e9846d 100644 --- a/filibuster/server/__init__.py +++ b/filibuster/server/__init__.py @@ -1068,23 +1068,25 @@ def wait_for_teardown_completed(period=0.25): global current_test_execution notice("Waiting for teardown completed: BLOCKED PYTHON WAITING FOR AFTEREACH.") - while True: - if teardown_completed: - notice("Teardown completed; nulling out current test execution: PYTHON UNBLOCKED.") - # This unblocks python. - teardown_completed = False - break + if server_only_mode: + while True: + if teardown_completed: + notice("Teardown completed. Marking teardown_completed.") + # This unblocks python. + teardown_completed = False + break - time.sleep(period) + time.sleep(period) def wait_until_current_test_execution(period=0.25): global current_test_execution notice("Waiting for current test execution.") - while True: - if current_test_execution is not None: - notice("Current test execution populated: UNBLOCKED JAVA.") - break + if server_only_mode: + while True: + if current_test_execution is not None: + notice("Current test execution populated: UNBLOCKED JAVA.") + break - time.sleep(period) + time.sleep(period) From 04d2eed5ccea6bb898967fbe0e178964f2b10aef Mon Sep 17 00:00:00 2001 From: "Christopher S. Meiklejohn" Date: Sat, 5 Nov 2022 01:41:56 -0400 Subject: [PATCH 06/14] Clarify comment. --- filibuster/server/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filibuster/server/__init__.py b/filibuster/server/__init__.py index 9e9846d..33bfa6e 100644 --- a/filibuster/server/__init__.py +++ b/filibuster/server/__init__.py @@ -708,7 +708,7 @@ def teardowns_completed(iteration): # to be set immediately and not asynchronously otherwise beforeEach will run before # we have swapped the test execution. if current_test_execution is not None: - notice("Nulling current test execution.") + notice("Nulling current test execution because teardown is completed.") current_test_execution = None teardown_completed = True From 80f8146330eb5114e14cc184e28a3d7098dccdf7 Mon Sep 17 00:00:00 2001 From: "Christopher S. Meiklejohn" Date: Sat, 5 Nov 2022 01:42:08 -0400 Subject: [PATCH 07/14] Ensure we have access to global. --- filibuster/server/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/filibuster/server/__init__.py b/filibuster/server/__init__.py index 33bfa6e..4c73f99 100644 --- a/filibuster/server/__init__.py +++ b/filibuster/server/__init__.py @@ -1066,6 +1066,8 @@ def wait_indefinitely_until_shutdown(period=0.25): def wait_for_teardown_completed(period=0.25): global teardown_completed global current_test_execution + global server_only_mode + notice("Waiting for teardown completed: BLOCKED PYTHON WAITING FOR AFTEREACH.") if server_only_mode: From f4e2e438a04ac1383b35e5f8ecdc6257a6b2cfff Mon Sep 17 00:00:00 2001 From: "Christopher S. Meiklejohn" Date: Sat, 5 Nov 2022 01:42:15 -0400 Subject: [PATCH 08/14] Add comment. --- filibuster/server/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/filibuster/server/__init__.py b/filibuster/server/__init__.py index 4c73f99..4e67d4f 100644 --- a/filibuster/server/__init__.py +++ b/filibuster/server/__init__.py @@ -167,6 +167,7 @@ def run_test(functional_test, only_initial_execution, disable_dynamic_reduction, requests_to_fail = next_test_execution.failures # Set current test execution. + notice("Setting current test execution.") current_test_execution = next_test_execution notice("Set current test execution to next execution.") From 5d1c4ea79e9185e1dc939060118fb4e0e3c6248f Mon Sep 17 00:00:00 2001 From: "Christopher S. Meiklejohn" Date: Sat, 5 Nov 2022 01:42:32 -0400 Subject: [PATCH 09/14] Switch logging. --- filibuster/server/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/filibuster/server/__init__.py b/filibuster/server/__init__.py index 4e67d4f..a1e4598 100644 --- a/filibuster/server/__init__.py +++ b/filibuster/server/__init__.py @@ -563,9 +563,9 @@ def has_next_iteration(iteration, caller): elif current_test_execution is None and test_executions_scheduled.size() > 0: # Wait until current test execution is set. - # print("current not yet set, waiting.") + print("current not yet set, waiting.") wait_until_current_test_execution() - # print("current now set, returning true") + print("current now set, returning true") print("has_next_iteration called: " + str(iteration) + " for caller " + str(caller)) return jsonify({"has-next-iteration": True}) From 8ffd5c7c201bb23fa3f58c4b914f45ac3dba0838 Mon Sep 17 00:00:00 2001 From: "Christopher S. Meiklejohn" Date: Sat, 5 Nov 2022 01:42:41 -0400 Subject: [PATCH 10/14] Make sure routes are under /filibuster. --- filibuster/server/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/filibuster/server/__init__.py b/filibuster/server/__init__.py index a1e4598..dbb1b16 100644 --- a/filibuster/server/__init__.py +++ b/filibuster/server/__init__.py @@ -692,7 +692,7 @@ def health_check(): return jsonify({"status": "OK"}) -@app.route("/terminate", methods=['GET']) +@app.route("/filibuster/terminate", methods=['GET']) def terminate(): global should_terminate_immediately should_terminate_immediately = True @@ -700,7 +700,7 @@ def terminate(): return jsonify({}) -@app.route("/teardowns-completed/", methods=['GET']) +@app.route("/filibuster/teardowns-completed/", methods=['GET']) def teardowns_completed(iteration): global teardown_completed global current_test_execution From 2424f6ba7a316eabcc84984acc5ec73c857406be Mon Sep 17 00:00:00 2001 From: "Christopher S. Meiklejohn" Date: Sat, 5 Nov 2022 01:42:46 -0400 Subject: [PATCH 11/14] Comment. --- filibuster/server/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filibuster/server/__init__.py b/filibuster/server/__init__.py index dbb1b16..14d9da5 100644 --- a/filibuster/server/__init__.py +++ b/filibuster/server/__init__.py @@ -713,7 +713,7 @@ def teardowns_completed(iteration): current_test_execution = None teardown_completed = True - # notice("Teardown completed for iteration: " + str(iteration)) + notice("Teardown completed for iteration: " + str(iteration)) return jsonify({}) From 30be0ebab1e5b38d2b4b7ec893f2ead4c4c13451 Mon Sep 17 00:00:00 2001 From: "Christopher S. Meiklejohn" Date: Sat, 5 Nov 2022 01:43:00 -0400 Subject: [PATCH 12/14] Add API for supplying analysis file. --- filibuster/server/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/filibuster/server/__init__.py b/filibuster/server/__init__.py index 14d9da5..d2ef006 100644 --- a/filibuster/server/__init__.py +++ b/filibuster/server/__init__.py @@ -74,6 +74,7 @@ server_only_mode = False should_terminate_immediately = False teardown_completed = False +instrumentation = None def run_test(functional_test, only_initial_execution, disable_dynamic_reduction, forced_failure, should_suppress_combinations, setup_script, teardown_script): @@ -735,6 +736,13 @@ def new_test_execution_check(service_name): return jsonify({"new-test-execution": new_test_execution}) +@app.route("/filibuster/analysis-file", methods=['POST']) +def analysis_file(): + global instrumentation + instrumentation = request.get_json() + return jsonify({}) + + @app.route("/filibuster/create", methods=['PUT']) def create(): try: From 51e056e0a86c9931e36914a0e88c9bc59ceee72c Mon Sep 17 00:00:00 2001 From: "Christopher S. Meiklejohn" Date: Sat, 5 Nov 2022 01:43:15 -0400 Subject: [PATCH 13/14] Don't read analysis file is provided by API. --- filibuster/server/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/filibuster/server/__init__.py b/filibuster/server/__init__.py index d2ef006..84a8d0e 100644 --- a/filibuster/server/__init__.py +++ b/filibuster/server/__init__.py @@ -285,6 +285,7 @@ def generate_additional_test_executions(generated_id, execution_index, instrumen global current_test_execution_batch global requests_to_fail global suppress_combinations + global instrumentation # List of additional test executions. additional_test_executions = [] @@ -314,7 +315,8 @@ def generate_additional_test_executions(generated_id, execution_index, instrumen break # Iterate list of faults. - instrumentation = read_analysis_file(analysis_file) + if instrumentation is None: + instrumentation = read_analysis_file(analysis_file) for module in instrumentation: pattern = instrumentation[module]['pattern'] From 5100e06fe8a8a5009853ff36eef7b3e9885ee7a4 Mon Sep 17 00:00:00 2001 From: "Christopher S. Meiklejohn" Date: Sat, 5 Nov 2022 01:43:38 -0400 Subject: [PATCH 14/14] Ensure we generate client specific errors on initial call not just termination. --- filibuster/server/__init__.py | 108 ++++++++++++++++++++++------------ 1 file changed, 72 insertions(+), 36 deletions(-) diff --git a/filibuster/server/__init__.py b/filibuster/server/__init__.py index 84a8d0e..e225efa 100644 --- a/filibuster/server/__init__.py +++ b/filibuster/server/__init__.py @@ -380,42 +380,78 @@ def generate_additional_test_executions(generated_id, execution_index, instrumen additional_test_executions.append(new_execution) # Error testing. - if instrumentation_type == 'request_received': - if 'errors' in instrumentation[module]: - for error in instrumentation[module]['errors']: - if 'target_service_name' in req and req['target_service_name'] is not None: - target_service_name = req['target_service_name'] - - service_pattern = error['service_name'] - service_matcher = re.compile(service_pattern) - service_matching = service_matcher.match(target_service_name) - - if service_matching is not None: - for type in error['types']: - # warning("Checking if we need to inject error: " + str(type)) - # warning("already_failed: " + str(already_failed)) - - if not already_failed: - # For this execution, we need to fail everything we did before to get here - # but, we also need to fail this additional one request as well. - # (also, add the exception so we know what to throw later.) - new_req = copy.deepcopy(req) - new_req['failure_metadata'] = {} - for key in type: - new_req['failure_metadata'][key] = type[key] - new_failures = copy.deepcopy(failures) - new_failures.append(TestExecution.filter_request_for_failures(new_req)) - new_failures = sorted(new_failures, key=lambda k: k['execution_index']) - - new_execution = TestExecution(log, new_failures) - if should_schedule(new_execution, additional_test_executions): - if new_execution not in additional_test_executions: - debug("Adding req failure for request: " + str( - req['execution_index'])) - debug("=> failure description: " + str(type)) - additional_test_executions.append(new_execution) - else: - warning("Request does not have a target service, it's made outside of the system.") + # if instrumentation_type == 'request_received': + # if 'errors' in instrumentation[module]: + # for error in instrumentation[module]['errors']: + # if 'target_service_name' in req and req['target_service_name'] is not None: + # target_service_name = req['target_service_name'] + # + # service_pattern = error['service_name'] + # service_matcher = re.compile(service_pattern) + # service_matching = service_matcher.match(target_service_name) + # + # if service_matching is not None: + # for type in error['types']: + # # warning("Checking if we need to inject error: " + str(type)) + # # warning("already_failed: " + str(already_failed)) + # + # if not already_failed: + # # For this execution, we need to fail everything we did before to get here + # # but, we also need to fail this additional one request as well. + # # (also, add the exception so we know what to throw later.) + # new_req = copy.deepcopy(req) + # new_req['failure_metadata'] = {} + # for key in type: + # new_req['failure_metadata'][key] = type[key] + # new_failures = copy.deepcopy(failures) + # new_failures.append(TestExecution.filter_request_for_failures(new_req)) + # new_failures = sorted(new_failures, key=lambda k: k['execution_index']) + # + # new_execution = TestExecution(log, new_failures) + # if should_schedule(new_execution, additional_test_executions): + # if new_execution not in additional_test_executions: + # debug("Adding req failure for request: " + str( + # req['execution_index'])) + # debug("=> failure description: " + str(type)) + # additional_test_executions.append(new_execution) + # else: + # warning("Request does not have a target service, it's made outside of the system.") + + if 'errors' in instrumentation[module]: + for error in instrumentation[module]['errors']: + if 'target_service_name' in req and req['target_service_name'] is not None: + target_service_name = req['target_service_name'] + else: + target_service_name = "" + + service_pattern = error['service_name'] + service_matcher = re.compile(service_pattern) + service_matching = service_matcher.match(target_service_name) + + if service_matching is not None: + for type in error['types']: + # warning("Checking if we need to inject error: " + str(type)) + # warning("already_failed: " + str(already_failed)) + + if not already_failed: + # For this execution, we need to fail everything we did before to get here + # but, we also need to fail this additional one request as well. + # (also, add the exception so we know what to throw later.) + new_req = copy.deepcopy(req) + new_req['failure_metadata'] = {} + for key in type: + new_req['failure_metadata'][key] = type[key] + new_failures = copy.deepcopy(failures) + new_failures.append(TestExecution.filter_request_for_failures(new_req)) + new_failures = sorted(new_failures, key=lambda k: k['execution_index']) + + new_execution = TestExecution(log, new_failures) + if should_schedule(new_execution, additional_test_executions): + if new_execution not in additional_test_executions: + debug("Adding req failure for request: " + str( + req['execution_index'])) + debug("=> failure description: " + str(type)) + additional_test_executions.append(new_execution) append_quantity = 0