From 3ea5363df5c1695a6446b63076708c2c3c1ff7da Mon Sep 17 00:00:00 2001 From: Michael Assad Date: Fri, 14 Jul 2023 00:38:39 -0400 Subject: [PATCH 01/10] Reverse add new transformer tests to stack/queue --- .../junit/server/core/FilibusterCore.java | 20 +++++++++++++------ .../server/core/TestExecutionCollection.java | 2 +- .../junit/server/core/TestExecutionQueue.java | 8 ++++++-- .../junit/server/core/TestExecutionStack.java | 8 ++++++-- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/src/main/java/cloud/filibuster/junit/server/core/FilibusterCore.java b/src/main/java/cloud/filibuster/junit/server/core/FilibusterCore.java index 8e22269c3..297cd7421 100644 --- a/src/main/java/cloud/filibuster/junit/server/core/FilibusterCore.java +++ b/src/main/java/cloud/filibuster/junit/server/core/FilibusterCore.java @@ -288,7 +288,7 @@ public synchronized JSONObject beginInvocation(JSONObject payload) { setNextAccumulator(newFaultObject.getJSONObject("transformer_fault"), transformationResult.getNextAccumulator()); generateAndSetTransformerValue(newFaultObject.getJSONObject("transformer_fault")); - createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, newFaultObject); + createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, newFaultObject, true); } logger.info("[FILIBUSTER-CORE]: beginInvocation, injecting faults using transformer_fault: " + transformerFaultObject.toString(4)); @@ -434,7 +434,7 @@ private void generateByzantineAndTransformerFaults(JSONObject payload, Distribut ) ); generateAndSetTransformerValue(transformer.getJSONObject("transformer_fault")); - createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, new JSONObject(transformer.toMap())); + createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, new JSONObject(transformer.toMap()), true); } catch (Throwable e) { logger.warning("[FILIBUSTER-CORE]: generateByzantineAndTransformerFaults, an exception occurred in generateByzantineAndTransformerFaults: " + e); throw new FilibusterFaultInjectionException("[FILIBUSTER-CORE]: generateByzantineAndTransformerFaults: ", e); @@ -562,14 +562,14 @@ public synchronized void teardownsCompleted(int currentIteration) { // Don't add to explored queue if it's already there. numberOfAbstractExecutionsExecuted++; - exploredTestExecutions.addTestExecution(currentAbstractTestExecution); + exploredTestExecutions.addTestExecution(currentAbstractTestExecution, false); } else { logger.severe("[FILIBUSTER-CORE]: teardownsCompleted called, currentAbstractTestExecution already exists in the explored queue, this could indicate a problem in Filibuster."); } } if (!exploredTestExecutions.containsTestExecution(currentConcreteTestExecution)) { - exploredTestExecutions.addTestExecution(currentConcreteTestExecution); + exploredTestExecutions.addTestExecution(currentConcreteTestExecution, false); } numberOfConcreteExecutionsExecuted++; @@ -1069,6 +1069,14 @@ private void createAndScheduleAbstractTestExecution( FilibusterConfiguration filibusterConfiguration, DistributedExecutionIndex distributedExecutionIndex, JSONObject faultObject) { + createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, faultObject, false); + } + + private void createAndScheduleAbstractTestExecution( + FilibusterConfiguration filibusterConfiguration, + DistributedExecutionIndex distributedExecutionIndex, + JSONObject faultObject, + boolean isTransformerFault) { logger.info("[FILIBUSTER-CORE]: createAndScheduleAbstractTestExecution called."); if (currentConcreteTestExecution != null) { @@ -1082,14 +1090,14 @@ private void createAndScheduleAbstractTestExecution( if (!abstractIsExploredExecution && !abstractIsScheduledExecution && !abstractIsCurrentExecution) { if (filibusterConfiguration.getSuppressCombinations()) { if (!(abstractTestExecution.getFaultsToInjectSize() > 1)) { - unexploredTestExecutions.addTestExecution(abstractTestExecution); + unexploredTestExecutions.addTestExecution(abstractTestExecution, isTransformerFault); logger.info("[FILIBUSTER-CORE]: createAndScheduleAbstractTestExecution, adding new execution to the queue."); } else { logger.info("[FILIBUSTER-CORE]: createAndScheduleAbstractTestExecution, not scheduling test execution because it contains > 1 fault."); } } else { logger.info("[FILIBUSTER-CORE]: createAndScheduleAbstractTestExecution, adding new execution to the queue."); - unexploredTestExecutions.addTestExecution(abstractTestExecution); + unexploredTestExecutions.addTestExecution(abstractTestExecution, isTransformerFault); } } } diff --git a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionCollection.java b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionCollection.java index 27107806c..56ca8e47b 100644 --- a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionCollection.java +++ b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionCollection.java @@ -13,5 +13,5 @@ public interface TestExecutionCollection { int size(); - void addTestExecution(T testExecution); + void addTestExecution(T testExecution, boolean isTransformerFault); } diff --git a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionQueue.java b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionQueue.java index da6b6b5a1..0c4294626 100644 --- a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionQueue.java +++ b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionQueue.java @@ -27,7 +27,11 @@ public T removeAndReturnNextTestExecution() { } @Override - public void addTestExecution(T testExecution) { - this.add(testExecution); + public void addTestExecution(T testExecution, boolean isTransformerFault) { + if (isTransformerFault) { + this.addFirst(testExecution); + } else { + this.add(testExecution); + } } } diff --git a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionStack.java b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionStack.java index c1960ab8a..1249b0ffe 100644 --- a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionStack.java +++ b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionStack.java @@ -27,7 +27,11 @@ public T removeAndReturnNextTestExecution() { } @Override - public void addTestExecution(T testExecution) { - this.addFirst(testExecution); + public void addTestExecution(T testExecution, boolean isTransformerFault) { + if (isTransformerFault) { + this.addLast(testExecution); + } else { + this.addFirst(testExecution); + } } } From 70a5b4b7001ba5fc7e7db400105961bc5f3a954d Mon Sep 17 00:00:00 2001 From: Michael Assad Date: Thu, 17 Aug 2023 15:04:54 -0400 Subject: [PATCH 02/10] Refactor --- .../junit/server/core/TestExecutionCollection.java | 2 +- .../filibuster/junit/server/core/TestExecutionQueue.java | 8 ++------ .../filibuster/junit/server/core/TestExecutionStack.java | 8 ++------ 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionCollection.java b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionCollection.java index 56ca8e47b..27107806c 100644 --- a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionCollection.java +++ b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionCollection.java @@ -13,5 +13,5 @@ public interface TestExecutionCollection { int size(); - void addTestExecution(T testExecution, boolean isTransformerFault); + void addTestExecution(T testExecution); } diff --git a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionQueue.java b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionQueue.java index 0c4294626..da6b6b5a1 100644 --- a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionQueue.java +++ b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionQueue.java @@ -27,11 +27,7 @@ public T removeAndReturnNextTestExecution() { } @Override - public void addTestExecution(T testExecution, boolean isTransformerFault) { - if (isTransformerFault) { - this.addFirst(testExecution); - } else { - this.add(testExecution); - } + public void addTestExecution(T testExecution) { + this.add(testExecution); } } diff --git a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionStack.java b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionStack.java index 1249b0ffe..c1960ab8a 100644 --- a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionStack.java +++ b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionStack.java @@ -27,11 +27,7 @@ public T removeAndReturnNextTestExecution() { } @Override - public void addTestExecution(T testExecution, boolean isTransformerFault) { - if (isTransformerFault) { - this.addLast(testExecution); - } else { - this.addFirst(testExecution); - } + public void addTestExecution(T testExecution) { + this.addFirst(testExecution); } } From 4b5195639f295d5a5b07276c531508b81377c29b Mon Sep 17 00:00:00 2001 From: Michael Assad Date: Thu, 17 Aug 2023 18:18:51 -0400 Subject: [PATCH 03/10] Fix test order for transformer faults - BFS --- .../junit/server/core/FilibusterCore.java | 23 ++++++++++--------- .../server/core/TestExecutionCollection.java | 2 +- .../junit/server/core/TestExecutionQueue.java | 8 +++++-- .../junit/server/core/TestExecutionStack.java | 3 ++- .../JUnitCockroachDBTransformerTest.java | 6 ++--- .../JUnitPostgresTransformerTest.java | 6 ++--- .../test_services/postgresql/BasicDAO.java | 4 ++-- 7 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/main/java/cloud/filibuster/junit/server/core/FilibusterCore.java b/src/main/java/cloud/filibuster/junit/server/core/FilibusterCore.java index 1f0c50fdf..58012fb75 100644 --- a/src/main/java/cloud/filibuster/junit/server/core/FilibusterCore.java +++ b/src/main/java/cloud/filibuster/junit/server/core/FilibusterCore.java @@ -309,7 +309,7 @@ public synchronized JSONObject beginInvocation(JSONObject payload) { setNextAccumulator(newFaultObject.getJSONObject("transformer_fault"), transformationResult.getNextAccumulator()); generateAndSetTransformerValue(newFaultObject.getJSONObject("transformer_fault")); - createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, newFaultObject); + createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, newFaultObject, /* isAdHoc= */true); } logger.info("[FILIBUSTER-CORE]: beginInvocation, injecting faults using transformer_fault: " + transformerFaultObject.toString(4)); @@ -464,7 +464,7 @@ private void generateByzantineAndTransformerFaults(JSONObject payload, Distribut initialAccumulator ); generateAndSetTransformerValue(handledTransformer.getJSONObject("transformer_fault")); - createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, new JSONObject(handledTransformer.toMap())); + createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, new JSONObject(handledTransformer.toMap()), /* isAdHoc= */false); } catch (Throwable e) { logger.warning("[FILIBUSTER-CORE]: generateByzantineAndTransformerFaults, an exception occurred in generateByzantineAndTransformerFaults: " + e); throw new FilibusterFaultInjectionException("[FILIBUSTER-CORE]: generateByzantineAndTransformerFaults: ", e); @@ -476,7 +476,7 @@ private void generateByzantineAndTransformerFaults(JSONObject payload, Distribut List byzantineFaultObjects = filibusterAnalysisConfiguration.getByzantineFaultObjects(); for (JSONObject faultObject : byzantineFaultObjects) { - createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, faultObject); + createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, faultObject, /* isAdHoc= */false); } } } @@ -650,14 +650,14 @@ public synchronized void teardownsCompleted(int currentIteration) { // Don't add to explored queue if it's already there. numberOfAbstractExecutionsExecuted++; - exploredTestExecutions.addTestExecution(currentAbstractTestExecution); + exploredTestExecutions.addTestExecution(currentAbstractTestExecution, /* isAdHoc= */false); } else { logger.severe("[FILIBUSTER-CORE]: teardownsCompleted called, currentAbstractTestExecution already exists in the explored queue, this could indicate a problem in Filibuster."); } } if (!exploredTestExecutions.containsTestExecution(currentConcreteTestExecution)) { - exploredTestExecutions.addTestExecution(currentConcreteTestExecution); + exploredTestExecutions.addTestExecution(currentConcreteTestExecution, /* isAdHoc= */false); } numberOfConcreteExecutionsExecuted++; @@ -1088,7 +1088,7 @@ private void generateFaultsUsingSpecificAnalysisConfiguration( } if (matcher.find()) { - createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, faultObject); + createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, faultObject, /* isAdHoc= */false); } } @@ -1098,7 +1098,7 @@ private void generateFaultsUsingSpecificAnalysisConfiguration( for (JSONObject faultObject : exceptionFaultObjects) { // If we shouldn't execute this, skip it. if (filibusterFaultInjectionFilter.shouldInjectFault(methodName)) { - createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, faultObject); + createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, faultObject, /* isAdHoc= */false); } } @@ -1119,7 +1119,7 @@ private void generateFaultsUsingSpecificAnalysisConfiguration( HashMap faultTypeMap = (HashMap) obj; JSONObject faultTypeObject = new JSONObject(); faultTypeObject.put("failure_metadata", faultTypeMap); - createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, faultTypeObject); + createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, faultTypeObject, /* isAdHoc= */false); } } } @@ -1204,7 +1204,8 @@ private void generateFaultsUsingAnalysisConfiguration( private void createAndScheduleAbstractTestExecution( FilibusterConfiguration filibusterConfiguration, DistributedExecutionIndex distributedExecutionIndex, - JSONObject faultObject) { + JSONObject faultObject, + boolean isAdHoc) { logger.info("[FILIBUSTER-CORE]: createAndScheduleAbstractTestExecution called."); if (currentConcreteTestExecution != null) { @@ -1218,14 +1219,14 @@ private void createAndScheduleAbstractTestExecution( if (!abstractIsExploredExecution && !abstractIsScheduledExecution && !abstractIsCurrentExecution) { if (filibusterConfiguration.getSuppressCombinations()) { if (!(abstractTestExecution.getFaultsToInjectSize() > 1)) { - unexploredTestExecutions.addTestExecution(abstractTestExecution); + unexploredTestExecutions.addTestExecution(abstractTestExecution, isAdHoc); logger.info("[FILIBUSTER-CORE]: createAndScheduleAbstractTestExecution, adding new execution to the queue."); } else { logger.info("[FILIBUSTER-CORE]: createAndScheduleAbstractTestExecution, not scheduling test execution because it contains > 1 fault."); } } else { logger.info("[FILIBUSTER-CORE]: createAndScheduleAbstractTestExecution, adding new execution to the queue."); - unexploredTestExecutions.addTestExecution(abstractTestExecution); + unexploredTestExecutions.addTestExecution(abstractTestExecution, isAdHoc); } } } diff --git a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionCollection.java b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionCollection.java index 27107806c..ae54240f3 100644 --- a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionCollection.java +++ b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionCollection.java @@ -13,5 +13,5 @@ public interface TestExecutionCollection { int size(); - void addTestExecution(T testExecution); + void addTestExecution(T testExecution, boolean isAdHoc); } diff --git a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionQueue.java b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionQueue.java index da6b6b5a1..0f58983fd 100644 --- a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionQueue.java +++ b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionQueue.java @@ -27,7 +27,11 @@ public T removeAndReturnNextTestExecution() { } @Override - public void addTestExecution(T testExecution) { - this.add(testExecution); + public void addTestExecution(T testExecution, boolean isAdHoc) { + if (isAdHoc) { + this.addFirst(testExecution); + } else { + this.add(testExecution); + } } } diff --git a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionStack.java b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionStack.java index c1960ab8a..4fcb1e721 100644 --- a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionStack.java +++ b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionStack.java @@ -27,7 +27,8 @@ public T removeAndReturnNextTestExecution() { } @Override - public void addTestExecution(T testExecution) { + public void addTestExecution(T testExecution, boolean isAdHoc) { + // TODO: Add transformer logic this.addFirst(testExecution); } } diff --git a/src/test/java/cloud/filibuster/functional/database/postgresql/JUnitCockroachDBTransformerTest.java b/src/test/java/cloud/filibuster/functional/database/postgresql/JUnitCockroachDBTransformerTest.java index 0b0d1ca52..8f05ba7ce 100644 --- a/src/test/java/cloud/filibuster/functional/database/postgresql/JUnitCockroachDBTransformerTest.java +++ b/src/test/java/cloud/filibuster/functional/database/postgresql/JUnitCockroachDBTransformerTest.java @@ -13,13 +13,13 @@ import org.junit.jupiter.api.TestMethodOrder; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; import static cloud.filibuster.junit.assertions.protocols.GenericAssertions.wasFaultInjected; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -59,8 +59,8 @@ public void testCockroachConnection() { assertEquals(initBalance2, cockroachDAO.getAccountBalance(id2)); // Assert that the correct account IDs are returned for the given balances - assertArrayEquals(new String[]{id1.toString()}, cockroachDAO.getAccountIdByBalance(initBalance1)); - assertArrayEquals(new String[]{id2.toString()}, cockroachDAO.getAccountIdByBalance(initBalance2)); + assertEquals(Collections.singletonList(id1.toString()), cockroachDAO.getAccountIdByBalance(initBalance1)); + assertEquals(Collections.singletonList(id2.toString()), cockroachDAO.getAccountIdByBalance(initBalance2)); // Remove all accounts from DB cockroachDAO.deleteAllAccounts(); diff --git a/src/test/java/cloud/filibuster/functional/database/postgresql/JUnitPostgresTransformerTest.java b/src/test/java/cloud/filibuster/functional/database/postgresql/JUnitPostgresTransformerTest.java index ffabb1a64..993bfc353 100644 --- a/src/test/java/cloud/filibuster/functional/database/postgresql/JUnitPostgresTransformerTest.java +++ b/src/test/java/cloud/filibuster/functional/database/postgresql/JUnitPostgresTransformerTest.java @@ -13,13 +13,13 @@ import org.junit.jupiter.api.TestMethodOrder; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; import static cloud.filibuster.junit.assertions.protocols.GenericAssertions.wasFaultInjected; -import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -59,8 +59,8 @@ public void testCockroachConnection() { assertEquals(initBalance2, postgresDAO.getAccountBalance(id2)); // Assert that the correct account IDs are returned for the given balances - assertArrayEquals(new String[]{id1.toString()}, postgresDAO.getAccountIdByBalance(initBalance1)); - assertArrayEquals(new String[]{id2.toString()}, postgresDAO.getAccountIdByBalance(initBalance2)); + assertEquals(Collections.singletonList(id1.toString()), postgresDAO.getAccountIdByBalance(initBalance1)); + assertEquals(Collections.singletonList(id2.toString()), postgresDAO.getAccountIdByBalance(initBalance2)); // Remove all accounts from DB postgresDAO.deleteAllAccounts(); diff --git a/src/test/java/cloud/filibuster/integration/examples/armeria/grpc/test_services/postgresql/BasicDAO.java b/src/test/java/cloud/filibuster/integration/examples/armeria/grpc/test_services/postgresql/BasicDAO.java index 0ab4acd85..0a9c154a3 100644 --- a/src/test/java/cloud/filibuster/integration/examples/armeria/grpc/test_services/postgresql/BasicDAO.java +++ b/src/test/java/cloud/filibuster/integration/examples/armeria/grpc/test_services/postgresql/BasicDAO.java @@ -307,7 +307,7 @@ public int getAccountBalance(UUID id) { * * @return balance (int) */ - public String[] getAccountIdByBalance(int balance) { + public List getAccountIdByBalance(int balance) { List id = new ArrayList<>(); try (Connection connection = this.getConnection()) { @@ -322,7 +322,7 @@ public String[] getAccountIdByBalance(int balance) { e.getSQLState(), e.getCause(), e.getMessage())); } - return id.toArray(new String[0]); + return id.stream().toList(); } } From 891a395b7cbd049fbba255db5380f8b1e915a734 Mon Sep 17 00:00:00 2001 From: Michael Assad Date: Thu, 17 Aug 2023 18:32:38 -0400 Subject: [PATCH 04/10] Fix failing test --- .../configuration/examples/db/redis/test.java | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 src/main/java/cloud/filibuster/junit/configuration/examples/db/redis/test.java diff --git a/src/main/java/cloud/filibuster/junit/configuration/examples/db/redis/test.java b/src/main/java/cloud/filibuster/junit/configuration/examples/db/redis/test.java new file mode 100644 index 000000000..74d62064d --- /dev/null +++ b/src/main/java/cloud/filibuster/junit/configuration/examples/db/redis/test.java @@ -0,0 +1,64 @@ +package cloud.filibuster.junit.configuration.examples.db.redis; + +import cloud.filibuster.junit.configuration.FilibusterAnalysisConfiguration; +import cloud.filibuster.junit.configuration.FilibusterAnalysisConfigurationFile; +import cloud.filibuster.junit.configuration.FilibusterCustomAnalysisConfigurationFile; +import cloud.filibuster.junit.server.core.transformers.StringTransformer; + +import java.util.HashMap; +import java.util.Map; + +public class test implements FilibusterAnalysisConfigurationFile { + private static final FilibusterCustomAnalysisConfigurationFile filibusterCustomAnalysisConfigurationFile; + + private static Map createErrorMap(String cause) { + Map myMap = new HashMap<>(); + myMap.put("cause", cause); + myMap.put("code", ""); + return myMap; + } + + static { + FilibusterCustomAnalysisConfigurationFile.Builder filibusterCustomAnalysisConfigurationFileBuilder = new FilibusterCustomAnalysisConfigurationFile.Builder(); + + // Redis timeout exception for string get + String[][] exceptions = new String[][]{ + {"io.lettuce.core.RedisCommandTimeoutException", + "(io.lettuce.core.api.StatefulRedisConnection/sync|" + + "io.lettuce.core.api.sync.RedisStringCommands/sasdet)\\b", + "Command timed out after 100 millisecond(s)"}, + {"io.lettuce.core.RedisBusyException", + "(io.lettuce.core.api.StatefulRedisConnection/sync|" + + "io.lettuce.core.api.sync.RedisStringCommands/sasdet)\\b", + "Command timed out after 100 millisecond(s)"} + }; + + for (String[] exception : exceptions) { + createException(filibusterCustomAnalysisConfigurationFileBuilder, exception[0], exception[1], exception[2]); + } + + // Transformer faults + FilibusterAnalysisConfiguration.Builder filibusterAnalysisConfigurationBuilderRedisExceptions = new FilibusterAnalysisConfiguration.Builder() + .name("java.transformers.transform_string.redis") + .pattern("io.lettuce.core.api.sync.RedisStringCommands/get\\b"); + filibusterAnalysisConfigurationBuilderRedisExceptions.transformer(StringTransformer.class); + + filibusterCustomAnalysisConfigurationFileBuilder.analysisConfiguration(filibusterAnalysisConfigurationBuilderRedisExceptions.build()); + + filibusterCustomAnalysisConfigurationFile = filibusterCustomAnalysisConfigurationFileBuilder.build(); + } + + private static void createException(FilibusterCustomAnalysisConfigurationFile.Builder filibusterCustomAnalysisConfigurationFileBuilder, String name, String pattern, String cause) { + + FilibusterAnalysisConfiguration.Builder filibusterAnalysisConfigurationBuilderRedisExceptions = new FilibusterAnalysisConfiguration.Builder().name(name).pattern(pattern); + + filibusterAnalysisConfigurationBuilderRedisExceptions.exception(name, createErrorMap(cause)); + + filibusterCustomAnalysisConfigurationFileBuilder.analysisConfiguration(filibusterAnalysisConfigurationBuilderRedisExceptions.build()); + } + + @Override + public FilibusterCustomAnalysisConfigurationFile toFilibusterCustomAnalysisConfigurationFile() { + return filibusterCustomAnalysisConfigurationFile; + } +} From d413e8cb81779d278b2e1b573ff397cbbfb7a0b9 Mon Sep 17 00:00:00 2001 From: Michael Assad Date: Thu, 17 Aug 2023 19:03:46 -0400 Subject: [PATCH 05/10] Fix failing test --- .../armeria/grpc/test_services/postgresql/BasicDAO.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/test/java/cloud/filibuster/integration/examples/armeria/grpc/test_services/postgresql/BasicDAO.java b/src/test/java/cloud/filibuster/integration/examples/armeria/grpc/test_services/postgresql/BasicDAO.java index 0a9c154a3..5e07a87d5 100644 --- a/src/test/java/cloud/filibuster/integration/examples/armeria/grpc/test_services/postgresql/BasicDAO.java +++ b/src/test/java/cloud/filibuster/integration/examples/armeria/grpc/test_services/postgresql/BasicDAO.java @@ -308,21 +308,20 @@ public int getAccountBalance(UUID id) { * @return balance (int) */ public List getAccountIdByBalance(int balance) { - List id = new ArrayList<>(); + List ids = new ArrayList<>(); try (Connection connection = this.getConnection()) { ResultSet res = connection.createStatement() .executeQuery(String.format("SELECT id FROM accounts WHERE balance = '%d'", balance)); while (res.next()) { - id.add(res.getString("id")); + ids.add(res.getString("id")); } } catch (SQLException e) { logger.log(Level.INFO, String.format("BasicDAO.getAccountIdByBalance ERROR: { state => %s, cause => %s, message => %s }\n", e.getSQLState(), e.getCause(), e.getMessage())); } - return id.stream().toList(); + return ids; } - } From b480e7c9b144b52f0df9ce1c1a2e5cd0187519fb Mon Sep 17 00:00:00 2001 From: Michael Assad Date: Thu, 17 Aug 2023 19:35:40 -0400 Subject: [PATCH 06/10] Fix test order for transformer faults - DFS --- .../configuration/examples/db/redis/test.java | 64 ------------------- .../junit/server/core/TestExecutionStack.java | 7 +- 2 files changed, 5 insertions(+), 66 deletions(-) delete mode 100644 src/main/java/cloud/filibuster/junit/configuration/examples/db/redis/test.java diff --git a/src/main/java/cloud/filibuster/junit/configuration/examples/db/redis/test.java b/src/main/java/cloud/filibuster/junit/configuration/examples/db/redis/test.java deleted file mode 100644 index 74d62064d..000000000 --- a/src/main/java/cloud/filibuster/junit/configuration/examples/db/redis/test.java +++ /dev/null @@ -1,64 +0,0 @@ -package cloud.filibuster.junit.configuration.examples.db.redis; - -import cloud.filibuster.junit.configuration.FilibusterAnalysisConfiguration; -import cloud.filibuster.junit.configuration.FilibusterAnalysisConfigurationFile; -import cloud.filibuster.junit.configuration.FilibusterCustomAnalysisConfigurationFile; -import cloud.filibuster.junit.server.core.transformers.StringTransformer; - -import java.util.HashMap; -import java.util.Map; - -public class test implements FilibusterAnalysisConfigurationFile { - private static final FilibusterCustomAnalysisConfigurationFile filibusterCustomAnalysisConfigurationFile; - - private static Map createErrorMap(String cause) { - Map myMap = new HashMap<>(); - myMap.put("cause", cause); - myMap.put("code", ""); - return myMap; - } - - static { - FilibusterCustomAnalysisConfigurationFile.Builder filibusterCustomAnalysisConfigurationFileBuilder = new FilibusterCustomAnalysisConfigurationFile.Builder(); - - // Redis timeout exception for string get - String[][] exceptions = new String[][]{ - {"io.lettuce.core.RedisCommandTimeoutException", - "(io.lettuce.core.api.StatefulRedisConnection/sync|" + - "io.lettuce.core.api.sync.RedisStringCommands/sasdet)\\b", - "Command timed out after 100 millisecond(s)"}, - {"io.lettuce.core.RedisBusyException", - "(io.lettuce.core.api.StatefulRedisConnection/sync|" + - "io.lettuce.core.api.sync.RedisStringCommands/sasdet)\\b", - "Command timed out after 100 millisecond(s)"} - }; - - for (String[] exception : exceptions) { - createException(filibusterCustomAnalysisConfigurationFileBuilder, exception[0], exception[1], exception[2]); - } - - // Transformer faults - FilibusterAnalysisConfiguration.Builder filibusterAnalysisConfigurationBuilderRedisExceptions = new FilibusterAnalysisConfiguration.Builder() - .name("java.transformers.transform_string.redis") - .pattern("io.lettuce.core.api.sync.RedisStringCommands/get\\b"); - filibusterAnalysisConfigurationBuilderRedisExceptions.transformer(StringTransformer.class); - - filibusterCustomAnalysisConfigurationFileBuilder.analysisConfiguration(filibusterAnalysisConfigurationBuilderRedisExceptions.build()); - - filibusterCustomAnalysisConfigurationFile = filibusterCustomAnalysisConfigurationFileBuilder.build(); - } - - private static void createException(FilibusterCustomAnalysisConfigurationFile.Builder filibusterCustomAnalysisConfigurationFileBuilder, String name, String pattern, String cause) { - - FilibusterAnalysisConfiguration.Builder filibusterAnalysisConfigurationBuilderRedisExceptions = new FilibusterAnalysisConfiguration.Builder().name(name).pattern(pattern); - - filibusterAnalysisConfigurationBuilderRedisExceptions.exception(name, createErrorMap(cause)); - - filibusterCustomAnalysisConfigurationFileBuilder.analysisConfiguration(filibusterAnalysisConfigurationBuilderRedisExceptions.build()); - } - - @Override - public FilibusterCustomAnalysisConfigurationFile toFilibusterCustomAnalysisConfigurationFile() { - return filibusterCustomAnalysisConfigurationFile; - } -} diff --git a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionStack.java b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionStack.java index 4fcb1e721..22f7cf73a 100644 --- a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionStack.java +++ b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionStack.java @@ -28,7 +28,10 @@ public T removeAndReturnNextTestExecution() { @Override public void addTestExecution(T testExecution, boolean isAdHoc) { - // TODO: Add transformer logic - this.addFirst(testExecution); + if (isAdHoc) { + this.addLast(testExecution); + } else { + this.addFirst(testExecution); + } } } From a58799eab246e20afea296974887043dfd249b1e Mon Sep 17 00:00:00 2001 From: Michael Assad Date: Thu, 17 Aug 2023 19:37:18 -0400 Subject: [PATCH 07/10] Update CHANGELOG and build.gradle --- CHANGELOG | 2 ++ build.gradle | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index 391e9d060..5c174f126 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,5 @@ +2.1.6: + - (MichaelA-ops) Fix bug where order of transformer tests is incorrect. 2.1.5: - (cmeiklejohn) Improve UX. - (MichaelA-ops) Attach transformer faults to Postgres/CockroachDB int, boolean, byte[] and string field types. diff --git a/build.gradle b/build.gradle index 023e2842b..3183ecf1f 100644 --- a/build.gradle +++ b/build.gradle @@ -239,7 +239,7 @@ javadoc { group = "cloud.filibuster" archivesBaseName = "instrumentation" -version = "2.1.5" +version = "2.1.6" java { withJavadocJar() From da3e89f5ab221c7915d92f764dc374652fe51217 Mon Sep 17 00:00:00 2001 From: Michael Assad Date: Thu, 17 Aug 2023 19:39:06 -0400 Subject: [PATCH 08/10] Update version to 2.1.7 --- CHANGELOG | 3 ++- build.gradle | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index ea1208115..a7ac60a87 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,6 +1,7 @@ +2.1.7: + - (MichaelA-ops) Fix bug where order of transformer tests is incorrect. 2.1.6: - (cmeiklejohn) Fix percentage error. - - (MichaelA-ops) Fix bug where order of transformer tests is incorrect. 2.1.5: - (cmeiklejohn) Improve UX. - (MichaelA-ops) Attach transformer faults to Postgres/CockroachDB int, boolean, byte[] and string field types. diff --git a/build.gradle b/build.gradle index 3183ecf1f..744cc9cf0 100644 --- a/build.gradle +++ b/build.gradle @@ -239,7 +239,7 @@ javadoc { group = "cloud.filibuster" archivesBaseName = "instrumentation" -version = "2.1.6" +version = "2.1.7" java { withJavadocJar() From 756d3ed8e0866697d150e42ba6dea027e8f6e938 Mon Sep 17 00:00:00 2001 From: Michael Assad Date: Thu, 17 Aug 2023 20:00:52 -0400 Subject: [PATCH 09/10] Rename boolean to isScheduledByTransformer --- .../junit/server/core/FilibusterCore.java | 22 +++++++++---------- .../server/core/TestExecutionCollection.java | 2 +- .../junit/server/core/TestExecutionQueue.java | 4 ++-- .../junit/server/core/TestExecutionStack.java | 4 ++-- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/main/java/cloud/filibuster/junit/server/core/FilibusterCore.java b/src/main/java/cloud/filibuster/junit/server/core/FilibusterCore.java index 58012fb75..a108fdab9 100644 --- a/src/main/java/cloud/filibuster/junit/server/core/FilibusterCore.java +++ b/src/main/java/cloud/filibuster/junit/server/core/FilibusterCore.java @@ -309,7 +309,7 @@ public synchronized JSONObject beginInvocation(JSONObject payload) { setNextAccumulator(newFaultObject.getJSONObject("transformer_fault"), transformationResult.getNextAccumulator()); generateAndSetTransformerValue(newFaultObject.getJSONObject("transformer_fault")); - createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, newFaultObject, /* isAdHoc= */true); + createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, newFaultObject, /* isScheduledByTransformer= */true); } logger.info("[FILIBUSTER-CORE]: beginInvocation, injecting faults using transformer_fault: " + transformerFaultObject.toString(4)); @@ -464,7 +464,7 @@ private void generateByzantineAndTransformerFaults(JSONObject payload, Distribut initialAccumulator ); generateAndSetTransformerValue(handledTransformer.getJSONObject("transformer_fault")); - createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, new JSONObject(handledTransformer.toMap()), /* isAdHoc= */false); + createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, new JSONObject(handledTransformer.toMap()), /* isScheduledByTransformer= */false); } catch (Throwable e) { logger.warning("[FILIBUSTER-CORE]: generateByzantineAndTransformerFaults, an exception occurred in generateByzantineAndTransformerFaults: " + e); throw new FilibusterFaultInjectionException("[FILIBUSTER-CORE]: generateByzantineAndTransformerFaults: ", e); @@ -476,7 +476,7 @@ private void generateByzantineAndTransformerFaults(JSONObject payload, Distribut List byzantineFaultObjects = filibusterAnalysisConfiguration.getByzantineFaultObjects(); for (JSONObject faultObject : byzantineFaultObjects) { - createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, faultObject, /* isAdHoc= */false); + createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, faultObject, /* isScheduledByTransformer= */false); } } } @@ -650,14 +650,14 @@ public synchronized void teardownsCompleted(int currentIteration) { // Don't add to explored queue if it's already there. numberOfAbstractExecutionsExecuted++; - exploredTestExecutions.addTestExecution(currentAbstractTestExecution, /* isAdHoc= */false); + exploredTestExecutions.addTestExecution(currentAbstractTestExecution, /* isScheduledByTransformer= */false); } else { logger.severe("[FILIBUSTER-CORE]: teardownsCompleted called, currentAbstractTestExecution already exists in the explored queue, this could indicate a problem in Filibuster."); } } if (!exploredTestExecutions.containsTestExecution(currentConcreteTestExecution)) { - exploredTestExecutions.addTestExecution(currentConcreteTestExecution, /* isAdHoc= */false); + exploredTestExecutions.addTestExecution(currentConcreteTestExecution, /* isScheduledByTransformer= */false); } numberOfConcreteExecutionsExecuted++; @@ -1088,7 +1088,7 @@ private void generateFaultsUsingSpecificAnalysisConfiguration( } if (matcher.find()) { - createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, faultObject, /* isAdHoc= */false); + createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, faultObject, /* isScheduledByTransformer= */false); } } @@ -1098,7 +1098,7 @@ private void generateFaultsUsingSpecificAnalysisConfiguration( for (JSONObject faultObject : exceptionFaultObjects) { // If we shouldn't execute this, skip it. if (filibusterFaultInjectionFilter.shouldInjectFault(methodName)) { - createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, faultObject, /* isAdHoc= */false); + createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, faultObject, /* isScheduledByTransformer= */false); } } @@ -1119,7 +1119,7 @@ private void generateFaultsUsingSpecificAnalysisConfiguration( HashMap faultTypeMap = (HashMap) obj; JSONObject faultTypeObject = new JSONObject(); faultTypeObject.put("failure_metadata", faultTypeMap); - createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, faultTypeObject, /* isAdHoc= */false); + createAndScheduleAbstractTestExecution(filibusterConfiguration, distributedExecutionIndex, faultTypeObject, /* isScheduledByTransformer= */false); } } } @@ -1205,7 +1205,7 @@ private void createAndScheduleAbstractTestExecution( FilibusterConfiguration filibusterConfiguration, DistributedExecutionIndex distributedExecutionIndex, JSONObject faultObject, - boolean isAdHoc) { + boolean isScheduledByTransformer) { logger.info("[FILIBUSTER-CORE]: createAndScheduleAbstractTestExecution called."); if (currentConcreteTestExecution != null) { @@ -1219,14 +1219,14 @@ private void createAndScheduleAbstractTestExecution( if (!abstractIsExploredExecution && !abstractIsScheduledExecution && !abstractIsCurrentExecution) { if (filibusterConfiguration.getSuppressCombinations()) { if (!(abstractTestExecution.getFaultsToInjectSize() > 1)) { - unexploredTestExecutions.addTestExecution(abstractTestExecution, isAdHoc); + unexploredTestExecutions.addTestExecution(abstractTestExecution, isScheduledByTransformer); logger.info("[FILIBUSTER-CORE]: createAndScheduleAbstractTestExecution, adding new execution to the queue."); } else { logger.info("[FILIBUSTER-CORE]: createAndScheduleAbstractTestExecution, not scheduling test execution because it contains > 1 fault."); } } else { logger.info("[FILIBUSTER-CORE]: createAndScheduleAbstractTestExecution, adding new execution to the queue."); - unexploredTestExecutions.addTestExecution(abstractTestExecution, isAdHoc); + unexploredTestExecutions.addTestExecution(abstractTestExecution, isScheduledByTransformer); } } } diff --git a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionCollection.java b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionCollection.java index ae54240f3..984cd5130 100644 --- a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionCollection.java +++ b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionCollection.java @@ -13,5 +13,5 @@ public interface TestExecutionCollection { int size(); - void addTestExecution(T testExecution, boolean isAdHoc); + void addTestExecution(T testExecution, boolean isScheduledByTransformer); } diff --git a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionQueue.java b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionQueue.java index 0f58983fd..1e570658c 100644 --- a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionQueue.java +++ b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionQueue.java @@ -27,8 +27,8 @@ public T removeAndReturnNextTestExecution() { } @Override - public void addTestExecution(T testExecution, boolean isAdHoc) { - if (isAdHoc) { + public void addTestExecution(T testExecution, boolean isScheduledByTransformer) { + if (isScheduledByTransformer) { this.addFirst(testExecution); } else { this.add(testExecution); diff --git a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionStack.java b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionStack.java index 22f7cf73a..7c9ea4cb0 100644 --- a/src/main/java/cloud/filibuster/junit/server/core/TestExecutionStack.java +++ b/src/main/java/cloud/filibuster/junit/server/core/TestExecutionStack.java @@ -27,8 +27,8 @@ public T removeAndReturnNextTestExecution() { } @Override - public void addTestExecution(T testExecution, boolean isAdHoc) { - if (isAdHoc) { + public void addTestExecution(T testExecution, boolean isScheduledByTransformer) { + if (isScheduledByTransformer) { this.addLast(testExecution); } else { this.addFirst(testExecution); From 72e0c8efb7142bb8d8d7f2f614d5deed1edca9a1 Mon Sep 17 00:00:00 2001 From: Michael Assad Date: Fri, 18 Aug 2023 14:28:15 -0400 Subject: [PATCH 10/10] Update build.gradle --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 744cc9cf0..68aff516d 100644 --- a/build.gradle +++ b/build.gradle @@ -239,7 +239,7 @@ javadoc { group = "cloud.filibuster" archivesBaseName = "instrumentation" -version = "2.1.7" +version = "2.1.8" java { withJavadocJar()