diff --git a/docs/getting_started.md b/docs/getting_started.md index 73bc463..99e65ab 100644 --- a/docs/getting_started.md +++ b/docs/getting_started.md @@ -162,3 +162,60 @@ This is entirely equivalent to having put this in basicModules.yaml instead (the AFLForkserverExecutor: sutArgv: ["test/haystackSUT/haystack"] ``` + +## Example Differential Fuzzing VMF Configuration +VMF's configuration-driven paradigm has driven new advancements in fuzzing capabilities. The following modules must be included to enable differential fuzzing of two Systems Under Test (SUTs): + +```yaml +vmfVariables: # ... no changes ... + +vmfFramework: # ... no changes ... + +vmfModules: + storage: # SimpleStorage MUST be specified + className: SimpleStorage + controller: + # DifferentialController MUST specify AT LEAST TWO AFLForkserverExecutor modules + className: DifferentialController + children: + # Each AFLForkserverExecutor MUST have a unique id + - id: knownGoodSutA + className: AFLForkserverExecutor + - id: unknownSutB + className: AFLForkserverExecutor + # DiffInputGenerator MUST specify their children + - className: DiffInputGenerator + # AFLDiffFeedback MUST specify module-specific params + - className: AFLDiffFeedback + # ComputeDiffStats MAY specify statsRateInSeconds (default is 1) + - className: ComputeDiffStats + statsRateInSeconds: 2 + # StatsDiffOutput MAY specify outputRateInSeconds (default is 5) + - className: StatsDiffOutput + outputRateInSeconds: 10 + + # DiffInputGenerator MUST have one or more Mutator module children + DiffInputGenerator: + children: + - className: # ex: AFLRandomByteMutator + - # ... + +### Module-specific parameters ### + +# Each AFLForkserverExecutor MUST specify their command-line arguments +sutA: + sutArgv: # ... +sutB: + sutArgv: # ... + +# AFLDiffFeedback MUST specify the ID of ONE trusted SUT, as a reference for the system +AFLDiffFeedback: + systemOfTruth: # ... + # The module MAY specify custom fitness weights to favor different test attributes. + # NOTE: not specifying customWeights will weigh feedback with regards to the SUT's + # average statistics over the campaign + useCustomWeights: # default is false + diffWeight: # default is 10.0 + sizeWeight: # default is 1.0 + speedWeight: # default is 5.0 +``` diff --git a/test/unittest/TestConfigInterface.hpp b/test/unittest/TestConfigInterface.hpp index 017d6f2..00ba38d 100644 --- a/test/unittest/TestConfigInterface.hpp +++ b/test/unittest/TestConfigInterface.hpp @@ -51,6 +51,9 @@ class TestConfigInterface : public ConfigInterface //Methods required by ConfigInterface -- these are just stubbed out to compile virtual std::string getAllParamsYAML(std::string moduleName); + //Methods required by ConfigInterface -- these are stubbed out to compile + virtual Module* getSuperModule(std::string subModuleName) {return nullptr;} + //Methods required by ConfigInterface -- these have reasonably real implementations virtual std::string getOutputDir(); virtual std::vector getSubModules(std::string parentModuleName); diff --git a/vmf/src/framework/app/ConfigManager.cpp b/vmf/src/framework/app/ConfigManager.cpp index a22429c..f4da1d3 100644 --- a/vmf/src/framework/app/ConfigManager.cpp +++ b/vmf/src/framework/app/ConfigManager.cpp @@ -567,6 +567,12 @@ std::vector ConfigManager::getSubModules(std::string parentModuleName) return list; } +//see ConfigInterface::getSuperModule +Module* ConfigManager::getSuperModule(std::string childName) +{ + return moduleManager->getRootModule(); +} + //see ConfigInterface::isParam bool ConfigManager::isParam(std::string moduleName, std::string paramName) { diff --git a/vmf/src/framework/app/ConfigManager.hpp b/vmf/src/framework/app/ConfigManager.hpp index ec9c8c1..ab41435 100644 --- a/vmf/src/framework/app/ConfigManager.hpp +++ b/vmf/src/framework/app/ConfigManager.hpp @@ -52,6 +52,7 @@ class ConfigManager : public ConfigInterface virtual std::string getOutputDir(); virtual void setOutputDir(std::string dir); virtual std::vector getSubModules(std::string parentModuleName); + virtual Module* getSuperModule(std::string subModuleName); virtual bool isParam(std::string moduleName, std::string paramName); virtual std::string getStringParam(std::string moduleName, std::string paramName); diff --git a/vmf/src/framework/baseclasses/FeedbackModule.hpp b/vmf/src/framework/baseclasses/FeedbackModule.hpp index 28a8312..271d470 100644 --- a/vmf/src/framework/baseclasses/FeedbackModule.hpp +++ b/vmf/src/framework/baseclasses/FeedbackModule.hpp @@ -52,6 +52,15 @@ class FeedbackModule: public StorageUserModule * @param entries */ virtual void evaluateTestCaseResults(StorageModule& storage, std::unique_ptr& entries) = 0; + + /** + * @brief Evaluate test case results of a differential fuzzing campaign + * The method is nearly identical to a regular evaluateTestCaseResults, except on N entries at once. + * + * @param storage + * @param entries - vector of list instead of an Iterator, one per executor in differential campaign + */ + virtual void evaluateDiffTestCaseResults(StorageModule& storage, std::vector>& entries) = 0; virtual ~FeedbackModule() {}; /** @@ -93,7 +102,7 @@ class FeedbackModule: public StorageUserModule * @brief Helper method to return a single Feedback submodule from config by name * This method will retrieve a single Feedback submodule by name for the specified parent modules. * If there are no Feedback submodules with the specified name, then an nullptr will be returned. - * + * * @param config the ConfigInterface object * @param parentName the name of the parent module * @param childName the name of the child module to finde @@ -129,7 +138,7 @@ class FeedbackModule: public StorageUserModule * @brief Helper method to get the Feedback Submodules from config * This method will retrieve all of the Feedback submodules for the specified parent modules. * If there are no Feedback submodules, then an empty list will be returned. - * + * * @param config the ConfigInterface object * @param parentName the name of the parent module * @return std::vector the list of submodules diff --git a/vmf/src/framework/baseclasses/SimpleStorage.cpp b/vmf/src/framework/baseclasses/SimpleStorage.cpp index 1f89ab3..7e27565 100644 --- a/vmf/src/framework/baseclasses/SimpleStorage.cpp +++ b/vmf/src/framework/baseclasses/SimpleStorage.cpp @@ -18,8 +18,12 @@ * @license GPL-2.0-only * ===========================================================================*/ #include "SimpleStorage.hpp" +#include "SimpleIterator.hpp" +#include "StorageEntry.hpp" #include "StorageKeyHelper.hpp" #include "Logging.hpp" +#include "plog/Log.h" +#include using namespace vmf; @@ -605,3 +609,79 @@ StorageEntry& SimpleStorage::getMetadata() throw RuntimeException("Storage must be initialized before use.", RuntimeException::USAGE_ERROR); } } + +std::unique_ptr SimpleStorage::getSavedEntriesByIntersection(int tagA, int tagB) +{ + checkThatTagIsValid(tagA, numTags); checkThatTagIsValid(tagB, numTags); + std::list intersection = {}; + auto thatList = tagList[tagB]; + for(auto const entryA : tagList[tagA]) + { + // DEV'S NOTE: find will compare by ADDRESS NOT ENTRY DATA + if(std::find(thatList.begin(), thatList.end(), entryA) != thatList.end()) + { + intersection.emplace_back(entryA); + } + } + + SimpleIterator* theIterator = new SimpleIterator(intersection); + std::unique_ptr returnPointer(theIterator); + return returnPointer; +} + +std::unique_ptr SimpleStorage::getNewEntriesByIntersection(int tagA, int tagB) +{ + checkThatTagIsValid(tagA, numTags); checkThatTagIsValid(tagB, numTags); + std::list intersection = {}; + auto thatList = newTagList[tagB]; + for(auto const entryA : newTagList[tagA]) + { + // DEV'S NOTE: find will compare by ADDRESS NOT ENTRY DATA + if(std::find(thatList.begin(), thatList.end(), entryA) != thatList.end()) + { + intersection.emplace_back(entryA); + } + } + + SimpleIterator* theIterator = new SimpleIterator(intersection); + std::unique_ptr returnPointer(theIterator); + return returnPointer; +} + +std::unique_ptr SimpleStorage::getKeySortedSavedEntriesByTag(int tagId, + std::function lessThanFunc) +{ + checkThatTagIsValid(tagId, numTags); + std::list& entries = tagList[tagId]; + entries.sort(lessThanFunc); + if(entries.size() > 0) + { + SimpleIterator* theIterator = new SimpleIterator(entries); + std::unique_ptr returnPointer(theIterator); + return returnPointer; + } + else + { + LOG_WARNING << "No entries with the tag \"" << tagNameMap[tagId] << "\""; + return nullptr; + } +} + +std::unique_ptr SimpleStorage::getKeySortedNewEntriesByTag(int tagId, + std::function lessThanFunc) +{ + checkThatTagIsValid(tagId, numTags); + std::list& newEntries = newTagList[tagId]; + newEntries.sort(lessThanFunc); + if(newEntries.size() > 0) + { + SimpleIterator* theIterator = new SimpleIterator(newEntries); + std::unique_ptr returnPointer(theIterator); + return returnPointer; + } + else + { + LOG_WARNING << "No new entries with the tag \"" << tagNameMap[tagId] << "\""; + return nullptr; + } +} diff --git a/vmf/src/framework/baseclasses/SimpleStorage.hpp b/vmf/src/framework/baseclasses/SimpleStorage.hpp index 61a9580..504227a 100644 --- a/vmf/src/framework/baseclasses/SimpleStorage.hpp +++ b/vmf/src/framework/baseclasses/SimpleStorage.hpp @@ -31,6 +31,7 @@ #include #include #include +#include namespace vmf{ /** @@ -90,6 +91,14 @@ class SimpleStorage: public StorageModule //This method returns the one and only metadata storage entry virtual StorageEntry& getMetadata(); + //These methods provide a way to retrieve all entries that are designated with both tags provided + virtual std::unique_ptr getSavedEntriesByIntersection(int tagA, int tagB); + virtual std::unique_ptr getNewEntriesByIntersection(int tagA, int tagB); + + //These methods return entries SORTED by the given lambda (which usually compares based on a key) + virtual std::unique_ptr getKeySortedSavedEntriesByTag(int tagId, std::function lessThanFunc); + virtual std::unique_ptr getKeySortedNewEntriesByTag(int tagId, std::function lessThanFunc); + private: static bool removeEntryIfPresent(std::list& list, StorageEntry* entry); static bool checkThatTagIsValid(int tagId, int numTags); diff --git a/vmf/src/framework/baseclasses/StorageModule.hpp b/vmf/src/framework/baseclasses/StorageModule.hpp index 06accd4..92bfd53 100644 --- a/vmf/src/framework/baseclasses/StorageModule.hpp +++ b/vmf/src/framework/baseclasses/StorageModule.hpp @@ -23,7 +23,9 @@ #include "StorageEntry.hpp" #include "StorageEntryListener.hpp" #include "Iterator.hpp" +#include #include +#include namespace vmf { @@ -256,6 +258,34 @@ class StorageModule : public Module, public StorageEntryListener { */ virtual StorageEntry& getMetadata() = 0; + /** + * @brief Get the saved entries that have been previously tagged with all the provided tags. + * + * Returns an iterator that can be used to step through all of the tagged entries. + * Entries are sorted using the sort by fields that were configured in the StorageRegistry. + * + * @param tagA the tag handle (as returned from a call to StoragRegistry.registerTag) + * @param tagB + * @return std::unique_ptr with entries (if any) + */ + virtual std::unique_ptr getSavedEntriesByIntersection(int tagA, int tagB) = 0; + + //These methods return entries SORTED by the corresponding key + virtual std::unique_ptr getKeySortedSavedEntriesByTag(int tagId, std::function lessThanFunc) = 0; + virtual std::unique_ptr getKeySortedNewEntriesByTag(int tagId, std::function lessThanFunc) = 0; + + /** + * @brief Get the new entries with all the provided tags. + * + * Returns an iterator that can be used to step through all of the tagged entries. + * Entries are sorted using the sort by fields that were configured in the StorageRegistry. + * + * @param tagA the tag handle (as returned from a call to StoragRegistry.registerTag) + * @param tagB + * @return std::unique_ptr with entries (if any) + */ + virtual std::unique_ptr getNewEntriesByIntersection(int tagA, int tagB) = 0; + /** * @brief Convenience method to determine if a module is actually a storage module * diff --git a/vmf/src/framework/util/ConfigInterface.hpp b/vmf/src/framework/util/ConfigInterface.hpp index 63538ae..75937de 100644 --- a/vmf/src/framework/util/ConfigInterface.hpp +++ b/vmf/src/framework/util/ConfigInterface.hpp @@ -79,6 +79,18 @@ class ConfigInterface */ virtual std::vector getSubModules(std::string parentModuleName) = 0; + /** + * @brief Retrieves the supermodule that is associated with this module in the config file(s) + * + * To use, any module can call getSuperModule(getModuleName()) + * Module* will need to be converted to their underlying type, using the convenience methods + * isAnInstance() and castTo() that are defined in each of the module base classes. + * + * @param subModuleName the name of the module + * @return Module* the supermodule + */ + virtual Module* getSuperModule(std::string subModuleName) = 0; + /** * @brief Check to see if a parameter is defined in a config file, without returning the value. * diff --git a/vmf/src/modules/CMakeLists.txt b/vmf/src/modules/CMakeLists.txt index f0d4fd4..4196eee 100644 --- a/vmf/src/modules/CMakeLists.txt +++ b/vmf/src/modules/CMakeLists.txt @@ -37,6 +37,9 @@ endif() list(APPEND CoreModules_SOURCES + common/controller/DifferentialController.cpp + common/feedback/AFLDiffFeedback.cpp + common/controller/AnalysisController.cpp common/controller/RunOnceController.cpp common/controller/IterativeController.cpp @@ -50,6 +53,7 @@ list(APPEND CoreModules_SOURCES common/initialization/ServerCorpusInitialization.cpp common/initialization/ServerSeedInitialization.cpp common/initialization/TrivialSeedInitialization.cpp + common/inputgeneration/DiffInputGenerator.cpp common/inputgeneration/GeneticAlgorithmInputGenerator.cpp common/inputgeneration/MOPTInputGenerator.cpp common/inputgeneration/MOPT.cpp @@ -78,6 +82,7 @@ list(APPEND CoreModules_SOURCES common/mutator/StackedMutator.cpp common/mutator/MutatorSelector.cpp common/output/ComputeStats.cpp + common/output/ComputeDiffStats.cpp common/output/CorpusMinimization.cpp common/output/CSVMetadataOutput.cpp common/output/LoggerMetadataOutput.cpp @@ -85,6 +90,7 @@ list(APPEND CoreModules_SOURCES common/output/ServerCorpusMinOutput.cpp common/output/ServerCorpusOutput.cpp common/output/StatsOutput.cpp + common/output/StatsDiffOutput.cpp ) add_library(CoreModules SHARED ${CoreModules_SOURCES}) diff --git a/vmf/src/modules/common/controller/DifferentialController.cpp b/vmf/src/modules/common/controller/DifferentialController.cpp new file mode 100644 index 0000000..0b4b44a --- /dev/null +++ b/vmf/src/modules/common/controller/DifferentialController.cpp @@ -0,0 +1,173 @@ +/* ============================================================================= + * Vader Modular Fuzzer (VMF) + * Copyright (c) 2021-2025 The Charles Stark Draper Laboratory, Inc. + * + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 (only) as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * @license GPL-2.0-only + * ===========================================================================*/ +#include "DifferentialController.hpp" +#include "Logging.hpp" +#include "RuntimeException.hpp" +#include "plog/Log.h" +#include +#include + +using namespace vmf; + +#include "ModuleFactory.hpp" +REGISTER_MODULE(DifferentialController); + +/** + * @brief Builder method to support the ModuleFactory + * Constructs an instance of this class + * @return Module* + */ +Module* DifferentialController::build(std::string name) +{ + return new DifferentialController(name); +} + +/** + * @brief Initialization method + * Reads in all configuration options for this class + * + * @param config + */ +void DifferentialController::init(ConfigInterface& config) +{ + ControllerModulePattern::init(config); + + if(1 >= executors.size()) + { + throw RuntimeException("DifferentialController requires at least two ExecutorModules", + RuntimeException::USAGE_ERROR); + } + + if(1 != feedbacks.size()) + { + throw RuntimeException("DifferentialController requires a single FeedbackModule", + RuntimeException::CONFIGURATION_ERROR); + } + + if(1 != inputGenerators.size()) + { + throw RuntimeException("DifferentialController requires a single InputGeneratorModule", + RuntimeException::CONFIGURATION_ERROR); + } + executorIdTags = {}; + batchNumIdKey = 0; + + //Initialization and output modules are optional, and any number are supported +} + +/** + * @brief Register differential controller to read the executor tags + * + * @param registry + */ +void DifferentialController::registerStorageNeeds(StorageRegistry& registry) +{ + for(auto const& e : executors) + { + executorIdTags.emplace_back( + registry.registerTag(e->getModuleName(), StorageRegistry::READ_ONLY) + ); + } + batchNumIdKey = registry.registerU64Key("ENTRY_BATCH_NUM", StorageRegistry::READ_ONLY,0); +} + +/** + * @brief Construct a new Differential Controller object + * + * @param name the name o the module + */ +DifferentialController::DifferentialController( + std::string name) : + ControllerModulePattern(name) +{ + +} + +DifferentialController::~DifferentialController() +{ + +} + + +bool DifferentialController::run(StorageModule& storage, bool firstPass) +{ + bool done = false; + if(firstPass) + { + performInitialSetupAndCalibration(storage); + } + + executeTestCases(firstPass, storage); + + analyzeResults(firstPass, storage); + + done = generateNewTestCases(firstPass, storage); //also clears the new list + if(done) + { + LOG_INFO << "Fuzzing complete -- our own input generator indicated completion"; + } + + done = hasExecutionTimeCompleted(); + + return done; +} + +/** + * @brief Execute test cases meant only for each specific executor + * + * Overwritten from ControllerModulePattern. + * + * @param firstPass true if this is the first pass through the fuzzing loop + * @param storage the storage module + */ +void DifferentialController::executeTestCases(bool firstPass, StorageModule& storage) +{ + std::unique_ptr storageIterator; + + for(size_t i=0; irunTestCases(storage, storageIterator); + } + + int& batchRef = batchNumIdKey; + + // Request new entries f/e executor, sorted by batch number + std::vector> newEntriesByExecutor; + for(int id : executorIdTags) + { + newEntriesByExecutor.emplace_back( + storage.getKeySortedNewEntriesByTag(id, [batchRef](StorageEntry* e, StorageEntry* f){ + return e->getU64Value(batchRef) < f->getU64Value(batchRef); + }) + ); + } + + if(std::any_of(newEntriesByExecutor.begin(), newEntriesByExecutor.end(), [](auto const& i){ return i == nullptr ;})) + { + return; + } + + for(FeedbackModule* feedback: feedbacks) + { + feedback->evaluateDiffTestCaseResults(storage, newEntriesByExecutor); + storageIterator->resetIndex(); + } +} \ No newline at end of file diff --git a/vmf/src/modules/common/controller/DifferentialController.hpp b/vmf/src/modules/common/controller/DifferentialController.hpp new file mode 100644 index 0000000..365c200 --- /dev/null +++ b/vmf/src/modules/common/controller/DifferentialController.hpp @@ -0,0 +1,55 @@ +/* ============================================================================= + * Vader Modular Fuzzer (VMF) + * Copyright (c) 2021-2025 The Charles Stark Draper Laboratory, Inc. + * + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 (only) as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * @license GPL-2.0-only + * + * ===========================================================================*/ +#pragma once + +// include common modules +#include "ControllerModulePattern.hpp" + + +namespace vmf +{ +/** + * @brief Controller that is capable of managing multiple storage, execution, and feedback modules + * for comparing two differential binaries. Campaign rounds are similar to IterativeController. + * This controller supports one InputGenerator and Feedback modules, two Executors, + * and any number of Initialization and Output modules. + */ +class DifferentialController : public ControllerModulePattern { +public: + + static Module* build(std::string name); + virtual void init(ConfigInterface& config); + virtual void executeTestCases(bool firstPass, StorageModule& storage); + + //This controller has no additional storage needs + virtual void registerStorageNeeds(StorageRegistry& registry); + //virtual void registerMetadataNeeds(StorageRegistry& registry); + + virtual bool run(StorageModule& storage, bool isFirstPass); + + DifferentialController(std::string name); + virtual ~DifferentialController(); + +private: + std::vector executorIdTags; + int batchNumIdKey; +}; +} \ No newline at end of file diff --git a/vmf/src/modules/common/feedback/AFLDiffFeedback.cpp b/vmf/src/modules/common/feedback/AFLDiffFeedback.cpp new file mode 100644 index 0000000..db1c81c --- /dev/null +++ b/vmf/src/modules/common/feedback/AFLDiffFeedback.cpp @@ -0,0 +1,432 @@ +/* ============================================================================= + * Vader Modular Fuzzer (VMF) + * Copyright (c) 2021-2025 The Charles Stark Draper Laboratory, Inc. + * + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 (only) as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * @license GPL-2.0-only + * + * ===========================================================================*/ +#include "AFLDiffFeedback.hpp" +#include "Iterator.hpp" +#include "Logging.hpp" +#include "RuntimeException.hpp" +#include "StorageEntry.hpp" +#include "StorageRegistry.hpp" +#include "VmfUtil.hpp" +#include "ExecutorModule.hpp" +#include "plog/Log.h" +#include +#include +#include +#include +#include +#include +#include + +using namespace vmf; + +#include "ModuleFactory.hpp" +REGISTER_MODULE(AFLDiffFeedback); + + + +Module* AFLDiffFeedback::build(std::string name) +{ + return new AFLDiffFeedback(name); +} + +void AFLDiffFeedback::init(ConfigInterface& config) +{ + outputDir = config.getOutputDir(); + useCustomWeights = config.getBoolParam(getModuleName(),"useCustomWeights", false); + sizeFitnessWeight = config.getFloatParam(getModuleName(), "sizeWeight", 1.0); + speedFitnessWeight = config.getFloatParam(getModuleName(), "speedWeight", 5.0); + diffFitnessWeight = config.getFloatParam(getModuleName(), "diffWeight", diffFitnessWeight); + expectedSUT = config.getStringParam(getModuleName(), "systemOfTruth"); + + auto execs = ExecutorModule::getExecutorSubmodules(config, + config.getSuperModule(getModuleName())->getModuleName()); + unsigned long numExecutors = execs.size(); + + for(auto const& e : execs) + { + auto n = e->getModuleName(); + execNameTags.emplace(std::make_pair(n, 0)); + if(n == expectedSUT) + isExpectedReal = true; + } + + if(!isExpectedReal) + { + if(numExecutors < 3) + { + throw RuntimeException("Differential Feedback between 2 SUTs must have a reference of correctness. YAML config: systemOfTruth", + RuntimeException::USAGE_ERROR); + } + else + { + LOG_WARNING << "Differential Feedback without a reference of correctness will default " + << "to majority voting, and \"RAN_SUCCESSFULLY\" upon voting failure. Is this your intent?"; + } + } + + + + // module name -> tagID -> pass result w/ tagId as src of truth + + avgExecTimePerExec.resize(numExecutors, 0); + maxExecTimePerExec.resize(numExecutors, 0); + avgTestCaseSizePerExec.resize(numExecutors, 0); + maxTestCaseSizePerExec.resize(numExecutors, 0); + + if(useCustomWeights) + { + if(sizeFitnessWeight < 0.0 || speedFitnessWeight < 0.0) + { + throw RuntimeException("One or more Custom Fitness Weights for feedback is invalid", + RuntimeException::USAGE_ERROR); + } + LOG_INFO << "Fitness weights: speed = " << speedFitnessWeight << ", size = " << sizeFitnessWeight; + } + else + LOG_INFO << "Using AFL++ style Fitness algorithm"; +} + +AFLDiffFeedback::AFLDiffFeedback(std::string name) : + FeedbackModule(name) +{ + avgExecTimePerExec = {}; + maxExecTimePerExec = {}; + avgTestCaseSizePerExec = {}; + maxTestCaseSizePerExec = {}; + numTestCases = 0; + + //These should all be set in config + sizeFitnessWeight = 0; + speedFitnessWeight = 0; + diffFitnessWeight = 10; + useCustomWeights = false; + isExpectedReal = false; + + //These should all be set during registration + testCaseKey = 0; + coverageByteCountKey = 0; + fitnessKey = 0; + hasNewCoverageTag = 0; + execTimeKey = 0; + crashedTag = 0; + hungTag = 0; + expectedSUTTag = 0; + + deviantTag = 0; +} + +AFLDiffFeedback::~AFLDiffFeedback() +{ + +} + +void AFLDiffFeedback::registerStorageNeeds(StorageRegistry& registry) +{ + //Inputs + testCaseKey = registry.registerKey("TEST_CASE", StorageRegistry::BUFFER, StorageRegistry::READ_ONLY); + execTimeKey = registry.registerKey("EXEC_TIME_US", StorageRegistry::UINT, StorageRegistry::READ_ONLY); + coverageByteCountKey = registry.registerKey("COVERAGE_COUNT", StorageRegistry::UINT, StorageRegistry::READ_ONLY); + batchNumIdKey = registry.registerKey("ENTRY_BATCH_NUM", StorageRegistry::U64, StorageRegistry::READ_ONLY); + hasNewCoverageTag = registry.registerTag("HAS_NEW_COVERAGE", StorageRegistry::READ_ONLY); + + crashedTag = registry.registerTag("CRASHED", StorageRegistry::READ_ONLY); + hungTag = registry.registerTag("HUNG", StorageRegistry::READ_ONLY); + deviantTag = registry.registerTag("DEVIATED", StorageRegistry::WRITE_ONLY); + + for(auto const& pair : execNameTags) + { + execNameTags[pair.first] = registry.registerTag(pair.first, StorageRegistry::READ_ONLY); + } + expectedSUTTag = isExpectedReal ? execNameTags[expectedSUT] : expectedSUTTag; + + //Outputs + fitnessKey = registry.registerKey("FITNESS", StorageRegistry::FLOAT, StorageRegistry::WRITE_ONLY); +} + +void AFLDiffFeedback::evaluateTestCaseResults(StorageModule& storage, std::unique_ptr& entries) +{ + LOG_ERROR << "Differrential feedback REQUIRES more than one iterator of Storage Entries. Please use evaluate-DIFF-TestCaseResults"; + throw RuntimeException("Differential Feedback module does not support single-iterator feedback.", + RuntimeException::USAGE_ERROR); +} + +void AFLDiffFeedback::evaluateDiffTestCaseResults(StorageModule& storage, std::vector>& entries) +{ + unsigned long numSUTs = entries.size(); + std::vector entryBatch(numSUTs); + // while( every Iterator still has entries ) + while( std::all_of(entries.begin(), entries.end(), [](std::unique_ptr& x){ return x->hasNext(); })) + { + std::transform(entries.begin(), entries.end(), entryBatch.begin(), [](std::unique_ptr& x){ + return x->getNext(); + }); + + if( !assertSharedBatch(entryBatch) ) + { + LOG_ERROR << "An unexpected usage exception was thrown, but not properly handled. Aborting feedback..."; + break; + } + + std::vector execTimes(numSUTs, 0); + std::transform(entryBatch.begin(), entryBatch.end(), execTimes.begin(), [this](StorageEntry* e){ + return getExecTimeMs(e); + }); + for(size_t i=0; i < maxExecTimePerExec.size(); i++) + { + if(execTimes[i] > maxExecTimePerExec[i]) + { + maxExecTimePerExec[i] = execTimes[i]; + } + } + for(size_t i=0; i < avgExecTimePerExec.size(); i++) + { + avgExecTimePerExec[i] = ((avgExecTimePerExec[i] * numTestCases/numSUTs) + execTimes[i]) + / (static_cast(numTestCases)/numSUTs + 1); + } + + int& tcKeyRef = testCaseKey; + std::vector entrySizes(numSUTs, 0); + std::transform(entryBatch.begin(), entryBatch.end(), entrySizes.begin(), [tcKeyRef](StorageEntry* e){ + return e->getBufferSize(tcKeyRef); + }); + for(size_t i=0; i < maxTestCaseSizePerExec.size(); i++) + { + if(entrySizes[i] > maxTestCaseSizePerExec[i]) + { + maxTestCaseSizePerExec[i] = entrySizes[i]; + } + } + for(size_t i=0; i < avgTestCaseSizePerExec.size(); i++) + { + avgTestCaseSizePerExec[i] = ((avgTestCaseSizePerExec[i] * numTestCases/numSUTs) + entrySizes[i]) / (static_cast(numTestCases)/numSUTs + 1); + } + + numTestCases += numSUTs; + + // Calculate fitness if any of the testcases have new coverage OR their end state differed + int& newCovgRef = hasNewCoverageTag; + end_state expected = safe; + for(auto const& e : entryBatch) + { + if(e->hasTag(expectedSUTTag)) + { + expected = getEndState(e); + } + } + + bool anyHasCovg = std::any_of(entryBatch.begin(), entryBatch.end(),[newCovgRef](StorageEntry* e){ return e->hasTag(newCovgRef); }); + auto const& devs = findEndStateDeviants(entryBatch, expected, anyHasCovg); + if( anyHasCovg || devs.size() > 0) + { + std::vector coverage(numSUTs, 0); + int& covgBCRef = coverageByteCountKey; + std::transform(entryBatch.begin(), entryBatch.end(), coverage.begin(), [covgBCRef](StorageEntry* e){ + return e->getUIntValue(covgBCRef); + }); + + std::vector execFitnesses = computeDiffFitness(entryBatch, coverage, execTimes, entrySizes, devs); + + for(size_t i=0; i < entryBatch.size(); i++) + { + if(execFitnesses[i] > 0) + { + entryBatch[i]->setValue(fitnessKey, execFitnesses[i]); + storage.saveEntry(entryBatch[i]); + } + } + } + } +} + +std::vector AFLDiffFeedback::computeDiffFitness(std::vector& entries, + std::vector& covg, std::vector& execT, std::vector& sizes, + std::vector deviants) +{ + std::vector fits(covg.size(), 1.0); + if(useCustomWeights) + { + std::transform(covg.begin(), covg.end(), fits.begin(), [](unsigned int c){ return (float)log10(c) + 1; }); + float normalizedSpeed; float normalizedSize; + for(size_t j=0; jgetUIntValue(execTimeKey); + unsigned int execTimeMs = 1; + if(execTimeUs > 1000) //This is needed to prevent an execution time of 0ms + { + execTimeMs = execTimeUs / 1000; + } + return execTimeMs; +} + +std::vector AFLDiffFeedback::findEndStateDeviants(std::vector& batch, AFLDiffFeedback::end_state def, bool hasNewCovg) +{ + std::vector ret = {}; + end_state decision = def; + + if(batch.size() > 2) + { + // Boyer-moore majority voting + int votes = 0; + for(size_t i=0; i 2 && ret.size() > batch.size()/2) + { + LOG_WARNING << "Differential Feedback could not determine majority execution result by voting - defaulting to trusted end_state"; + + decision = def; + for(unsigned long i=0; i 0) + { + LOG_DEBUG << "Following SUTs did not match the expected state, " << end_state_str[decision] << ":"; + for(unsigned long& i : ret) + { + batch[i]->addTag(deviantTag); + LOG_DEBUG << '\t' << getExecName(batch[i]) << ": " << end_state_str[getEndState(batch[i])]; + } + + + // write out file iff one of the states had new coverage + if(hasNewCovg) + { + char* buffer = batch[0]->getBufferPointer(testCaseKey); + int size = batch[0]->getBufferSize(testCaseKey); + std::string filename = std::to_string(batch[0]->getID()); + std::string deviant_dir = outputDir+"/testcases/deviants/"; + for(auto const& e : batch) + { + deviant_dir = deviant_dir+getExecName(e)+"_"+end_state_str[getEndState(e)]+"/"; + } + + // create a file name with id + LOG_DEBUG << "\tthe input buffer, ID " << filename << " will be written to disk."; + VmfUtil::createDirectory(deviant_dir.c_str()); + VmfUtil::writeBufferToFile(deviant_dir, filename, buffer, size); + } + } + + return ret; +} + +AFLDiffFeedback::end_state AFLDiffFeedback::getEndState(StorageEntry* e) +{ + if(e->hasTag(crashedTag)) + return crash; + else if(e->hasTag(hungTag)) + return hang; + else + return safe; +} + +std::string AFLDiffFeedback::getExecName(StorageEntry* e) +{ + for(auto const& pair : execNameTags) + { + if(e->hasTag(pair.second)) + return pair.first; + } + throw RuntimeException("Uh oh. How did this entry make it through without a Executor Name Tag?", + RuntimeException::UNEXPECTED_ERROR); + return ""; +} + +bool AFLDiffFeedback::assertSharedBatch(std::vector& batch) +{ + unsigned long long batch_num = batch[0]->getU64Value(batchNumIdKey); + int cpy = batchNumIdKey; + if(std::all_of(batch.begin(), batch.end(), [&batch_num, &cpy](StorageEntry* e){ return (e->getU64Value(cpy) == batch_num); })) + { + return true; + } + else + { + std::cout << "Entry IDs: "; + for(auto const& e : batch){ std::cout << e->getID() << " "; } + std::cout << "\n"; + throw RuntimeException("[ FATAL ] Feedback pulled a batch from the StorageIterator whose IDs did not all match.", + RuntimeException::UNEXPECTED_ERROR); + } +} diff --git a/vmf/src/modules/common/feedback/AFLDiffFeedback.hpp b/vmf/src/modules/common/feedback/AFLDiffFeedback.hpp new file mode 100644 index 0000000..2b45df5 --- /dev/null +++ b/vmf/src/modules/common/feedback/AFLDiffFeedback.hpp @@ -0,0 +1,184 @@ +/* ============================================================================= + * Vader Modular Fuzzer (VMF) + * Copyright (c) 2021-2025 The Charles Stark Draper Laboratory, Inc. + * + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 (only) as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * @license GPL-2.0-only + * + * ===========================================================================*/ + +#pragma once + +#include "FeedbackModule.hpp" +#include "StorageRegistry.hpp" +#include +#include + +namespace vmf +{ + +/** + * @brief FeedbackModule to examine results from a differential campaign of AFLForkserverExecutors. + * AFLDiffFeedback requires as inputs the TEST_CASE buffer as well as some of the + * execution results. The module outputs a FITNESS value in storage. + */ +class AFLDiffFeedback : public FeedbackModule { +public: + /** + * @brief Builder method to support the ModuleFactory + * Constructs an instance of this class + * @return Module* + */ + static Module* build(std::string name); + + /** + * @brief Initialization method + * Reads in all configuration options for this class, including the given Executor names. + * + * @param config + */ + virtual void init(ConfigInterface& config); + + /** + * @brief Notify Storage module of necessary data needs + * Create output keys and copies of essential input keys and tags. + * + * @param registry + */ + virtual void registerStorageNeeds(StorageRegistry& registry); + + /** + * @warning This module does not support feedback on single testcase results. + */ + virtual void evaluateTestCaseResults(StorageModule& storage, std::unique_ptr& entries); + + /** + * @brief Evaluate the test case results from all executors + * This method: + * 1) computes and saves the fitness to the storage entry + * 2) saves any other values of interest to the storage entry, including tagging the entry if relevant + * 3) determines if the test case shared across the executors is interesting enough to save in + * long term storage (and save the entry if it is) + * + * @param storage: storage module + * @param entries: a map from each executor to respective entries + */ + virtual void evaluateDiffTestCaseResults(StorageModule& storage, std::vector>& entries); + + /** + * @brief Construct a new AFLDiffFeedback object + * + * @param name the module name + */ + AFLDiffFeedback(std::string name); + + virtual ~AFLDiffFeedback(); +protected: + enum end_state {crash, hang, safe}; + const std::string end_state_str[3] = {"CRASH", "HANG", "SUCCESS"}; + + /** + * @brief Computes the fitness for the provided differential test cases + * + * @param storage the storage module + * @param entries the test cases that were just executed + * @param covg testcase coverage + * @param execT testcase execution time + * @param deviants testcases that exhibited differential behavior + * + * @return float the fitness + */ + virtual std::vector computeDiffFitness(std::vector& entries, + std::vector& covg, std::vector& execT, std::vector& sizes, + std::vector deviants); + + /** + * @brief Helper method to convert microsecond execution time to milliseconds + * This method is used to ensure consistency with the AFL++ fitness algorithm, + * which uses millisecond time precision. The minimum returned execution time + * from this method is 1ms. + * + * @param e the test case to examine + * @return unsigned int the execution time in milliseconds + */ + unsigned int getExecTimeMs(StorageEntry* e); + + /** + * @brief Helper method to determine which entries, if any, resulted DIFFERENTLY from the + * voted on output. + * + * @param batch one test case per executor + * @param def default value in case of voting failure, aka end state of trusted executor + * @return vector representing the index of deviant entries in the batch + */ + std::vector findEndStateDeviants(std::vector& batch, AFLDiffFeedback::end_state def, bool anyHasCovg); + + /** + * @brief Helper method to promote DRY lookups of an entry's endState tag + * + * @param e StorageEntry to lookup + * @return end_state enum of CRASH, HUNG, or SAFE (ran successfully) + */ + AFLDiffFeedback::end_state getEndState(StorageEntry* e); + + /** + * @brief Helper method to find an entry's corresponding ExecutorModule name + * + * @param e StorageEntry to lookup + * @return ExecutorModule name + */ + std::string getExecName(StorageEntry* e); + + /** + * @brief Helper method to ensure that only Entries from the same input buffer are compared + * Compares Entry batch ID numbers to ensure that differential end states originate from + * a shared input. + * + * @param batch comparable storage entries + * @return if entries are from the same batch + * @throws RuntimeException: StorageIterator returned testcases from the same round that did not + * have matching inputs + */ + bool assertSharedBatch(std::vector& batch); + +protected: + std::string outputDir; ///< Location of output directory + std::string expectedSUT; ///< Name of reference SUT for end-state source of truth + + int testCaseKey; ///< Handle for the "TEST_CASE" field + int execTimeKey; ///< Handle for the "EXEC_TIME_US" field + int coverageByteCountKey; ///< Handle for the "COVERAGE_COUNT" field + int batchNumIdKey; ///< Handle for the "ENTRY_BATCH_NUM" field + int fitnessKey; ///< Handle for the "FITNESS" field + int hasNewCoverageTag; ///< Handle for the "HAS_NEW_COVERAGE" tag + int crashedTag; ///< Testcase crashed + int hungTag; ///< Testcase did not exit + int expectedSUTTag; ///< Designated source of truth + int deviantTag; ///< Testcase did not exit with expected code + std::map execNameTags; + + std::vector avgExecTimePerExec; ///< The average execution time per executor (for test cases that have been eval'd) + std::vector maxExecTimePerExec; + std::vector avgTestCaseSizePerExec; + std::vector maxTestCaseSizePerExec; + float sizeFitnessWeight; ///< A configurable weight to apply to the size factor in computing fitness. Must be >=0.0 + float speedFitnessWeight; ///< A configurable weight to apply to the speed factor in computing fitness. Must be >=0.0 + float diffFitnessWeight; ///< A configurable weight to apply to the differential factor in computing fitness. Must be >=0.0 + int numTestCases; ///< The total number of test cases that have been evaluated + + bool useCustomWeights; ///< Whether or not custom weights are enabled + bool isExpectedReal; ///< Whether or not the user provided a valid reference SUT in the config +}; +} \ No newline at end of file diff --git a/vmf/src/modules/common/feedback/AFLFeedback.cpp b/vmf/src/modules/common/feedback/AFLFeedback.cpp index 5c6bf0b..c8f61e6 100644 --- a/vmf/src/modules/common/feedback/AFLFeedback.cpp +++ b/vmf/src/modules/common/feedback/AFLFeedback.cpp @@ -103,7 +103,8 @@ void AFLFeedback::registerStorageNeeds(StorageRegistry& registry) fitnessKey = registry.registerKey("FITNESS", StorageRegistry::FLOAT, StorageRegistry::WRITE_ONLY); } - +void AFLFeedback::evaluateDiffTestCaseResults(StorageModule& storage, std::vector>& entries) +{ LOG_ERROR << "AFLFeedback does not support differential fuzzing feedback."; return; } void AFLFeedback::evaluateTestCaseResults(StorageModule& storage, std::unique_ptr& entries) { diff --git a/vmf/src/modules/common/feedback/AFLFeedback.hpp b/vmf/src/modules/common/feedback/AFLFeedback.hpp index 7491380..7dd90df 100644 --- a/vmf/src/modules/common/feedback/AFLFeedback.hpp +++ b/vmf/src/modules/common/feedback/AFLFeedback.hpp @@ -39,7 +39,13 @@ class AFLFeedback : public FeedbackModule { virtual void init(ConfigInterface& config); virtual void registerStorageNeeds(StorageRegistry& registry); - virtual void evaluateTestCaseResults(StorageModule& storage, std::unique_ptr& entries); + virtual void evaluateTestCaseResults(StorageModule& storage, std::unique_ptr& entries); + + /** + * @warning This function does nothing. + * AFLFeedback does not support fitness feedback on Differential testcases / executors + */ + virtual void evaluateDiffTestCaseResults(StorageModule& storage, std::vector>& entries); /** * @brief Construct a new AFLFeedback object diff --git a/vmf/src/modules/common/inputgeneration/DiffInputGenerator.cpp b/vmf/src/modules/common/inputgeneration/DiffInputGenerator.cpp new file mode 100644 index 0000000..0c9750c --- /dev/null +++ b/vmf/src/modules/common/inputgeneration/DiffInputGenerator.cpp @@ -0,0 +1,198 @@ +/* ============================================================================= + * Vader Modular Fuzzer (VMF) + * Copyright (c) 2021-2025 The Charles Stark Draper Laboratory, Inc. + * + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 (only) as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * @license GPL-2.0-only + * ===========================================================================*/ +#include "DiffInputGenerator.hpp" +#include "Logging.hpp" +#include "RuntimeException.hpp" +#include "StorageRegistry.hpp" +#include "VmfUtil.hpp" + +using namespace vmf; + +#include "ModuleFactory.hpp" +REGISTER_MODULE(DiffInputGenerator); + +Module* DiffInputGenerator::build(std::string name) +{ + return new DiffInputGenerator(name); +} + +void DiffInputGenerator::init(ConfigInterface& config) +{ + mutators = MutatorModule::getMutatorSubmodules(config,getModuleName()); + int size = (int)mutators.size(); + if(0 == size) + { + throw RuntimeException("DiffInputGenerator must be configured with at least one child mutator", + RuntimeException::CONFIGURATION_ERROR); + } + + for(int i=0; igetModuleName()); + + int numSwarms = config.getIntParam(getModuleName(), "numSwarms", 5); + int pilotPeriod = config.getIntParam(getModuleName(), "pilotPeriodLength", 50000); + int corePeriod = config.getIntParam(getModuleName(), "corePeriodLength", 500000); + double pMin = config.getFloatParam(getModuleName(), "pMin", 0); + + testCasesRan = 0; + batchNum = 0; + + // Create a new MOPT object. We must provide it with our mutators and the desired + // number of swarms and period lengths. + LOG_INFO << "MOPT swarms: " << numSwarms; + LOG_INFO << "pilotPeriodLength: " << pilotPeriod; + LOG_INFO << "corePeriodLength: " << corePeriod; + LOG_INFO << "pMin: " << pMin; + + mopt = new MOPT(&mutators, numSwarms, pilotPeriod, corePeriod, pMin); + batchNumIdKey = 0; +} + +DiffInputGenerator::DiffInputGenerator(std::string name) : + InputGeneratorModule(name) +{ + +} + + +DiffInputGenerator::~DiffInputGenerator() +{ + delete mopt; +} + + +void DiffInputGenerator::registerStorageNeeds(StorageRegistry& registry) +{ + normalTag = registry.registerTag("RAN_SUCCESSFULLY", StorageRegistry::READ_ONLY); + + for(auto const& e : executors) + { + executorIdTags.emplace_back(registry.registerTag(e->getModuleName(), StorageRegistry::WRITE_ONLY)); + } + + moptMutatorIdKey = registry.registerIntKey("MOPT_MUTATOR_ID", StorageRegistry::READ_WRITE, -1); + mutatorIdKey = registry.registerIntKey("MUTATOR_ID", StorageRegistry::WRITE_ONLY, 1); + testCaseKey = registry.registerKey("TEST_CASE", StorageRegistry::BUFFER, StorageRegistry::READ_WRITE); + batchNumIdKey = registry.registerU64Key("ENTRY_BATCH_NUM", StorageRegistry::WRITE_ONLY, 0); +} + + +void DiffInputGenerator::addNewTestCases(StorageModule& storage) +{ + + StorageEntry* baseTestCase = selectBaseEntry(storage); + + if(nullptr != baseTestCase) + { + // Generate N testcases per call to AddNewTestCases() + for(size_t i=0; i< 32; i++) + { + int pickedMutator = mopt -> pickMutator(); + MutatorModule* mutator = mutators[pickedMutator]; + StorageEntry* commonEntry = storage.createNewEntry(); + mutator->mutateTestCase(storage, baseTestCase, commonEntry, testCaseKey); + + // Copy the old buffer before modifying it for safety + char* oldBuff = commonEntry->getBufferPointer(testCaseKey); + int newSize = commonEntry->getBufferSize(testCaseKey); + + if(executorIdTags.size() > 1) + { + for(size_t i=1; iallocateBuffer(testCaseKey, newSize); + + memcpy((void*)newBuff, (void*)oldBuff, newSize); + + newEntry->setValue(moptMutatorIdKey, pickedMutator + 1); // Id is the index into the mutators vector plus 1 + newEntry->setValue(mutatorIdKey, mutator->getID()); + newEntry->setValue(batchNumIdKey, batchNum); + newEntry->addTag(tag); + mopt->updateExecCount(pickedMutator); + testCasesRan++; + } + } + // Edit the commonEntry last, based on a gut feeling + commonEntry->setValue(moptMutatorIdKey, pickedMutator + 1); + commonEntry->setValue(mutatorIdKey, mutator->getID()); + commonEntry->setValue(batchNumIdKey, batchNum); + commonEntry->addTag(executorIdTags[0]); + mopt->updateExecCount(pickedMutator); + testCasesRan++; + batchNum++; + } + } +} + +bool DiffInputGenerator::examineTestCaseResults(StorageModule& storage) +{ + + std::unique_ptr interestingEntries = storage.getNewEntriesThatWillBeSaved(); + + while(interestingEntries->hasNext()) + { + StorageEntry* entry = interestingEntries->getNext(); + int id = entry->getIntValue(mutatorIdKey); + if(id > 0 && id <= (int)mutatorStats.size()) + { + int mutator = id - 1; + mopt->updateFindingsCount(mutator); + } + } + + mopt -> ranTestCases(testCasesRan, true); + testCasesRan = 0; + + return false; //This input generator is never "complete" +} + + +StorageEntry* DiffInputGenerator::selectBaseEntry(StorageModule& storage) +{ + StorageEntry* baseTestCase = nullptr; + std::unique_ptr entries; + + //Use only the entries in the corpus that ran normally + entries = storage.getSavedEntriesByTag(normalTag); + + int maxIndex = entries->getSize(); + if(0 == maxIndex) { + //This should only occur on the first run. It either indicates that we are not receiving feedback + //from the executor, causing it to never flag any entries to be saved (and tagged as "RAN_SUCCESSFULLY"), + //or VMF was configured without a seed generator, so there are no initial test cases to run. + throw RuntimeException("No executed test cases in storage. Either something is wrong with the executor feedback, or there is no seed generator.", + RuntimeException::USAGE_ERROR); + } + + int randIndex = VmfUtil::selectWeightedRandomValue(0, maxIndex); + baseTestCase = entries->setIndexTo(randIndex); + return baseTestCase; +} + diff --git a/vmf/src/modules/common/inputgeneration/DiffInputGenerator.hpp b/vmf/src/modules/common/inputgeneration/DiffInputGenerator.hpp new file mode 100644 index 0000000..1ee9696 --- /dev/null +++ b/vmf/src/modules/common/inputgeneration/DiffInputGenerator.hpp @@ -0,0 +1,111 @@ +/* ============================================================================= + * Vader Modular Fuzzer (VMF) + * Copyright (c) 2021-2025 The Charles Stark Draper Laboratory, Inc. + * + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 (only) as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * @license GPL-2.0-only + * ===========================================================================*/ +#pragma once + +#include "InputGeneratorModule.hpp" +#include "MOPT.hpp" +#include "ExecutorModule.hpp" + +namespace vmf +{ + +/** + * @brief This InputGeneratorModule is an optimized mutator selection approach that is based on the MOpt algorithm. + * + * See https://www.usenix.org/system/files/sec19-lyu.pdf + * + * This module uses the RAN_SUCCESSFULLY tag to select only test cases with a normal execution + * pattern as the basis of mutation. It uses MUTATOR_ID to track which MutatorModule submodule + * was used to create each TEST_CASE, and adjusts how frequently it uses each mutator based on + * the observed performance of the resulting test cases. + * @image html CoreModuleDataModel_4.png width=800px + * @image latex CoreModuleDataModel_4.png width=6in + */ +class DiffInputGenerator: public InputGeneratorModule +{ +public: + /** + * @brief Builder method to support the ModuleFactory + * Constructs an instance of this class + * @return Module* + */ + static Module* build(std::string name); + + /** + * @brief Initialization method + * Reads in all configuration options for this class + * + * @param config + */ + virtual void init(ConfigInterface& config); + + /** + * @brief Notify Storage module of necessary data needs + * Create output keys and copies of essential input keys and tags. + * + * @param registry + */ + virtual void registerStorageNeeds(StorageRegistry& registry); + + /** + * @brief Generate new testcases and copy them for each testcase + * + * @param storage + */ + virtual void addNewTestCases(StorageModule& storage); + + virtual bool examineTestCaseResults(StorageModule& storage); + + /** + * @brief Construct a new Genetic Algorithm Input Generator module + * + * @param name the name of the module + */ + DiffInputGenerator(std::string name); + virtual ~DiffInputGenerator(); +private: + + /** + * @brief Helper method to select the base entry to mutate + * + * This implementation uses a weighted random selection that favors entries with lower indices + * + * @param storage the storage module + * @return StorageEntry* the base entry to use + */ + StorageEntry* selectBaseEntry(StorageModule& storage); + + MOPT* mopt; + unsigned int testCasesRan; ///< Number of test cases run across all executors + unsigned long long batchNum; ///< Batch number for a uniquely generated test case + int moptMutatorIdKey; + int mutatorIdKey; + std::vector executorIdTags; ///< ID Tags for Executor modules + int normalTag; ///< Test case RAN_SUCCESSFULLY + int testCaseKey; ///< Input buffer key for StorageEntry + int batchNumIdKey; ///< Batch number key for StorageEntry + + std::vector mutators; ///< The list of mutators being managed by this input generator + std::vector executors; ///< The list of executors present in this fuzzing campaign + + std::vector mutatorStats; + std::vector mutatorStatsTotalTestCases; +}; +} diff --git a/vmf/src/modules/common/output/ComputeDiffStats.cpp b/vmf/src/modules/common/output/ComputeDiffStats.cpp new file mode 100644 index 0000000..430eff3 --- /dev/null +++ b/vmf/src/modules/common/output/ComputeDiffStats.cpp @@ -0,0 +1,285 @@ +/* ============================================================================= + * Vader Modular Fuzzer (VMF) + * Copyright (c) 2021-2025 The Charles Stark Draper Laboratory, Inc. + * + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 (only) as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * @license GPL-2.0-only + * ===========================================================================*/ +#include "ComputeDiffStats.hpp" +#include "StorageRegistry.hpp" +#include "plog/Log.h" + +using namespace vmf; + +#include "ModuleFactory.hpp" +REGISTER_MODULE(ComputeDiffStats); + +Module* ComputeDiffStats::build(std::string name) +{ + return new ComputeDiffStats(name); +} + +void ComputeDiffStats::init(ConfigInterface& config) +{ + outputRate = config.getIntParam(getModuleName(),"statsRateInSeconds", 1); + + executors = ExecutorModule::getExecutorSubmodules( + config, config.getSuperModule(getModuleName())->getModuleName()); + for(auto const& e : executors) + { + executorNames.emplace_back(e->getModuleName()); + } + + executorAllTestCases.resize(executors.size()); + executorAllCrashes.resize(executors.size()); + executorAllHangs.resize(executors.size()); + executorAllDiffs.resize(executors.size()); + + executorUQTestCases.resize(executors.size()); + executorUQCrashes.resize(executors.size()); + executorUQHangs.resize(executors.size()); + executorUQDiffs.resize(executors.size()); + + executorAverageCPs.resize(executors.size()); + executorLatestCPs.resize(executors.size()); + executorPrevTCTotal.resize(executors.size()); + executorLastFindTS.resize(executors.size()); + executorStaleDuration.resize(executors.size()); +} + + +ComputeDiffStats::ComputeDiffStats(std::string name) : + OutputModule(name) +{ + // Output rate variables + outputRate = 0; + timeLastComputedStats = time(0); + + // Statistics varaibles + executorAllTestCases = {}; // vector variables will have to be re-sized during init + executorAllCrashes = {}; + executorAllHangs = {}; + executorAllDiffs = {}; + + executorUQTestCases = {}; + executorUQCrashes = {}; + executorUQHangs = {}; + executorUQDiffs = {}; + + total_time = 0; + executorAverageCPs = {}; + executorLatestCPs = {}; + executorPrevTCTotal = {}; + + executorLastFindTS = {}; + executorStaleDuration = {}; + + // Storage(-related) Tags + hungTag = 0; + crashedTag = 0; + deviantTag = 0; + executorNames = {}; + executorTagIDs = {}; + + // Single Output Keys + grandUQTotalMetadataKey = 0; + grandUQCrashedMetadataKey = 0; + grandUQHungMetadataKey = 0; + grandUQDiffMetadataKey = 0; + + grandTotalMetadataKey = 0; + grandCrashedMetadataKey = 0; + grandHungMetadataKey = 0; + grandDiffMetadataKey = 0; + + // Per-executor Keylists + executorAllTCMetadataKeys = {}; + executorAllCrashMetadataKeys = {}; + executorAllHungMetadataKeys = {}; + executorAllDiffMetadataKeys = {}; + + executorUQTCMetadataKeys = {}; + executorUQCrashMetadataKeys = {}; + executorUQHungMetadataKeys = {}; + executorUQDiffMetadataKeys = {}; + + executorAverageEPSMetadataKeys = {}; + executorLatestEPSMetadataKeys = {}; + executorStaleDurationMetadataKeys = {}; +} + +ComputeDiffStats::~ComputeDiffStats() +{ + +} + +void ComputeDiffStats::registerStorageNeeds(StorageRegistry& registry) +{ + crashedTag = registry.registerTag("CRASHED", StorageRegistry::READ_ONLY); + hungTag = registry.registerTag("HUNG", StorageRegistry::READ_ONLY); + deviantTag = registry.registerTag("DEVIATED", StorageRegistry::READ_ONLY); + + // input executor data tag keys + for(std::string& e : executorNames) + { + executorTagIDs.emplace_back(registry.registerTag(e, StorageRegistry::READ_ONLY)); + } +} + +void ComputeDiffStats::registerMetadataNeeds(StorageRegistry& registry) +{ + // Single Output Keys + grandUQTotalMetadataKey = registry.registerKey("GRAND_UQ_TEST_CASES", StorageRegistry::UINT, StorageRegistry::WRITE_ONLY); + grandUQCrashedMetadataKey = registry.registerKey("GRAND_UQ_CRASHED_CASES", StorageRegistry::UINT, StorageRegistry::WRITE_ONLY); + grandUQHungMetadataKey = registry.registerKey("GRAND_UQ_HUNG_CASES", StorageRegistry::UINT, StorageRegistry::WRITE_ONLY); + grandUQDiffMetadataKey = registry.registerKey("GRAND_UQ_DIFF_CASES", StorageRegistry::UINT, StorageRegistry::WRITE_ONLY); + + grandTotalMetadataKey = registry.registerKey("GRAND_TEST_CASES", StorageRegistry::U64, StorageRegistry::WRITE_ONLY); + grandCrashedMetadataKey = registry.registerKey("GRAND_CRASHED_CASES", StorageRegistry::UINT, StorageRegistry::WRITE_ONLY); + grandHungMetadataKey = registry.registerKey("GRAND_HUNG_CASES", StorageRegistry::UINT, StorageRegistry::WRITE_ONLY); + grandDiffMetadataKey = registry.registerKey("GRAND_DIFF_CASES", StorageRegistry::UINT, StorageRegistry::WRITE_ONLY); + + // Per-executor Output KeyLists + for(std::string& e : executorNames) + { + executorAllTCMetadataKeys.emplace_back(registry.registerKey( + "TOTAL_TEST_CASES_" + e, StorageRegistry::U64, StorageRegistry::WRITE_ONLY)); + executorAllCrashMetadataKeys.emplace_back(registry.registerKey( + "TOTAL_CRASHED_CASES_" + e, StorageRegistry::UINT, StorageRegistry::WRITE_ONLY)); + executorAllHungMetadataKeys.emplace_back(registry.registerKey( + "TOTAL_HUNG_CASES_" + e, StorageRegistry::UINT, StorageRegistry::WRITE_ONLY)); + executorAllDiffMetadataKeys.emplace_back(registry.registerKey( + "TOTAL_DIFF_CASES_" + e, StorageRegistry::UINT, StorageRegistry::WRITE_ONLY)); + + executorUQTCMetadataKeys.emplace_back(registry.registerKey( + "UQ_TEST_CASES_" + e, StorageRegistry::UINT, StorageRegistry::WRITE_ONLY)); + executorUQCrashMetadataKeys.emplace_back(registry.registerKey( + "UQ_CRASHED_CASES_" + e, StorageRegistry::UINT, StorageRegistry::WRITE_ONLY)); + executorUQHungMetadataKeys.emplace_back(registry.registerKey( + "UQ_HUNG_CASES_" + e, StorageRegistry::UINT, StorageRegistry::WRITE_ONLY)); + executorUQDiffMetadataKeys.emplace_back(registry.registerKey( + "UQ_DIFF_CASES_" + e, StorageRegistry::UINT, StorageRegistry::WRITE_ONLY)); + + executorAverageEPSMetadataKeys.emplace_back(registry.registerKey( + "AVERAGE_EPS_" + e, StorageRegistry::FLOAT, StorageRegistry::WRITE_ONLY)); + executorLatestEPSMetadataKeys.emplace_back(registry.registerKey( + "LATEST_EPS_" + e, StorageRegistry::FLOAT, StorageRegistry::WRITE_ONLY)); + executorStaleDurationMetadataKeys.emplace_back(registry.registerKey( + "DUR_LAST_FIND_" + e, StorageRegistry::FLOAT, StorageRegistry::WRITE_ONLY)); + } +} + + +void ComputeDiffStats::run(StorageModule& storage) +{ + StorageEntry& metadata = storage.getMetadata(); + + //These statistics have to be counted on every pass through the fuzzing loop + //because they require examining the newEntries (which change each time) + unsigned long long grandTotalTests = 0; + unsigned int grandTotalCrashes = 0; + unsigned int grandTotalHangs = 0; + unsigned int grandTotalDeviants = 0; + + for(size_t i=0; igetSize(); + executorAllCrashes[i] += storage.getNewEntriesByIntersection(executorTagIDs[i], crashedTag)->getSize(); + executorAllHangs[i] += storage.getNewEntriesByIntersection(executorTagIDs[i], hungTag)->getSize(); + executorAllDiffs[i] += storage.getNewEntriesByIntersection(executorTagIDs[i], deviantTag)->getSize(); + + metadata.setValue(executorAllTCMetadataKeys[i], executorAllTestCases[i]); + metadata.setValue(executorAllCrashMetadataKeys[i], executorAllCrashes[i]); + metadata.setValue(executorAllHungMetadataKeys[i], executorAllHangs[i]); + metadata.setValue(executorAllDiffMetadataKeys[i], executorAllDiffs[i]); + + grandTotalTests += executorAllTestCases[i]; + grandTotalCrashes += executorAllCrashes[i]; + grandTotalHangs += executorAllHangs[i]; + grandTotalDeviants += executorAllDiffs[i]; + } + metadata.setValue(grandTotalMetadataKey, grandTotalTests); + metadata.setValue(grandCrashedMetadataKey, grandTotalCrashes); + metadata.setValue(grandHungMetadataKey, grandTotalHangs); + metadata.setValue(grandDiffMetadataKey, grandTotalDeviants); + + //These statistics are computed at the configured rate + time_t now = time(0); + double elapsed = difftime(now, timeLastComputedStats); + if(elapsed > outputRate) + { + timeLastComputedStats = now; + total_time += elapsed; + // Reset the grandTotalUQ fields to be re-totaled later + unsigned int grandUQTotalTests = 0; + unsigned int grandUQTotalCrashes = 0; + unsigned int grandUQTotalHangs = 0; + unsigned int grandUQTotalDiffs = 0; + + // Compute unique statistics by executors + unsigned int newUniqueTotal = 0; + for(size_t i=0; igetSize(); + now = time(0); + if(newUniqueTotal <= executorUQTestCases[i]) + { + // Nothing new; update how long we've waited for this executor to find someting new + executorStaleDuration[i] = (float)difftime(now, executorLastFindTS[i]); + } + else + { + // Found something new; reset StaleDuration and set LastFindTS + executorLastFindTS[i] = now; + executorStaleDuration[i] = 0.0; + } + // TOTAL UQ for CASES, CRASHES, and HANGS per executor: + executorUQTestCases[i] = newUniqueTotal; + executorUQCrashes[i] = storage.getSavedEntriesByIntersection(crashedTag, executorTagIDs[i])->getSize(); + executorUQHangs[i] = storage.getSavedEntriesByIntersection(hungTag, executorTagIDs[i])->getSize(); + executorUQDiffs[i] = storage.getSavedEntriesByIntersection(deviantTag, executorTagIDs[i])->getSize(); + + // GRAND TOTAL UQ for CASES, CRASHES, and HANGS + grandUQTotalTests += executorUQTestCases[i]; + grandUQTotalCrashes += executorUQCrashes[i]; + grandUQTotalHangs += executorUQHangs[i]; + grandUQTotalDiffs += executorUQDiffs[i]; + + // SUT EXEC/SEC per executor: + executorAverageCPs[i] = static_cast(executorAllTestCases[i] / total_time); + executorLatestCPs[i] = static_cast( + (executorAllTestCases[i] - executorPrevTCTotal[i]) / elapsed); + executorPrevTCTotal[i] = executorAllTestCases[i]; + + // Write Unique cases for this executor to metadata + metadata.setValue(executorUQTCMetadataKeys[i], executorUQTestCases[i]); + metadata.setValue(executorUQCrashMetadataKeys[i], executorUQCrashes[i]); + metadata.setValue(executorUQHungMetadataKeys[i], executorUQHangs[i]); + metadata.setValue(executorUQDiffMetadataKeys[i], executorUQDiffs[i]); + + // Write Timing data for this executor to metadata + metadata.setValue(executorLatestEPSMetadataKeys[i], executorLatestCPs[i]); + metadata.setValue(executorAverageEPSMetadataKeys[i], executorAverageCPs[i]); + metadata.setValue(executorStaleDurationMetadataKeys[i], executorStaleDuration[i]); + } + + // Write Grand Total number of Unique cases to metadata + metadata.setValue(grandUQTotalMetadataKey, grandUQTotalTests); + metadata.setValue(grandUQCrashedMetadataKey, grandUQTotalCrashes); + metadata.setValue(grandUQHungMetadataKey, grandUQTotalHangs); + metadata.setValue(grandUQDiffMetadataKey, grandUQTotalDiffs); + } +} \ No newline at end of file diff --git a/vmf/src/modules/common/output/ComputeDiffStats.hpp b/vmf/src/modules/common/output/ComputeDiffStats.hpp new file mode 100644 index 0000000..42be0bb --- /dev/null +++ b/vmf/src/modules/common/output/ComputeDiffStats.hpp @@ -0,0 +1,146 @@ + +/* ============================================================================= + * Vader Modular Fuzzer (VMF) + * Copyright (c) 2021-2025 The Charles Stark Draper Laboratory, Inc. + * + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 (only) as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * @license GPL-2.0-only + * ===========================================================================*/ +#pragma once + + +#include "OutputModule.hpp" +#include "ExecutorModule.hpp" + +namespace vmf +{ +/** + * @brief OutputModule that computes execution statistics and publishes them + * to metadata. + * A number of fields are written, for usage by other modules (such as StatsDiffOutput). + * @image html CoreModuleDataModel_6.png width=800px + * @image latex CoreModuleDataModel_6.png width=6in + */ +class ComputeDiffStats : public OutputModule { +public: + /** + * @brief Builder method to support the ModuleFactory + * Constructs an instance of this class + * @return Module* + */ + static Module* build(std::string name); + + /** + * @brief Initialization method + * Reads in all configuration options for this class + * + * @param config + */ + virtual void init(ConfigInterface& config); + + /** + * @brief Notify Storage module of necessary data needs + * Create copies of essential input tags. + * + * @param registry + */ + virtual void registerStorageNeeds(StorageRegistry& registry); + + /** + * @brief Register metadata about group and individual fuzzing statistics + * Register campaign-wide statistics as _Grand_ metadata statistics and per-executor metadata + * under their respective metadata type and ExecutorName. + * + * @param registry + */ + virtual void registerMetadataNeeds(StorageRegistry& registry); + + /** + * @brief Compute rolling statistics about each ExecutorModule and the campaign as a whole + */ + virtual void run(StorageModule& storage); + + /** + * @brief Construct a new StatsOutput object for Differential Fuzzing + * + * @param name the name of the module + */ + ComputeDiffStats(std::string name); + virtual ~ComputeDiffStats(); +private: + // control variables + int outputRate; ///< Elapsed time before entire statistics are reset + time_t timeLastComputedStats; ///< Timestamp for accurate average calculation + + // Groupings of statistics variables per executor + std::vectorexecutorAllTestCases; + std::vectorexecutorAllCrashes; + std::vectorexecutorAllHangs; + std::vectorexecutorAllDiffs; + + std::vectorexecutorUQTestCases; + std::vectorexecutorUQCrashes; + std::vectorexecutorUQHangs; + std::vectorexecutorUQDiffs; + + double total_time; ///< Elapsed time of the campaign + // Rolling average of... + std::vectorexecutorAverageCPs; ///< Executions Per Second + std::vectorexecutorLatestCPs; + std::vectorexecutorPrevTCTotal; ///< Total Testcases + std::vectorexecutorLastFindTS; ///< Time since last finding + std::vectorexecutorStaleDuration; + + + // storage tags + int hungTag; ///< SUT hung + int crashedTag; ///< SUT crashed + int deviantTag; ///< SUTs resulted in differential behavior + std::vector executorTagIDs; ///< Executor IDs + // Differential Executor variables for registration + operation + std::vector executorNames; + std::vector executors; + + + // Campaign-wide metadata keys + int grandUQTotalMetadataKey; + int grandUQCrashedMetadataKey; + int grandUQHungMetadataKey; + int grandUQDiffMetadataKey; + + int grandTotalMetadataKey; + int grandCrashedMetadataKey; + int grandHungMetadataKey; + int grandDiffMetadataKey; + + // Groupings for per-executor metadata keys + // total (end state) + std::vector executorAllTCMetadataKeys; + std::vector executorAllCrashMetadataKeys; + std::vector executorAllHungMetadataKeys; + std::vector executorAllDiffMetadataKeys; + + // number of (end state) with new coverage + std::vector executorUQTCMetadataKeys; + std::vector executorUQCrashMetadataKeys; + std::vector executorUQHungMetadataKeys; + std::vector executorUQDiffMetadataKeys; + + // timing statistics + std::vector executorAverageEPSMetadataKeys; + std::vector executorLatestEPSMetadataKeys; + std::vector executorStaleDurationMetadataKeys; +}; +} \ No newline at end of file diff --git a/vmf/src/modules/common/output/StatsDiffOutput.cpp b/vmf/src/modules/common/output/StatsDiffOutput.cpp new file mode 100644 index 0000000..e74b7a5 --- /dev/null +++ b/vmf/src/modules/common/output/StatsDiffOutput.cpp @@ -0,0 +1,265 @@ +/* ============================================================================= + * Vader Modular Fuzzer (VMF) + * Copyright (c) 2021-2025 The Charles Stark Draper Laboratory, Inc. + * + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 (only) as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * @license GPL-2.0-only + * ===========================================================================*/ +#include +#include +#include +#include "StatsDiffOutput.hpp" +#include "Logging.hpp" +#include "ExecutorModule.hpp" + +using namespace vmf; +#define MAGIC_SPACE 6 + +#include "ModuleFactory.hpp" +REGISTER_MODULE(StatsDiffOutput); + +/** + * @brief Builder method to support the ModuleFactory + * Constructs an instance of this class + * @return Module* + */ +Module* StatsDiffOutput::build(std::string name) +{ + return new StatsDiffOutput(name); +} + +/** + * @brief Initialization method + * Reads in all configuration options for this class + * + * @param config + */ +void StatsDiffOutput::init(ConfigInterface& config) +{ + int defaultRate = 5; + outputRate = config.getIntParam(getModuleName(),"outputRateInSeconds", defaultRate); + + for(auto const& e : ExecutorModule::getExecutorSubmodules( + config, config.getSuperModule(getModuleName())->getModuleName())) + { + executorNames.emplace_back(e->getModuleName()); + } +} + +/** + * @brief Construct a new Differential Statics Outputobject + * + * @param name the name of the module + */ +StatsDiffOutput::StatsDiffOutput(std::string name) : + OutputModule(name) +{ + outputRate = 0; + format_rspace = 6; + executorNames = {}; + + // single metadata keys + grandUQTotalMetadataKey = 0; + grandUQCrashedMetadataKey = 0; + grandUQHungMetadataKey = 0; + grandUQDiffMetadataKey = 0; + + grandTotalMetadataKey = 0; + grandCrashedMetadataKey = 0; + grandHungMetadataKey = 0; + grandDiffMetadataKey = 0; + + // per-executor metadata key + executorAllTCMetadataKeys = {}; + executorAllCrashMetadataKeys = {}; + executorAllHungMetadataKeys = {}; + executorAllDiffMetadataKeys = {}; + executorUQTCMetadataKeys = {}; + executorUQCrashMetadataKeys = {}; + executorUQHungMetadataKeys = {}; + executorUQDiffMetadataKeys = {}; + + executorAverageEPSMetadataKeys = {}; + executorLatestEPSMetadataKeys = {}; + executorStaleDurationMetadataKeys = {}; +} + +StatsDiffOutput::~StatsDiffOutput() +{ + +} + +void StatsDiffOutput::registerStorageNeeds(StorageRegistry& registry) +{ + // NO TAGS NEEDED +} + +void StatsDiffOutput::registerMetadataNeeds(StorageRegistry& registry) +{ + // Single Output Keys + grandUQTotalMetadataKey = registry.registerKey("GRAND_UQ_TEST_CASES", StorageRegistry::UINT, StorageRegistry::READ_ONLY); + grandUQCrashedMetadataKey = registry.registerKey("GRAND_UQ_CRASHED_CASES", StorageRegistry::UINT, StorageRegistry::READ_ONLY); + grandUQHungMetadataKey = registry.registerKey("GRAND_UQ_HUNG_CASES", StorageRegistry::UINT, StorageRegistry::READ_ONLY); + grandUQDiffMetadataKey = registry.registerKey("GRAND_UQ_DIFF_CASES", StorageRegistry::UINT, StorageRegistry::READ_ONLY); + + grandTotalMetadataKey = registry.registerKey("GRAND_TEST_CASES", StorageRegistry::U64, StorageRegistry::READ_ONLY); + grandCrashedMetadataKey = registry.registerKey("GRAND_CRASHED_CASES", StorageRegistry::UINT, StorageRegistry::READ_ONLY); + grandHungMetadataKey = registry.registerKey("GRAND_HUNG_CASES", StorageRegistry::UINT, StorageRegistry::READ_ONLY); + grandDiffMetadataKey = registry.registerKey("GRAND_DIFF_CASES", StorageRegistry::UINT, StorageRegistry::READ_ONLY); + + // Per-executor Output KeyLists + for(std::string& e : executorNames) + { + executorAllTCMetadataKeys.emplace_back(registry.registerKey( + "TOTAL_TEST_CASES_" + e, StorageRegistry::U64, StorageRegistry::READ_ONLY)); + executorAllCrashMetadataKeys.emplace_back(registry.registerKey( + "TOTAL_CRASHED_CASES_" + e, StorageRegistry::UINT, StorageRegistry::READ_ONLY)); + executorAllHungMetadataKeys.emplace_back(registry.registerKey( + "TOTAL_HUNG_CASES_" + e, StorageRegistry::UINT, StorageRegistry::READ_ONLY)); + executorAllDiffMetadataKeys.emplace_back(registry.registerKey( + "TOTAL_DIFF_CASES_" + e, StorageRegistry::UINT, StorageRegistry::READ_ONLY)); + + executorUQTCMetadataKeys.emplace_back(registry.registerKey( + "UQ_TEST_CASES_" + e, StorageRegistry::UINT, StorageRegistry::READ_ONLY)); + executorUQCrashMetadataKeys.emplace_back(registry.registerKey( + "UQ_CRASHED_CASES_" + e, StorageRegistry::UINT, StorageRegistry::READ_ONLY)); + executorUQHungMetadataKeys.emplace_back(registry.registerKey( + "UQ_HUNG_CASES_" + e, StorageRegistry::UINT, StorageRegistry::READ_ONLY)); + executorUQDiffMetadataKeys.emplace_back(registry.registerKey( + "UQ_DIFF_CASES_" + e, StorageRegistry::UINT, StorageRegistry::READ_ONLY)); + + executorAverageEPSMetadataKeys.emplace_back(registry.registerKey( + "AVERAGE_EPS_" + e, StorageRegistry::FLOAT, StorageRegistry::READ_ONLY)); + executorLatestEPSMetadataKeys.emplace_back(registry.registerKey( + "LATEST_EPS_" + e, StorageRegistry::FLOAT, StorageRegistry::READ_ONLY)); + executorStaleDurationMetadataKeys.emplace_back(registry.registerKey( + "DUR_LAST_FIND_" + e, StorageRegistry::FLOAT, StorageRegistry::READ_ONLY)); + } +} + +OutputModule::ScheduleTypeEnum StatsDiffOutput::getDesiredScheduleType() +{ + return OutputModule::CALL_ON_NUM_SECONDS; +} + +int StatsDiffOutput::getDesiredScheduleRate() +{ + return outputRate; +} + +void StatsDiffOutput::run(StorageModule& storage) +{ + StorageEntry& metadata = storage.getMetadata(); + + // Get Statistics from metadata + unsigned long long grandTotalTests = metadata.getU64Value(grandTotalMetadataKey); + unsigned int grandTotalCrashes = metadata.getUIntValue(grandCrashedMetadataKey); + unsigned int grandTotalHangs = metadata.getUIntValue(grandHungMetadataKey); + unsigned int grandTotalDiffs = metadata.getUIntValue(grandDiffMetadataKey); + + unsigned int grandUQTotalTests = metadata.getUIntValue(grandUQTotalMetadataKey); + unsigned int grandUQTotalCrashes = metadata.getUIntValue(grandUQCrashedMetadataKey); + unsigned int grandUQTotalHangs = metadata.getUIntValue(grandUQHungMetadataKey); + unsigned int grandUQTotalDiffs = metadata.getUIntValue(grandUQDiffMetadataKey); + + std::vectorexecutorAllTestCases = {}; + std::vectorexecutorAllCrashes = {}; + std::vectorexecutorAllHangs = {}; + std::vectorexecutorAllDiffs = {}; + + std::vectorexecutorUQTestCases = {}; + std::vectorexecutorUQCrashes = {}; + std::vectorexecutorUQHangs = {}; + std::vectorexecutorUQDiffs = {}; + + std::vectorexecutorAverageCPs = {}; + std::vectorexecutorLatestCPs = {}; + std::vectorexecutorStaleDuration = {}; + + for(size_t i=0; i 0 ? static_cast( std::log10(grandTotalTests) ) + 1 : 1; + format_rspace = num_digits > format_rspace ? num_digits : format_rspace; + + ogl(h_len, format_rspace, grandUQTotalTests, grandTotalTests, "TEST CASES"); + odl(h_len, executorUQTestCases, executorAllTestCases, " TOTAL"); + + ogl(h_len, format_rspace, grandUQTotalCrashes, grandTotalCrashes, "CRASHES"); + odl(h_len, executorUQCrashes, executorAllCrashes, " TOTAL"); + + ogl(h_len, format_rspace, grandUQTotalHangs, grandTotalHangs, "HANGS"); + odl(h_len, executorUQHangs, executorAllHangs, " TOTAL"); + + ogl(h_len, format_rspace, grandUQTotalDiffs, grandTotalDiffs, "DEVIANTS"); + odl(h_len, executorUQDiffs, executorAllDiffs, " TOTAL"); + + LOG_INFO << std::setw(h_len) << std::right << "SUT EXEC/SEC: "; + for(size_t i=0; i // outputDiffLine is shortened to ODL to save on h-space in logging +void StatsDiffOutput::odl(int l_width, std::vector dataCenter, std::vector dataRight, std::string postfix) +{ + for(size_t i=0; i // outputGrandLine +void StatsDiffOutput::ogl(int l_width, int r_width, T uqData, V allData, std::string statName) +{ + LOG_INFO << std::setw(l_width) << std::right << "GRAND TOTAL UNIQUE " + statName + ": " + << std::setw(MAGIC_SPACE) << std::left << uqData + << " | " + << std::setw(r_width) << std::left<< allData << " TOTAL"; +} diff --git a/vmf/src/modules/common/output/StatsDiffOutput.hpp b/vmf/src/modules/common/output/StatsDiffOutput.hpp new file mode 100644 index 0000000..522ec57 --- /dev/null +++ b/vmf/src/modules/common/output/StatsDiffOutput.hpp @@ -0,0 +1,85 @@ +/* ============================================================================= + * Vader Modular Fuzzer (VMF) + * Copyright (c) 2021-2025 The Charles Stark Draper Laboratory, Inc. + * + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 (only) as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + * + * @license GPL-2.0-only + * ===========================================================================*/ +#pragma once + + +#include "OutputModule.hpp" + +namespace vmf +{ +/** + * @brief OutputModule that logs high level execution statistics for the operator. + * This module requires ComputeDiffStats (or an equivalent module) to be present, such + * that a number of required differential metadata inputs are available. + * Statistics will be provided to the logger. + * @image html CoreModuleDataModel_6.png width=800px + * @image latex CoreModuleDataModel_6.png width=6in + */ +class StatsDiffOutput : public OutputModule { +public: + static Module* build(std::string name); + virtual void init(ConfigInterface& config); + + virtual void registerStorageNeeds(StorageRegistry& registry); + virtual void registerMetadataNeeds(StorageRegistry& registry); + virtual OutputModule::ScheduleTypeEnum getDesiredScheduleType(); + virtual int getDesiredScheduleRate(); + + virtual void run(StorageModule& storage); + + StatsDiffOutput(std::string name); + virtual ~StatsDiffOutput(); +private: + template + void odl(int l_width, std::vector dataCenter, std::vector dataRight, std::string postfix); + template + void ogl(int l_width, int r_width, T uqData, V allData, std::string statName); + + int outputRate; + unsigned int format_rspace; + std::vector executorNames; + + // single metadata keys + int grandUQTotalMetadataKey; + int grandUQCrashedMetadataKey; + int grandUQHungMetadataKey; + int grandUQDiffMetadataKey; + + int grandTotalMetadataKey; + int grandCrashedMetadataKey; + int grandHungMetadataKey; + int grandDiffMetadataKey; + + // per-executor metadata keys + std::vector executorAllTCMetadataKeys; + std::vector executorAllCrashMetadataKeys; + std::vector executorAllHungMetadataKeys; + std::vector executorAllDiffMetadataKeys; + + std::vector executorUQTCMetadataKeys; + std::vector executorUQCrashMetadataKeys; + std::vector executorUQHungMetadataKeys; + std::vector executorUQDiffMetadataKeys; + + std::vector executorAverageEPSMetadataKeys; + std::vector executorLatestEPSMetadataKeys; + std::vector executorStaleDurationMetadataKeys; +}; +}