Add Differential Fuzzing + Supporting Capabilities to VMF

docs/getting_started.md

@@ -162,3 +162,60 @@ This is entirely equivalent to having put this in basicModules.yaml instead (the
 AFLForkserverExecutor:
   sutArgv: ["test/haystackSUT/haystack"]
 ```
+
+## Example Differential Fuzzing VMF Configuration 
+VMF's configuration-driven paradigm has driven new advancements in fuzzing capabilities. The following modules must be included to enable differential fuzzing of two Systems Under Test (SUTs):
+
+```yaml
+vmfVariables: # ... no changes ...
+
+vmfFramework: # ... no changes ...
+
+vmfModules:
+  storage: # SimpleStorage MUST be specified 
+    className: SimpleStorage
+  controller: 
+    # DifferentialController MUST specify AT LEAST TWO AFLForkserverExecutor modules
+    className: DifferentialController
+    children:
+        # Each AFLForkserverExecutor MUST have a unique id
+      - id: knownGoodSutA
+        className: AFLForkserverExecutor
+      - id: unknownSutB
+        className: AFLForkserverExecutor
+        # DiffInputGenerator MUST specify their children
+      - className: DiffInputGenerator
+        # AFLDiffFeedback MUST specify module-specific params
+      - className: AFLDiffFeedback
+        # ComputeDiffStats MAY specify statsRateInSeconds (default is 1)
+      - className: ComputeDiffStats
+        statsRateInSeconds: 2
+        # StatsDiffOutput MAY specify outputRateInSeconds (default is 5)
+      - className: StatsDiffOutput
+        outputRateInSeconds: 10
+
+  # DiffInputGenerator MUST have one or more Mutator module children
+  DiffInputGenerator:
+    children:
+      - className: # ex: AFLRandomByteMutator
+      - # ... 
+
+### Module-specific parameters ###
+
+# Each AFLForkserverExecutor MUST specify their command-line arguments
+sutA:
+  sutArgv: # ...
+sutB:
+  sutArgv: # ...
+
+# AFLDiffFeedback MUST specify the ID of ONE trusted SUT, as a reference for the system
+AFLDiffFeedback:
+  systemOfTruth: # ...
+  # The module MAY specify custom fitness weights to favor different test attributes. 
+  # NOTE: not specifying customWeights will weigh feedback with regards to the SUT's 
+  #   average statistics over the campaign
+  useCustomWeights: # default is false
+  diffWeight: # default is 10.0
+  sizeWeight: # default is 1.0
+  speedWeight: # default is 5.0
+```

test/unittest/TestConfigInterface.hpp

@@ -51,6 +51,9 @@ class TestConfigInterface : public ConfigInterface
     //Methods required by ConfigInterface -- these are just stubbed out to compile
     virtual std::string getAllParamsYAML(std::string moduleName);
 
+    //Methods required by ConfigInterface -- these are stubbed out to compile
+    virtual Module* getSuperModule(std::string subModuleName) {return nullptr;}
+
     //Methods required by ConfigInterface -- these have reasonably real implementations
     virtual std::string getOutputDir();
     virtual std::vector<Module*> getSubModules(std::string parentModuleName);

vmf/src/framework/app/ConfigManager.cpp

@@ -567,6 +567,12 @@ std::vector<Module*> ConfigManager::getSubModules(std::string parentModuleName)
     return list;
 }
 
+//see ConfigInterface::getSuperModule
+Module* ConfigManager::getSuperModule(std::string childName)
+{
+    return moduleManager->getRootModule();
+}
+
 //see ConfigInterface::isParam
 bool ConfigManager::isParam(std::string moduleName, std::string paramName)
 {

vmf/src/framework/app/ConfigManager.hpp

@@ -52,6 +52,7 @@ class ConfigManager : public ConfigInterface
     virtual std::string getOutputDir();
     virtual void setOutputDir(std::string dir);
     virtual std::vector<Module*> getSubModules(std::string parentModuleName);
+    virtual Module* getSuperModule(std::string subModuleName);
 
     virtual bool isParam(std::string moduleName, std::string paramName);
     virtual std::string getStringParam(std::string moduleName, std::string paramName);

vmf/src/framework/baseclasses/FeedbackModule.hpp

@@ -52,6 +52,15 @@ class FeedbackModule: public StorageUserModule
      * @param entries 
      */
     virtual void evaluateTestCaseResults(StorageModule& storage, std::unique_ptr<Iterator>& entries) = 0;
+
+    /**
+     * @brief Evaluate test case results of a differential fuzzing campaign
+     * The method is nearly identical to a regular evaluateTestCaseResults, except on N entries at once.
+     *
+     * @param storage
+     * @param entries - vector of list<Entries> instead of an Iterator, one per executor in differential campaign 
+     */
+    virtual void evaluateDiffTestCaseResults(StorageModule& storage, std::vector<std::unique_ptr<Iterator>>& entries) = 0;
     virtual ~FeedbackModule() {};
 
       /**
@@ -93,7 +102,7 @@ class FeedbackModule: public StorageUserModule
      * @brief Helper method to return a single Feedback submodule from config by name
      * This method will retrieve a single Feedback submodule by name for the specified parent modules.
      * If there are no Feedback submodules with the specified name, then an nullptr will be returned.  
-     * 
+     *
      * @param config the ConfigInterface object
      * @param parentName the name of the parent module
      * @param childName the name of the child module to finde
@@ -129,7 +138,7 @@ class FeedbackModule: public StorageUserModule
      * @brief Helper method to get the Feedback Submodules from config
      * This method will retrieve all of the Feedback submodules for the specified parent modules.
      * If there are no Feedback submodules, then an empty list will be returned.
-     * 
+     *
      * @param config the ConfigInterface object
      * @param parentName the name of the parent module
      * @return std::vector<FeedbackModule*> the list of submodules

vmf/src/framework/baseclasses/SimpleStorage.cpp

@@ -18,8 +18,12 @@
  * @license GPL-2.0-only <https://spdx.org/licenses/GPL-2.0-only.html>
  * ===========================================================================*/
 #include "SimpleStorage.hpp"
+#include "SimpleIterator.hpp"
+#include "StorageEntry.hpp"
 #include "StorageKeyHelper.hpp"
 #include "Logging.hpp"
+#include "plog/Log.h"
+#include <memory>
 
 using namespace vmf;
 
@@ -605,3 +609,79 @@ StorageEntry& SimpleStorage::getMetadata()
         throw RuntimeException("Storage must be initialized before use.", RuntimeException::USAGE_ERROR);
     }
 }
+
+std::unique_ptr<Iterator> SimpleStorage::getSavedEntriesByIntersection(int tagA, int tagB)
+{
+    checkThatTagIsValid(tagA, numTags); checkThatTagIsValid(tagB, numTags);
+    std::list<vmf::StorageEntry*> intersection = {};
+    auto thatList = tagList[tagB];
+    for(auto const entryA : tagList[tagA])
+    {
+        // DEV'S NOTE: find will compare by ADDRESS NOT ENTRY DATA
+        if(std::find(thatList.begin(), thatList.end(), entryA) != thatList.end())
+        {
+            intersection.emplace_back(entryA);
+        }
+    }
+
+    SimpleIterator* theIterator = new SimpleIterator(intersection);
+    std::unique_ptr<Iterator> returnPointer(theIterator);
+    return returnPointer;
+}
+
+std::unique_ptr<Iterator> SimpleStorage::getNewEntriesByIntersection(int tagA, int tagB)
+{
+    checkThatTagIsValid(tagA, numTags); checkThatTagIsValid(tagB, numTags);
+    std::list<vmf::StorageEntry*> intersection = {};
+    auto thatList = newTagList[tagB];
+    for(auto const entryA : newTagList[tagA])
+    {
+        // DEV'S NOTE: find will compare by ADDRESS NOT ENTRY DATA
+        if(std::find(thatList.begin(), thatList.end(), entryA) != thatList.end())
+        {
+            intersection.emplace_back(entryA);
+        }
+    }
+
+    SimpleIterator* theIterator = new SimpleIterator(intersection);
+    std::unique_ptr<Iterator> returnPointer(theIterator);
+    return returnPointer;
+}
+
+std::unique_ptr<Iterator> SimpleStorage::getKeySortedSavedEntriesByTag(int tagId, 
+    std::function<bool(StorageEntry*,StorageEntry*)> lessThanFunc)
+{
+    checkThatTagIsValid(tagId, numTags);
+    std::list<vmf::StorageEntry*>& entries = tagList[tagId];
+    entries.sort(lessThanFunc);
+    if(entries.size() > 0)
+    {
+        SimpleIterator* theIterator = new SimpleIterator(entries);
+        std::unique_ptr<Iterator> returnPointer(theIterator);
+        return returnPointer;
+    }
+    else
+    {
+        LOG_WARNING << "No entries with the tag \"" << tagNameMap[tagId] << "\"";
+        return nullptr;
+    }
+}
+
+std::unique_ptr<Iterator> SimpleStorage::getKeySortedNewEntriesByTag(int tagId,
+    std::function<bool(StorageEntry*,StorageEntry*)> lessThanFunc)
+{
+    checkThatTagIsValid(tagId, numTags);
+    std::list<vmf::StorageEntry*>& newEntries = newTagList[tagId];
+    newEntries.sort(lessThanFunc);
+    if(newEntries.size() > 0)
+    {
+        SimpleIterator* theIterator = new SimpleIterator(newEntries);
+        std::unique_ptr<Iterator> returnPointer(theIterator);
+        return returnPointer;
+    }
+    else
+    {
+        LOG_WARNING << "No new entries with the tag \"" << tagNameMap[tagId] << "\"";
+        return nullptr;
+    }
+}

vmf/src/framework/baseclasses/SimpleStorage.hpp

@@ -31,6 +31,7 @@
 #include <vector>
 #include <list>
 #include <unordered_map>
+#include <functional>
 
 namespace vmf{
 /**
@@ -90,6 +91,14 @@ class SimpleStorage: public StorageModule
     //This method returns the one and only metadata storage entry
     virtual StorageEntry& getMetadata();
 
+    //These methods provide a way to retrieve all entries that are designated with both tags provided
+    virtual std::unique_ptr<Iterator> getSavedEntriesByIntersection(int tagA, int tagB);
+    virtual std::unique_ptr<Iterator> getNewEntriesByIntersection(int tagA, int tagB);
+
+    //These methods return entries SORTED by the given lambda (which usually compares based on a key)
+    virtual std::unique_ptr<Iterator> getKeySortedSavedEntriesByTag(int tagId, std::function<bool(StorageEntry*,StorageEntry*)> lessThanFunc);
+    virtual std::unique_ptr<Iterator> getKeySortedNewEntriesByTag(int tagId, std::function<bool(StorageEntry*,StorageEntry*)> lessThanFunc);
+
 private:
     static bool removeEntryIfPresent(std::list<StorageEntry*>& list, StorageEntry* entry);
     static bool checkThatTagIsValid(int tagId, int numTags);

vmf/src/framework/baseclasses/StorageModule.hpp

@@ -23,7 +23,9 @@
 #include "StorageEntry.hpp"
 #include "StorageEntryListener.hpp"
 #include "Iterator.hpp"
+#include <list>
 #include <memory>
+#include <functional>
 
 namespace vmf
 {
@@ -256,6 +258,34 @@ class StorageModule : public Module, public StorageEntryListener {
      */
     virtual StorageEntry& getMetadata() = 0;
 
+    /**
+     * @brief Get the saved entries that have been previously tagged with all the provided tags.
+     *
+     * Returns an iterator that can be used to step through all of the tagged entries.
+     * Entries are sorted using the sort by fields that were configured in the StorageRegistry.
+     *
+     * @param tagA the tag handle (as returned from a call to StoragRegistry.registerTag)
+     * @param tagB 
+     * @return std::unique_ptr<Iterator> with entries (if any)
+     */
+    virtual std::unique_ptr<Iterator> getSavedEntriesByIntersection(int tagA, int tagB) = 0;
+
+    //These methods return entries SORTED by the corresponding key
+    virtual std::unique_ptr<Iterator> getKeySortedSavedEntriesByTag(int tagId, std::function<bool(StorageEntry*,StorageEntry*)> lessThanFunc) = 0;
+    virtual std::unique_ptr<Iterator> getKeySortedNewEntriesByTag(int tagId, std::function<bool(StorageEntry*,StorageEntry*)> lessThanFunc) = 0;
+
+    /**
+     * @brief Get the new entries with all the provided tags.
+     *
+     * Returns an iterator that can be used to step through all of the tagged entries.
+     * Entries are sorted using the sort by fields that were configured in the StorageRegistry.
+     *
+     * @param tagA the tag handle (as returned from a call to StoragRegistry.registerTag)
+     * @param tagB 
+     * @return std::unique_ptr<Iterator> with entries (if any)
+     */
+    virtual std::unique_ptr<Iterator> getNewEntriesByIntersection(int tagA, int tagB) = 0;
+
     /**
      * @brief Convenience method to determine if a module is actually a storage module
      * 

vmf/src/framework/util/ConfigInterface.hpp

@@ -79,6 +79,18 @@ class ConfigInterface
      */
     virtual std::vector<Module*> getSubModules(std::string parentModuleName) = 0;
 
+    /**
+     * @brief Retrieves the supermodule that is associated with this module in the config file(s)
+     * 
+     * To use, any module can call getSuperModule(getModuleName())
+     * Module* will need to be converted to their underlying type, using the convenience methods
+     * isAnInstance() and castTo() that are defined in each of the module base classes.
+     * 
+     * @param subModuleName the name of the module
+     * @return Module* the supermodule
+     */
+    virtual Module* getSuperModule(std::string subModuleName) = 0;
+
     /**
      * @brief Check to see if a parameter is defined in a config file, without returning the value.
      * 

vmf/src/modules/CMakeLists.txt

@@ -37,6 +37,9 @@ endif()
 
 
 list(APPEND CoreModules_SOURCES
+    common/controller/DifferentialController.cpp
+    common/feedback/AFLDiffFeedback.cpp
+
     common/controller/AnalysisController.cpp
     common/controller/RunOnceController.cpp
     common/controller/IterativeController.cpp
@@ -50,6 +53,7 @@ list(APPEND CoreModules_SOURCES
     common/initialization/ServerCorpusInitialization.cpp
     common/initialization/ServerSeedInitialization.cpp
     common/initialization/TrivialSeedInitialization.cpp
+    common/inputgeneration/DiffInputGenerator.cpp
     common/inputgeneration/GeneticAlgorithmInputGenerator.cpp
     common/inputgeneration/MOPTInputGenerator.cpp    
     common/inputgeneration/MOPT.cpp
@@ -78,13 +82,15 @@ list(APPEND CoreModules_SOURCES
     common/mutator/StackedMutator.cpp
     common/mutator/MutatorSelector.cpp
     common/output/ComputeStats.cpp
+    common/output/ComputeDiffStats.cpp
     common/output/CorpusMinimization.cpp
     common/output/CSVMetadataOutput.cpp
     common/output/LoggerMetadataOutput.cpp
     common/output/SaveCorpusOutput.cpp
     common/output/ServerCorpusMinOutput.cpp
     common/output/ServerCorpusOutput.cpp
     common/output/StatsOutput.cpp
+    common/output/StatsDiffOutput.cpp
 )
 
 add_library(CoreModules SHARED ${CoreModules_SOURCES})