diff --git a/resources/examples/testfiles/pomdp/simple.prism b/resources/examples/testfiles/pomdp/simple.prism index 24c19bb76a..82134f1f3a 100644 --- a/resources/examples/testfiles/pomdp/simple.prism +++ b/resources/examples/testfiles/pomdp/simple.prism @@ -11,6 +11,7 @@ module main [alpha] s>0 & s<5 -> (1-slippery): (s'=s+2) + slippery: true; [beta] s=3 -> (1-slippery): (s'=6) + slippery: true; [beta] s=4 -> (1-slippery): (s'=5) + slippery: true; + [alpha] s=5 | s=6 -> 1: true; endmodule rewards diff --git a/src/storm-pomdp-cli/settings/modules/BeliefExplorationSettings.cpp b/src/storm-pomdp-cli/settings/modules/BeliefExplorationSettings.cpp index 844c0d3cd3..7cc4ba3b6c 100644 --- a/src/storm-pomdp-cli/settings/modules/BeliefExplorationSettings.cpp +++ b/src/storm-pomdp-cli/settings/modules/BeliefExplorationSettings.cpp @@ -1,66 +1,35 @@ #include "storm-pomdp-cli/settings/modules/BeliefExplorationSettings.h" -#include "storm-pomdp/modelchecker/BeliefExplorationPomdpModelCheckerOptions.h" -#include "storm/adapters/RationalNumberAdapter.h" #include "storm/settings/ArgumentBuilder.h" #include "storm/settings/OptionBuilder.h" -#include "storm/utility/NumberTraits.h" -namespace storm { -namespace settings { -namespace modules { +namespace storm::settings::modules { const std::string BeliefExplorationSettings::moduleName = "beliefExploration"; -const std::string refineOption = "refine"; const std::string explorationTimeLimitOption = "exploration-time"; const std::string resolutionOption = "resolution"; const std::string clipGridResolutionOption = "clip-resolution"; const std::string sizeThresholdOption = "size-threshold"; -const std::string gapThresholdOption = "gap-threshold"; -const std::string optimalChoiceValueThresholdOption = "optimal-choice-value-threshold"; -const std::string observationThresholdOption = "obs-threshold"; -const std::string numericPrecisionOption = "numeric-precision"; const std::string triangulationModeOption = "triangulationmode"; const std::string clippingOption = "use-clipping"; const std::string cutZeroGapOption = "cut-zero-gap"; -const std::string stateEliminationCutoffOption = "state-elimination-cutoff"; +const std::string inexactPreprocessingOption = "inexact-preprocessing"; +const std::string beliefMdpNumberTypeOption = "belief-mdp-number-type"; +std::vector const beliefMdpNumberTypes = {"double", "rational", "match"}; BeliefExplorationSettings::BeliefExplorationSettings() : ModuleSettings(moduleName) { - this->addOption( - storm::settings::OptionBuilder(moduleName, refineOption, false, - "Refines the result bounds until reaching either the goal precision or the refinement step limit") - .addArgument(storm::settings::ArgumentBuilder::createDoubleArgument("prec", "The goal precision.") - .setDefaultValueDouble(1e-4) - .makeOptional() - .addValidatorDouble(storm::settings::ArgumentValidatorFactory::createDoubleGreaterEqualValidator(0.0)) - .build()) - .addArgument(storm::settings::ArgumentBuilder::createUnsignedIntegerArgument("steps", "The number of allowed refinement steps (0 means no limit).") - .setDefaultValueUnsignedInteger(0) - .makeOptional() - .build()) - .build()); - this->addOption( storm::settings::OptionBuilder(moduleName, explorationTimeLimitOption, false, "Sets after which time no further states shall be explored.") .addArgument(storm::settings::ArgumentBuilder::createUnsignedIntegerArgument("time", "In seconds.").setDefaultValueUnsignedInteger(0).build()) .build()); - this->addOption( - storm::settings::OptionBuilder(moduleName, resolutionOption, false, - "Sets the resolution of the discretization and how it is increased in case of refinement") - .setIsAdvanced() - .addArgument(storm::settings::ArgumentBuilder::createUnsignedIntegerArgument("init", "the initial resolution (higher means more precise)") - .setDefaultValueUnsignedInteger(2) - .addValidatorUnsignedInteger(storm::settings::ArgumentValidatorFactory::createUnsignedGreaterValidator(0)) - .build()) - .addArgument(storm::settings::ArgumentBuilder::createDoubleArgument( - "factor", "Multiplied to the resolution of refined observations (higher means more precise).") - .setDefaultValueDouble(2) - .makeOptional() - .addValidatorDouble(storm::settings::ArgumentValidatorFactory::createDoubleGreaterValidator(1)) - .build()) - .build()); + this->addOption(storm::settings::OptionBuilder(moduleName, resolutionOption, false, "Sets the resolution of the discretization") + .addArgument(storm::settings::ArgumentBuilder::createUnsignedIntegerArgument("resolution", "the resolution (higher means more precise)") + .setDefaultValueUnsignedInteger(2) + .addValidatorUnsignedInteger(storm::settings::ArgumentValidatorFactory::createUnsignedGreaterValidator(0)) + .build()) + .build()); this->addOption(storm::settings::OptionBuilder(moduleName, clipGridResolutionOption, false, "Sets the resolution of the clipping grid") .addArgument(storm::settings::ArgumentBuilder::createUnsignedIntegerArgument("resolution", "the resolution (higher means more precise)") @@ -69,79 +38,14 @@ BeliefExplorationSettings::BeliefExplorationSettings() : ModuleSettings(moduleNa .build()) .build()); - this->addOption( - storm::settings::OptionBuilder(moduleName, observationThresholdOption, false, "Only observations whose score is below this threshold will be refined.") - .setIsAdvanced() - .addArgument(storm::settings::ArgumentBuilder::createDoubleArgument("init", "initial threshold (higher means more precise") - .setDefaultValueDouble(0.1) - .addValidatorDouble(storm::settings::ArgumentValidatorFactory::createDoubleRangeValidatorIncluding(0, 1)) - .build()) - .addArgument(storm::settings::ArgumentBuilder::createDoubleArgument( - "factor", "Controlls how fast the threshold is increased in each refinement step (higher means more precise).") - .setDefaultValueDouble(0.1) - .makeOptional() - .addValidatorDouble(storm::settings::ArgumentValidatorFactory::createDoubleRangeValidatorIncluding(0, 1)) - .build()) - .build()); - - this->addOption( - storm::settings::OptionBuilder( - moduleName, sizeThresholdOption, false, - "Sets how many new states are explored or rewired in a refinement step and how this value is increased in case of refinement.") - .setIsAdvanced() - .addArgument( - storm::settings::ArgumentBuilder::createUnsignedIntegerArgument("init", "initial limit (higher means more precise, 0 means automatic choice)") - .setDefaultValueUnsignedInteger(0) - .build()) - .addArgument(storm::settings::ArgumentBuilder::createDoubleArgument( - "factor", "Before each step the new threshold is set to the current state count times this number (higher means more precise).") - .setDefaultValueDouble(4) - .makeOptional() - .addValidatorDouble(storm::settings::ArgumentValidatorFactory::createDoubleGreaterEqualValidator(1)) - .build()) - .build()); - - this->addOption( - storm::settings::OptionBuilder(moduleName, gapThresholdOption, false, - "Sets how large the gap between known lower- and upper bounds at a beliefstate needs to be in order to explore") - .setIsAdvanced() - .addArgument(storm::settings::ArgumentBuilder::createDoubleArgument("init", "initial threshold (higher means less precise") - .setDefaultValueDouble(0.1) - .addValidatorDouble(storm::settings::ArgumentValidatorFactory::createDoubleGreaterEqualValidator(0)) - .build()) - .addArgument( - storm::settings::ArgumentBuilder::createDoubleArgument("factor", "Multiplied to the gap in each refinement step (higher means less precise).") - .setDefaultValueDouble(0.25) - .makeOptional() - .addValidatorDouble(storm::settings::ArgumentValidatorFactory::createDoubleRangeValidatorIncluding(0, 1)) - .build()) - .build()); - - this->addOption(storm::settings::OptionBuilder(moduleName, optimalChoiceValueThresholdOption, false, - "Sets how much worse a sub-optimal choice can be in order to be included in the relevant explored fragment") - .setIsAdvanced() - .addArgument(storm::settings::ArgumentBuilder::createDoubleArgument("init", "initial threshold (higher means more precise") - .setDefaultValueDouble(1e-3) - .addValidatorDouble(storm::settings::ArgumentValidatorFactory::createDoubleGreaterEqualValidator(0)) - .build()) - .addArgument(storm::settings::ArgumentBuilder::createDoubleArgument( - "factor", "Multiplied to the threshold in each refinement step (higher means more precise).") - .setDefaultValueDouble(1) - .makeOptional() - .addValidatorDouble(storm::settings::ArgumentValidatorFactory::createDoubleGreaterEqualValidator(1)) + this->addOption(storm::settings::OptionBuilder(moduleName, sizeThresholdOption, false, + "Sets how many beliefs are explored in the unfolding before approximations are applied.") + .addArgument(storm::settings::ArgumentBuilder::createUnsignedIntegerArgument( + "threshold", "number of beliefs to explore (higher means more precise, 0 means automatic/heuristic choice)") + .setDefaultValueUnsignedInteger(0) .build()) .build()); - this->addOption( - storm::settings::OptionBuilder(moduleName, numericPrecisionOption, false, "Sets the precision used to determine whether two belief-states are equal.") - .setIsAdvanced() - .addArgument(storm::settings::ArgumentBuilder::createDoubleArgument("value", "the precision") - .setDefaultValueDouble(1e-9) - .makeOptional() - .addValidatorDouble(storm::settings::ArgumentValidatorFactory::createDoubleRangeValidatorIncluding(0, 1)) - .build()) - .build()); - this->addOption(storm::settings::OptionBuilder(moduleName, triangulationModeOption, false, "Sets how to triangulate beliefs when discretizing.") .setIsAdvanced() .addArgument(storm::settings::ArgumentBuilder::createStringArgument("value", "the triangulation mode") @@ -150,29 +54,19 @@ BeliefExplorationSettings::BeliefExplorationSettings() : ModuleSettings(moduleNa .build()) .build()); this->addOption( - storm::settings::OptionBuilder(moduleName, clippingOption, false, "If this is set, unfolding will use (grid) clipping instead of cut-offs only.") - .build()); + storm::settings::OptionBuilder(moduleName, clippingOption, false, "If this is set, unfolding will use grid clipping in addition to cut-offs.").build()); this->addOption( storm::settings::OptionBuilder(moduleName, cutZeroGapOption, false, "Cut beliefs where the gap between over- and underapproximation is 0.").build()); - this->addOption(storm::settings::OptionBuilder(moduleName, stateEliminationCutoffOption, false, - "If this is set, an additional unfolding step for cut-off beliefs is performed.") + this->addOption(storm::settings::OptionBuilder(moduleName, inexactPreprocessingOption, false, + "If this is set, the POMDP will be analysed using floating point arithmetic for preprocessing. This speeds " + "up computations, but can lead to inaccurate results.") + .build()); + this->addOption(storm::settings::OptionBuilder(moduleName, beliefMdpNumberTypeOption, false, "Sets the number type to use for generated belief MDPs") + .addArgument(storm::settings::ArgumentBuilder::createStringArgument("type", "Type to use.") + .addValidatorString(ArgumentValidatorFactory::createMultipleChoiceValidator(beliefMdpNumberTypes)) + .setDefaultValueString("match") + .build()) .build()); -} - -bool BeliefExplorationSettings::isRefineSet() const { - return this->getOption(refineOption).getHasOptionBeenSet(); -} - -bool BeliefExplorationSettings::isStateEliminationCutoffSet() const { - return this->getOption(stateEliminationCutoffOption).getHasOptionBeenSet(); -} - -double BeliefExplorationSettings::getRefinePrecision() const { - return this->getOption(refineOption).getArgumentByName("prec").getValueAsDouble(); -} - -uint64_t BeliefExplorationSettings::getRefineStepLimit() const { - return this->getOption(refineOption).getArgumentByName("steps").getValueAsUnsignedInteger(); } uint64_t BeliefExplorationSettings::getExplorationTimeLimit() const { @@ -180,63 +74,34 @@ uint64_t BeliefExplorationSettings::getExplorationTimeLimit() const { } uint64_t BeliefExplorationSettings::getResolutionInit() const { - return this->getOption(resolutionOption).getArgumentByName("init").getValueAsUnsignedInteger(); + return this->getOption(resolutionOption).getArgumentByName("resolution").getValueAsUnsignedInteger(); } uint64_t BeliefExplorationSettings::getClippingGridResolution() const { return this->getOption(clipGridResolutionOption).getArgumentByName("resolution").getValueAsUnsignedInteger(); } -double BeliefExplorationSettings::getResolutionFactor() const { - return this->getOption(resolutionOption).getArgumentByName("factor").getValueAsDouble(); -} - uint64_t BeliefExplorationSettings::getSizeThresholdInit() const { - return this->getOption(sizeThresholdOption).getArgumentByName("init").getValueAsUnsignedInteger(); -} - -double BeliefExplorationSettings::getSizeThresholdFactor() const { - return this->getOption(sizeThresholdOption).getArgumentByName("factor").getValueAsDouble(); -} - -double BeliefExplorationSettings::getGapThresholdInit() const { - return this->getOption(gapThresholdOption).getArgumentByName("init").getValueAsDouble(); -} - -double BeliefExplorationSettings::getGapThresholdFactor() const { - return this->getOption(gapThresholdOption).getArgumentByName("factor").getValueAsDouble(); -} - -double BeliefExplorationSettings::getOptimalChoiceValueThresholdInit() const { - return this->getOption(optimalChoiceValueThresholdOption).getArgumentByName("init").getValueAsDouble(); -} - -double BeliefExplorationSettings::getOptimalChoiceValueThresholdFactor() const { - return this->getOption(optimalChoiceValueThresholdOption).getArgumentByName("factor").getValueAsDouble(); + return this->getOption(sizeThresholdOption).getArgumentByName("threshold").getValueAsUnsignedInteger(); } -double BeliefExplorationSettings::getObservationScoreThresholdInit() const { - return this->getOption(observationThresholdOption).getArgumentByName("init").getValueAsDouble(); +bool BeliefExplorationSettings::isDynamicTriangulationModeSet() const { + return this->getOption(triangulationModeOption).getArgumentByName("value").getValueAsString() == "dynamic"; } - -double BeliefExplorationSettings::getObservationScoreThresholdFactor() const { - return this->getOption(observationThresholdOption).getArgumentByName("factor").getValueAsDouble(); +bool BeliefExplorationSettings::isStaticTriangulationModeSet() const { + return this->getOption(triangulationModeOption).getArgumentByName("value").getValueAsString() == "static"; } -bool BeliefExplorationSettings::isNumericPrecisionSetFromDefault() const { - return !this->getOption(numericPrecisionOption).getHasOptionBeenSet() || - this->getOption(numericPrecisionOption).getArgumentByName("value").wasSetFromDefaultValue(); +bool BeliefExplorationSettings::isBeliefMDPNumberTypeDouble() const { + return this->getOption(beliefMdpNumberTypeOption).getArgumentByName("type").getValueAsString() == "double"; } -double BeliefExplorationSettings::getNumericPrecision() const { - return this->getOption(numericPrecisionOption).getArgumentByName("value").getValueAsDouble(); +bool BeliefExplorationSettings::isBeliefMDPNumberTypeRational() const { + return this->getOption(beliefMdpNumberTypeOption).getArgumentByName("type").getValueAsString() == "rational"; } -bool BeliefExplorationSettings::isDynamicTriangulationModeSet() const { - return this->getOption(triangulationModeOption).getArgumentByName("value").getValueAsString() == "dynamic"; -} -bool BeliefExplorationSettings::isStaticTriangulationModeSet() const { - return this->getOption(triangulationModeOption).getArgumentByName("value").getValueAsString() == "static"; +bool BeliefExplorationSettings::isBeliefMDPNumberTypeMatch() const { + return this->getOption(beliefMdpNumberTypeOption).getArgumentByName("type").getValueAsString() == "match"; } bool BeliefExplorationSettings::isUseClippingSet() const { @@ -247,46 +112,7 @@ bool BeliefExplorationSettings::isCutZeroGapSet() const { return this->getOption(cutZeroGapOption).getHasOptionBeenSet(); } -template -void BeliefExplorationSettings::setValuesInOptionsStruct(storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions& options) const { - options.refine = isRefineSet(); - options.refinePrecision = storm::utility::convertNumber(getRefinePrecision()); - options.refineStepLimit = getRefineStepLimit(); - options.explorationTimeLimit = getExplorationTimeLimit(); - - options.clippingGridRes = getClippingGridResolution(); - options.resolutionInit = getResolutionInit(); - options.resolutionFactor = storm::utility::convertNumber(getResolutionFactor()); - options.sizeThresholdInit = getSizeThresholdInit(); - options.sizeThresholdFactor = storm::utility::convertNumber(getSizeThresholdFactor()); - options.gapThresholdInit = storm::utility::convertNumber(getGapThresholdInit()); - options.gapThresholdFactor = storm::utility::convertNumber(getGapThresholdFactor()); - options.optimalChoiceValueThresholdInit = storm::utility::convertNumber(getOptimalChoiceValueThresholdInit()); - options.optimalChoiceValueThresholdFactor = storm::utility::convertNumber(getOptimalChoiceValueThresholdFactor()); - options.obsThresholdInit = storm::utility::convertNumber(getObservationScoreThresholdInit()); - options.obsThresholdIncrementFactor = storm::utility::convertNumber(getObservationScoreThresholdFactor()); - options.useClipping = isUseClippingSet(); - options.useStateEliminationCutoff = isStateEliminationCutoffSet(); - - options.numericPrecision = storm::utility::convertNumber(getNumericPrecision()); - if (storm::NumberTraits::IsExact) { - if (isNumericPrecisionSetFromDefault()) { - STORM_LOG_WARN_COND(storm::utility::isZero(options.numericPrecision), "Setting numeric precision to zero because exact arithmethic is used."); - options.numericPrecision = storm::utility::zero(); - } else { - STORM_LOG_WARN_COND(storm::utility::isZero(options.numericPrecision), - "A non-zero numeric precision was set although exact arithmethic is used. Results might be inexact."); - } - } - options.dynamicTriangulation = isDynamicTriangulationModeSet(); - options.cutZeroGap = isCutZeroGapSet(); +bool BeliefExplorationSettings::isInexactPreprocessingSet() const { + return this->getOption(inexactPreprocessingOption).getHasOptionBeenSet(); } - -template void BeliefExplorationSettings::setValuesInOptionsStruct( - storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions& options) const; -template void BeliefExplorationSettings::setValuesInOptionsStruct( - storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions& options) const; - -} // namespace modules -} // namespace settings -} // namespace storm +} // namespace storm::settings::modules diff --git a/src/storm-pomdp-cli/settings/modules/BeliefExplorationSettings.h b/src/storm-pomdp-cli/settings/modules/BeliefExplorationSettings.h index 476a456c59..bb26550c3f 100644 --- a/src/storm-pomdp-cli/settings/modules/BeliefExplorationSettings.h +++ b/src/storm-pomdp-cli/settings/modules/BeliefExplorationSettings.h @@ -1,24 +1,8 @@ #pragma once -#include "storm-config.h" #include "storm/settings/modules/ModuleSettings.h" -namespace storm { -namespace builder { -template -class BeliefMdpExplorer; -} -namespace pomdp { -namespace modelchecker { -template -struct BeliefExplorationPomdpModelCheckerOptions; -} - -enum BeliefNumberType { Default, Float, Rational }; -} // namespace pomdp - -namespace settings { -namespace modules { +namespace storm::settings::modules { /*! * This class represents the settings for POMDP model checking. @@ -33,49 +17,29 @@ class BeliefExplorationSettings : public ModuleSettings { virtual ~BeliefExplorationSettings() = default; bool isCutZeroGapSet() const; - bool isRefineSet() const; - double getRefinePrecision() const; - uint64_t getRefineStepLimit() const; uint64_t getExplorationTimeLimit() const; /// Discretization Resolution uint64_t getResolutionInit() const; - double getResolutionFactor() const; /// Clipping Grid Resolution uint64_t getClippingGridResolution() const; /// The maximal number of newly expanded MDP states in a refinement step uint64_t getSizeThresholdInit() const; - double getSizeThresholdFactor() const; - - /// Controls how large the gap between known lower- and upper bounds at a beliefstate needs to be in order to explore - double getGapThresholdInit() const; - double getGapThresholdFactor() const; - - /// Controls whether "almost optimal" choices will be considered optimal - double getOptimalChoiceValueThresholdInit() const; - double getOptimalChoiceValueThresholdFactor() const; - - /// Controls which observations are refined. - double getObservationScoreThresholdInit() const; - double getObservationScoreThresholdFactor() const; - - /// Used to determine whether two beliefs are equal - bool isNumericPrecisionSetFromDefault() const; - double getNumericPrecision() const; bool isDynamicTriangulationModeSet() const; bool isStaticTriangulationModeSet() const; - /// Controls if (grid) clipping is to be used + /// Controls if grid clipping is to be used bool isUseClippingSet() const; - bool isStateEliminationCutoffSet() const; + bool isBeliefMDPNumberTypeDouble() const; + bool isBeliefMDPNumberTypeRational() const; + bool isBeliefMDPNumberTypeMatch() const; - template - void setValuesInOptionsStruct(storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions& options) const; + bool isInexactPreprocessingSet() const; // The name of the module. static const std::string moduleName; @@ -83,6 +47,4 @@ class BeliefExplorationSettings : public ModuleSettings { private: }; -} // namespace modules -} // namespace settings -} // namespace storm +} // namespace storm::settings::modules diff --git a/src/storm-pomdp-cli/settings/modules/POMDPSettings.cpp b/src/storm-pomdp-cli/settings/modules/POMDPSettings.cpp index fd52ec7c54..b9c45b29b0 100644 --- a/src/storm-pomdp-cli/settings/modules/POMDPSettings.cpp +++ b/src/storm-pomdp-cli/settings/modules/POMDPSettings.cpp @@ -24,6 +24,8 @@ const std::string memoryPatternOption = "memorypattern"; const std::vector memoryPatterns = {"trivial", "fixedcounter", "selectivecounter", "ring", "fixedring", "settablebits", "full"}; const std::string checkFullyObservableOption = "check-fully-observable"; const std::string isQualitativeOption = "qualitative-analysis"; +const std::string isBoundedToUnboundedReachabilityTransformationOption = "unfold-reward-bound"; +const std::string isRewardObservableOption = "reward-aware"; POMDPSettings::POMDPSettings() : ModuleSettings(moduleName) { this->addOption(storm::settings::OptionBuilder(moduleName, noCanonicOption, false, @@ -52,17 +54,28 @@ POMDPSettings::POMDPSettings() : ModuleSettings(moduleName) { .setDefaultValueString("full") .build()) .build()); - this->addOption( - storm::settings::OptionBuilder(moduleName, beliefExplorationOption, false, "Analyze the POMDP by exploring the belief state-space.") - .addArgument(storm::settings::ArgumentBuilder::createStringArgument("mode", "Sets whether lower, upper, or interval result bounds are computed.") - .addValidatorString(ArgumentValidatorFactory::createMultipleChoiceValidator(beliefExplorationModes)) - .setDefaultValueString("both") - .makeOptional() - .build()) - .build()); + this->addOption(storm::settings::OptionBuilder(moduleName, beliefExplorationOption, false, "Analyze the POMDP by exploring the belief space.") + .addArgument(storm::settings::ArgumentBuilder::createStringArgument( + "mode", "Sets whether lower bounds, upper bounds, or interval bounds are computed.") + .addValidatorString(ArgumentValidatorFactory::createMultipleChoiceValidator(beliefExplorationModes)) + .setDefaultValueString("both") + .makeOptional() + .build()) + .build()); this->addOption( storm::settings::OptionBuilder(moduleName, checkFullyObservableOption, false, "Performs standard model checking on the underlying MDP").build()); this->addOption(storm::settings::OptionBuilder(moduleName, isQualitativeOption, false, "Sets the option qualitative analysis").build()); + this->addOption(storm::settings::OptionBuilder( + moduleName, isBoundedToUnboundedReachabilityTransformationOption, false, + "Sets the option that reward bounded reachability properties are transformed to an unbounded problem on an unfolded POMDP.") + .build()); + this->addOption(storm::settings::OptionBuilder(moduleName, isRewardObservableOption, false, + "Sets the option that rewards are observable for bounded reachability properties.") + .addArgument(storm::settings::ArgumentBuilder::createStringArgument("levelwidths", "comma separated list of width of reward levels.") + .setDefaultValueString("") + .makeOptional() + .build()) + .build()); } bool POMDPSettings::isNoCanonicSet() const { @@ -111,6 +124,25 @@ bool POMDPSettings::isQualitativeAnalysisSet() const { return this->getOption(isQualitativeOption).getHasOptionBeenSet(); } +bool POMDPSettings::isBoundedToUnboundedReachabilityTransformationSet() const { + return this->getOption(isBoundedToUnboundedReachabilityTransformationOption).getHasOptionBeenSet(); +} + +bool POMDPSettings::isRewardObservableSet() const { + return this->getOption(isRewardObservableOption).getHasOptionBeenSet(); +} + +std::vector POMDPSettings::getLevelWidthForBoundedReachability() const { + auto const input = this->getOption(isRewardObservableOption).getArgumentByName("levelwidths").getValueAsString(); + if (input.empty()) { + return {}; + } + // split the string by comma + auto result = input | std::ranges::views::split(',') | + std::ranges::views::transform([](auto&& r) -> uint64_t { return std::stoull(std::string(r.begin(), r.end())); }); + return {result.begin(), result.end()}; +} + uint64_t POMDPSettings::getMemoryBound() const { return this->getOption(memoryBoundOption).getArgumentByName("bound").getValueAsUnsignedInteger(); } diff --git a/src/storm-pomdp-cli/settings/modules/POMDPSettings.h b/src/storm-pomdp-cli/settings/modules/POMDPSettings.h index 11f5070aba..a6db737a38 100644 --- a/src/storm-pomdp-cli/settings/modules/POMDPSettings.h +++ b/src/storm-pomdp-cli/settings/modules/POMDPSettings.h @@ -33,6 +33,9 @@ class POMDPSettings : public ModuleSettings { bool isSelfloopReductionSet() const; bool isCheckFullyObservableSet() const; bool isQualitativeAnalysisSet() const; + bool isBoundedToUnboundedReachabilityTransformationSet() const; + std::vector getLevelWidthForBoundedReachability() const; + bool isRewardObservableSet() const; uint64_t getMemoryBound() const; storm::storage::PomdpMemoryPattern getMemoryPattern() const; diff --git a/src/storm-pomdp-cli/storm-pomdp.cpp b/src/storm-pomdp-cli/storm-pomdp.cpp index d7f6fa18e6..6162195cd2 100644 --- a/src/storm-pomdp-cli/storm-pomdp.cpp +++ b/src/storm-pomdp-cli/storm-pomdp.cpp @@ -7,19 +7,27 @@ #include "storm-pomdp-cli/settings/modules/POMDPSettings.h" #include "storm-pomdp-cli/settings/modules/QualitativePOMDPAnalysisSettings.h" #include "storm-pomdp-cli/settings/modules/ToParametricSettings.h" + +#include "storm-pomdp/analysis/FiniteBeliefMdpDetection.h" #include "storm-pomdp/analysis/FormulaInformation.h" #include "storm-pomdp/analysis/IterativePolicySearch.h" #include "storm-pomdp/analysis/JaniBeliefSupportMdpGenerator.h" #include "storm-pomdp/analysis/OneShotPolicySearch.h" #include "storm-pomdp/analysis/QualitativeAnalysisOnGraphs.h" #include "storm-pomdp/analysis/UniqueObservationStates.h" -#include "storm-pomdp/modelchecker/BeliefExplorationPomdpModelChecker.h" +#include "storm-pomdp/beliefs/storage/Belief.h" +#include "storm-pomdp/beliefs/verification/BeliefBasedModelChecker.h" +#include "storm-pomdp/modelchecker/PreprocessingPomdpValueBoundsModelChecker.h" +#include "storm-pomdp/storage/BeliefExplorationResult.h" #include "storm-pomdp/transformer/ApplyFiniteSchedulerToPomdp.h" #include "storm-pomdp/transformer/BinaryPomdpTransformer.h" #include "storm-pomdp/transformer/GlobalPOMDPSelfLoopEliminator.h" #include "storm-pomdp/transformer/GlobalPomdpMecChoiceEliminator.h" #include "storm-pomdp/transformer/KnownProbabilityTransformer.h" +#include "storm-pomdp/transformer/MakeStateSetObservationClosed.h" #include "storm-pomdp/transformer/PomdpMemoryUnfolder.h" +#include "storm-pomdp/transformer/RewardBoundUnfolder.h" +#include "storm-pomdp/transformer/ToStateBasedObservationTransformer.h" #include "storm/analysis/GraphConditions.h" #include "storm/api/storm.h" #include "storm/exceptions/InvalidPropertyException.h" @@ -27,12 +35,12 @@ #include "storm/exceptions/UnexpectedException.h" #include "storm/exceptions/WrongFormatException.h" #include "storm/modelchecker/results/ExplicitQualitativeCheckResult.h" -#include "storm/settings/modules/DebugSettings.h" -#include "storm/settings/modules/GeneralSettings.h" #include "storm/transformer/MakePOMDPCanonic.h" +#include "storm/transformer/SparseModelValueTypeTransformer.h" #include "storm/utility/NumberTraits.h" #include "storm/utility/SignalHandler.h" #include "storm/utility/Stopwatch.h" +#include "storm/utility/graph.h" namespace storm { namespace pomdp { @@ -60,7 +68,6 @@ bool performPreprocessing(std::shared_ptr qualitativeAnalysis(*pomdp); STORM_PRINT_AND_LOG("Computing states with probability 0 ..."); storm::storage::BitVector prob0States = qualitativeAnalysis.analyseProb0(formula.asProbabilityOperatorFormula()); - std::cout << prob0States << '\n'; STORM_PRINT_AND_LOG(" done. " << prob0States.getNumberOfSetBits() << " states found.\n"); STORM_PRINT_AND_LOG("Computing states with probability 1 ..."); storm::storage::BitVector prob1States = qualitativeAnalysis.analyseProb1(formula.asProbabilityOperatorFormula()); @@ -76,36 +83,61 @@ bool performPreprocessing(std::shared_ptr -void printResult(ValueType const& lowerBound, ValueType const& upperBound) { - if (lowerBound == upperBound) { - if (storm::utility::isInfinity(lowerBound)) { - STORM_PRINT_AND_LOG("inf"); - } else { - STORM_PRINT_AND_LOG(lowerBound); - } - } else if (storm::utility::isInfinity(-lowerBound)) { - if (storm::utility::isInfinity(upperBound)) { - STORM_PRINT_AND_LOG("[-inf, inf] (width=inf)"); +void printResult(std::optional const& lowerBound, std::optional const& upperBound) { + if (lowerBound.has_value() && upperBound.has_value()) { + if (*lowerBound == *upperBound) { + if (storm::utility::isInfinity(*lowerBound)) { + STORM_PRINT_AND_LOG("inf"); + } else { + STORM_PRINT_AND_LOG(*lowerBound); + } + } else if (storm::utility::isInfinity(-*lowerBound)) { + if (storm::utility::isInfinity(*upperBound)) { + STORM_PRINT_AND_LOG("[-inf, inf] (width=inf)"); + } } else { - // Only upper bound is known - STORM_PRINT_AND_LOG("≤ " << upperBound); + STORM_PRINT_AND_LOG("[" << *lowerBound << ", " << *upperBound << "] (width=" << ValueType(*upperBound - *lowerBound) << ")"); } - } else if (storm::utility::isInfinity(upperBound)) { - STORM_PRINT_AND_LOG("≥ " << lowerBound); - } else { - STORM_PRINT_AND_LOG("[" << lowerBound << ", " << upperBound << "] (width=" << ValueType(upperBound - lowerBound) << ")"); + } else if (lowerBound.has_value()) { + STORM_PRINT_AND_LOG("≥ " << *lowerBound); + } else if (upperBound.has_value()) { + STORM_PRINT_AND_LOG("≤ " << *upperBound); } - if (storm::NumberTraits::IsExact) { + if constexpr (storm::NumberTraits::IsExact) { STORM_PRINT_AND_LOG(" (approx. "); - double roundedLowerBound = - storm::utility::isInfinity(-lowerBound) ? -storm::utility::infinity() : storm::utility::convertNumber(lowerBound); - double roundedUpperBound = - storm::utility::isInfinity(upperBound) ? storm::utility::infinity() : storm::utility::convertNumber(upperBound); + std::optional roundedLowerBound = std::nullopt; + std::optional roundedUpperBound = std::nullopt; + if (lowerBound.has_value()) { + roundedLowerBound = + storm::utility::isInfinity(-*lowerBound) ? -storm::utility::infinity() : storm::utility::convertNumber(*lowerBound); + } + if (upperBound.has_value()) { + roundedUpperBound = + storm::utility::isInfinity(*upperBound) ? storm::utility::infinity() : storm::utility::convertNumber(*upperBound); + } printResult(roundedLowerBound, roundedUpperBound); STORM_PRINT_AND_LOG(")"); } } +template +void printBeliefExplorationStatistics(Statistics const& statistics) { + if (!statistics.available) { + return; + } + STORM_PRINT_AND_LOG("Belief exploration " << (statistics.completedExploration ? "completed" : "stopped early") << ": " << statistics.discoveredBeliefs + << " beliefs discovered, " << statistics.exploredBeliefs << " beliefs explored.\n"); + STORM_PRINT_AND_LOG("Constructed belief MDP: " << statistics.beliefMdpStates << " states, " << statistics.beliefMdpChoices << " choices, " + << statistics.beliefMdpTransitions << " transitions.\n"); + if (statistics.processedMdpStates && statistics.processedMdpChoices && statistics.processedMdpTransitions) { + STORM_PRINT_AND_LOG("Processed belief MDP: " << *statistics.processedMdpStates << " states, " << *statistics.processedMdpChoices << " choices, " + << *statistics.processedMdpTransitions << " transitions.\n"); + } + STORM_PRINT_AND_LOG("Time for exploring beliefs: " << statistics.explorationTimeMilliseconds << "ms.\n"); + STORM_PRINT_AND_LOG("Time for building the belief MDP: " << statistics.beliefMdpBuildTimeMilliseconds << "ms.\n"); + STORM_PRINT_AND_LOG("Time for analyzing the belief MDP: " << statistics.beliefMdpAnalysisTimeMilliseconds << "ms.\n"); +} + MemlessSearchOptions fillMemlessSearchOptionsFromSettings() { storm::pomdp::MemlessSearchOptions options; auto const& qualSettings = storm::settings::getModule(); @@ -242,27 +274,247 @@ void performQualitativeAnalysis(std::shared_ptr +bool performBeliefExploration(std::shared_ptr> const& pomdp, + storm::pomdp::analysis::FormulaInformation const& formulaInfo, storm::logic::Formula const& formula) { + auto const& pomdpSettings = storm::settings::getModule(); + auto const& belExplSettings = storm::settings::getModule(); + storm::Environment env; + + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions revisedOptions; + // We hard-code this to FIFO for now to mimic the legacy behaviour + revisedOptions.explorationQueueOrder = beliefs::ExplorationQueueOrder::FIFO; + if (belExplSettings.getExplorationTimeLimit() != 0) { + revisedOptions.maxExplorationTime = belExplSettings.getExplorationTimeLimit(); + } + if (belExplSettings.isCutZeroGapSet()) { + revisedOptions.maxGapToCut = storm::utility::zero(); + } + + std::shared_ptr> preprocessedPomdpPtr = pomdp; + + std::optional rewardModelName; + std::set targetObservations; + if (formulaInfo.isNonNestedReachabilityProbability() || formulaInfo.isNonNestedExpectedRewardFormula()) { + if (formulaInfo.getTargetStates().observationClosed) { + targetObservations = formulaInfo.getTargetStates().observations; + } else { + storm::transformer::MakeStateSetObservationClosed obsCloser(pomdp); + std::tie(preprocessedPomdpPtr, targetObservations) = obsCloser.transform(formulaInfo.getTargetStates().states); + } + if (formulaInfo.isNonNestedReachabilityProbability()) { + if (!formulaInfo.getSinkStates().empty()) { + storm::storage::sparse::ModelComponents components; + components.stateLabeling = preprocessedPomdpPtr->getStateLabeling(); + components.rewardModels = preprocessedPomdpPtr->getRewardModels(); + auto matrix = preprocessedPomdpPtr->getTransitionMatrix(); + matrix.makeRowGroupsAbsorbing(formulaInfo.getSinkStates().states, true); + STORM_LOG_ASSERT(matrix.isProbabilistic(storm::utility::zero()), "Resulting transition matrix is not a probability matrix."); + STORM_LOG_ASSERT(matrix.hasOnlyPositiveEntries(), "Resulting transition matrix has non-positive entries."); + components.transitionMatrix = matrix; + components.observabilityClasses = preprocessedPomdpPtr->getObservations(); + if (preprocessedPomdpPtr->hasChoiceLabeling()) { + components.choiceLabeling = preprocessedPomdpPtr->getChoiceLabeling(); + } + if (preprocessedPomdpPtr->hasObservationValuations()) { + components.observationValuations = preprocessedPomdpPtr->getObservationValuations(); + } + preprocessedPomdpPtr = std::make_shared>(std::move(components), true); + auto reachableFromSinkStates = + storm::utility::graph::getReachableStates(preprocessedPomdpPtr->getTransitionMatrix(), formulaInfo.getSinkStates().states, + formulaInfo.getSinkStates().states, ~formulaInfo.getSinkStates().states); + reachableFromSinkStates &= ~formulaInfo.getSinkStates().states; + STORM_LOG_THROW(reachableFromSinkStates.empty(), storm::exceptions::NotSupportedException, + "There are sink states that can reach non-sink states. This is currently not supported"); + } + } else { + // Expected reward formula! + rewardModelName = formulaInfo.getRewardModelName(); + } + } else { + STORM_LOG_THROW(false, storm::exceptions::NotSupportedException, "Unsupported formula '" << formula << "'."); + } + std::optional optionalTargetStates; + optionalTargetStates = formulaInfo.getTargetStates().states; + if (storm::pomdp::detectFiniteBeliefMdp(*preprocessedPomdpPtr, optionalTargetStates)) { + STORM_LOG_INFO("Detected that the belief MDP is finite."); + } + + storm::pomdp::storage::BeliefExplorationBounds beliefExplorationBounds; + + if (!formulaInfo.isBounded()) { + // Precompute initial bounds used for cut-offs and clipping + if (belExplSettings.isInexactPreprocessingSet()) { + STORM_LOG_WARN("Using inexact preprocessing for belief exploration can lead to inaccurate results."); + auto preprocessedPomdpDouble = storm::transformer::SparseModelValueTypeTransformer().transformModel(preprocessedPomdpPtr); + auto inExactPreProcessingMC = modelchecker::PreprocessingPomdpValueBoundsModelChecker>( + *preprocessedPomdpDouble->template as>()); + beliefExplorationBounds.preprocessingBounds = inExactPreProcessingMC.getValueBounds(env, formula).template toValueType(); + if (belExplSettings.isUseClippingSet() && rewardModelName) { + beliefExplorationBounds.extremeBounds = inExactPreProcessingMC.getExtremeValueBound(env, formula).template toValueType(); + } + } else { + auto preProcessingMC = modelchecker::PreprocessingPomdpValueBoundsModelChecker>(*preprocessedPomdpPtr); + beliefExplorationBounds.preprocessingBounds = preProcessingMC.getValueBounds(env, formula); + if (belExplSettings.isUseClippingSet() && rewardModelName) { + beliefExplorationBounds.extremeBounds = preProcessingMC.getExtremeValueBound(env, formula); + } + } + } else { + // We only consider bounded probability formulae, so we can use 0-1 bounds + // TODO make smarter pre-computed value bounds + storm::pomdp::storage::PreprocessingPomdpValueBounds zeroOneValueBound; + zeroOneValueBound.lower.push_back(std::vector(preprocessedPomdpPtr->getNumberOfStates(), storm::utility::zero())); + zeroOneValueBound.upper.push_back(std::vector(preprocessedPomdpPtr->getNumberOfStates(), storm::utility::one())); + + beliefExplorationBounds.preprocessingBounds = zeroOneValueBound; + } + + uint64_t initialPomdpState = preprocessedPomdpPtr->getInitialStates().getNextSetIndex(0); + storage::BeliefExplorationResult result( + beliefExplorationBounds.preprocessingBounds->template getHighestLowerBound(initialPomdpState), + beliefExplorationBounds.preprocessingBounds->template getSmallestUpperBound(initialPomdpState)); + STORM_LOG_INFO("Initial value bounds are [" << *result.lowerBound << ", " << *result.upperBound << "]"); + + storm::pomdp::beliefs::PropertyInformation propertyInfo; + if (rewardModelName) { + propertyInfo.kind = storm::pomdp::beliefs::PropertyInformation::Kind::ExpectedTotalReachabilityReward; + propertyInfo.rewardModelName = rewardModelName; + } else if (formulaInfo.isBounded()) { + propertyInfo.kind = storm::pomdp::beliefs::PropertyInformation::Kind::RewardBoundedReachabilityProbability; + // Collect reward bounds from bounded formula + auto boundedFormula = formula.asProbabilityOperatorFormula().getSubformula().asBoundedUntilFormula(); + for (uint64_t i = 0; i < boundedFormula.getDimension(); ++i) { + const auto& tbRef = boundedFormula.getTimeBoundReference(i); + if (tbRef.isRewardBound()) { + propertyInfo.rewardBounds.push_back( + {tbRef.getRewardName(), boundedFormula.getLowerBoundAsOptionalTimeBound(i), boundedFormula.getUpperBoundAsOptionalTimeBound(i)}); + } + } + } else { + propertyInfo.kind = storm::pomdp::beliefs::PropertyInformation::Kind::ReachabilityProbability; + } + propertyInfo.dir = formulaInfo.getOptimizationDirection(); + propertyInfo.targetObservations = targetObservations; + + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefType, BeliefMDPType> checker(*preprocessedPomdpPtr); + BeliefMDPType overResultValue; + BeliefMDPType underResultValue; + bool isOverApproximation{false}; + bool isUnderApproximation{false}; + bool completedExploration{false}; + if (pomdpSettings.isBeliefExplorationDiscretizeSet()) { + STORM_PRINT_AND_LOG("Computing an over-approximation via belief MDP discretization...\n"); + isOverApproximation = true; + if (belExplSettings.getSizeThresholdInit() == 0) { + revisedOptions.maxExplorationSize.reset(); + } else { + revisedOptions.maxExplorationSize = belExplSettings.getSizeThresholdInit(); + } + if (propertyInfo.kind == beliefs::PropertyInformation::Kind::RewardBoundedReachabilityProbability) { + std::vector relevantRewardModelNames; + for (auto const& rewardBound : propertyInfo.rewardBounds) { + relevantRewardModelNames.push_back(rewardBound.rewardModelName); + } + auto checkResult = + checker.checkRewardAwareDiscretize(env, propertyInfo, revisedOptions, belExplSettings.getResolutionInit(), + belExplSettings.isDynamicTriangulationModeSet(), beliefExplorationBounds, relevantRewardModelNames); + overResultValue = checkResult.first; + printBeliefExplorationStatistics(checker.getLastRunStatistics()); + } else { + auto checkResult = checker.checkDiscretize(env, propertyInfo, revisedOptions, belExplSettings.getResolutionInit(), + belExplSettings.isDynamicTriangulationModeSet(), beliefExplorationBounds); + overResultValue = checkResult.first; + printBeliefExplorationStatistics(checker.getLastRunStatistics()); + } + } + + if (pomdpSettings.isBeliefExplorationUnfoldSet()) { + STORM_PRINT_AND_LOG("Computing an under-approximation via belief MDP unfolding...\n"); + if (belExplSettings.getSizeThresholdInit() == 0) { + revisedOptions.maxExplorationSize = preprocessedPomdpPtr->getNumberOfStates() * preprocessedPomdpPtr->getMaxNrStatesWithSameObservation(); + STORM_PRINT_AND_LOG("Heuristically selected an under-approximation MDP size threshold of " << revisedOptions.maxExplorationSize.value() << ".\n"); + } else { + revisedOptions.maxExplorationSize = belExplSettings.getSizeThresholdInit(); + } + if (belExplSettings.isUseClippingSet()) { + revisedOptions.useClipping = true; + revisedOptions.clippingResolutions = std::vector(preprocessedPomdpPtr->getNrObservations(), belExplSettings.getClippingGridResolution()); + } + isUnderApproximation = true; + if (propertyInfo.kind == beliefs::PropertyInformation::Kind::RewardBoundedReachabilityProbability) { + std::vector relevantRewardModelNames; + for (auto const& rewardBound : propertyInfo.rewardBounds) { + relevantRewardModelNames.push_back(rewardBound.rewardModelName); + } + std::tie(underResultValue, completedExploration) = + checker.checkRewardAwareUnfold(env, propertyInfo, revisedOptions, beliefExplorationBounds, relevantRewardModelNames); + printBeliefExplorationStatistics(checker.getLastRunStatistics()); + } else { + std::tie(underResultValue, completedExploration) = checker.checkUnfold(env, propertyInfo, revisedOptions, beliefExplorationBounds); + printBeliefExplorationStatistics(checker.getLastRunStatistics()); + } + isOverApproximation = (completedExploration && !belExplSettings.isUseClippingSet()) || isOverApproximation; + } + + if (completedExploration && !belExplSettings.isUseClippingSet()) { + result.updateLowerBound(underResultValue); + result.updateUpperBound(underResultValue); + } else { + if (isOverApproximation) { + if (storm::solver::maximize(propertyInfo.dir)) { + result.updateUpperBound(overResultValue); + if (!isUnderApproximation) { + result.removeLowerBound(); + } + } else { + result.updateLowerBound(overResultValue); + if (!isUnderApproximation) { + result.removeUpperBound(); + } + } + } + if (isUnderApproximation) { + if (storm::solver::maximize(propertyInfo.dir)) { + result.updateLowerBound(underResultValue); + if (!isOverApproximation) { + result.removeUpperBound(); + } + } else { + result.updateUpperBound(underResultValue); + if (!isOverApproximation) { + result.removeLowerBound(); + } + } + } + } + + if (storm::utility::resources::isTerminate()) { + STORM_PRINT_AND_LOG("\nResult till abort: "); + } else { + STORM_PRINT_AND_LOG("\nResult: "); + } + printResult(result.lowerBound, result.upperBound); + STORM_PRINT_AND_LOG('\n'); + return true; +} + template bool performAnalysis(std::shared_ptr> const& pomdp, storm::pomdp::analysis::FormulaInformation const& formulaInfo, storm::logic::Formula const& formula) { auto const& pomdpSettings = storm::settings::getModule(); bool analysisPerformed = false; if (pomdpSettings.isBeliefExplorationSet()) { - STORM_PRINT_AND_LOG("Exploring the belief MDP... \n"); - auto options = storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions(pomdpSettings.isBeliefExplorationDiscretizeSet(), - pomdpSettings.isBeliefExplorationUnfoldSet()); auto const& beliefExplorationSettings = storm::settings::getModule(); - beliefExplorationSettings.setValuesInOptionsStruct(options); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker, BeliefType> checker(pomdp, options); - auto result = checker.check(formula); - checker.printStatisticsToStream(std::cout); - if (storm::utility::resources::isTerminate()) { - STORM_PRINT_AND_LOG("\nResult till abort: "); - } else { - STORM_PRINT_AND_LOG("\nResult: "); + if (beliefExplorationSettings.isBeliefMDPNumberTypeDouble()) { + performBeliefExploration(pomdp, formulaInfo, formula); + } else if (beliefExplorationSettings.isBeliefMDPNumberTypeRational()) { + performBeliefExploration(pomdp, formulaInfo, formula); + } else if (beliefExplorationSettings.isBeliefMDPNumberTypeMatch()) { + STORM_LOG_ASSERT(beliefExplorationSettings.isBeliefMDPNumberTypeMatch(), + "Expected belief MDP number type to be set to match the POMDP, but it is not."); + performBeliefExploration(pomdp, formulaInfo, formula); } - printResult(result.lowerBound, result.upperBound); - STORM_PRINT_AND_LOG('\n'); analysisPerformed = true; } if (pomdpSettings.isQualitativeAnalysisSet()) { @@ -281,7 +533,7 @@ bool performAnalysis(std::shared_ptr> co } else { STORM_PRINT_AND_LOG("\nResult: "); } - printResult(result.getMin(), result.getMax()); + printResult(std::optional(result.getMin()), std::optional(result.getMax())); STORM_PRINT_AND_LOG('\n'); } else { STORM_PRINT_AND_LOG("\nResult: Not available.\n"); @@ -388,7 +640,54 @@ void processPomdp(std::shared_ptr>& pomd } template -void processFormula(std::shared_ptr>&& pomdp, std::shared_ptr const& formula) { +void processFormula(std::shared_ptr>&& pomdp, std::shared_ptr formula) { + if (formula->asOperatorFormula().getSubformula().isBoundedUntilFormula()) { + auto const& pomdpSettings = storm::settings::getModule(); + // Process bounded until formulas + // If level widths are given, unfold the levels and make sure that the level rewards (and only those) are made observable + // If explicit unfolding is requested, unfold all reward bounds (if levels were unfolded before, this means a second round of unfolding) + // If reward observability is set and no level widths are given, make all rewards occurring in the formula observable. This also happens if no + // unfolding was requested. + storm::utility::Stopwatch boundedUntilProcessingWatch(true); + auto const levelWidths = pomdpSettings.getLevelWidthForBoundedReachability(); + if (!levelWidths.empty()) { + STORM_PRINT_AND_LOG("Perform unfolding for observation levels.\n"); + // Unfold the levels (includes dimensions with level-width 0) + typename transformer::RewardBoundUnfolder::UnfoldingOptions options; + options.levelWidths = levelWidths; + auto const unfoldingResult = transformer::RewardBoundUnfolder::transform(*pomdp, *formula, options); + pomdp = unfoldingResult.model->template as>(); + formula = unfoldingResult.formula; + // make sure the levels are observable + std::set levelRewardModels; + formula->gatherReferencedRewardModels(levelRewardModels); + pomdp = storm::pomdp::transformer::ToStateBasedObservationTransformer::transformRewardAware(*pomdp, levelRewardModels); + } + std::set rewardModelsToObserve; + if (pomdpSettings.isRewardObservableSet() && levelWidths.empty()) { + formula->gatherReferencedRewardModels(rewardModelsToObserve); // keep rewards to make them observable later + } + if (pomdpSettings.isBoundedToUnboundedReachabilityTransformationSet()) { + STORM_PRINT_AND_LOG("Perform explicit unfolding of reward bounds.\n"); + typename transformer::RewardBoundUnfolder::UnfoldingOptions options; + options.preservedRewardModels = rewardModelsToObserve; + auto const unfoldingResult = transformer::RewardBoundUnfolder::transform(*pomdp, *formula, options); + pomdp = unfoldingResult.model->template as>(); + formula = unfoldingResult.formula; + } + if (pomdpSettings.isRewardObservableSet() && levelWidths.empty()) { + STORM_PRINT_AND_LOG("Extend observation function to become reward aware.\n"); + pomdp = storm::pomdp::transformer::ToStateBasedObservationTransformer::transformRewardAware(*pomdp, rewardModelsToObserve); + } + STORM_LOG_THROW(!levelWidths.empty() || pomdpSettings.isRewardObservableSet() || pomdpSettings.isBoundedToUnboundedReachabilityTransformationSet(), + storm::exceptions::InvalidSettingsException, + "No handling of bounded until formulas specified. Consider setting --unfold-reward-bound and/or --reward-aware."); + STORM_PRINT_AND_LOG("bounded reachability processing done. POMDP Information:\n"); + pomdp->printModelInformationToStream(std::cout); + STORM_PRINT_AND_LOG("Transformed formula: " << *formula << "\n"); + boundedUntilProcessingWatch.stop(); + STORM_PRINT_AND_LOG("Time for pre-processing: " << boundedUntilProcessingWatch << ".\n"); + } auto formulaInfo = storm::pomdp::analysis::getFormulaInformation(*pomdp, *formula); STORM_LOG_THROW(!formulaInfo.isUnsupported(), storm::exceptions::InvalidPropertyException, "The formula '" << *formula << "' is not supported by storm-pomdp."); diff --git a/src/storm-pomdp/analysis/FormulaInformation.cpp b/src/storm-pomdp/analysis/FormulaInformation.cpp index b1b6e0121b..a9233209e6 100644 --- a/src/storm-pomdp/analysis/FormulaInformation.cpp +++ b/src/storm-pomdp/analysis/FormulaInformation.cpp @@ -7,7 +7,6 @@ #include "storm/modelchecker/propositional/SparsePropositionalModelChecker.h" #include "storm/modelchecker/results/ExplicitQualitativeCheckResult.h" #include "storm/models/sparse/Pomdp.h" -#include "storm/models/sparse/StandardRewardModel.h" #include "storm/utility/macros.h" namespace storm { @@ -50,6 +49,10 @@ bool FormulaInformation::isUnsupported() const { return type == Type::Unsupported; } +bool FormulaInformation::isBounded() const { + return rewardBounded; +} + typename FormulaInformation::StateSet const& FormulaInformation::getTargetStates() const { STORM_LOG_ASSERT(this->type == Type::NonNestedExpectedRewardFormula || this->type == Type::NonNestedReachabilityProbability, "Target states requested for unexpected formula type."); @@ -71,6 +74,10 @@ storm::solver::OptimizationDirection const& FormulaInformation::getOptimizationD return optimizationDirection; } +std::vector const& FormulaInformation::getRewardBoundReferences() const { + return rewardBoundReferences; +} + bool FormulaInformation::minimize() const { return storm::solver::minimize(optimizationDirection); } @@ -78,6 +85,9 @@ bool FormulaInformation::minimize() const { bool FormulaInformation::maximize() const { return storm::solver::maximize(optimizationDirection); } +void FormulaInformation::setRewardBounded(bool newValue) { + rewardBounded = newValue; +} template FormulaInformation::StateSet getStateSet(PomdpType const& pomdp, storm::storage::BitVector&& inputStates) { @@ -109,6 +119,10 @@ void FormulaInformation::updateSinkStates(PomdpType const& pomdp, storm::storage STORM_LOG_ASSERT(this->type == Type::NonNestedReachabilityProbability, "Sink states requested for unexpected formula type."); sinkStates = getStateSet(pomdp, std::move(newSinkStates)); } +void FormulaInformation::setRewardBoundReferences(std::vector& newRewardBoundReferences) { + STORM_LOG_ASSERT(this->type == Type::NonNestedReachabilityProbability && this->isBounded(), "RewardBoundReference requested for unexpected formula type."); + rewardBoundReferences = newRewardBoundReferences; +} template storm::storage::BitVector getStates(storm::logic::Formula const& propositionalFormula, bool formulaInverted, PomdpType const& pomdp) { @@ -127,6 +141,9 @@ FormulaInformation getFormulaInformation(PomdpType const& pomdp, storm::logic::P "The property does not specify an optimization direction (min/max)."); STORM_LOG_WARN_COND(!formula.hasBound(), "The probability threshold for the given property will be ignored."); auto const& subformula = formula.getSubformula(); + bool bounded = false; + std::vector rewardBoundReferences; + std::shared_ptr targetStatesFormula, constraintsStatesFormula; if (subformula.isEventuallyFormula()) { targetStatesFormula = subformula.asEventuallyFormula().getSubformula().asSharedPointer(); @@ -135,12 +152,25 @@ FormulaInformation getFormulaInformation(PomdpType const& pomdp, storm::logic::P storm::logic::UntilFormula const& untilFormula = subformula.asUntilFormula(); targetStatesFormula = untilFormula.getRightSubformula().asSharedPointer(); constraintsStatesFormula = untilFormula.getLeftSubformula().asSharedPointer(); + } else if (subformula.isBoundedUntilFormula()) { + storm::logic::BoundedUntilFormula const& boundedUntilFormula = subformula.asBoundedUntilFormula(); + targetStatesFormula = boundedUntilFormula.getRightSubformula().asSharedPointer(); + constraintsStatesFormula = boundedUntilFormula.getLeftSubformula().asSharedPointer(); + bounded = true; + for (uint64_t i = 0; i < boundedUntilFormula.getDimension(); ++i) { + STORM_LOG_ASSERT(boundedUntilFormula.getTimeBoundReference(i).isRewardBound(), "Expected a reward bound reference."); + rewardBoundReferences.push_back(boundedUntilFormula.getTimeBoundReference(i)); + } } if (targetStatesFormula && targetStatesFormula->isInFragment(storm::logic::propositional()) && constraintsStatesFormula && constraintsStatesFormula->isInFragment(storm::logic::propositional())) { FormulaInformation result(FormulaInformation::Type::NonNestedReachabilityProbability, formula.getOptimalityType()); result.updateTargetStates(pomdp, getStates(*targetStatesFormula, false, pomdp)); result.updateSinkStates(pomdp, getStates(*constraintsStatesFormula, true, pomdp)); + result.setRewardBounded(bounded); + if (bounded) { + result.setRewardBoundReferences(rewardBoundReferences); + } return result; } return FormulaInformation(); @@ -162,6 +192,10 @@ FormulaInformation getFormulaInformation(PomdpType const& pomdp, storm::logic::R rewardModelName = pomdp.getUniqueRewardModelName(); } auto const& subformula = formula.getSubformula(); + if (subformula.isDiscountedTotalRewardFormula()) { + FormulaInformation result(FormulaInformation::Type::DiscountedTotalRewardFormula, formula.getOptimalityType(), rewardModelName); + return result; + } std::shared_ptr targetStatesFormula; if (subformula.isEventuallyFormula()) { targetStatesFormula = subformula.asEventuallyFormula().getSubformula().asSharedPointer(); diff --git a/src/storm-pomdp/analysis/FormulaInformation.h b/src/storm-pomdp/analysis/FormulaInformation.h index 4e2853c482..327baa61f7 100644 --- a/src/storm-pomdp/analysis/FormulaInformation.h +++ b/src/storm-pomdp/analysis/FormulaInformation.h @@ -3,6 +3,7 @@ #include #include #include +#include "storm/logic/TimeBoundType.h" #include "storm/solver/OptimizationDirection.h" #include "storm/storage/BitVector.h" @@ -39,11 +40,13 @@ class FormulaInformation { bool isNonNestedReachabilityProbability() const; bool isNonNestedExpectedRewardFormula() const; bool isDiscountedTotalRewardFormula() const; + bool isBounded() const; bool isUnsupported() const; StateSet const& getTargetStates() const; StateSet const& getSinkStates() const; // Shall not be called for reward formulas std::string const& getRewardModelName() const; // Shall not be called for probability formulas storm::solver::OptimizationDirection const& getOptimizationDirection() const; + std::vector const& getRewardBoundReferences() const; bool minimize() const; bool maximize() const; @@ -53,12 +56,18 @@ class FormulaInformation { template void updateSinkStates(PomdpType const& pomdp, storm::storage::BitVector&& newSinkStates); + void setRewardBounded(bool newValue); + + void setRewardBoundReferences(std::vector& newRewardBoundReferences); + private: Type type; storm::solver::OptimizationDirection optimizationDirection; std::optional targetStates; std::optional sinkStates; std::optional rewardModelName; + std::vector rewardBoundReferences; + bool rewardBounded = false; }; template diff --git a/src/storm-pomdp/api/verification.h b/src/storm-pomdp/api/verification.h deleted file mode 100644 index 8e9d5c7962..0000000000 --- a/src/storm-pomdp/api/verification.h +++ /dev/null @@ -1,147 +0,0 @@ -#pragma once - -#include -#include -#include - -#include "storm-pomdp/modelchecker/BeliefExplorationPomdpModelChecker.h" -#include "storm/environment/Environment.h" -#include "storm/modelchecker/CheckTask.h" -#include "storm/models/sparse/Model.h" -#include "storm/models/sparse/Pomdp.h" -#include "storm/storage/Scheduler.h" -#include "storm/utility/constants.h" - -namespace storm { -namespace pomdp { -namespace api { - -/** - * Uses the belief exploration with cut-offs to under-approximate the given objective on a POMDP. - * @tparam ValueType number type to be used - * @param pomdp the input pomdp to be checked - * @param task the check task to be performed - * @param sizeThreshold number of states up to which the belief MDP should be unfolded - * @param pomdpStateValues additional values that can be used for cut-offs in the under-approximation (generated by finite memory schedulers). - * Each element of the outer vector represents a scheduler. Each scheduler itself is represented by a vector of maps representing (memory node x state) -> value - * @return the result structure - */ -template -typename storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker>::Result underapproximateWithCutoffs( - std::shared_ptr> pomdp, storm::modelchecker::CheckTask const& task, - uint64_t sizeThreshold, - std::vector>> additionalPomdpStateValues = - std::vector>>()) { - storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions options(false, true); - options.useClipping = false; - options.useStateEliminationCutoff = false; - options.sizeThresholdInit = sizeThreshold; - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> modelchecker(pomdp, options); - return modelchecker.check(task.getFormula(), additionalPomdpStateValues); -} - -/** - * Uses the belief exploration with cut-offs *without* the pre-processing to generate cut-off values to under-approximate the given objective on a POMDP. - * Cut-off values need to be provided in the form of a vector of vectors representing finite memory schedulers. - * @tparam ValueType number type to be used - * @param pomdp the input pomdp to be checked - * @param task the check task to be performed - * @param sizeThreshold number of states up to which the belief MDP should be unfolded - * @param pomdpStateValues additional values that can be used for cut-offs in the under-approximation (generated by finite memory schedulers). - * Each element of the outer vector represents a scheduler. Each scheduler itself is represented by a vector of maps representing (memory node x state) -> value - * @return the result structure - */ -template -typename storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker>::Result underapproximateWithoutHeuristicValues( - std::shared_ptr> pomdp, storm::modelchecker::CheckTask const& task, - uint64_t sizeThreshold, std::vector>> pomdpStateValues) { - storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions options(false, true); - options.skipHeuristicSchedulers = true; - options.useClipping = false; - options.useStateEliminationCutoff = false; - options.sizeThresholdInit = sizeThreshold; - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> modelchecker(pomdp, options); - return modelchecker.check(task.getFormula(), pomdpStateValues); -} - -// Interactive Interface -/** - * Create a model checker with the correct settings for an interactive unfolding. Needs to be called first to set-up the unfolding. - * @tparam ValueType number type to be used - * @param env the environment to use - * @param pomdp the input pomdp to be checked - * @param useClipping true if clipping is to be used in addition to cut-offs - * @return the model checker object, configured for an interactive unfolding - */ -template -storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> createInteractiveUnfoldingModelChecker( - storm::Environment const& env, std::shared_ptr> pomdp, bool useClipping) { - storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions options(false, true); - options.skipHeuristicSchedulers = false; - options.useClipping = useClipping; - options.useStateEliminationCutoff = false; - options.sizeThresholdInit = storm::utility::infinity(); - options.interactiveUnfolding = true; - options.refine = false; - options.gapThresholdInit = 0; - options.cutZeroGap = false; - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> modelchecker(pomdp, options); - return modelchecker; -} - -/** - * Start an interactive unfolding to under approximate the given objective - * @tparam ValueType number type to be used - * @param modelchecker the model checker object configured for the interactive unfolding - * @param task the check task to be performed - * @param additionalPomdpStateValues additional values that can be used for cut-offs in the under-approximation (generated by finite memory schedulers). - * Each element of the outer vector represents a scheduler. Each scheduler itself is represented by a vector of maps representing (memory node x state) -> value - */ -template -void startInteractiveExploration(storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker>& modelchecker, - storm::modelchecker::CheckTask const& task, - std::vector>> additionalPomdpStateValues = - std::vector>>()) { - modelchecker.check(task.getFormula(), additionalPomdpStateValues); -} - -/** - * Extract the scheduler generated by an under-approximation from the given result struct. The scheduler is represented by a Markov chain. - * @tparam ValueType number type to be used - * @param modelcheckingResult the result struct containing the scheduler - * @return the scheduler represented by a Markov chain. - */ -template -std::shared_ptr> extractSchedulerAsMarkovChain( - typename storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker>::Result modelcheckingResult) { - return modelcheckingResult.schedulerAsMarkovChain; -} - -/** - * Get a specific scheduler used to generate cut-off values from the result struct. - * @tparam ValueType number type to be used - * @param modelcheckingResult the result struct - * @param schedId the ID of the scheduler used during the exploration. This corresponds to the labels in the scheduler MC. - * @return the desired scheduler - */ -template -storm::storage::Scheduler getCutoffScheduler( - typename storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker>::Result modelcheckingResult, - uint64_t schedId) { - return modelcheckingResult.cutoffSchedulers.at(schedId); -} - -/** - * Get the overall number of schedulers generated by the pre-processing for the under-approximation from the result struct. - * @tparam ValueType number type to be used - * @param modelcheckingResult the result struct - * @return the number of schedulers - */ -template -uint64_t getNumberOfPreprocessingSchedulers( - typename storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker>::Result modelcheckingResult) { - return modelcheckingResult.cutoffSchedulers.size(); -} -} // namespace api -} // namespace pomdp -} // namespace storm diff --git a/src/storm-pomdp/beliefs/abstraction/ClippingBeliefAbstraction.cpp b/src/storm-pomdp/beliefs/abstraction/ClippingBeliefAbstraction.cpp new file mode 100644 index 0000000000..75f8374fc0 --- /dev/null +++ b/src/storm-pomdp/beliefs/abstraction/ClippingBeliefAbstraction.cpp @@ -0,0 +1,192 @@ +#include "storm-pomdp/beliefs/abstraction/ClippingBeliefAbstraction.h" + +#include "storm-pomdp/beliefs/storage/Belief.h" + +#include "storm-pomdp/beliefs/storage/BeliefBuilder.h" +#include "storm-pomdp/beliefs/utility/BeliefNumerics.h" +#include "storm/adapters/RationalNumberAdapter.h" +#include "storm/solver/LpSolver.h" +#include "storm/storage/expressions/Expression.h" +#include "storm/storage/expressions/ExpressionManager.h" +#include "storm/utility/solver.h" + +namespace storm::pomdp::beliefs { + +template +ClippingBeliefAbstraction::ClippingBeliefAbstraction(storm::Environment const& env, std::vector&& observationResolutions) + : observationResolutions(std::forward>(observationResolutions)) { + STORM_LOG_ASSERT(std::all_of(this->observationResolutions.begin(), this->observationResolutions.end(), [](auto o) { return o > 0; }), + "Expected that the resolutions are positive."); + lpSolver = storm::utility::solver::getLpSolver(env, "POMDP LP Solver"); + lpSolver->push(); +} + +template +ClippingBeliefAbstraction::ClippingBeliefAbstraction(storm::Environment const& env, std::vector&& observationResolutions, + std::vector&& extremalRewardValues) + : observationResolutions(std::forward>(observationResolutions)), + extremalRewardValues(std::forward>(extremalRewardValues)) { + STORM_LOG_ASSERT(std::all_of(this->observationResolutions.begin(), this->observationResolutions.end(), [](auto o) { return o > 0; }), + "Expected that the resolutions are positive."); + lpSolver = storm::utility::solver::getLpSolver(env, "POMDP LP Solver"); + lpSolver->push(); +} + +template +typename ClippingBeliefAbstraction::BeliefClipping ClippingBeliefAbstraction::clipBeliefToGrid(const BeliefType& belief, + const uint64_t resolution) { + lpSolver->pop(); + lpSolver->push(); + + auto const resolutionConverted = storm::utility::convertNumber(resolution); + + std::vector helper(belief.size(), storm::utility::zero()); + helper[0] = resolutionConverted; + bool done = false; + // Set-up Variables + std::vector decisionVariables; + // Add variable for the clipping value, it is to be minimized + auto bigDelta = lpSolver->addBoundedContinuousVariable("D", storm::utility::zero(), storm::utility::one(), + storm::utility::one()); + // State clipping values + std::vector deltas; + uint64_t i = 0; + belief.forEach([this, &deltas, &i](BeliefStateType const& state, BeliefValueType const& beliefValue) { + auto localDelta = lpSolver->addBoundedContinuousVariable("d_" + std::to_string(i), storm::utility::zero(), beliefValue); + deltas.push_back(storm::expressions::Expression(localDelta)); + ++i; + }); + lpSolver->update(); + std::vector gridCandidates; + while (!done) { + BeliefBuilder candidateBuilder; + candidateBuilder.setObservation(belief.observation()); + + uint64_t j{0}; + uint64_t const jMax = belief.size() - 1; + belief.forEach([&helper, &j, &resolutionConverted, &candidateBuilder, &jMax](BeliefStateType const& state, BeliefValueType const& beliefValue) { + if (j < jMax) { + if (!BeliefNumerics::isZero(helper[j] - helper[j + 1])) { + candidateBuilder.addValue(state, (helper[j] - helper[j + 1]) / resolutionConverted); + } + } else { + if (!BeliefNumerics::isZero(helper[jMax])) { + candidateBuilder.addValue(state, helper[jMax] / resolutionConverted); + } + } + ++j; + }); + auto candidate = candidateBuilder.build(); + if (candidate == belief) { + STORM_LOG_TRACE(belief.toString() << " on clipping grid."); + // TODO Improve handling of successors which are already on the grid + return BeliefClipping{false, std::move(candidate), storm::utility::zero(), {}, true}; + } else { + gridCandidates.push_back(candidate); + + // Add variables a_j + auto decisionVar = lpSolver->addBinaryVariable("a_" + std::to_string(gridCandidates.size() - 1)); + decisionVariables.push_back(storm::expressions::Expression(decisionVar)); + lpSolver->update(); + + i = 0; + belief.forEachCombine(candidate, [&](BeliefStateType const& state, BeliefValueType const& beliefValue, BeliefValueType const& candidateValue) { + // Add the constraint to describe the transformation between the state values in the beliefs + // Add d_i >= b(s_i) - b_j(s_i) + D * b_j(s_i) - 1 + a_j + lpSolver->addConstraint("state_eq_" + std::to_string(i) + "_" + std::to_string(gridCandidates.size() - 1), + deltas.at(i) >= lpSolver->getConstant(beliefValue) - lpSolver->getConstant(candidateValue) + + storm::expressions::Expression(bigDelta) * lpSolver->getConstant(candidateValue) - + lpSolver->getConstant(storm::utility::one()) + + storm::expressions::Expression(decisionVar)); + ++i; + lpSolver->update(); + }); + } + if (helper.back() == storm::utility::convertNumber(resolution)) { + // If the last entry of helper is the gridResolution, we have enumerated all necessary distributions + done = true; + } else { + // Update helper by finding the index to increment + auto helperIt = helper.end() - 1; + while (*helperIt == *(helperIt - 1)) { + --helperIt; + } + STORM_LOG_ASSERT(helperIt != helper.begin(), "Error in grid clipping - index wrong"); + // Increment the value at the index + *helperIt += 1; + // Reset all indices greater than the changed one to 0 + ++helperIt; + while (helperIt != helper.end()) { + *helperIt = 0; + ++helperIt; + } + } + } + + // Only one target belief should be chosen + lpSolver->addConstraint("choice", storm::expressions::sum(decisionVariables) == lpSolver->getConstant(storm::utility::one())); + // Link D and d_i + lpSolver->addConstraint("delta", storm::expressions::Expression(bigDelta) == storm::expressions::sum(deltas)); + // Exclude D = 0 (self-loop) + lpSolver->addConstraint("not_zero", storm::expressions::Expression(bigDelta) > lpSolver->getConstant(storm::utility::zero())); + + lpSolver->update(); + + lpSolver->optimize(); + // Get the optimal belief for clipping + // Not a belief but has the same type + BeliefFlatMap deltaValues; + auto optDelta = storm::utility::zero(); + auto deltaSum = storm::utility::zero(); + if (lpSolver->isOptimal()) { + uint64_t targetBeliefIndex = std::numeric_limits::max(); + optDelta = lpSolver->getObjectiveValue(); + for (uint64_t dist = 0; dist < gridCandidates.size(); ++dist) { + if (lpSolver->getBinaryValue(lpSolver->getManager().getVariable("a_" + std::to_string(dist)))) { + targetBeliefIndex = dist; + break; + } + } + STORM_LOG_ASSERT(targetBeliefIndex < gridCandidates.size(), "LP optimal but no belief selected"); + auto targetBelief = gridCandidates.at(targetBeliefIndex); + i = 0; + belief.forEachStateInSupport([this, &i, &deltaValues, &deltaSum](BeliefStateType const& state) { + auto val = lpSolver->getContinuousValue(lpSolver->getManager().getVariable("d_" + std::to_string(i))); + if (!BeliefNumerics::lessOrEqual(val, storm::utility::zero())) { + deltaValues.emplace(state, val); + deltaSum += val; + } + ++i; + }); + + if (BeliefNumerics::isZero(optDelta)) { + // If we get an optimal value of 0, the LP solver considers two beliefs to be equal, possibly due to numerical instability + // For a sound result, we consider the state to not be clippable + STORM_LOG_WARN("LP solver returned an optimal value of 0. This should definitely not happen when using a grid"); + STORM_LOG_WARN("Origin" << belief.toString()); + STORM_LOG_WARN("Target [Bel " << targetBelief.toString()); + return BeliefClipping{false, std::move(targetBelief), storm::utility::zero(), {}, false}; + } + + if (optDelta == storm::utility::one()) { + STORM_LOG_WARN("LP solver returned an optimal value of 1. Sum of state clipping values is " << deltaSum); + // If we get an optimal value of 1, we cannot clip the belief as by definition this would correspond to a division by 0. + STORM_LOG_DEBUG("Origin " << belief.toString()); + STORM_LOG_DEBUG("Target " << targetBelief.toString()); + + if (deltaSum == storm::utility::one()) { + return BeliefClipping{false, std::move(targetBelief), storm::utility::zero(), {}, false}; + } + optDelta = deltaSum; + } + STORM_LOG_TRACE("Clip " << belief.toString() << " to " << targetBelief.toString() << " with value " << optDelta); + return BeliefClipping{true, std::move(targetBelief), optDelta, deltaValues, false}; + } + STORM_LOG_TRACE("Clipping " << belief.toString() << " not possible. LP not optimal."); + return BeliefClipping{false, belief, optDelta, deltaValues, false}; +} + +template class ClippingBeliefAbstraction>; +template class ClippingBeliefAbstraction>; + +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/beliefs/abstraction/ClippingBeliefAbstraction.h b/src/storm-pomdp/beliefs/abstraction/ClippingBeliefAbstraction.h new file mode 100644 index 0000000000..4e33fc30b8 --- /dev/null +++ b/src/storm-pomdp/beliefs/abstraction/ClippingBeliefAbstraction.h @@ -0,0 +1,84 @@ +#pragma once + +#include +#include +#include + +#include "storm-pomdp/beliefs/utility/types.h" +#include "storm/solver/LpSolver.h" +#include "storm/utility/constants.h" + +namespace storm { +class Environment; +} +namespace storm::pomdp::beliefs { + +/** + * Approximates a belief by clipping it to an observation-specific grid point. + * @see 10.1007/978-3-030-99527-0_2 + * + * Clipping preserves the probability mass represented by the grid belief and reports the removed mass separately to + * the callback. For expected rewards, optional extremal state values yield the reward correction for removed mass. + */ +template +class ClippingBeliefAbstraction { + public: + using BeliefValueType = typename BeliefType::ValueType; + + /** Result of clipping one belief to one grid. */ + struct BeliefClipping { + bool isClippable; + BeliefType targetBelief; + BeliefValueType delta; + BeliefFlatMap deltaValues; + bool onGrid = false; + }; + + /** Creates clipping grids with one resolution per POMDP observation. */ + explicit ClippingBeliefAbstraction(storm::Environment const& env, std::vector&& observationResolutions); + + /** Creates clipping grids and enables expected-reward corrections using an extremal value for every POMDP state. */ + ClippingBeliefAbstraction(storm::Environment const& env, std::vector&& observationResolutions, + std::vector&& extremalRewardValues); + + /** + * Clips a belief and passes its representation to @p callback. + * + * The callback receives the target grid belief, retained transition probability, optional removed probability, + * and optional reward adjustment, respectively. + */ + template + void abstract(BeliefType&& belief, BeliefValueType&& probabilityFactor, AbstractCallback const& callback) { + BeliefClipping clipping = clipBeliefToGrid(belief, observationResolutions[belief.observation()]); + if (clipping.isClippable) { + BeliefValueType a = (storm::utility::one() - clipping.delta) * probabilityFactor; + BeliefValueType b = clipping.delta * probabilityFactor; + if (extremalRewardValues.has_value()) { + // We compute the reward adjustment necessary for clipping (see https://doi.org/10.48550/arXiv.2201.08772) + // Because we don't add clipped beliefs into the abstraction MDP, we compute the influence of the clipping transition (based on the expected + // reward value in the implicit clipped belief) here such that we can simply add the value to the state-action reward of the transition + auto rewardAdjustment = storm::utility::zero(); + for (auto const& [state, deltaValue] : clipping.deltaValues) { + rewardAdjustment += deltaValue * extremalRewardValues->at(state); + } + rewardAdjustment = probabilityFactor * rewardAdjustment; + callback(std::move(clipping.targetBelief), std::move(a), std::move(b), std::move(rewardAdjustment)); + } else { + callback(std::move(clipping.targetBelief), std::move(a), std::move(b), std::nullopt); + } + } else { + // Belief on Grid + callback(std::move(belief), std::move(probabilityFactor), std::nullopt, std::nullopt); + } + } + + /** @return the grid clipping of @p belief at the given resolution. */ + BeliefClipping clipBeliefToGrid(BeliefType const& belief, uint64_t resolution); + + private: + std::vector observationResolutions; + std::shared_ptr> lpSolver; + std::optional> extremalRewardValues = std::nullopt; +}; + +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/beliefs/abstraction/FreudenthalTriangulationBeliefAbstraction.cpp b/src/storm-pomdp/beliefs/abstraction/FreudenthalTriangulationBeliefAbstraction.cpp new file mode 100644 index 0000000000..42bc54435e --- /dev/null +++ b/src/storm-pomdp/beliefs/abstraction/FreudenthalTriangulationBeliefAbstraction.cpp @@ -0,0 +1,18 @@ +#include "storm-pomdp/beliefs/abstraction/FreudenthalTriangulationBeliefAbstraction.h" +#include "storm-pomdp/beliefs/storage/Belief.h" +#include "storm/adapters/RationalNumberAdapter.h" + +namespace storm::pomdp::beliefs { + +template +FreudenthalTriangulationBeliefAbstraction::FreudenthalTriangulationBeliefAbstraction(BeliefValueType const& initialResolution, + FreudenthalTriangulationMode mode) + : defaultResolution(storm::utility::ceil(initialResolution)), mode(mode) { + STORM_LOG_ASSERT(defaultResolution > storm::utility::zero(), + "Expected that the resolution is a positive integer. Got " << defaultResolution << " instead."); +} + +template class FreudenthalTriangulationBeliefAbstraction>; +template class FreudenthalTriangulationBeliefAbstraction>; + +} // namespace storm::pomdp::beliefs \ No newline at end of file diff --git a/src/storm-pomdp/beliefs/abstraction/FreudenthalTriangulationBeliefAbstraction.h b/src/storm-pomdp/beliefs/abstraction/FreudenthalTriangulationBeliefAbstraction.h new file mode 100644 index 0000000000..b2cdac0dc1 --- /dev/null +++ b/src/storm-pomdp/beliefs/abstraction/FreudenthalTriangulationBeliefAbstraction.h @@ -0,0 +1,168 @@ +#pragma once + +#include + +#include "storm-pomdp/beliefs/storage/BeliefBuilder.h" +#include "storm-pomdp/beliefs/utility/BeliefNumerics.h" +#include "storm-pomdp/beliefs/utility/types.h" +#include "storm/utility/constants.h" +#include "storm/utility/macros.h" + +namespace storm::pomdp::beliefs { +/** Selects a fixed grid resolution or a suitable coarser resolution per belief. */ +enum class FreudenthalTriangulationMode { Static, Dynamic }; + +/*! + * Abstracts a belief by triangulating it using the Freudenthal triangulation. + * Intuitively, the Freudenthal triangulation considers a grid (or: foundation) of beliefs that assign probabilities from the set {0/N, 1/N, 2/N, ..., N/N} for + * some resolution N. The freudenthal belief abstraction then represents a given belief by a convex combination over nearby grid-beliefs. + * In the static mode, the resolution is fixed. + * In the dynamic mode, a lower resolution is chosen, if it fits the belief significantly better. For example, if the original belief is {state1: 1/3, state2: + * 2/3 } and the given resolution is N=4, it will be triangulated using N'=3 instead. + * + * @see 10.1007/978-3-030-59152-6_16 + * + * @tparam BeliefType Sparse belief representation to abstract. + */ +template +class FreudenthalTriangulationBeliefAbstraction { + public: + using BeliefValueType = typename BeliefType::ValueType; + + /** Creates an abstraction with the default grid resolution and resolution-selection mode. */ + FreudenthalTriangulationBeliefAbstraction(BeliefValueType const& initialResolution, FreudenthalTriangulationMode mode); + + /** + * Emits the grid beliefs and weights whose convex combination represents @p belief. + * + * The emitted weights include @p probabilityFactor. + */ + template + void abstract(BeliefType&& belief, BeliefValueType&& probabilityFactor, AbstractCallback const& callback) const { + // Quickly triangulate Dirac beliefs + if (belief.size() == 1u) { + callback(std::move(belief), std::move(probabilityFactor)); + } else { + auto observationResolutionFindRes = observationResolutions.find(belief.observation()); + auto observationResolution = + observationResolutionFindRes != observationResolutions.end() ? observationResolutionFindRes->second : defaultResolution; + switch (mode) { + case FreudenthalTriangulationMode::Static: + abstractStatic(probabilityFactor, belief, callback, observationResolution); + break; + case FreudenthalTriangulationMode::Dynamic: + abstractDynamic(probabilityFactor, belief, callback, observationResolution); + break; + default: + STORM_LOG_ASSERT(false, "Invalid triangulation mode."); + } + } + } + + private: + template + void abstractStatic(BeliefValueType const& probabilityFactor, BeliefType const& belief, AbstractCallback& callback, + BeliefValueType const& staticResolution) const { + struct FreudenthalDiff { + BeliefValueType diff; // d[i] + BeliefStateType dimension; // i + bool operator>(FreudenthalDiff const& other) const { + if (diff != other.diff) { + return diff > other.diff; + } else { + return dimension < other.dimension; + } + } + }; + + STORM_LOG_ASSERT(storm::utility::isInteger(staticResolution), "Expected an integer resolution"); + STORM_LOG_ASSERT(staticResolution > 0, "Expected a positive resolution"); + BeliefStateType numEntries = belief.size(); + // This is the Freudenthal Triangulation as described in Lovejoy (a whole lotta math) + // Probabilities will be triangulated to values in 0/N, 1/N, 2/N, ..., N/N + // Variable names are mostly based on the paper + // However, we speed this up a little by exploiting that belief states usually have sparse support (i.e. numEntries is much smaller than + // pomdp.getNumberOfStates()). Initialize diffs and the first row of the 'qs' matrix (aka v) + std::set> sorted_diffs; // d (and p?) in the paper + std::vector qsRow; // Row of the 'qs' matrix from the paper (initially corresponds to v + qsRow.reserve(numEntries); + std::vector toOriginalIndicesMap; // Maps 'local' indices to the original pomdp state indices + toOriginalIndicesMap.reserve(numEntries); + BeliefValueType x = staticResolution; + belief.forEach([&qsRow, &sorted_diffs, &toOriginalIndicesMap, &x, &staticResolution](auto const& state, auto const& value) { + qsRow.push_back(storm::utility::floor(x)); // v + sorted_diffs.insert(FreudenthalDiff({x - qsRow.back(), toOriginalIndicesMap.size()})); // x-v + toOriginalIndicesMap.push_back(state); + x -= value * staticResolution; + }); + // Insert a dummy 0 column in the qs matrix so the loops below are a bit simpler + qsRow.push_back(storm::utility::zero()); + + auto currentSortedDiff = sorted_diffs.begin(); + auto previousSortedDiff = sorted_diffs.end(); + --previousSortedDiff; + for (BeliefStateType i = 0; i < numEntries; ++i) { + // Compute the weight for the grid points + BeliefValueType weight = previousSortedDiff->diff - currentSortedDiff->diff; + if (i == 0) { + // The first weight is a bit different + weight += storm::utility::one(); + } else { + // 'compute' the next row of the qs matrix + qsRow[previousSortedDiff->dimension] += storm::utility::one(); + } + if (!BeliefNumerics::isZero(weight)) { + // build the grid point + BeliefBuilder builder; + builder.reserve(numEntries); + builder.setObservation(belief.observation()); + for (BeliefStateType j = 0; j < numEntries; ++j) { + BeliefValueType gridPointEntry = qsRow[j] - qsRow[j + 1]; + if (!BeliefNumerics::isZero(gridPointEntry)) { + builder.addValue(toOriginalIndicesMap[j], gridPointEntry / staticResolution); + } + } + callback(builder.build(), static_cast(weight * probabilityFactor)); + } + previousSortedDiff = currentSortedDiff++; + } + } + + template + void abstractDynamic(BeliefValueType const& probabilityFactor, BeliefType const& belief, AbstractCallback& callback, + BeliefValueType const& maxResolution) const { + // Find the best resolution for this belief, i.e., N such that the largest distance between one of the belief values to a value in {i/N | 0 ≤ i ≤ N} is + // minimal + STORM_LOG_ASSERT(storm::utility::isInteger(maxResolution), "Expected an integer resolution"); + BeliefValueType const halfResolution = maxResolution / storm::utility::convertNumber(2); + BeliefValueType const initialDist = storm::utility::one() / maxResolution; + // We take 1/maxResolution as initial distance. This means that coarser maxResolution are only chosen if they are clearly better suited for the given + // belief. + + BeliefValueType resolution = maxResolution; + BeliefValueType resolutionDist = initialDist; + // We don't need to check resolutions that are smaller than the maximal resolution divided by 2 as we already checked multiples of these + for (BeliefValueType currResolution = maxResolution; currResolution > halfResolution; --currResolution) { + BeliefValueType currDist = storm::utility::zero(); + bool const newBest = belief.allOf([&currDist, &currResolution, &resolutionDist](BeliefStateType const&, BeliefValueType const& val) { + currDist += storm::utility::abs(val - storm::utility::round(val * currResolution) / currResolution); + return currDist <= resolutionDist; // continue as long as the current dist is still smaller than the smallest dist + }); + if (newBest) { + STORM_LOG_ASSERT(currDist <= resolutionDist, "Expected a smaller resolution"); + resolution = currResolution; + resolutionDist = currDist; + if (BeliefNumerics::isZero(resolutionDist)) { + break; + } + } + } + abstractStatic(probabilityFactor, belief, callback, resolution); + } + + private: + BeliefValueType const defaultResolution; + FreudenthalTriangulationMode const mode; + std::map observationResolutions; +}; +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/beliefs/abstraction/NoAbstraction.h b/src/storm-pomdp/beliefs/abstraction/NoAbstraction.h new file mode 100644 index 0000000000..71c436ceb4 --- /dev/null +++ b/src/storm-pomdp/beliefs/abstraction/NoAbstraction.h @@ -0,0 +1,19 @@ +#pragma once + +#include + +namespace storm::pomdp::beliefs { + +/*! + * Auxiliary struct used to identify cases where no abstraction of beliefs shall be applied. + * Inspired by std::nullopt, see https://en.cppreference.com/w/cpp/utility/optional/nullopt_t + */ +struct NoAbstractionType { + constexpr explicit NoAbstractionType(int) {} +}; +inline constexpr NoAbstractionType NoAbstraction{0}; + +template +inline constexpr bool isNoAbstraction = std::is_same_v, NoAbstractionType>; + +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/beliefs/abstraction/RewardBoundedBeliefSplitter.cpp b/src/storm-pomdp/beliefs/abstraction/RewardBoundedBeliefSplitter.cpp new file mode 100644 index 0000000000..f46c9a2918 --- /dev/null +++ b/src/storm-pomdp/beliefs/abstraction/RewardBoundedBeliefSplitter.cpp @@ -0,0 +1,61 @@ +#include "storm-pomdp/beliefs/abstraction/RewardBoundedBeliefSplitter.h" + +#include "storm-pomdp/beliefs/storage/Belief.h" +#include "storm/adapters/RationalNumberAdapter.h" +#include "storm/exceptions/NotSupportedException.h" +#include "storm/models/sparse/Pomdp.h" +#include "storm/utility/macros.h" + +namespace storm::pomdp::beliefs { + +template +RewardBoundedBeliefSplitter::RewardBoundedBeliefSplitter(PomdpType const& pomdp) : pomdp(pomdp) {} + +template +void RewardBoundedBeliefSplitter::setRewardModel(std::string const& rewardModelName) { + setRewardModels({rewardModelName}); +} + +template +void RewardBoundedBeliefSplitter::setRewardModels(std::vector const& rewardModelNames) { + actionRewardVectors.assign(pomdp.getNumberOfChoices(), {}); + for (auto const& rewardModelName : rewardModelNames) { + auto const& rewardModel = pomdp.getRewardModel(rewardModelName); + STORM_LOG_THROW(!rewardModel.hasTransitionRewards(), storm::exceptions::NotSupportedException, + "POMDPs with transition rewards are currently not supported."); + for (uint64_t state = 0; state < pomdp.getNumberOfStates(); ++state) { + for (auto const& choice : pomdp.getTransitionMatrix().getRowGroupIndices(state)) { + auto val = rewardModel.hasStateActionRewards() ? rewardModel.getStateActionReward(choice) : storm::utility::zero(); + if (rewardModel.hasStateRewards()) { + val += rewardModel.getStateReward(state); + } + actionRewardVectors[choice].push_back(storm::utility::convertNumber(val)); + } + } + } +} + +template +void RewardBoundedBeliefSplitter::unsetRewardModels() { + actionRewardVectors.clear(); +} + +template +std::size_t RewardBoundedBeliefSplitter::getNumberOfSetRewardModels() const { + if (actionRewardVectors.empty()) { + return 0; + } else { + return actionRewardVectors.front().size(); + } +} + +template class RewardBoundedBeliefSplitter, Belief>; +template class RewardBoundedBeliefSplitter, Belief>; +template class RewardBoundedBeliefSplitter, Belief>; +template class RewardBoundedBeliefSplitter, Belief>; +template class RewardBoundedBeliefSplitter, Belief>; +template class RewardBoundedBeliefSplitter, Belief>; +template class RewardBoundedBeliefSplitter, Belief>; +template class RewardBoundedBeliefSplitter, Belief>; + +} // namespace storm::pomdp::beliefs \ No newline at end of file diff --git a/src/storm-pomdp/beliefs/abstraction/RewardBoundedBeliefSplitter.h b/src/storm-pomdp/beliefs/abstraction/RewardBoundedBeliefSplitter.h new file mode 100644 index 0000000000..0b77f4309c --- /dev/null +++ b/src/storm-pomdp/beliefs/abstraction/RewardBoundedBeliefSplitter.h @@ -0,0 +1,71 @@ +#pragma once + +#include +#include + +#include "storm-pomdp/beliefs/storage/BeliefBuilder.h" +#include "storm-pomdp/beliefs/utility/types.h" +#include "storm/utility/constants.h" + +namespace storm::pomdp::beliefs { + +/*! + * Abstracts a given belief by splitting it into sub-beliefs based on the (state-action) reward of the POMDP. + * Multiple reward models can be set. + * States in the support of the given belief where all action rewards are equal will be grouped together. + * When applying this as a pre-abstraction, this intuitively means that the collected rewards are observable. + */ +template +class RewardBoundedBeliefSplitter { + public: + using PomdpValueType = typename PomdpType::ValueType; + using BeliefValueType = typename BeliefType::ValueType; + using RewardVectorType = std::vector; + + /** Creates a splitter for @p pomdp. The POMDP must outlive the splitter. */ + RewardBoundedBeliefSplitter(PomdpType const& pomdp); + /** Selects one reward model; an empty name selects Storm's default reward model. */ + void setRewardModel(std::string const& rewardModelName = ""); + /** Selects the reward models used to form reward vectors. */ + void setRewardModels(std::vector const& rewardModelNames); + /** Clears the selected reward models and all generated reward-vector observation indices. */ + void unsetRewardModels(); + /** @return the number of selected reward models. */ + std::size_t getNumberOfSetRewardModels() const; + + /** + * Splits @p belief into normalized sub-beliefs with equal reward vectors for @p localActionIndex. + * + * The callback receives a sub-belief, its probability mass, a stable reward-vector observation index, and the + * reward vector itself. + */ + template + void abstract(BeliefType const& belief, uint64_t localActionIndex, ExpandCallback const& callback) { + // gather the occurring reward vectors and build the sub-beliefs + std::map> splitBeliefs; + belief.forEach([&localActionIndex, &splitBeliefs, this](BeliefStateType const& state, BeliefValueType const& beliefValue) { + auto const globalActionIndex = pomdp.getTransitionMatrix().getRowGroupIndices()[state] + localActionIndex; + auto const& rewVector = actionRewardVectors[globalActionIndex]; + splitBeliefs[rewVector].addValue(state, beliefValue); + }); + + for (auto& [rewardVector, builder] : splitBeliefs) { + BeliefActionObservationType const freshIndex = rewardVectorToIndex.size(); + auto const rewVectorIndex = rewardVectorToIndex.emplace(rewardVector, freshIndex).first->second; + builder.setObservation(belief.observation()); + if (splitBeliefs.size() == 1u) { + // Fix the distribution to diminish numerical issues a bit + callback(builder.build(), storm::utility::one(), rewVectorIndex, rewardVector); + } else { + auto val = builder.normalize(); + callback(builder.build(), std::move(val), rewVectorIndex, rewardVector); + } + } + } + + private: + PomdpType const& pomdp; + std::vector actionRewardVectors; + std::map rewardVectorToIndex; +}; +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/beliefs/exploration/BeliefExploration.cpp b/src/storm-pomdp/beliefs/exploration/BeliefExploration.cpp new file mode 100644 index 0000000000..72ba523d04 --- /dev/null +++ b/src/storm-pomdp/beliefs/exploration/BeliefExploration.cpp @@ -0,0 +1,18 @@ +#include "storm-pomdp/beliefs/exploration/BeliefExploration.h" + +#include "storm-pomdp/beliefs/storage/Belief.h" +#include "storm/adapters/RationalNumberAdapter.h" +#include "storm/models/sparse/Pomdp.h" + +namespace storm::pomdp::beliefs { +template +BeliefExploration::BeliefExploration(PomdpType const& pomdp) : firstStateNextStateGenerator(pomdp) { + // Intentionally left empty. +} + +template class BeliefExploration, Belief>; +template class BeliefExploration, Belief>; +template class BeliefExploration, Belief>; +template class BeliefExploration, Belief>; +template class BeliefExploration, Belief>; +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/beliefs/exploration/BeliefExploration.h b/src/storm-pomdp/beliefs/exploration/BeliefExploration.h new file mode 100644 index 0000000000..4cb362ffca --- /dev/null +++ b/src/storm-pomdp/beliefs/exploration/BeliefExploration.h @@ -0,0 +1,232 @@ +#pragma once + +#include +#include +#include + +#include "storm-pomdp/beliefs/exploration/ExplorationInformation.h" +#include "storm-pomdp/beliefs/exploration/FirstStateNextStateGenerator.h" +#include "storm/utility/OptionalRef.h" +#include "storm/utility/SignalHandler.h" + +namespace storm::pomdp::beliefs { + +template +class FreudenthalTriangulationBeliefAbstraction; + +template +class RewardBoundedBeliefSplitter; + +template +/** Stores ordinary successor transitions in exploration information. */ +struct StandardDiscoverCallback { + StandardExplorationInformation& info; + + explicit StandardDiscoverCallback(StandardExplorationInformation& info) : info(info) { + // Intentionally left empty + } + void operator()(BeliefType&& bel, typename BeliefType::ValueType&& val) { + auto const belId = info.discoveredBeliefs.getIdOrAddBelief(std::move(bel)); + if (info.exploredBeliefs.count(belId) == 0u && info.terminalBeliefValues.count(belId) == 0u) { + info.queue.push(belId); + } + info.matrix.transitions.push_back({storm::utility::convertNumber(val), belId}); + } +}; + +template +/** Stores successor transitions together with clipping metadata. */ +struct ClippingDiscoverCallback { + ClippingExplorationInformation& info; + + explicit ClippingDiscoverCallback(ClippingExplorationInformation& info) : info(info) { + // Intentionally left empty + } + void operator()(BeliefType&& bel, typename BeliefType::ValueType&& val, std::optional optionalWeightedClippingValue, + std::optional optionalRewardAdjustment) { + auto const belId = info.discoveredBeliefs.getIdOrAddBelief(std::move(bel)); + if (info.exploredBeliefs.count(belId) == 0u && info.terminalBeliefValues.count(belId) == 0u) { + info.queue.push(belId); + } + if (optionalWeightedClippingValue) { + if (optionalRewardAdjustment) { + STORM_LOG_TRACE("Add transition to belief " << belId << " with val " << val << ", weighted clipping value " << *optionalWeightedClippingValue + << " and reward adjustment " << *optionalRewardAdjustment << "."); + info.matrix.transitions.push_back({storm::utility::convertNumber(val), + belId, + {storm::utility::convertNumber(*optionalWeightedClippingValue), + storm::utility::convertNumber(*optionalRewardAdjustment)}}); + } else { + info.matrix.transitions.push_back({storm::utility::convertNumber(val), + belId, + {storm::utility::convertNumber(*optionalWeightedClippingValue), std::nullopt}}); + } + } else { + info.matrix.transitions.push_back({storm::utility::convertNumber(val), belId, {std::nullopt, std::nullopt}}); + } + } + void operator()(BeliefType&& bel, typename BeliefType::ValueType&& val) { + auto const belId = info.discoveredBeliefs.getIdOrAddBelief(std::move(bel)); + if (info.exploredBeliefs.count(belId) == 0u && info.terminalBeliefValues.count(belId) == 0u) { + info.queue.push(belId); + } + info.matrix.transitions.push_back({storm::utility::convertNumber(val), belId, {std::nullopt, std::nullopt}}); + } +}; + +template +/** Stores reward-aware successor transitions and their reward vectors. */ +struct RewardAwareDiscoverCallback { + RewardAwareExplorationInformation& info; + + RewardAwareDiscoverCallback(RewardAwareExplorationInformation& info) : info(info) { + // Intentionally left empty + } + void operator()(BeliefType&& bel, typename BeliefType::ValueType&& val, std::vector const& rewards) { + auto const belId = info.discoveredBeliefs.getIdOrAddBelief(std::move(bel)); + if (info.exploredBeliefs.count(belId) == 0u && info.terminalBeliefValues.count(belId) == 0u) { + info.queue.push(belId); + } + info.matrix.transitions.push_back({storm::utility::convertNumber(val), belId, rewards}); + } +}; + +/** + * Class to perform belief exploration. Heavily templated to allow for different belief, value, abstraction, etc. types. + * Therefore, implementations are in the header file. + * @tparam BeliefMdpValueType Value type used in the constructed belief MDP. + * @tparam PomdpType POMDP type used for successor generation. + * @tparam BeliefType Sparse belief representation. + */ +template +class BeliefExploration { + public: + using TerminationCallback = std::function; + using TerminalBeliefCallback = std::function(BeliefType const&)>; + + /** Creates an explorer for @p pomdp. The POMDP must outlive the explorer. */ + explicit BeliefExploration(PomdpType const& pomdp); + + /** + * Initializes an exploration with the initial belief as its only queued frontier belief. + * + * @tparam InfoType A standard, clipping, or reward-aware exploration-information type. + */ + template + InfoType initializeExploration(uint64_t nrObservationsInPomdp, ExplorationQueueOrder const explorationQueueOrder = ExplorationQueueOrder::Unordered) { + InfoType info; + info.queue.changeOrder(explorationQueueOrder); + info.initialBeliefId = info.discoveredBeliefs.addBelief(firstStateNextStateGenerator.computeInitialBelief()); + info.queue.push(info.initialBeliefId); + info.nrObservationsInPomdp = nrObservationsInPomdp; + return info; + } + + /** + * Continues an ordinary exploration until the queue is empty, a terminal callback applies, or termination is requested. + * + * A post-abstraction is applied to successor beliefs when provided. + */ + template + void resumeExploration(StandardExplorationInformation& info, TerminalBeliefCallback const& terminalBeliefCallback, + TerminationCallback const& terminationCallback, storm::OptionalRef rewardModelName, + storm::OptionalRef abstraction) { + if (rewardModelName.has_value()) { + firstStateNextStateGenerator.setRewardModel(rewardModelName.value()); + } + StandardDiscoverCallback discoverCallback(info); + if (abstraction) { + performExploration(info, firstStateNextStateGenerator.getPostAbstractionHandle(abstraction.value(), discoverCallback), terminalBeliefCallback, + terminationCallback); + } else { + performExploration(info, firstStateNextStateGenerator.getHandle(discoverCallback), terminalBeliefCallback, terminationCallback); + } + } + + /** + * Continues reward-aware exploration, splitting a belief by reward vector before generating successors. + */ + template + void resumeRewardAwareExploration(RewardAwareExplorationInformation& info, + TerminalBeliefCallback const& terminalBeliefCallback, TerminationCallback const& terminationCallback, + RewardBoundedBeliefSplitter rewardSplitter, + storm::OptionalRef abstraction) { + RewardAwareDiscoverCallback discoverCallback(info); + if (abstraction) { + performExploration(info, firstStateNextStateGenerator.getPrePostAbstractionHandle(rewardSplitter, abstraction.value(), discoverCallback), + terminalBeliefCallback, terminationCallback); + } else { + performExploration(info, firstStateNextStateGenerator.getPreAbstractionHandle(rewardSplitter, discoverCallback), terminalBeliefCallback, + terminationCallback); + } + } + + /** + * Continues exploration with clipping metadata on the generated transitions. + */ + template + void resumeClippingExploration(ClippingExplorationInformation& info, TerminalBeliefCallback const& terminalBeliefCallback, + TerminationCallback const& terminationCallback, storm::OptionalRef rewardModelName, + storm::OptionalRef abstraction) { + if (rewardModelName.has_value()) { + firstStateNextStateGenerator.setRewardModel(rewardModelName.value()); + } + ClippingDiscoverCallback discoverCallback(info); + if (abstraction) { + performExploration(info, firstStateNextStateGenerator.getPostAbstractionHandle(abstraction.value(), discoverCallback), terminalBeliefCallback, + terminationCallback); + } else { + performExploration(info, firstStateNextStateGenerator.getHandle(discoverCallback), terminalBeliefCallback, terminationCallback); + } + } + + private: + template + bool performExploration(InfoType& info, NextStateHandleType&& exploreNextStates, TerminalBeliefCallback const& terminalBeliefCallback, + TerminationCallback const& terminationCallback) { + while (info.queue.hasNext()) { + // Check if we terminate prematurely + if ((terminationCallback && terminationCallback()) || storm::utility::resources::isTerminate()) { + STORM_LOG_ASSERT(storm::utility::resources::isTerminate() || info.queue.getContents() == info.getFrontierBeliefs(), + "Frontier beliefs inconsistent."); + return false; // Terminate prematurely + } + + // Get the next belief to explore and perform some checks + auto const currentBeliefId = info.queue.popNext(); + STORM_LOG_ASSERT(info.discoveredBeliefs.containsId(currentBeliefId), "Unknown belief id"); + STORM_LOG_ASSERT(info.exploredBeliefs.count(currentBeliefId) == 0, "Belief #" << currentBeliefId << " already explored."); + STORM_LOG_ASSERT(info.terminalBeliefValues.count(currentBeliefId) == 0, "Belief #" << currentBeliefId << " already found to be terminal."); + // do not take the current belief as reference since it will be invalidated when collecting more beliefs + auto const currentBelief = info.discoveredBeliefs.getBeliefFromId(currentBeliefId); + STORM_LOG_TRACE("Explore belief " << currentBeliefId << " : " << currentBelief.toString()); + // Check if the current belief is terminal + if (terminalBeliefCallback) { + if (auto terminal = terminalBeliefCallback(currentBelief); terminal.has_value()) { + info.terminalBeliefValues.emplace(currentBeliefId, std::move(terminal.value())); + continue; + } + } + + // Explore for each action the successors of the current belief with that action. Potentially also add rewards. + info.exploredBeliefs.emplace(currentBeliefId, info.matrix.groups()); + auto const numActions = firstStateNextStateGenerator.getBeliefNumberOfActions(currentBelief); + for (uint64_t localActionIndex = 0; localActionIndex < numActions; ++localActionIndex) { + exploreNextStates(currentBelief, localActionIndex); + info.matrix.endCurrentRow(); + if (firstStateNextStateGenerator.hasRewardModel()) { + info.actionRewards.emplace_back( + storm::utility::convertNumber(firstStateNextStateGenerator.getBeliefActionReward(currentBelief, localActionIndex))); + } + if (info.generateChoiceLabeling) { + info.matrix.choiceLabels.push_back(firstStateNextStateGenerator.getBeliefActionChoiceLabels(currentBelief, localActionIndex)); + } + } + info.matrix.endCurrentRowGroup(); + } + return true; + } + + FirstStateNextStateGenerator firstStateNextStateGenerator; +}; +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/beliefs/exploration/BeliefExplorationMatrix.cpp b/src/storm-pomdp/beliefs/exploration/BeliefExplorationMatrix.cpp new file mode 100644 index 0000000000..3bb2d365a3 --- /dev/null +++ b/src/storm-pomdp/beliefs/exploration/BeliefExplorationMatrix.cpp @@ -0,0 +1,46 @@ +#include "storm-pomdp/beliefs/exploration/BeliefExplorationMatrix.h" + +#include +#include "storm/adapters/RationalNumberAdapter.h" + +namespace storm::pomdp::beliefs { + +template +BeliefExplorationMatrix::BeliefExplorationMatrix() { + rowIndications.push_back(0u); + rowGroupIndices.push_back(0u); +} + +template +void BeliefExplorationMatrix::endCurrentRow() { + rowIndications.push_back(transitions.size()); +}; + +template +void BeliefExplorationMatrix::endCurrentRowGroup() { + rowGroupIndices.push_back(rowIndications.size() - 1); +}; + +template +std::size_t BeliefExplorationMatrix::rows() const { + return rowIndications.size() - 1; +} + +template +std::size_t BeliefExplorationMatrix::groups() const { + return rowGroupIndices.size() - 1; +} + +template +bool BeliefExplorationMatrix::hasChoiceLabels() const { + return !choiceLabels.empty(); +} + +template class BeliefExplorationMatrix; +template class BeliefExplorationMatrix>; +template class BeliefExplorationMatrix; +template class BeliefExplorationMatrix>; +template class BeliefExplorationMatrix, std::optional>; +template class BeliefExplorationMatrix, std::optional>; + +} // namespace storm::pomdp::beliefs \ No newline at end of file diff --git a/src/storm-pomdp/beliefs/exploration/BeliefExplorationMatrix.h b/src/storm-pomdp/beliefs/exploration/BeliefExplorationMatrix.h new file mode 100644 index 0000000000..b6710eecbf --- /dev/null +++ b/src/storm-pomdp/beliefs/exploration/BeliefExplorationMatrix.h @@ -0,0 +1,81 @@ +#pragma once + +#include +#include +#include +#include + +#include "storm-pomdp/beliefs/utility/types.h" + +namespace storm::pomdp::beliefs { + +/** A successor entry recorded while exploring one action of a belief. */ +template +struct BeliefExplorationTransition { + ValueType probability; + storm::pomdp::beliefs::BeliefId targetBelief; +}; + +template +struct BeliefExplorationTransition { + ValueType probability; + storm::pomdp::beliefs::BeliefId targetBelief; + ExtraTransitionData data; +}; + +template +struct BeliefExplorationTransition { + ValueType probability; + storm::pomdp::beliefs::BeliefId targetBelief; + std::tuple data; +}; + +/** + * Sparse transition data for the explored part of a belief MDP. + * + * Rows correspond to actions and row groups to explored beliefs. Extra transition data is stored alongside each + * probability, for example reward-bound updates or clipping corrections. + */ +template +class BeliefExplorationMatrix { + public: + /*! + * Initializes a new (empty) belief exploration matrix. + */ + BeliefExplorationMatrix(); + + /*! + * While building the matrix, ends the current row in the matrix. + */ + void endCurrentRow(); + + /*! + * While building the matrix, ends the current row group in the matrix. + * @note This function should be called after endCurrentRow() has been called. + */ + void endCurrentRowGroup(); + + /*! + * @return the current number of rows in the matrix + */ + std::size_t rows() const; + + /*! + * @return the current number of row groups in the matrix + */ + std::size_t groups() const; + + /** @return whether labels have been recorded for the matrix choices. */ + bool hasChoiceLabels() const; + + /** Successor entries, grouped by rowIndications. */ + std::vector> transitions; + /** Start indices of action rows in transitions. */ + std::vector rowIndications; + /** Start indices of belief row groups in the action rows. */ + std::vector rowGroupIndices; + /** Optional labels for the action rows. */ + std::vector> choiceLabels; +}; + +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/beliefs/exploration/BeliefMdpBuilder.cpp b/src/storm-pomdp/beliefs/exploration/BeliefMdpBuilder.cpp new file mode 100644 index 0000000000..0c7dcb3917 --- /dev/null +++ b/src/storm-pomdp/beliefs/exploration/BeliefMdpBuilder.cpp @@ -0,0 +1,391 @@ +#include "storm-pomdp/beliefs/exploration/BeliefMdpBuilder.h" + +#include "storm-pomdp/beliefs/storage/Belief.h" + +#include "storm/adapters/RationalNumberAdapter.h" +#include "storm/models/sparse/Mdp.h" +#include "storm/models/sparse/StandardRewardModel.h" +#include "storm/storage/SparseMatrix.h" +#include "storm/storage/sparse/ModelComponents.h" + +#include "storm/exceptions/UnexpectedException.h" + +namespace storm::pomdp::beliefs { + +std::shared_ptr createFormulaForBeliefMdp(PropertyInformation const& propertyInformation) { + STORM_LOG_ASSERT(propertyInformation.kind == PropertyInformation::Kind::ReachabilityProbability || + propertyInformation.kind == PropertyInformation::Kind::ExpectedTotalReachabilityReward || + propertyInformation.kind == PropertyInformation::Kind::RewardBoundedReachabilityProbability, + "Unexpected kind of property."); + switch (propertyInformation.kind) { + case PropertyInformation::Kind::ReachabilityProbability: { + auto target = std::make_shared("target"); + auto eventuallyTarget = std::make_shared(target, storm::logic::FormulaContext::Probability); + return std::make_shared(eventuallyTarget, + storm::logic::OperatorInformation(propertyInformation.dir)); + } + case PropertyInformation::Kind::ExpectedTotalReachabilityReward: { + auto bottom = std::make_shared("target"); + auto eventuallyBottom = std::make_shared(bottom, storm::logic::FormulaContext::Reward, + storm::logic::RewardAccumulation(true, false, false)); + return std::make_shared(eventuallyBottom, propertyInformation.rewardModelName.value(), + storm::logic::OperatorInformation(propertyInformation.dir)); + } + case PropertyInformation::Kind::RewardBoundedReachabilityProbability: { + auto target = std::make_shared("target"); + auto trueFormula = std::make_shared(true); + + std::vector> lowerBounds; + std::vector> upperBounds; + std::vector timeBoundReferences; + + for (auto const& rewardBound : propertyInformation.rewardBounds) { + if (rewardBound.rewardModelName.empty()) { + timeBoundReferences.emplace_back(); + } else { + timeBoundReferences.emplace_back(rewardBound.rewardModelName); + } + if (rewardBound.lowerBound.has_value()) { + lowerBounds.emplace_back(rewardBound.lowerBound.value()); + } else { + lowerBounds.emplace_back(boost::none); + } + if (rewardBound.upperBound.has_value()) { + upperBounds.emplace_back(rewardBound.upperBound.value()); + } else { + upperBounds.emplace_back(boost::none); + } + } + auto eventuallyTarget = + std::make_shared(trueFormula, target, lowerBounds, upperBounds, timeBoundReferences); + return std::make_shared(eventuallyTarget, + storm::logic::OperatorInformation(propertyInformation.dir)); + } + } + STORM_LOG_THROW(false, storm::exceptions::UnexpectedException, "Unhandled case."); +} + +template +std::pair>, std::unordered_map> buildBeliefMdp( + ExplorationInformation const& explorationInformation, + PropertyInformation const& propertyInformation, + std::function(BeliefType const&)> const& computeCutOffValueMap) { + bool const isReachProb = propertyInformation.kind == PropertyInformation::Kind::ReachabilityProbability; + bool const isTotRew = propertyInformation.kind == PropertyInformation::Kind::ExpectedTotalReachabilityReward; + bool const isRewBndReachProb = propertyInformation.kind == PropertyInformation::Kind::RewardBoundedReachabilityProbability; + STORM_LOG_ASSERT(isReachProb || isTotRew || isRewBndReachProb, "Unexpected kind of property."); + + bool constexpr extraDataCompatibleWithRewardAwareness = + sizeof...(ExtraTransitionData) == 1 && (std::is_same_v, ExtraTransitionData> || ...); + + // First gather all cut-off information + uint64_t nrCutOffChoices = 0ull; + std::unordered_map> cutOffInformationMap; + for (auto const& frontierBeliefId : explorationInformation.getFrontierBeliefs()) { + auto const& frontierBelief = explorationInformation.discoveredBeliefs.getBeliefFromId(frontierBeliefId); + cutOffInformationMap[frontierBeliefId] = computeCutOffValueMap(frontierBelief); + nrCutOffChoices += cutOffInformationMap[frontierBeliefId].size(); + } + + constexpr bool clippingUsed = std::is_same_v, + ClippingExplorationInformation>; + + // (unbounded) reachability probabilities get a dedicated target state as an extra state + // This is not done for reward-bounded reachability probabilities because target states are not terminal for those (e.g. because of lower reward bounds) + // Possible optimisation: check if bottom is really needed for clipping + uint64_t const numBottomTargetStates = isReachProb || clippingUsed ? 2ull : 1ull; + uint64_t const numExtraStates = numBottomTargetStates + explorationInformation.getFrontierBeliefs().size(); + uint64_t const numStates = explorationInformation.matrix.groups() + numExtraStates; + uint64_t const numChoices = explorationInformation.matrix.rows() + numBottomTargetStates + nrCutOffChoices; + uint64_t const targetState = numStates - numBottomTargetStates; + uint64_t const bottomState = numStates - 1; + std::optional optionalChoiceLabeling; + if (explorationInformation.matrix.hasChoiceLabels()) { + optionalChoiceLabeling = models::sparse::ChoiceLabeling(numChoices); + } + + std::vector actionRewards; + if (isTotRew) { + actionRewards.reserve(numChoices); + actionRewards.insert(actionRewards.end(), explorationInformation.actionRewards.begin(), explorationInformation.actionRewards.end()); + // Insert 0 for all cut-off choices and bottom state + actionRewards.insert(actionRewards.end(), nrCutOffChoices + numBottomTargetStates, storm::utility::zero()); + STORM_LOG_ASSERT(numChoices == actionRewards.size(), + "Unexpected size of action rewards: Expected " << numChoices << " got " << actionRewards.size() << "."); + } + + std::unordered_map frontierBeliefToStateMap; + std::unordered_map stateToFrontierBeliefMap; + uint64_t nextStateId = numStates - numExtraStates; + + std::vector> transitionRewardBuilderVector; + + if constexpr (extraDataCompatibleWithRewardAwareness) { + if (isRewBndReachProb) { + for (uint64_t i = 0; i < propertyInformation.rewardBounds.size(); ++i) { + transitionRewardBuilderVector.emplace_back(numChoices, numStates, 0, true, true, numStates); + } + } + } + + storm::storage::SparseMatrixBuilder transitionBuilder(numChoices, numStates, 0, true, true, numStates); + // Treat explored beliefs + for (uint64_t state = 0; state < numStates - numExtraStates; ++state) { + uint64_t choice = explorationInformation.matrix.rowGroupIndices[state]; + transitionBuilder.newRowGroup(choice); + for (auto& transitionRewardBuilder : transitionRewardBuilderVector) { + transitionRewardBuilder.newRowGroup(choice); + } + for (uint64_t const groupEnd = explorationInformation.matrix.rowGroupIndices[state + 1]; choice < groupEnd; ++choice) { + auto probabilityToBottom = storm::utility::zero(); + auto probabilityToTarget = storm::utility::zero(); + for (uint64_t entryIndex = explorationInformation.matrix.rowIndications[choice]; + entryIndex < explorationInformation.matrix.rowIndications[choice + 1]; ++entryIndex) { + auto const& entry = explorationInformation.matrix.transitions[entryIndex]; + if (auto explIt = explorationInformation.exploredBeliefs.find(entry.targetBelief); explIt != explorationInformation.exploredBeliefs.end()) { + // Transition to explored belief + transitionBuilder.addNextValue(choice, explIt->second, entry.probability); + if constexpr (extraDataCompatibleWithRewardAwareness) { + if (isRewBndReachProb) { + for (uint64_t i = 0; i < propertyInformation.rewardBounds.size(); ++i) { + if (!storm::utility::isZero(entry.data[i])) { + transitionRewardBuilderVector.at(i).addNextValue(choice, explIt->second, entry.data[i]); + } + } + } + } + if constexpr (clippingUsed) { + // In case of clipping exploration, we have extra data that indicates whether the transition is a clipping transition + auto const& clippingProbability = std::get<0>(entry.data); + auto const& rewardPenalty = std::get<1>(entry.data); + + if (clippingProbability) { + if (isReachProb) { + probabilityToBottom += *clippingProbability; + } else if (rewardPenalty) { + if (storm::utility::isInfinity(*rewardPenalty)) { + /* Infinite reward on transitions is not correctly handled by the model checker. Therefore, we treat it by adding a + * transition to the bottom state which due to the semantics of expected reward until reaching a target has infinite + * expected reward.This causes the expected reward for the transition to become infinite. */ + probabilityToBottom += *clippingProbability; + } else { + actionRewards[choice] += *rewardPenalty; + probabilityToTarget += *clippingProbability; + } + } else { + probabilityToTarget += *clippingProbability; + } + } + } + } else { + // Transition to unexplored belief (either terminal or cut-off) + BeliefMdpValueType successorValue; + if (auto terminalIt = explorationInformation.terminalBeliefValues.find(entry.targetBelief); + // Transition to terminal belief + terminalIt != explorationInformation.terminalBeliefValues.end()) { + successorValue = entry.probability * terminalIt->second; // terminal value determined during exploration + if (isReachProb) { + probabilityToTarget += successorValue; + probabilityToBottom += entry.probability - successorValue; + } else { + probabilityToTarget += entry.probability; + actionRewards[choice] += successorValue; + } + } else { + // Transition to frontier belief + auto [insertIterator, inserted] = frontierBeliefToStateMap.insert({entry.targetBelief, nextStateId}); + if (inserted) { + stateToFrontierBeliefMap[nextStateId] = entry.targetBelief; + ++nextStateId; + } + transitionBuilder.addNextValue(choice, insertIterator->second, entry.probability); + if constexpr (extraDataCompatibleWithRewardAwareness) { + if (isRewBndReachProb) { + for (uint64_t i = 0; i < propertyInformation.rewardBounds.size(); ++i) { + if (!storm::utility::isZero(entry.data[i])) { + transitionRewardBuilderVector.at(i).addNextValue(choice, insertIterator->second, entry.data[i]); + } + } + } + } + } + } + } + // Add transition to bottom/target state if necessary + if (!storm::utility::isZero(probabilityToTarget)) { + transitionBuilder.addNextValue(choice, targetState, probabilityToTarget); + } + if (!storm::utility::isZero(probabilityToBottom)) { + transitionBuilder.addNextValue(choice, bottomState, probabilityToBottom); + } + if (optionalChoiceLabeling.has_value()) { + for (auto const& label : explorationInformation.matrix.choiceLabels.at(choice)) { + if (!optionalChoiceLabeling.value().containsLabel(label)) { + optionalChoiceLabeling.value().addLabel(label); + } + optionalChoiceLabeling.value().addLabelToChoice(label, choice); + } + } + } + } + // Treat frontier beliefs + uint64_t choice = explorationInformation.matrix.rows(); + for (uint64_t state = numStates - numExtraStates; state < numStates - numBottomTargetStates; ++state) { + transitionBuilder.newRowGroup(choice); + for (auto& transitionRewardBuilder : transitionRewardBuilderVector) { + transitionRewardBuilder.newRowGroup(choice); + } + std::unordered_map cutOffInformationForBelief = cutOffInformationMap.at(stateToFrontierBeliefMap.at(state)); + for (auto const& entry : cutOffInformationForBelief) { + if (isReachProb) { + transitionBuilder.addNextValue(choice, targetState, entry.second); + transitionBuilder.addNextValue(choice, bottomState, storm::utility::one() - entry.second); + } else { + transitionBuilder.addNextValue(choice, targetState, storm::utility::one()); + if (isTotRew) { + actionRewards[choice] += entry.second; + } + } + if (optionalChoiceLabeling.has_value()) { + if (!optionalChoiceLabeling.value().containsLabel(entry.first)) { + optionalChoiceLabeling.value().addLabel(entry.first); + } + optionalChoiceLabeling.value().addLabelToChoice(entry.first, choice); + } + ++choice; + } + } + + // Treat extra states + transitionBuilder.newRowGroup(numChoices - numBottomTargetStates); + if (optionalChoiceLabeling.has_value()) { + if (!optionalChoiceLabeling.value().containsLabel("__loop__")) { + optionalChoiceLabeling.value().addLabel("__loop__"); + } + optionalChoiceLabeling.value().addLabelToChoice("__loop__", numChoices - numBottomTargetStates); + } + transitionBuilder.addNextValue(numChoices - numBottomTargetStates, targetState, storm::utility::one()); + for (auto& transitionRewardBuilder : transitionRewardBuilderVector) { + transitionRewardBuilder.newRowGroup(numChoices - 1); + } + if (isReachProb || clippingUsed) { + transitionBuilder.newRowGroup(numChoices - 1); + transitionBuilder.addNextValue(numChoices - 1, bottomState, storm::utility::one()); + if (optionalChoiceLabeling.has_value()) { + if (!optionalChoiceLabeling.value().containsLabel("__loop__")) { + optionalChoiceLabeling.value().addLabel("__loop__"); + } + optionalChoiceLabeling.value().addLabelToChoice("__loop__", numChoices - 1); + } + } + + storm::models::sparse::StateLabeling stateLabeling(numStates); + stateLabeling.addLabel("target"); + if (isRewBndReachProb) { + for (auto const& [belId, state] : explorationInformation.exploredBeliefs) { + if (propertyInformation.targetObservations.count(explorationInformation.discoveredBeliefs.getBeliefFromId(belId).observation() % + explorationInformation.nrObservationsInPomdp) > 0) { + stateLabeling.addLabelToState("target", state); + } + } + for (auto const& belId : explorationInformation.getFrontierBeliefs()) { + if (propertyInformation.targetObservations.count(explorationInformation.discoveredBeliefs.getBeliefFromId(belId).observation() % + explorationInformation.nrObservationsInPomdp) > 0) { + stateLabeling.addLabelToState("target", frontierBeliefToStateMap.at(belId)); + } + } + } else { + stateLabeling.addLabelToState("target", targetState); + } + stateLabeling.addLabel("init"); + stateLabeling.addLabelToState("init", explorationInformation.exploredBeliefs.at(explorationInformation.initialBeliefId)); + stateLabeling.addLabel("truncated"); + for (uint64_t state = numStates - numExtraStates; state < numStates - numBottomTargetStates; ++state) { + stateLabeling.addLabelToState("truncated", state); + } + + if (isReachProb || isRewBndReachProb || clippingUsed) { + stateLabeling.addLabel("bottom"); + stateLabeling.addLabelToState("bottom", bottomState); + } + storm::storage::sparse::ModelComponents components(transitionBuilder.build(), std::move(stateLabeling)); + + if (isTotRew) { + storm::models::sparse::StandardRewardModel rewardModel(std::nullopt, std::move(actionRewards)); + components.rewardModels.emplace(propertyInformation.rewardModelName.value(), std::move(rewardModel)); + } else if (isRewBndReachProb) { + uint64_t i = 0ul; + for (auto& transitionRewardBuilder : transitionRewardBuilderVector) { + storm::models::sparse::StandardRewardModel rewardModel(std::nullopt, std::nullopt, transitionRewardBuilder.build()); + components.rewardModels.emplace(propertyInformation.rewardBounds.at(i).rewardModelName, std::move(rewardModel)); + ++i; + } + } + if (optionalChoiceLabeling.has_value()) { + components.choiceLabeling = optionalChoiceLabeling.value(); + } + + // If requested, populate the stateToBeliefMap for the generic buildBeliefMdp variant + std::unordered_map stateToBeliefMap; + // Explored beliefs + for (auto const& [beliefId, stateIndex] : explorationInformation.exploredBeliefs) { + stateToBeliefMap[stateIndex] = beliefId; + } + // Frontier beliefs: mapping was built as stateToFrontierBeliefMap + for (auto const& [stateIndex, beliefId] : stateToFrontierBeliefMap) { + stateToBeliefMap[stateIndex] = beliefId; + } + + return std::make_pair(std::make_shared>(std::move(components)), std::move(stateToBeliefMap)); +} + +template std::pair>, std::unordered_map> buildBeliefMdp( + ExplorationInformation> const& explorationInformation, PropertyInformation const& propertyInformation, + std::function(Belief const&)> const& computeCutOffValueMap); + +template std::pair>, std::unordered_map> buildBeliefMdp( + ExplorationInformation> const& explorationInformation, PropertyInformation const& propertyInformation, + std::function(Belief const&)> const& computeCutOffValueMap); + +template std::pair>, std::unordered_map> buildBeliefMdp( + ExplorationInformation> const& explorationInformation, PropertyInformation const& propertyInformation, + std::function(Belief const&)> const& computeCutOffValueMap); + +template std::pair>, std::unordered_map> buildBeliefMdp( + ExplorationInformation> const& explorationInformation, PropertyInformation const& propertyInformation, + std::function(Belief const&)> const& computeCutOffValueMap); + +template std::pair>, std::unordered_map> buildBeliefMdp( + ClippingExplorationInformation> const& explorationInformation, PropertyInformation const& propertyInformation, + std::function(Belief const&)> const& computeCutOffValueMap); + +template std::pair>, std::unordered_map> buildBeliefMdp( + ClippingExplorationInformation> const& explorationInformation, PropertyInformation const& propertyInformation, + std::function(Belief const&)> const& computeCutOffValueMap); + +template std::pair>, std::unordered_map> buildBeliefMdp( + ClippingExplorationInformation> const& explorationInformation, PropertyInformation const& propertyInformation, + std::function(Belief const&)> const& computeCutOffValueMap); + +template std::pair>, std::unordered_map> buildBeliefMdp( + ClippingExplorationInformation> const& explorationInformation, + PropertyInformation const& propertyInformation, + std::function(Belief const&)> const& computeCutOffValueMap); + +template std::pair>, std::unordered_map> buildBeliefMdp( + RewardAwareExplorationInformation> const& explorationInformation, PropertyInformation const& propertyInformation, + std::function(Belief const&)> const& computeCutOffValueMap); + +template std::pair>, std::unordered_map> buildBeliefMdp( + RewardAwareExplorationInformation> const& explorationInformation, PropertyInformation const& propertyInformation, + std::function(Belief const&)> const& computeCutOffValueMap); + +template std::pair>, std::unordered_map> buildBeliefMdp( + RewardAwareExplorationInformation> const& explorationInformation, PropertyInformation const& propertyInformation, + std::function(Belief const&)> const& computeCutOffValueMap); + +template std::pair>, std::unordered_map> buildBeliefMdp( + RewardAwareExplorationInformation> const& explorationInformation, + PropertyInformation const& propertyInformation, + std::function(Belief const&)> const& computeCutOffValueMap); +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/beliefs/exploration/BeliefMdpBuilder.h b/src/storm-pomdp/beliefs/exploration/BeliefMdpBuilder.h new file mode 100644 index 0000000000..6ca0295586 --- /dev/null +++ b/src/storm-pomdp/beliefs/exploration/BeliefMdpBuilder.h @@ -0,0 +1,32 @@ +#pragma once + +#include +#include + +#include "storm-pomdp/beliefs/exploration/ExplorationInformation.h" +#include "storm-pomdp/beliefs/verification/PropertyInformation.h" +#include "storm/logic/Formulas.h" +#include "storm/models/sparse/Mdp.h" + +namespace storm::pomdp::beliefs { + +std::shared_ptr createFormulaForBeliefMdp(PropertyInformation const& propertyInformation); + +/** + * Builds a belief MDP from the given exploration information and property information. + * Variant with implicit cut-offs, i.e. in frontier beliefs we consider all actions, add transitions to already explored beliefs and cut off the rest. + * @tparam BeliefMdpValueType ValueType of the belief MDP + * @tparam BeliefType Type of the belief + * @tparam ExtraTransitionData Types of additional data to store for transitions (e.g. reward vectors) + * @param explorationInformation object containing information about the exploration of the belief space (explored beliefs, transitions, etc.) + * @param propertyInformation object containing information about the property to verify + * @param computeCutOffValueMap function to compute all cut-off values for a belief given the provided information. A separate cut-off action is added for each + * value, allowing the model checker to choose the best one. This choice can later be retraced. + * @return the belief MDP + */ +template +std::pair>, std::unordered_map> buildBeliefMdp( + ExplorationInformation const& explorationInformation, + PropertyInformation const& propertyInformation, + std::function(BeliefType const&)> const& computeCutOffValueMap); +} // namespace storm::pomdp::beliefs \ No newline at end of file diff --git a/src/storm-pomdp/beliefs/exploration/ExplorationInformation.h b/src/storm-pomdp/beliefs/exploration/ExplorationInformation.h new file mode 100644 index 0000000000..07458321ef --- /dev/null +++ b/src/storm-pomdp/beliefs/exploration/ExplorationInformation.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include +#include +#include + +#include "storm-pomdp/beliefs/exploration/BeliefExplorationMatrix.h" +#include "storm-pomdp/beliefs/exploration/ExplorationQueue.h" +#include "storm-pomdp/beliefs/storage/BeliefCollector.h" +#include "storm-pomdp/beliefs/utility/types.h" + +namespace storm::pomdp::beliefs { +/** + * Mutable state of one belief-MDP exploration. + * + * A discovered belief is either explored, terminal, or on the frontier. The matrix contains rows only for explored + * beliefs; frontier beliefs are completed with explicit cut-off choices when the belief MDP is built. + */ +template +struct ExplorationInformation { + BeliefExplorationMatrix matrix; + std::vector actionRewards; + storm::pomdp::beliefs::BeliefCollector discoveredBeliefs; + std::unordered_map exploredBeliefs; + std::unordered_map terminalBeliefValues; + BeliefId initialBeliefId; + ExplorationQueue queue; + uint64_t nrObservationsInPomdp; + bool generateChoiceLabeling = false; + + /** @return discovered beliefs that have neither been explored nor classified as terminal. */ + [[nodiscard]] std::unordered_set getFrontierBeliefs() const { + std::unordered_set resFrontierBeliefs; + for (uint64_t id = 0; id < discoveredBeliefs.getNumberOfBeliefIds(); id++) { + if (!exploredBeliefs.contains(id) && !terminalBeliefValues.contains(id)) { + resFrontierBeliefs.insert(id); + } + } + return resFrontierBeliefs; + } +}; + +template +/** Exploration information without transition metadata. */ +using StandardExplorationInformation = ExplorationInformation; + +template +/** Exploration information that stores a reward vector on each transition. */ +using RewardAwareExplorationInformation = ExplorationInformation>; + +template +/** Exploration information that stores optional clipping probability and reward-adjustment metadata. */ +using ClippingExplorationInformation = + ExplorationInformation, std::optional>; +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/beliefs/exploration/ExplorationQueue.cpp b/src/storm-pomdp/beliefs/exploration/ExplorationQueue.cpp new file mode 100644 index 0000000000..7d15f35029 --- /dev/null +++ b/src/storm-pomdp/beliefs/exploration/ExplorationQueue.cpp @@ -0,0 +1,68 @@ +#include "storm-pomdp/beliefs/exploration/ExplorationQueue.h" + +#include "storm/utility/constants.h" +#include "storm/utility/macros.h" + +namespace storm::pomdp::beliefs { + +ExplorationQueue::ExplorationQueue(ExplorationQueueOrder const order) : order(order) { + // Intentionally left empty. +} + +void ExplorationQueue::changeOrder(ExplorationQueueOrder const newOrder) { + STORM_LOG_ASSERT((order == ExplorationQueueOrder::Unordered) ? queue.empty() : (contents.size() == queue.size()), "inconsistent queue state"); + if (newOrder == order) { + return; // nothing to do. + } + if (newOrder == ExplorationQueueOrder::Unordered) { + queue.clear(); // just keep the contents, drop the queue order + } else if (order == ExplorationQueueOrder::Unordered) { + // since newOrder != order, the newOrder will need the queue + for (auto const id : contents) { + queue.push_back(id); + } + } + order = newOrder; + STORM_LOG_ASSERT((order == ExplorationQueueOrder::Unordered) ? queue.empty() : (contents.size() == queue.size()), "inconsistent queue state"); +} + +bool ExplorationQueue::hasNext() const { + STORM_LOG_ASSERT((order == ExplorationQueueOrder::Unordered) ? queue.empty() : (contents.size() == queue.size()), "inconsistent queue state"); + return !contents.empty(); +} + +bool ExplorationQueue::push(BeliefId const id) { + if (contents.insert(id).second) { + if (order != ExplorationQueueOrder::Unordered) { + queue.push_back(id); + } + return true; + } + return false; +} + +BeliefId ExplorationQueue::popNext() { + STORM_LOG_ASSERT(hasNext(), "Trying to pop from empty queue."); + if (order == ExplorationQueueOrder::Unordered) { + BeliefId const id = *contents.begin(); + contents.erase(contents.begin()); + return id; + } else { + BeliefId id; + if (order == ExplorationQueueOrder::LIFO) { + id = queue.back(); + queue.pop_back(); + } else { + id = queue.front(); + queue.pop_front(); + } + STORM_LOG_ASSERT(contents.count(id) == 1, "Queue contains belief that is not in contents."); + contents.erase(id); + return id; + } +} + +std::unordered_set ExplorationQueue::getContents() const { + return contents; +} +} // namespace storm::pomdp::beliefs \ No newline at end of file diff --git a/src/storm-pomdp/beliefs/exploration/ExplorationQueue.h b/src/storm-pomdp/beliefs/exploration/ExplorationQueue.h new file mode 100644 index 0000000000..50d39d0a29 --- /dev/null +++ b/src/storm-pomdp/beliefs/exploration/ExplorationQueue.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include + +#include "storm-pomdp/beliefs/exploration/ExplorationQueueOrder.h" +#include "storm-pomdp/beliefs/utility/types.h" + +namespace storm::pomdp::beliefs { + +class ExplorationQueue { + public: + ExplorationQueue(ExplorationQueueOrder const order = ExplorationQueueOrder::Unordered); + + /*! + * Changes the order in which the queue processes the elements. + * @note if the queue is currently non-empty, all elements will remain in the queue. However, if there is more than one element in the queue, we do not give + * any guarantees about the order in which they will be processed. + * @param newOrder + */ + void changeOrder(ExplorationQueueOrder const newOrder); + + /*! + * @return true if the queue is not empty. + */ + bool hasNext() const; + + /*! + * Adds the given belief id to the queue. + */ + bool push(BeliefId const id); + + /*! + * Removes and returns the next belief id from the queue. + */ + BeliefId popNext(); + + /*! + * @return the contents of the queue + */ + std::unordered_set getContents() const; + + private: + /// The order in which to process the elements. + ExplorationQueueOrder order; + + /// The set of belief ids in the queue. + std::unordered_set contents; + + /// the order in which elements were inserted (empty if unordered) + std::deque queue; +}; + +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/beliefs/exploration/ExplorationQueueOrder.h b/src/storm-pomdp/beliefs/exploration/ExplorationQueueOrder.h new file mode 100644 index 0000000000..1b8c7f14c6 --- /dev/null +++ b/src/storm-pomdp/beliefs/exploration/ExplorationQueueOrder.h @@ -0,0 +1,5 @@ +#pragma once +namespace storm::pomdp::beliefs { +/** Order used to select the next discovered belief for exploration. */ +enum class ExplorationQueueOrder { Unordered, FIFO, LIFO }; +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/beliefs/exploration/FirstStateNextStateGenerator.cpp b/src/storm-pomdp/beliefs/exploration/FirstStateNextStateGenerator.cpp new file mode 100644 index 0000000000..8c399dd575 --- /dev/null +++ b/src/storm-pomdp/beliefs/exploration/FirstStateNextStateGenerator.cpp @@ -0,0 +1,83 @@ +#include "storm-pomdp/beliefs/exploration/FirstStateNextStateGenerator.h" +#include "storm-pomdp/beliefs/storage/Belief.h" + +#include "storm/adapters/RationalNumberAdapter.h" +#include "storm/models/sparse/Pomdp.h" +#include "storm/utility/macros.h" + +namespace storm::pomdp::beliefs { + +template +FirstStateNextStateGenerator::FirstStateNextStateGenerator(PomdpType const& pomdp) : pomdp(pomdp) { + // Intentionally left empty. +} + +template +void FirstStateNextStateGenerator::setRewardModel(std::string const& rewardModelName) { + auto const& rewardModel = pomdp.getRewardModel(rewardModelName); + actionRewards = rewardModel.getTotalRewardVector(pomdp.getTransitionMatrix()); +} + +template +bool FirstStateNextStateGenerator::hasRewardModel() const { + return !actionRewards.empty(); +} + +template +void FirstStateNextStateGenerator::unsetRewardModel() { + actionRewards.clear(); +} + +template +BeliefType FirstStateNextStateGenerator::computeInitialBelief() const { + STORM_LOG_ASSERT(pomdp.getInitialStates().getNumberOfSetBits() == 1, "Only a single initial state is supported, but the given POMDP contains " + << pomdp.getInitialStates().getNumberOfSetBits() << " initial states."); + BeliefStateType const init = *pomdp.getInitialStates().begin(); + BeliefBuilder builder; + builder.addValue(init, storm::utility::one()); + builder.setObservation(pomdp.getObservation(init)); + return builder.build(); +} + +template +uint64_t FirstStateNextStateGenerator::getBeliefNumberOfActions(BeliefType const& belief) const { + auto result = pomdp.getTransitionMatrix().getRowGroupSize(belief.representativeState()); + // Assert consistency with other states in the support + STORM_LOG_ASSERT(belief.allOf([&result, this](BeliefStateType const& state, typename BeliefType::ValueType const&) { + return pomdp.getTransitionMatrix().getRowGroupSize(state) == result; + }), + "Belief considers states with inconsistent number of choices."); + return result; +} + +template +typename PomdpType::ValueType FirstStateNextStateGenerator::getBeliefActionReward(BeliefType const& belief, + uint64_t const& localActionIndex) const { + using PomdpValueType = typename PomdpType::ValueType; + using BeliefValueType = typename BeliefType::ValueType; + + STORM_LOG_ASSERT(hasRewardModel(), "Requested a reward although no reward model was specified."); + STORM_LOG_ASSERT(localActionIndex < getBeliefNumberOfActions(belief), "Invalid action index " << localActionIndex << "."); + auto result = storm::utility::zero(); + auto const& actionIndices = pomdp.getTransitionMatrix().getRowGroupIndices(); + belief.forEach([&localActionIndex, &actionIndices, &result, this](BeliefStateType const& state, BeliefValueType const& val) { + uint64_t const actionIndex = actionIndices[state] + localActionIndex; + result += storm::utility::convertNumber(val) * this->actionRewards[actionIndex]; + }); + return result; +} + +template +std::set FirstStateNextStateGenerator::getBeliefActionChoiceLabels(BeliefType const& belief, + uint64_t const& localActionIndex) const { + STORM_LOG_ASSERT(pomdp.hasChoiceLabeling(), "Requested a choice label although no choice labeling was specified in the POMDP."); + STORM_LOG_ASSERT(localActionIndex < getBeliefNumberOfActions(belief), "Invalid action index " << localActionIndex << "."); + return pomdp.getChoiceLabeling().getLabelsOfChoice(pomdp.getTransitionMatrix().getRowGroupIndices().at(belief.representativeState()) + localActionIndex); +} + +template class FirstStateNextStateGenerator, Belief>; +template class FirstStateNextStateGenerator, Belief>; +template class FirstStateNextStateGenerator, Belief>; +template class FirstStateNextStateGenerator, Belief>; + +} // namespace storm::pomdp::beliefs \ No newline at end of file diff --git a/src/storm-pomdp/beliefs/exploration/FirstStateNextStateGenerator.h b/src/storm-pomdp/beliefs/exploration/FirstStateNextStateGenerator.h new file mode 100644 index 0000000000..382640903a --- /dev/null +++ b/src/storm-pomdp/beliefs/exploration/FirstStateNextStateGenerator.h @@ -0,0 +1,228 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "storm-pomdp/beliefs/abstraction/NoAbstraction.h" + +#include "storm-pomdp/beliefs/storage/BeliefBuilder.h" +#include "storm-pomdp/beliefs/utility/types.h" + +#include "storm/utility/constants.h" + +namespace storm::pomdp::beliefs { + +// Forward declare handle +namespace detail { +template +struct NextStateGeneratorHandle; +} + +/*! + * This class implements a first-state-next-state generator interface for exploring the belief MDP of a given POMDP. + * It provides methods to compute the initial belief, the number of actions available in a belief, and the reward of a given action in a belief. + * + * Furthermore, this class can configure a handle for the generation of successor beliefs in the belief MDP. + * Such a handle can be called with a belief and an action, which will then compute all the successors of the given belief w.r.t. the given action + * In its simplest form, the handle is configured using a callback function, which will be invoked on any discovered successor. + * One can additionally provide a pre-abstraction and/or a post-abstraction when configuring the handle. + * An abstraction is a class that provides a method `abstract` that takes a belief as input and returns a distribution over abstracted beliefs as output. + * When providing a pre-abstraction, the handle will first apply the abstraction to the given belief before computing the successors of each belief resulting + * from that abstraction When providing a post-abstraction, the handle will abstract each computed successor and will invoke the discover callback function on + * any abstracted successor. + * + * Further notes on the discover callback: + * The discover callback is invoked for each calculated (and potentially abstracted) successor belief, using the resulting successor belief and the transition + * probability to it as arguments (both by rvalue reference). If there is a pre- and/or post-abstraction, the transition probability with which the callback is + * invoked will be multiplied by the "abstraction weights", i.e., the corresponding probability of the distribution returned by the abstraction. + * + * Example: + * Assume `b` is the current belief and in the belief MDP there would be a transition to a successor belief `c` with probability 0.3 and a transition + * to a successor belief `d` with probability 0.7. Furthermore, suppose that the handle was configured with a post abstraction that, given `c`, returns the + * distribution `{0.4: c_1, 0.6: c_2}` and, given `d` returns the distribution `{0.1: d_1, 0.9: d_2}` . Then the discover callback would be invoked four times + * with the arguments `(c_1, 0.12)`, `(c_2, 0.18)`, `(d_1, 0.07)`, and `(d_2, 0.63)`. + * + * Further notes on the abstractions: + * A pre-abstraction gets as input the current belief (by const reference) and the executed action. + * A typical application for a pre-abstraction would be to incorporate observations based on a given (belief, action) pair. + * A post-abstraction gets as input a successor belief and the transition probability to it (both by rvalue reference). + * A typical application for a post-abstraction would be to discretize the found successor beliefs. + * For efficiency considerations, the distributions over abstracted beliefs calculated by the abstractions are not explicitly constructed. Instead, yet another + * callback is used. For example, instead of returning a map-like object for `{0.1: d_1, 0.9: d_2}`, the + * abstraction-callback will be invoked twice with the arguments `(d_1, 0.1)` and `(d_2, 0.9)`, respectively. + * + * Passing further information to the discover-callback: + * To forward information to the discover-callback, additional arguments can be provided + * - when invoking the handle, and + * - by the pre- and/or post-abstraction when they call the abstraction-callback. + * This is implemented using parameter packs / variadic templates. + * The signature of the discover callback must then consider those additional arguments: first those from the handle invocation, then those from the + * pre-abstraction, and then those from the post-abstraction. + * + * @tparam PomdpType The type of the POMDP model + * @tparam BeliefType The type of the beliefs of the POMDP + */ +template +class FirstStateNextStateGenerator { + public: + template + using Handle = detail::NextStateGeneratorHandle; + + /** Creates a successor generator for @p pomdp. The POMDP must outlive the generator and all of its handles. */ + FirstStateNextStateGenerator(PomdpType const& pomdp); + + /** Selects the reward model whose expected action rewards are queried; an empty name selects the default model. */ + void setRewardModel(std::string const& rewardModelName = ""); + /** @return whether a reward model is currently selected. */ + bool hasRewardModel() const; + void unsetRewardModel(); + + /** @return the initial belief induced by the POMDP's initial-state distribution. */ + BeliefType computeInitialBelief() const; + + /** @return the number of observation-compatible actions in @p belief. */ + uint64_t getBeliefNumberOfActions(BeliefType const& belief) const; + + /** @return the expected selected-model reward of the local action at @p belief. */ + typename PomdpType::ValueType getBeliefActionReward(BeliefType const& belief, uint64_t const& localActionIndex) const; + + /** @return labels of the POMDP action represented by @p localActionIndex at @p belief. */ + std::set getBeliefActionChoiceLabels(BeliefType const& belief, uint64_t const& localActionIndex) const; + + /** Returns a handle that generates unabstracted successor beliefs. */ + template + auto getHandle(DiscoverCallbackType& discoverCallback) { + return Handle{this->pomdp, NoAbstraction, NoAbstraction, discoverCallback}; + } + + /** Returns a handle that applies @p preAbstraction before successor generation. */ + template + auto getPreAbstractionHandle(PreAbstractionType& preAbstraction, DiscoverCallbackType& discoverCallback) { + return Handle{this->pomdp, preAbstraction, NoAbstraction, discoverCallback}; + } + + /** Returns a handle that applies @p postAbstraction to each successor belief. */ + template + auto getPostAbstractionHandle(PostAbstractionType& postAbstraction, DiscoverCallbackType& discoverCallback) { + return Handle{this->pomdp, NoAbstraction, postAbstraction, discoverCallback}; + } + + /** Returns a handle that applies both abstractions around successor generation. */ + template + auto getPrePostAbstractionHandle(PreAbstractionType& preAbstraction, PostAbstractionType& postAbstraction, DiscoverCallbackType& discoverCallback) { + return Handle{this->pomdp, preAbstraction, postAbstraction, discoverCallback}; + } + + private: + PomdpType const& pomdp; + std::vector actionRewards; +}; + +namespace detail { + +/*! + * Implementation of the NextStateGeneratorHandle + * @note as this is heavily templated, the implementation is intentionally put in the header file. + */ +template +struct NextStateGeneratorHandle { + using BeliefValueType = typename BeliefType::ValueType; + + template + /** Generates all observation-distinguished successors for one belief/action pair. */ + void operator()(BeliefType const& belief, uint64_t localActionIndex, CallBackArgs const&... additionalCallbackArgs) { + applyPreAbstraction(belief, localActionIndex, std::forward(additionalCallbackArgs)...); + } + + PomdpType const& pomdp; + PreAbstractionType& preAbstraction; + PostAbstractionType& postAbstraction; + DiscoverCallback& discoverCallback; + + private: + template + void applyPreAbstraction(BeliefType const& belief, uint64_t localActionIndex, CallBackArgs const&... additionalCallbackArgs) { + if constexpr (isNoAbstraction) { + computeSuccessorBeliefs(belief, localActionIndex, storm::utility::one(), DefaultActionObservation, additionalCallbackArgs...); + } else { + preAbstraction.abstract( + belief, localActionIndex, + [this, &localActionIndex, &additionalCallbackArgs...]( + BeliefType&& preBel, BeliefValueType&& preVal, BeliefActionObservationType actionObservation, auto const&... additionalPreAbstractionArgs) { + this->computeSuccessorBeliefs(preBel, localActionIndex, std::move(preVal), actionObservation, + std::forward(additionalCallbackArgs)..., + std::forward(additionalPreAbstractionArgs)...); + }); + } + } + + /*! + * @return the probability we go to each observation when starting in the given belief and performing the given action + */ + std::unordered_map computeSuccessorObservations(BeliefType const& belief, uint64_t localActionIndex, + BeliefActionObservationType actionObservation) { + std::unordered_map successorObservations; + belief.forEach([&localActionIndex, &successorObservations, &actionObservation, this](BeliefStateType const& state, BeliefValueType const& beliefValue) { + for (auto const& pomdpTransition : pomdp.getTransitionMatrix().getRow(state, localActionIndex)) { + if (!storm::utility::isZero(pomdpTransition.getValue())) { + auto const obs = (pomdp.getNrObservations() * actionObservation) + pomdp.getObservation(pomdpTransition.getColumn()); + BeliefValueType const val = beliefValue * storm::utility::convertNumber(pomdpTransition.getValue()); + if (auto [insertionIt, inserted] = successorObservations.emplace(obs, val); !inserted) { + insertionIt->second += val; + } + } + } + }); + + // Adjust the distribution to diminish numerical inaccuracies a bit + if constexpr (!storm::NumberTraits::IsExact || !storm::NumberTraits::IsExact) { + if (successorObservations.size() == 1) { + successorObservations.begin()->second = storm::utility::one(); + } + } + return successorObservations; + } + + template + void computeSuccessorBeliefs(BeliefType const& belief, uint64_t localActionIndex, BeliefValueType const& transitionProbability, + BeliefActionObservationType actionObservation, CallBackArgs const&... additionalCallbackArgs) { + // For each successor observation we build the successor belief + auto const successorObservations = computeSuccessorObservations(belief, localActionIndex, actionObservation); + for (auto const& successorObsValue : successorObservations) { + BeliefBuilder builder; + builder.setObservation(successorObsValue.first); + belief.forEach([&builder, &localActionIndex, &successorObsValue, this](BeliefStateType const& state, BeliefValueType const& beliefValue) { + for (auto const& pomdpTransition : pomdp.getTransitionMatrix().getRow(state, localActionIndex)) { + if (pomdp.getObservation(pomdpTransition.getColumn()) == (successorObsValue.first % pomdp.getNrObservations())) { + BeliefValueType const prob = + beliefValue * storm::utility::convertNumber(pomdpTransition.getValue()) / successorObsValue.second; + STORM_LOG_ASSERT(prob > storm::utility::zero(), "Invalid belief probability " << prob << "."); + builder.addValue(pomdpTransition.getColumn(), prob); + } + } + }); + applyPostAbstraction(builder.build(), static_cast(successorObsValue.second * transitionProbability), + std::forward(additionalCallbackArgs)...); + } + } + + template + void applyPostAbstraction(BeliefType&& belief, BeliefValueType&& transitionProbability, CallBackArgs const&... additionalCallbackArgs) { + if constexpr (isNoAbstraction) { + discoverCallback(std::move(belief), std::move(transitionProbability), additionalCallbackArgs...); + } else { + postAbstraction.abstract( + std::move(belief), std::move(transitionProbability), + [this, &additionalCallbackArgs...](BeliefType&& postBel, BeliefValueType&& postVal, auto&&... additionalPostAbstractionArgs) { + discoverCallback(std::move(postBel), std::move(postVal), std::forward(additionalCallbackArgs)..., + std::forward(additionalPostAbstractionArgs)...); + }); + } + } +}; + +} // namespace detail +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/beliefs/storage/Belief.cpp b/src/storm-pomdp/beliefs/storage/Belief.cpp new file mode 100644 index 0000000000..2cc02ad831 --- /dev/null +++ b/src/storm-pomdp/beliefs/storage/Belief.cpp @@ -0,0 +1,83 @@ +#include "storm-pomdp/beliefs/storage/Belief.h" + +#include "storm-pomdp/beliefs/storage/BeliefBuilder.h" +#include "storm-pomdp/beliefs/utility/BeliefNumerics.h" + +#include "storm/adapters/RationalNumberAdapter.h" +#include "storm/utility/NumberTraits.h" + +#include + +namespace storm::pomdp::beliefs { + +template +Belief::Belief(BeliefFlatMap&& data, BeliefObservationType&& obs) : data(std::move(data)), obs(std::move(obs)) { + // Intentionally empty +} + +template +std::size_t Belief::size() const { + return data.size(); +} + +template +BeliefStateType Belief::representativeState() const { + STORM_LOG_ASSERT(!data.empty(), "Empty belief"); + return data.begin()->first; +} + +template +BeliefObservationType const& Belief::observation() const { + return obs; +} + +template +bool Belief::operator==(Belief const& other) const { + if (obs != other.obs) { + return false; + } + if (data.size() != other.size()) { + return false; + } + static_assert(BeliefFlatMapIsOrdered); + auto secondIt = other.data.cbegin(); + for (auto const& [state, value] : data) { + if (state != secondIt->first) { + return false; + } + if (!BeliefNumerics::equal(value, secondIt->second)) { + return false; + } + ++secondIt; + } + return true; +} + +template +std::string Belief::toString(bool convertToDouble) const { + std::stringstream ss; + ss << "Belief{ obs:" << obs; + if (convertToDouble) { + forEach([&ss](auto const& state, auto const& val) { ss << ", " << state << ":" << storm::utility::convertNumber(val); }); + } else { + forEach([&ss](auto const& state, auto const& val) { ss << ", " << state << ":" << val; }); + } + ss << " }"; + return ss.str(); +} + +template +std::size_t Belief::BeliefHash::operator()(Belief const& belief) const { + auto seed = static_cast(belief.obs); + static_assert(BeliefFlatMapIsOrdered); + belief.forEach([&seed](auto const& state, auto const& val) { + boost::hash_combine(seed, state); + boost::hash_combine(seed, BeliefNumerics::valueForHash(val)); + }); + return seed; +} + +template class Belief; +template class Belief; + +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/beliefs/storage/Belief.h b/src/storm-pomdp/beliefs/storage/Belief.h new file mode 100644 index 0000000000..3047e1ff9b --- /dev/null +++ b/src/storm-pomdp/beliefs/storage/Belief.h @@ -0,0 +1,183 @@ +#pragma once + +#include "storm/utility/constants.h" +#include "storm/utility/macros.h" + +#include "storm-pomdp/beliefs/utility/types.h" + +namespace storm::pomdp::beliefs { + +template +class BeliefBuilder; + +/*! + * Represents a belief of a Pomdp, i.e. a probability distribution over the states of a POMDP. + * A belief also knows its observation. + * @note A belief is immutable. Use the BeliefBuilder class to construct new beliefs. + * @tparam ValueTypeArg the type of the values (probabilities) of the belief. + */ +template +class Belief { + public: + using ValueType = ValueTypeArg; + friend class BeliefBuilder>; + + Belief() = delete; + Belief(Belief const& other) = default; + Belief(Belief&& other) = default; + Belief& operator=(Belief const& other) = default; + Belief& operator=(Belief&& other) = default; + + /*! + * @return the number of states in the support of this belief. + */ + std::size_t size() const; + + /*! + * @return a representative state of this belief. + * @pre the belief is valid, in particular not empty. + */ + BeliefStateType representativeState() const; + + /*! + * @return the observation of this belief. + */ + BeliefObservationType const& observation() const; + + /*! + * @return true if this belief is equal to the other belief. + */ + bool operator==(Belief const& other) const; + + /*! + * A (human-readable) string representation of this belief + * @param convertToDouble if true, numbers are converted to double before printing. If this has ValueType=RationalNumber, the output as double is readable + * but potentially imprecise + */ + std::string toString(bool convertToDouble = true) const; + + /*! + * @param summands a vector containing a value for each state of the underlying POMDP. + * @return the sum of the POMDP state values, each multiplied by the value assigned to that state in this belief + * @pre for every state in the support of this belief, the corresponding index in summands must be valid + */ + template + SummandsType getWeightedSum(std::vector const& summands) const { + auto sum = storm::utility::zero(); + forEach([&sum, &summands](auto const& state, auto const& val) { + STORM_LOG_ASSERT(state < summands.size(), "State " << state << " is out of range for the given summands."); + // If SummandsType is the same as ValueType, we can directly multiply the value, otherwise convert + if constexpr (std::is_same_v) { + sum += summands[state] * val; + } else { + sum += summands[state] * storm::utility::convertNumber(val); + } + }); + return sum; + }; + + /*! + * @param f a function that is called for each (state, value)-pair in the support of this belief. + */ + template + void forEach(FunctionType const& f) const { + for (auto const& [state, value] : data) { + f(state, value); + } + } + + /*! + * @return true if f returns true for all (state, value)-pairs in the support of this belief. + */ + template + bool allOf(FunctionType const& f) const { + for (auto const& [state, value] : data) { + if (!f(state, value)) { + return false; + } + } + return true; + } + + /*! + * @param f a function that is called for each state in the support of this belief. + */ + template + void forEachStateInSupport(FunctionType const& f) const { + for (auto const& [state, value] : data) { + f(state); + } + } + + /*! + * Calls the function f for each state in X with the arguments (state, value1, value2), where + * - value1 is the value for state of this belief + * - value2 is the value for state of the other belief + * - if considerOnlyThisSupport is true, X is the set of states in the support of this belief + * - otherwise, X is the set of states in the support of either this or the other belief + */ + template + void forEachCombine(Belief const& other, FunctionType const& f, bool considerOnlyThisSupport = false) const { + static_assert(BeliefFlatMapIsOrdered); + auto const zero = storm::utility::zero(); + auto it2 = other.data.cbegin(); + auto const it2End = other.data.cend(); + if (considerOnlyThisSupport) { + for (auto const& [state1, value1] : data) { + while (it2 != it2End && it2->first < state1) { + ++it2; + } + if (it2 == it2End || it2->first > state1) { + f(state1, value1, zero); + } else { + STORM_LOG_ASSERT(state1 == it2->first, "Unexpected state."); + f(state1, value1, it2->second); + } + } + } else { + auto it1 = data.cbegin(); + auto const it1End = data.cend(); + while (it1 != it1End && it2 != it2End) { + auto const state1 = it1->first; + auto const state2 = it2->first; + if (state1 == state2) { + f(state1, it1->second, it2->second); + ++it1; + ++it2; + } else if (state1 < state2) { + f(state1, it1->second, zero); + ++it1; + } else { + STORM_LOG_ASSERT(state2 < state1, "unexpected states."); + f(state2, zero, it2->second); + ++it2; + } + } + for (; it1 != it1End; ++it1) { + f(it1->first, it1->second, zero); + } + for (; it2 != it2End; ++it2) { + f(it2->first, zero, it2->second); + } + } + } + + /*! + * @return provides a hash value for the given belief + */ + struct BeliefHash { + std::size_t operator()(Belief const& belief) const; + }; + + private: + /*! + * Constructs a belief from the given Data. + * @note Use the BeliefBuilder class to create a belief. + */ + Belief(BeliefFlatMap&& data, BeliefObservationType&& obs); + + BeliefFlatMap const data; + BeliefObservationType const obs; +}; + +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/beliefs/storage/BeliefBuilder.cpp b/src/storm-pomdp/beliefs/storage/BeliefBuilder.cpp new file mode 100644 index 0000000000..3114aa50fa --- /dev/null +++ b/src/storm-pomdp/beliefs/storage/BeliefBuilder.cpp @@ -0,0 +1,97 @@ +#include "storm-pomdp/beliefs/storage/BeliefBuilder.h" +#include "storm-pomdp/beliefs/storage/Belief.h" +#include "storm-pomdp/beliefs/utility/BeliefNumerics.h" + +#include "storm/adapters/RationalNumberAdapter.h" +#include "storm/utility/constants.h" +#include "storm/utility/macros.h" + +namespace storm::pomdp::beliefs { + +template +void BeliefBuilder::reserve(uint64_t size) { + data.reserve(size); +} + +template +void BeliefBuilder::addValue(BeliefStateType const& state, ValueType const& value) { + STORM_LOG_ASSERT(value > storm::utility::zero() && value <= storm::utility::one(), "Invalid belief value " << value << "."); + if (auto [insertionIt, success] = data.emplace(state, value); !success) { + insertionIt->second += value; + } +} + +template +void BeliefBuilder::setObservation(BeliefObservationType const& observation) { + obs = observation; +} + +template +typename BeliefBuilder::ValueType BeliefBuilder::normalize() { + auto sum = storm::utility::zero(); + for (auto const& [state, value] : data) { + STORM_LOG_ASSERT(value >= storm::utility::zero(), "Invalid value: " << value); + sum += value; + } + STORM_LOG_ASSERT(sum > storm::utility::zero(), "Invalid sum: " << sum); + if (!storm::utility::isOne(sum)) { + for (auto& [state, value] : data) { + value /= sum; + } + } + return sum; +} + +template +BeliefType BeliefBuilder::build() { + data.shrink_to_fit(); + if constexpr (!storm::NumberTraits::IsExact) { + if (data.size() == 1 && BeliefNumerics::isOne(data.begin()->second)) { + // If the distribution consists of only one entry and its value is sufficiently close to 1, make it exactly 1 to avoid numerical problems + data.begin()->second = storm::utility::one(); + } + } + STORM_LOG_ASSERT(assertBelief(), "Trying to build invalid belief."); + return BeliefType{std::move(data), std::move(obs)}; +} + +template +void BeliefBuilder::reset() { + data.clear(); + obs = InvalidObservation; +} + +template +bool BeliefBuilder::assertBelief() { + using ValueType = typename BeliefType::ValueType; + if (obs == InvalidObservation) { + STORM_LOG_ERROR("Observation of Belief not set."); + return false; + } + auto sum = storm::utility::zero(); + for (auto const& [state, value] : data) { + if (value <= storm::utility::zero()) { + STORM_LOG_ERROR("Non-positive belief value " << value << " at state " << state << "."); + return false; + } + if (!BeliefNumerics::lessOrEqual(value, storm::utility::one())) { + STORM_LOG_ERROR("Invalid belief value " << value << " at state " << state << "."); + return false; + } + sum += value; + } + if (!BeliefNumerics::lessOrEqual(sum, storm::utility::one())) { + STORM_LOG_ERROR("belief value sum " << sum << " is larger than 1 (sum-1=" << sum - storm::utility::one() << ")."); + return false; + } + if (!BeliefNumerics::lessOrEqual(storm::utility::one(), sum)) { + STORM_LOG_ERROR("belief value sum " << sum << " is smaller than 1 (1-sum=" << storm::utility::one() - sum << ")."); + return false; + } + return true; +} + +template class BeliefBuilder>; +template class BeliefBuilder>; + +} // namespace storm::pomdp::beliefs \ No newline at end of file diff --git a/src/storm-pomdp/beliefs/storage/BeliefBuilder.h b/src/storm-pomdp/beliefs/storage/BeliefBuilder.h new file mode 100644 index 0000000000..a1cea4a9b7 --- /dev/null +++ b/src/storm-pomdp/beliefs/storage/BeliefBuilder.h @@ -0,0 +1,65 @@ +#pragma once +#include + +#include "storm-pomdp/beliefs/utility/types.h" + +namespace storm::pomdp::beliefs { + +/*! + * Constructs a belief + * @tparam BeliefType the type of belief to construct + */ +template +class BeliefBuilder { + public: + using ValueType = typename BeliefType::ValueType; + + /*! + * Reserves space to build the belief more efficiently + * @param size the number of states in the support of the belief + * @note providing this is optional, but can improve performance + */ + void reserve(uint64_t size); + + /*! + * Adds a probability value to the given state. + * If a value for the state already exists, its new value is the sum of the existing one and the given value. + */ + void addValue(BeliefStateType const& state, ValueType const& value); + + /*! + * Sets the observation of the belief. + * @param observation the observation to set + */ + void setObservation(BeliefObservationType const& observation); + + /*! + * Normalizes the belief, i.e., if the sum of the values is not 1, it divides all values by the sum. + * Hence, after calling this, the sum of the values is 1. + * @pre the sum is must be strictly greater than 0. + * @return the sum of the values (before normalization) + */ + ValueType normalize(); + + /*! + * Builds the belief and returns it. + * @pre the belief must be a valid probability distribution, i.e., all values are in [0,1] and sum up to 1. Furthermore, an observation must have been set + * before calling this. + * @note After calling this, the builder is in an invalid state. Before building another belief with this builder, the reset() method must be called. + */ + BeliefType build(); + + /*! + * Resets the builder to its initial state. + * After build() has been called, this method must be called before building another belief with this builder. + */ + void reset(); + + private: + bool assertBelief(); + + BeliefFlatMap data; + BeliefObservationType obs{InvalidObservation}; +}; + +} // namespace storm::pomdp::beliefs \ No newline at end of file diff --git a/src/storm-pomdp/beliefs/storage/BeliefCollector.cpp b/src/storm-pomdp/beliefs/storage/BeliefCollector.cpp new file mode 100644 index 0000000000..42c74ac86a --- /dev/null +++ b/src/storm-pomdp/beliefs/storage/BeliefCollector.cpp @@ -0,0 +1,75 @@ +#include "storm-pomdp/beliefs/storage/BeliefCollector.h" +#include "storm-pomdp/beliefs/storage/Belief.h" +#include "storm/adapters/RationalNumberAdapter.h" + +namespace storm::pomdp::beliefs { + +template +bool BeliefCollector::isEqual(BeliefId const& first, BeliefId const& second) const { + return first == second || getBeliefFromId(first) == getBeliefFromId(second); +} + +template +BeliefId BeliefCollector::getNumberOfBeliefIds() const { + return gatheredBeliefs.size(); +} + +template +BeliefType const& BeliefCollector::getBeliefFromId(BeliefId const& id) const { + STORM_LOG_ASSERT(id < gatheredBeliefs.size(), "Unexpected belief id " << id << ". Ids are in [0," << getNumberOfBeliefIds() << ")."); + return gatheredBeliefs[id]; +} + +template +BeliefId BeliefCollector::getIdFromBelief(BeliefType const& belief) const { + STORM_LOG_ASSERT(belief.observation() < beliefToIdMap.size(), + "Unknown belief observation " << belief.observation() << ". Obervations are in [0," << beliefToIdMap.size()); + STORM_LOG_ASSERT(containsBelief(belief), "Belief " << belief.toString() << " is not present in this collector."); + return beliefToIdMap[belief.observation()].at(belief); +} + +template +bool BeliefCollector::containsBelief(BeliefType const& belief) const { + return belief.observation() < beliefToIdMap.size() && beliefToIdMap[belief.observation()].count(belief) > 0; +} + +template +bool BeliefCollector::containsId(BeliefId const& id) const { + return id < gatheredBeliefs.size(); +} + +template +BeliefId BeliefCollector::getIdOptional(BeliefType const& belief) const { + if (auto const& obs = belief.observation(); obs < beliefToIdMap.size()) { + if (auto findRes = beliefToIdMap[obs].find(belief); findRes != beliefToIdMap[obs].end()) { + return findRes->second; + } + } + return InvalidBeliefId; +} + +template +BeliefId BeliefCollector::getIdOrAddBelief(BeliefType&& belief) { + if (auto id = getIdOptional(belief); id != InvalidBeliefId) { + return id; + } + return addBelief(std::move(belief)); +} + +template +BeliefId BeliefCollector::addBelief(BeliefType&& inputBelief) { + auto const id = gatheredBeliefs.size(); + gatheredBeliefs.push_back(std::move(inputBelief)); + auto const& belief = gatheredBeliefs.back(); + auto const& obs = belief.observation(); + if (obs >= beliefToIdMap.size()) { + beliefToIdMap.resize(obs + 1); + } + beliefToIdMap[obs].emplace(belief, id); + return id; +} + +template class BeliefCollector>; +template class BeliefCollector>; + +} // namespace storm::pomdp::beliefs \ No newline at end of file diff --git a/src/storm-pomdp/beliefs/storage/BeliefCollector.h b/src/storm-pomdp/beliefs/storage/BeliefCollector.h new file mode 100644 index 0000000000..62c5cfe112 --- /dev/null +++ b/src/storm-pomdp/beliefs/storage/BeliefCollector.h @@ -0,0 +1,78 @@ +#pragma once + +#include +#include + +#include "storm-pomdp/beliefs/utility/types.h" + +namespace storm::pomdp::beliefs { + +/** + * Interns beliefs and assigns them consecutive identifiers. + * + * IDs are stable for the collector's lifetime. References returned from the collector may be invalidated when new + * beliefs are inserted. + */ +template +class BeliefCollector { + public: + /*! + * Indicates that this collector gives IDs in a consecutive way, ensuring that all Ids are less than `getNumberOfBeliefIds()` + */ + static constexpr bool HasConsecutiveIds = true; + + /*! + * @return true, if either the IDs are equal or the represented beliefs are equal. + */ + bool isEqual(BeliefId const& first, BeliefId const& second) const; + + /*! + * @return The number of beliefs that have been collected. + */ + BeliefId getNumberOfBeliefIds() const; + + /*! + * @return The belief that corresponds to the given ID. + * @note the returned belief is a reference to the internal storage which might be invalidated when further beliefs are added. + */ + BeliefType const& getBeliefFromId(BeliefId const& id) const; + + /*! + * @return The ID of the given belief. + */ + BeliefId getIdFromBelief(BeliefType const& belief) const; + + /*! + * @return true, if the given belief is present in the collector. + */ + bool containsBelief(BeliefType const& belief) const; + + /*! + * @return true, if the given ID is present in the collector. + */ + bool containsId(BeliefId const& id) const; + + /*! + * @return The Id of the given belief if it is present in the collector. `InvalidBeliefId` otherwise. + */ + BeliefId getIdOptional(BeliefType const& belief) const; + + /*! + * Checks if the given belief has already been collected before. + * - If yes, the Id is returned. + * - If not, the belief is collected and a fresh Id is allocated. + */ + BeliefId getIdOrAddBelief(BeliefType&& belief); + + /*! + * Adds a fresh belief and returns its ID. + * @pre The belief must not be present in this collector. + */ + BeliefId addBelief(BeliefType&& inputBelief); + + private: + std::vector gatheredBeliefs; + std::vector> beliefToIdMap; +}; + +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/beliefs/utility/BeliefNumerics.cpp b/src/storm-pomdp/beliefs/utility/BeliefNumerics.cpp new file mode 100644 index 0000000000..ac6bd871b7 --- /dev/null +++ b/src/storm-pomdp/beliefs/utility/BeliefNumerics.cpp @@ -0,0 +1,56 @@ +#include "storm-pomdp/beliefs/utility/BeliefNumerics.h" + +#include +#include "storm/adapters/RationalNumberAdapter.h" +#include "storm/utility/NumberTraits.h" +#include "storm/utility/constants.h" + +namespace storm::pomdp::beliefs { + +namespace detail { +/*! + * Used to decide whether two belief values are equal. + * The interpretation is that two values x,y are equal if round(x*2^NumericPrecision)==round(y*2^NumericPrecision). + * Thus, a high value means that we are more precise. + * This is only relevant if beliefs are represented using an inexact data type like double + */ +constexpr double NumericPrecisionFactor = 1e14; + +double reprValue(double const& val) { + return round(val * NumericPrecisionFactor); +} + +storm::RationalNumber reprValue(storm::RationalNumber const& val) { + return val; +} +} // namespace detail + +template +bool BeliefNumerics::lessOrEqual(ValueType const& lhs, ValueType const& rhs) { + return detail::reprValue(lhs) <= detail::reprValue(rhs); +} + +template +bool BeliefNumerics::equal(ValueType const& lhs, ValueType const& rhs) { + return detail::reprValue(lhs) == detail::reprValue(rhs); +} + +template +bool BeliefNumerics::isZero(const ValueType& val) { + return storm::utility::isZero(detail::reprValue(val)); +} + +template +bool BeliefNumerics::isOne(const ValueType& val) { + return storm::utility::isOne(detail::reprValue(val)); +} + +template +ValueType BeliefNumerics::valueForHash(const ValueType& val) { + return detail::reprValue(val); +} + +template struct BeliefNumerics; +template struct BeliefNumerics; + +} // namespace storm::pomdp::beliefs \ No newline at end of file diff --git a/src/storm-pomdp/beliefs/utility/BeliefNumerics.h b/src/storm-pomdp/beliefs/utility/BeliefNumerics.h new file mode 100644 index 0000000000..221c033974 --- /dev/null +++ b/src/storm-pomdp/beliefs/utility/BeliefNumerics.h @@ -0,0 +1,23 @@ +#pragma once + +namespace storm::pomdp::beliefs { + +/** + * Numeric predicates used when comparing and hashing beliefs. + * + * Floating-point specializations use the configured tolerance, while exact number types preserve exact semantics. + */ +template +struct BeliefNumerics { + static bool lessOrEqual(ValueType const& lhs, ValueType const& rhs); + static bool equal(ValueType const& lhs, ValueType const& rhs); + static bool isZero(ValueType const& val); + static bool isOne(ValueType const& val); + + /*! + * @return a representative value that shall be used for hashing. + */ + static ValueType valueForHash(ValueType const& val); +}; + +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/beliefs/utility/types.h b/src/storm-pomdp/beliefs/utility/types.h new file mode 100644 index 0000000000..1f21a9edb6 --- /dev/null +++ b/src/storm-pomdp/beliefs/utility/types.h @@ -0,0 +1,23 @@ +#pragma once +#include + +#include +#include + +namespace storm::pomdp::beliefs { + +/** Shared identifiers and sparse-container types used by the belief subsystem. */ +using BeliefStateType = uint64_t; +using BeliefObservationType = uint32_t; +using BeliefId = uint64_t; +using BeliefActionObservationType = uint32_t; +constexpr BeliefActionObservationType DefaultActionObservation = 0; + +template +using BeliefFlatMap = boost::container::flat_map; +constexpr bool BeliefFlatMapIsOrdered = true; /// Required e.g. for comparing two beliefs + +constexpr BeliefObservationType InvalidObservation = std::numeric_limits::max(); +constexpr BeliefId InvalidBeliefId = std::numeric_limits::max(); + +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/beliefs/verification/BeliefBasedModelChecker.cpp b/src/storm-pomdp/beliefs/verification/BeliefBasedModelChecker.cpp new file mode 100644 index 0000000000..561898b80c --- /dev/null +++ b/src/storm-pomdp/beliefs/verification/BeliefBasedModelChecker.cpp @@ -0,0 +1,469 @@ +#include "storm-pomdp/beliefs/verification/BeliefBasedModelChecker.h" + +#include "storm-pomdp/beliefs/abstraction/ClippingBeliefAbstraction.h" +#include "storm-pomdp/beliefs/abstraction/FreudenthalTriangulationBeliefAbstraction.h" +#include "storm-pomdp/beliefs/abstraction/RewardBoundedBeliefSplitter.h" +#include "storm-pomdp/beliefs/exploration/BeliefExploration.h" +#include "storm-pomdp/beliefs/exploration/BeliefMdpBuilder.h" +#include "storm-pomdp/beliefs/storage/Belief.h" +#include "storm-pomdp/beliefs/verification/BeliefBasedModelCheckerOptions.h" +#include "storm/api/verification.h" +#include "storm/modelchecker/results/ExplicitQuantitativeCheckResult.h" +#include "storm/models/sparse/Pomdp.h" +#include "storm/transformer/GoalStateMerger.h" +#include "storm/transformer/TransitionToActionRewardTransformer.h" +#include "storm/utility/OptionalRef.h" +#include "storm/utility/Stopwatch.h" +#include "storm/utility/constants.h" +#include "storm/utility/graph.h" +#include "storm/utility/macros.h" + +#include + +namespace storm::pomdp::beliefs { + +template +BeliefBasedModelChecker::BeliefBasedModelChecker(PomdpModelType const& pomdp) : inputPomdp(pomdp) { + STORM_LOG_ERROR_COND(inputPomdp.isCanonic(), "Input Pomdp is not known to be canonic. This might lead to unexpected verification results."); +} + +/** Creates the callback that ends exploration and turns the queued beliefs into the frontier. */ +template +typename BeliefExploration::TerminationCallback getTerminationCallback( + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions const& options, InfoType& info, storm::utility::Stopwatch& swExplore) { + switch (options.getTerminationCriterion()) { + case MAX_EXPLORATION_SIZE: + return [&info, maxSize = options.maxExplorationSize.value()]() { return info.discoveredBeliefs.getNumberOfBeliefIds() > maxSize; }; + case MAX_EXPLORATION_TIME: + return [&swExplore, maxDuration = options.maxExplorationTime.value()]() { return swExplore.getTimeInSeconds() > static_cast(maxDuration); }; + case MAX_EXPLORATION_SIZE_AND_TIME: + return [&info, &swExplore, maxSize = options.maxExplorationSize.value(), maxDuration = options.maxExplorationTime.value()]() { + return info.discoveredBeliefs.getNumberOfBeliefIds() > maxSize || swExplore.getTimeInSeconds() > static_cast(maxDuration); + }; + case NONE: + // Unlimited unfolding (useful for known finite belief MDPs) + return []() { return false; }; + default: + STORM_LOG_ERROR("Unknown termination criterion for belief exploration."); + return []() { return false; }; + } +} + +/** + * Creates a callback that recognizes property targets and optional small-gap cut-offs as terminal beliefs. + * + * Terminal beliefs receive their fixed value directly; all other unfinished beliefs remain explicit frontier states. + */ +template +typename BeliefExploration::TerminalBeliefCallback getTerminalBeliefCallback( + PropertyInformation const& propertyInformation, storm::pomdp::beliefs::BeliefBasedModelCheckerOptions const& options, + storm::pomdp::storage::PreprocessingPomdpValueBounds const& valueBounds) { + using PomdpValueType = PomdpModelType::ValueType; + if (propertyInformation.kind == PropertyInformation::Kind::ExpectedTotalReachabilityReward) { + if (options.maxGapToCut.has_value()) { + // Terminate if the gap is small enough + auto const maxGapToCut = storm::utility::convertNumber(options.maxGapToCut.value()); + return [&propertyInformation, &valueBounds, maxGapToCut](BeliefType const& belief) -> std::optional { + if (propertyInformation.targetObservations.contains(belief.observation())) { + return storm::utility::zero(); + } else { + auto smallestUpper = storm::utility::infinity(); + for (auto const& valueList : valueBounds.upper) { + smallestUpper = std::min(smallestUpper, belief.template getWeightedSum(valueList)); + } + PomdpValueType largestLower = -storm::utility::infinity(); + for (auto const& valueList : valueBounds.lower) { + largestLower = storm::utility::max(largestLower, belief.template getWeightedSum(valueList)); + } + if (storm::utility::abs(smallestUpper - largestLower) <= maxGapToCut) { + if constexpr (std::is_same_v) { + return propertyInformation.dir == solver::OptimizationDirection::Maximize ? largestLower : smallestUpper; + } else { + return storm::utility::convertNumber( + propertyInformation.dir == solver::OptimizationDirection::Maximize ? largestLower : smallestUpper); + } + } + return std::nullopt; + } + }; + } else { + return [&propertyInformation](BeliefType const& belief) -> std::optional { + if (propertyInformation.targetObservations.contains(belief.observation())) { + return storm::utility::zero(); + } else { + return std::nullopt; + } + }; + } + } else if (propertyInformation.kind == PropertyInformation::Kind::RewardBoundedReachabilityProbability) { + return [](BeliefType const& belief) -> std::optional { + // For reward-bounded properties, we cannot be sure that a target belief is terminal as we are not bound-aware at this point + return std::nullopt; + }; + } else if (options.maxGapToCut.has_value()) { + // Terminate if the gap is small enough + auto const maxGapToCut = storm::utility::convertNumber(options.maxGapToCut.value()); + return [&propertyInformation, &valueBounds, maxGapToCut](BeliefType const& belief) -> std::optional { + if (propertyInformation.targetObservations.contains(belief.observation())) { + return storm::utility::one(); + } else { + auto smallestUpper = storm::utility::infinity(); + for (auto const& valueList : valueBounds.upper) { + smallestUpper = std::min(smallestUpper, belief.template getWeightedSum(valueList)); + } + PomdpValueType largestLower = -storm::utility::infinity(); + for (auto const& valueList : valueBounds.lower) { + largestLower = storm::utility::max(largestLower, belief.template getWeightedSum(valueList)); + } + if (storm::utility::abs(smallestUpper - largestLower) <= maxGapToCut) { + if constexpr (std::is_same_v) { + return propertyInformation.dir == solver::OptimizationDirection::Maximize ? largestLower : smallestUpper; + } else { + return storm::utility::convertNumber( + propertyInformation.dir == solver::OptimizationDirection::Maximize ? largestLower : smallestUpper); + } + } + return std::nullopt; + } + }; + } else { + return [&propertyInformation](BeliefType const& belief) -> std::optional { + if (propertyInformation.targetObservations.contains(belief.observation())) { + return storm::utility::one(); + } else { + return std::nullopt; + }; + }; + } +} + +/** + * Converts exploration information into a finite MDP. + * + * Every frontier belief becomes an explicit state with one selectable cut-off action per preprocessing bound. + */ +template +std::pair>, std::unordered_map> buildBeliefMdpFromInfo( + PropertyInformation const& propertyInformation, storm::pomdp::storage::PreprocessingPomdpValueBounds const& valueBounds, + InfoType const& info) { + using PomdpValueType = PomdpModelType::ValueType; + std::function(BeliefType const&)> computeCutOffValueMap = + [&valueBounds, &propertyInformation](BeliefType const& belief) { + // Add addtional cut-off sources here + uint64_t const nrCutoffPolicies = + propertyInformation.dir == storm::OptimizationDirection::Minimize ? valueBounds.upper.size() : valueBounds.lower.size(); + std::unordered_map result; + for (uint64_t i = 0; i < nrCutoffPolicies; ++i) { + auto val = belief.template getWeightedSum( + propertyInformation.dir == storm::OptimizationDirection::Minimize ? valueBounds.upper.at(i) : valueBounds.lower.at(i)); + if constexpr (std::is_same_v) { + result["__sched_" + std::to_string(i)] = val; + } else { + result["__sched_" + std::to_string(i)] = storm::utility::convertNumber(val); + } + } + return result; + }; + return buildBeliefMdp(info, propertyInformation, computeCutOffValueMap); +} + +template> +std::pair checkUnfoldOrDiscretize( + storm::Environment const& env, PomdpModelType const& pomdp, PropertyInformation const& propertyInformation, + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions const& options, + storage::BeliefExplorationBounds const& valueBounds, storm::OptionalRef abstraction = {}, + typename BeliefBasedModelChecker::RunStatistics* statistics = nullptr) { + STORM_LOG_ASSERT(propertyInformation.kind == PropertyInformation::Kind::ReachabilityProbability || + propertyInformation.kind == PropertyInformation::Kind::ExpectedTotalReachabilityReward, + "Unexpected kind of property."); + + STORM_LOG_INFO("Exploring the belief space."); + + // First, explore the beliefs and its successors + using BeliefExplorationType = BeliefExploration; + storm::utility::Stopwatch swExplore(true); + BeliefExplorationType exploration(pomdp); + + auto info = exploration.template initializeExploration(pomdp.getNrObservations(), options.explorationQueueOrder); + info.generateChoiceLabeling = options.buildChoiceLabeling; + + // Determine terminationCallback based on options + typename BeliefExplorationType::TerminationCallback terminationCallback = + getTerminationCallback(options, info, swExplore); + + // Determine terminalBeliefCallback based on options + typename BeliefExplorationType::TerminalBeliefCallback terminalBeliefCallback = + getTerminalBeliefCallback(propertyInformation, options, *valueBounds.preprocessingBounds); + + if constexpr (std::is_same_v>) { + STORM_LOG_ASSERT(options.useClipping, "Clipping exploration information requires clipping to be enabled."); + if (propertyInformation.kind == PropertyInformation::Kind::ExpectedTotalReachabilityReward) { + exploration.resumeClippingExploration(info, terminalBeliefCallback, terminationCallback, propertyInformation.rewardModelName.value(), abstraction); + } else { + exploration.resumeClippingExploration(info, terminalBeliefCallback, terminationCallback, storm::NullRef, abstraction); + } + STORM_LOG_TRACE("Starting clipping phase."); + STORM_LOG_ASSERT(options.clippingResolutions.has_value(), "Clipping requested, but no resolution vector given."); + std::vector resolutions(options.clippingResolutions.value()); + if (propertyInformation.kind == PropertyInformation::Kind::ExpectedTotalReachabilityReward) { + STORM_LOG_ASSERT(valueBounds.extremeBounds.has_value(), + "Clipping for expected total reachability reward requires extreme value bounds to be given."); + ClippingBeliefAbstraction clippingAbstraction( + env, std::move(resolutions), std::move(valueBounds.extremeBounds->template copyValues())); + exploration.resumeClippingExploration( + info, terminalBeliefCallback, []() { return false; }, propertyInformation.rewardModelName.value(), storm::OptionalRef(clippingAbstraction)); + } else { + ClippingBeliefAbstraction clippingAbstraction(env, std::move(resolutions)); + exploration.resumeClippingExploration( + info, terminalBeliefCallback, []() { return false; }, storm::NullRef, storm::OptionalRef(clippingAbstraction)); + } + STORM_LOG_TRACE("Finished clipping phase."); + } else { + if (propertyInformation.kind == PropertyInformation::Kind::ExpectedTotalReachabilityReward) { + exploration.resumeExploration(info, terminalBeliefCallback, terminationCallback, propertyInformation.rewardModelName.value(), abstraction); + } else { + exploration.resumeExploration(info, terminalBeliefCallback, terminationCallback, storm::NullRef, abstraction); + } + } + swExplore.stop(); + bool earlyExplorationStop = info.queue.hasNext(); + if (statistics) { + statistics->available = true; + statistics->completedExploration = !earlyExplorationStop; + statistics->discoveredBeliefs = info.discoveredBeliefs.getNumberOfBeliefIds(); + statistics->exploredBeliefs = info.exploredBeliefs.size(); + statistics->explorationTimeMilliseconds = swExplore.getTimeInMilliseconds(); + } + if (earlyExplorationStop) { + STORM_LOG_INFO("Exploration stopped before all beliefs were explored. " << info.discoveredBeliefs.getNumberOfBeliefIds() << " beliefs discovered. " + << info.exploredBeliefs.size() << " beliefs explored."); + } + + // Second, build the Belief MDP from the exploration information + STORM_LOG_INFO("Constructing the belief MDP."); + storm::utility::Stopwatch swBuild(true); + auto [beliefMdp, stateToBeliefMap] = + buildBeliefMdpFromInfo(propertyInformation, *valueBounds.preprocessingBounds, info); + swBuild.stop(); + { + std::stringstream stream; + beliefMdp->printModelInformationToStream(stream); + STORM_LOG_INFO("Constructed belief MDP:\n" << stream.str()); + } + if (statistics) { + statistics->beliefMdpStates = beliefMdp->getNumberOfStates(); + statistics->beliefMdpChoices = beliefMdp->getNumberOfChoices(); + statistics->beliefMdpTransitions = beliefMdp->getNumberOfTransitions(); + statistics->beliefMdpBuildTimeMilliseconds = swBuild.getTimeInMilliseconds(); + } + + // Finally, perform model checking on the belief MDP. + storm::utility::Stopwatch swCheck(true); + auto formula = createFormulaForBeliefMdp(propertyInformation); + storm::modelchecker::CheckTask task(*formula, true); + std::unique_ptr res(storm::api::verifyWithSparseEngine(env, beliefMdp, task)); + swCheck.stop(); + if (statistics) { + statistics->beliefMdpAnalysisTimeMilliseconds = swCheck.getTimeInMilliseconds(); + } + STORM_LOG_INFO("Time for exploring beliefs: " << swExplore << "."); + STORM_LOG_INFO("Time for building the belief MDP: " << swBuild << "."); + STORM_LOG_INFO("Time for analyzing the belief MDP: " << swCheck << "."); + STORM_LOG_ASSERT(res, "Model checking of belief MDP did not return any result."); + STORM_LOG_ASSERT(res->isExplicitQuantitativeCheckResult(), "Model checking of belief MDP did not return result of expected type."); + STORM_LOG_ASSERT(beliefMdp->getInitialStates().getNumberOfSetBits() == 1, "Unexpected number of initial states for belief Mdp."); + auto const initState = beliefMdp->getInitialStates().getNextSetIndex(0); + return {res->asExplicitQuantitativeCheckResult()[initState], !earlyExplorationStop}; +} + +template +std::pair checkRewardAwareUnfoldOrDiscretize( + storm::Environment const& env, PomdpModelType const& pomdp, PropertyInformation const& propertyInformation, + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions const& options, + storage::BeliefExplorationBounds const& valueBounds, + RewardBoundedBeliefSplitter& rewardSplitter, + storm::OptionalRef> abstraction = {}, + typename BeliefBasedModelChecker::RunStatistics* statistics = nullptr) { + STORM_LOG_ASSERT(propertyInformation.kind == PropertyInformation::Kind::RewardBoundedReachabilityProbability, "Unexpected kind of property."); + STORM_LOG_ASSERT(rewardSplitter.getNumberOfSetRewardModels() != 0, "rewardSplitter must have a reward model set for reward-aware belief MDP construction."); + + STORM_LOG_INFO("Exploring the belief space."); + + // First, explore the beliefs and its successors + using BeliefExplorationType = BeliefExploration; + storm::utility::Stopwatch swExplore(true); + BeliefExplorationType exploration(pomdp); + using InfoType = RewardAwareExplorationInformation; + auto info = exploration.template initializeExploration(pomdp.getNrObservations(), options.explorationQueueOrder); + + // Determine terminationCallback based on options + typename BeliefExplorationType::TerminationCallback terminationCallback = + getTerminationCallback(options, info, swExplore); + + // Determine terminalBeliefCallback based on options + typename BeliefExplorationType::TerminalBeliefCallback terminalBeliefCallback = + getTerminalBeliefCallback(propertyInformation, options, *valueBounds.preprocessingBounds); + + exploration.resumeRewardAwareExploration(info, terminalBeliefCallback, terminationCallback, rewardSplitter, abstraction); + swExplore.stop(); + bool const earlyExplorationStop = info.queue.hasNext(); + if (statistics) { + statistics->available = true; + statistics->completedExploration = !earlyExplorationStop; + statistics->discoveredBeliefs = info.discoveredBeliefs.getNumberOfBeliefIds(); + statistics->exploredBeliefs = info.exploredBeliefs.size(); + statistics->explorationTimeMilliseconds = swExplore.getTimeInMilliseconds(); + } + if (earlyExplorationStop) { + STORM_LOG_INFO("Exploration stopped before all beliefs were explored. " << info.discoveredBeliefs.getNumberOfBeliefIds() << " beliefs discovered. " + << info.exploredBeliefs.size() << " beliefs explored."); + } + + // Second, build the Belief MDP from the exploration information + STORM_LOG_INFO("Constructing the belief MDP."); + storm::utility::Stopwatch swBuild(true); + auto [beliefMdp, stateToBeliefMap] = + buildBeliefMdpFromInfo(propertyInformation, *valueBounds.preprocessingBounds, info); + swBuild.stop(); + { + std::stringstream stream; + beliefMdp->printModelInformationToStream(stream); + STORM_LOG_INFO("Constructed belief MDP:\n" << stream.str()); + } + if (statistics) { + statistics->beliefMdpStates = beliefMdp->getNumberOfStates(); + statistics->beliefMdpChoices = beliefMdp->getNumberOfChoices(); + statistics->beliefMdpTransitions = beliefMdp->getNumberOfTransitions(); + statistics->beliefMdpBuildTimeMilliseconds = swBuild.getTimeInMilliseconds(); + } + + // Finally, perform model checking on the belief MDP. + auto formula = createFormulaForBeliefMdp(propertyInformation); + STORM_LOG_INFO("Analyzing property '" << *formula << "' on the belief MDP."); + storm::utility::Stopwatch swCheck(true); + std::shared_ptr> processedMdp = beliefMdp; + if (propertyInformation.kind == PropertyInformation::Kind::RewardBoundedReachabilityProbability) { + std::vector rewardModelNames; + for (auto const& bnd : propertyInformation.rewardBounds) { + rewardModelNames.push_back(bnd.rewardModelName); + } + processedMdp = storm::transformer::transformTransitionToActionRewards(processedMdp, rewardModelNames) + .model->template as>(); + double increase = static_cast(processedMdp->getNumberOfStates()) / static_cast(beliefMdp->getNumberOfStates()); + STORM_LOG_INFO("Transformation of transition rewards resulted in a model with " << processedMdp->getNumberOfStates() << " states. " << increase + << " times more states than the original belief MDP."); + + // Cut away states that can not reach the target + auto targetStates = processedMdp->getStateLabeling().getStates("target"); + storm::storage::BitVector allStates(targetStates.size(), true); + storm::storage::BitVector probGreaterZeroStates; + if (storm::solver::maximize(propertyInformation.dir)) { + probGreaterZeroStates = storm::utility::graph::performProbGreater0E(processedMdp->getBackwardTransitions(), allStates, targetStates); + } else { + probGreaterZeroStates = + storm::utility::graph::performProbGreater0A(processedMdp->getTransitionMatrix(), processedMdp->getTransitionMatrix().getRowGroupIndices(), + processedMdp->getBackwardTransitions(), allStates, targetStates); + } + auto mergingResult = storm::transformer::GoalStateMerger(*processedMdp) + .mergeTargetAndSinkStates(probGreaterZeroStates, ~probGreaterZeroStates, ~allStates, rewardModelNames); + processedMdp = mergingResult.model; + STORM_LOG_INFO("Merging of sink states resulted in a model with " << processedMdp->getNumberOfStates() << " states."); + } + if (statistics && processedMdp != beliefMdp) { + statistics->processedMdpStates = processedMdp->getNumberOfStates(); + statistics->processedMdpChoices = processedMdp->getNumberOfChoices(); + statistics->processedMdpTransitions = processedMdp->getNumberOfTransitions(); + } + storm::modelchecker::CheckTask task(*formula, true); + std::unique_ptr res(storm::api::verifyWithSparseEngine(env, processedMdp, task)); + swCheck.stop(); + if (statistics) { + statistics->beliefMdpAnalysisTimeMilliseconds = swCheck.getTimeInMilliseconds(); + } + STORM_LOG_INFO("Time for exploring beliefs: " << swExplore << "."); + STORM_LOG_INFO("Time for building the belief MDP: " << swBuild << "."); + STORM_LOG_INFO("Time for analyzing the belief MDP: " << swCheck << "."); + STORM_LOG_ASSERT(res, "Model checking of belief MDP did not return any result."); + STORM_LOG_ASSERT(res->isExplicitQuantitativeCheckResult(), "Model checking of belief MDP did not return result of expected type."); + STORM_LOG_ASSERT(processedMdp->getInitialStates().getNumberOfSetBits() == 1, "Unexpected number of initial states for (processed) belief Mdp."); + auto const initState = processedMdp->getInitialStates().getNextSetIndex(0); + return {res->asExplicitQuantitativeCheckResult()[initState], !earlyExplorationStop}; +} + +template +std::pair BeliefBasedModelChecker::checkUnfold( + storm::Environment const& env, PropertyInformation const& propertyInformation, + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions const& options, + storm::pomdp::storage::BeliefExplorationBounds const& valueBounds) { + lastRunStatistics = RunStatistics(); + if (options.useClipping) { + return checkUnfoldOrDiscretize, BeliefMdpValueType, NoAbstractionType, + ClippingExplorationInformation>>( + env, inputPomdp, propertyInformation, options, valueBounds, {}, &lastRunStatistics); + } else { + return checkUnfoldOrDiscretize, BeliefMdpValueType, NoAbstractionType>( + env, inputPomdp, propertyInformation, options, valueBounds, {}, &lastRunStatistics); + } +} + +template +std::pair BeliefBasedModelChecker::checkDiscretize( + storm::Environment const& env, PropertyInformation const& propertyInformation, + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions const& options, uint64_t resolution, bool useDynamic, + storage::BeliefExplorationBounds const& valueBounds) { + lastRunStatistics = RunStatistics(); + auto mode = useDynamic ? FreudenthalTriangulationMode::Dynamic : FreudenthalTriangulationMode::Static; + FreudenthalTriangulationBeliefAbstraction> abstraction(storm::utility::convertNumber(resolution), mode); + return checkUnfoldOrDiscretize, BeliefMdpValueType>(env, inputPomdp, propertyInformation, options, valueBounds, + storm::OptionalRef(abstraction), &lastRunStatistics); +} + +template +std::pair BeliefBasedModelChecker::checkRewardAwareUnfold( + storm::Environment const& env, PropertyInformation const& propertyInformation, + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions const& options, + storage::BeliefExplorationBounds const& valueBounds, std::vector const& relevantRewardModelNames) { + lastRunStatistics = RunStatistics(); + RewardBoundedBeliefSplitter> rewardBoundedBeliefSplitter(inputPomdp); + if (relevantRewardModelNames.empty()) { + rewardBoundedBeliefSplitter.setRewardModel(); + } else { + rewardBoundedBeliefSplitter.setRewardModels(relevantRewardModelNames); + } + return checkRewardAwareUnfoldOrDiscretize, BeliefMdpValueType>( + env, inputPomdp, propertyInformation, options, valueBounds, rewardBoundedBeliefSplitter, {}, &lastRunStatistics); +} + +template +std::pair BeliefBasedModelChecker::checkRewardAwareDiscretize( + storm::Environment const& env, PropertyInformation const& propertyInformation, + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions const& options, uint64_t resolution, bool useDynamic, + storage::BeliefExplorationBounds const& valueBounds, std::vector const& relevantRewardModelNames) { + lastRunStatistics = RunStatistics(); + auto mode = useDynamic ? FreudenthalTriangulationMode::Dynamic : FreudenthalTriangulationMode::Static; + FreudenthalTriangulationBeliefAbstraction> abstraction(storm::utility::convertNumber(resolution), mode); + RewardBoundedBeliefSplitter> rewardBoundedBeliefSplitter(inputPomdp); + if (relevantRewardModelNames.empty()) { + rewardBoundedBeliefSplitter.setRewardModel(); + } else { + rewardBoundedBeliefSplitter.setRewardModels(relevantRewardModelNames); + } + return checkRewardAwareUnfoldOrDiscretize, BeliefMdpValueType>( + env, inputPomdp, propertyInformation, options, valueBounds, rewardBoundedBeliefSplitter, abstraction, &lastRunStatistics); +} + +template +typename BeliefBasedModelChecker::RunStatistics const& +BeliefBasedModelChecker::getLastRunStatistics() const { + return lastRunStatistics; +} + +template class BeliefBasedModelChecker, double, double>; +template class BeliefBasedModelChecker, storm::RationalNumber, double>; +template class BeliefBasedModelChecker, storm::RationalNumber, storm::RationalNumber>; +template class BeliefBasedModelChecker, storm::RationalNumber, double>; +template class BeliefBasedModelChecker, double, storm::RationalNumber>; +// Currently we don't consider this combination as having rational numbers in models, but floats in beliefs does not really help us +// template class BeliefBasedModelChecker, double, storm::RationalNumber>; +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/beliefs/verification/BeliefBasedModelChecker.h b/src/storm-pomdp/beliefs/verification/BeliefBasedModelChecker.h new file mode 100644 index 0000000000..036ec879c0 --- /dev/null +++ b/src/storm-pomdp/beliefs/verification/BeliefBasedModelChecker.h @@ -0,0 +1,97 @@ +#pragma once + +#include "BeliefBasedModelCheckerOptions.h" +#include "storm-pomdp/beliefs/verification/PropertyInformation.h" +#include "storm-pomdp/storage/BeliefExplorationBounds.h" + +#include + +namespace storm { +class Environment; + +namespace pomdp::beliefs { + +template +/** + * Builds and checks a finite belief MDP approximation of a POMDP. + * + * The POMDP, belief, and generated MDP may use different value types. Callers provide preprocessing value bounds; + * these are used to value explicit frontier cut-offs when exploration is incomplete. + */ +class BeliefBasedModelChecker { + public: + using PomdpValueType = PomdpModelType::ValueType; + + /** Statistics recorded for the most recently completed checking run. */ + struct RunStatistics { + bool available = false; + bool completedExploration = false; + uint64_t discoveredBeliefs = 0; + uint64_t exploredBeliefs = 0; + uint64_t beliefMdpStates = 0; + uint64_t beliefMdpChoices = 0; + uint64_t beliefMdpTransitions = 0; + std::optional processedMdpStates; + std::optional processedMdpChoices; + std::optional processedMdpTransitions; + uint64_t explorationTimeMilliseconds = 0; + uint64_t beliefMdpBuildTimeMilliseconds = 0; + uint64_t beliefMdpAnalysisTimeMilliseconds = 0; + }; + + /** Creates a checker for a canonic POMDP. The POMDP must outlive the checker. */ + explicit BeliefBasedModelChecker(PomdpModelType const& pomdp); + + /** + * Explores the belief MDP by unfolding the belief space. Exploration may stop early and use cut-offs or clipping. + * + * @return the value of the constructed MDP at the initial belief and whether exploration completed. + */ + std::pair checkUnfold(storm::Environment const& env, PropertyInformation const& propertyInformation, + BeliefBasedModelCheckerOptions const& options, + storage::BeliefExplorationBounds const& valueBounds); + + /** + * Explores the belief space and discretises beliefs using the Freudenthal triangualtion approximation. + * + * @param resolution Grid resolution used for the discretisation. + * @param useDynamic Selects a per-belief resolution when a coarser grid represents the belief more accurately. + * @return the value of the constructed MDP at the initial belief and whether exploration completed. + */ + std::pair checkDiscretize(storm::Environment const& env, PropertyInformation const& propertyInformation, + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions const& options, + uint64_t resolution, bool useDynamic, + storage::BeliefExplorationBounds const& valueBounds); + + /** + * Explores a reward-aware belief MDP, splitting beliefs before successor generation by their reward vectors. + * + * @param relevantRewardModelNames Reward models whose accumulated rewards become part of the belief observation. + * @return the value of the constructed MDP at the initial belief and whether exploration completed. + */ + std::pair checkRewardAwareUnfold(storm::Environment const& env, PropertyInformation const& propertyInformation, + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions const& options, + storage::BeliefExplorationBounds const& valueBounds, + std::vector const& relevantRewardModelNames = {}); + + /** + * Combines reward-aware exploration with Freudenthal triangulation discretization. + * + * @return the value of the constructed MDP at the initial belief and whether exploration completed. + */ + std::pair checkRewardAwareDiscretize(storm::Environment const& env, PropertyInformation const& propertyInformation, + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions const& options, + uint64_t resolution, bool useDynamic, + storage::BeliefExplorationBounds const& valueBounds, + std::vector const& relevantRewardModelNames = {}); + + /** @return statistics for the last checking invocation. */ + RunStatistics const& getLastRunStatistics() const; + + private: + PomdpModelType const& inputPomdp; + RunStatistics lastRunStatistics; +}; +} // namespace pomdp::beliefs +} // namespace storm diff --git a/src/storm-pomdp/beliefs/verification/BeliefBasedModelCheckerOptions.h b/src/storm-pomdp/beliefs/verification/BeliefBasedModelCheckerOptions.h new file mode 100644 index 0000000000..43ddc9c189 --- /dev/null +++ b/src/storm-pomdp/beliefs/verification/BeliefBasedModelCheckerOptions.h @@ -0,0 +1,45 @@ +#pragma once + +#include +#include +#include + +#include + +namespace storm::pomdp::beliefs { +/** Criterion that ends exploration and leaves the queued beliefs as the explicit frontier. */ +enum explorationTerminationCriterion { MAX_EXPLORATION_SIZE, MAX_EXPLORATION_TIME, MAX_EXPLORATION_SIZE_AND_TIME, NONE }; + +template +struct BeliefBasedModelCheckerOptions { + bool buildChoiceLabeling = true; + bool useClipping = false; + + ExplorationQueueOrder explorationQueueOrder = ExplorationQueueOrder::Unordered; + + // Clipping abstraction parameters + std::optional> clippingResolutions; + + // Termination criteria + std::optional maxExplorationSize = std::nullopt; + std::optional maxExplorationTime = std::nullopt; + std::optional maxGapToCut = std::nullopt; + + /** + * Get the termination criterion for the exploration + * @return the termination criterion + */ + [[nodiscard]] explorationTerminationCriterion getTerminationCriterion() const { + if (maxExplorationSize.has_value() && maxExplorationTime.has_value()) { + return explorationTerminationCriterion::MAX_EXPLORATION_SIZE_AND_TIME; + } + if (maxExplorationSize.has_value()) { + return explorationTerminationCriterion::MAX_EXPLORATION_SIZE; + } + if (maxExplorationTime.has_value()) { + return explorationTerminationCriterion::MAX_EXPLORATION_TIME; + } + return explorationTerminationCriterion::NONE; + } +}; +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/beliefs/verification/PropertyInformation.h b/src/storm-pomdp/beliefs/verification/PropertyInformation.h new file mode 100644 index 0000000000..171a80b943 --- /dev/null +++ b/src/storm-pomdp/beliefs/verification/PropertyInformation.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include +#include + +#include "storm-pomdp/beliefs/utility/types.h" +#include "storm/logic/TimeBound.h" +#include "storm/solver/OptimizationDirection.h" + +namespace storm::pomdp::beliefs { +/** One reward dimension of a reward-bounded reachability property. */ +struct RewardBound { + std::string rewardModelName; + std::optional lowerBound; + std::optional upperBound; +}; + +/** + * Property data consumed by the belief MDP construction. + * + * Target states are represented by observations because a belief is terminal only when its observation identifies + * the target set. Reward bounds are used only for reward-bounded reachability properties. + */ +struct PropertyInformation { + enum class Kind { ReachabilityProbability, ExpectedTotalReachabilityReward, RewardBoundedReachabilityProbability }; + Kind kind; + std::set targetObservations; + std::optional rewardModelName; + storm::OptimizationDirection dir; + std::vector rewardBounds; +}; +} // namespace storm::pomdp::beliefs diff --git a/src/storm-pomdp/builder/BeliefMdpExplorer.cpp b/src/storm-pomdp/builder/BeliefMdpExplorer.cpp deleted file mode 100644 index 9f0f159293..0000000000 --- a/src/storm-pomdp/builder/BeliefMdpExplorer.cpp +++ /dev/null @@ -1,1399 +0,0 @@ -#include "storm-pomdp/builder/BeliefMdpExplorer.h" - -#include "storm-parsers/api/properties.h" -#include "storm/api/properties.h" -#include "storm/api/verification.h" - -#include "storm/modelchecker/hints/ExplicitModelCheckerHint.h" -#include "storm/modelchecker/results/CheckResult.h" -#include "storm/modelchecker/results/ExplicitQualitativeCheckResult.h" -#include "storm/modelchecker/results/ExplicitQuantitativeCheckResult.h" -#include "storm/models/sparse/Pomdp.h" -#include "storm/storage/SparseMatrix.h" -#include "storm/storage/jani/Property.h" -#include "storm/utility/SignalHandler.h" -#include "storm/utility/constants.h" -#include "storm/utility/graph.h" -#include "storm/utility/macros.h" -#include "storm/utility/vector.h" - -namespace storm { -namespace builder { -template -BeliefMdpExplorer::SuccessorObservationInformation::SuccessorObservationInformation(ValueType const &obsProb, - ValueType const &maxProb, uint64_t const &count) - : observationProbability(obsProb), maxProbabilityToSuccessorWithObs(maxProb), successorWithObsCount(count) { - // Intentionally left empty. -} - -template -void BeliefMdpExplorer::SuccessorObservationInformation::join( - SuccessorObservationInformation other) { /// Does not join support (for performance reasons) - observationProbability += other.observationProbability; - maxProbabilityToSuccessorWithObs = std::max(maxProbabilityToSuccessorWithObs, other.maxProbabilityToSuccessorWithObs); - successorWithObsCount += other.successorWithObsCount; -} - -template -BeliefMdpExplorer::BeliefMdpExplorer(std::shared_ptr beliefManager, - storm::pomdp::storage::PreprocessingPomdpValueBounds const &pomdpValueBounds, - ExplorationHeuristic explorationHeuristic) - : beliefManager(beliefManager), pomdpValueBounds(pomdpValueBounds), explHeuristic(explorationHeuristic), status(Status::Uninitialized) { - // Intentionally left empty -} - -template -typename BeliefMdpExplorer::BeliefManagerType const &BeliefMdpExplorer::getBeliefManager() const { - return *beliefManager; -} - -template -void BeliefMdpExplorer::startNewExploration(std::optional extraTargetStateValue, - std::optional extraBottomStateValue) { - status = Status::Exploring; - // Reset data from potential previous explorations - prio = storm::utility::zero(); - nextId = 0; - mdpStateToBeliefIdMap.clear(); - beliefIdToMdpStateMap.clear(); - exploredBeliefIds.clear(); - exploredBeliefIds.grow(beliefManager->getNumberOfBeliefIds(), false); - mdpStatesToExplorePrioState.clear(); - mdpStatesToExploreStatePrio.clear(); - stateRemapping.clear(); - lowerValueBounds.clear(); - upperValueBounds.clear(); - values.clear(); - exploredMdpTransitions.clear(); - exploredChoiceIndices.clear(); - previousChoiceIndices.clear(); - probabilityEstimation.clear(); - mdpActionRewards.clear(); - targetStates.clear(); - truncatedStates.clear(); - clippedStates.clear(); - delayedExplorationChoices.clear(); - clippingTransitionRewards.clear(); - mdpStateToChoiceLabelsMap.clear(); - optimalChoices = std::nullopt; - optimalChoicesReachableMdpStates = std::nullopt; - scheduler = nullptr; - exploredMdp = nullptr; - internalAddRowGroupIndex(); // Mark the start of the first row group - - // Add some states with special treatment (if requested) - if (extraBottomStateValue) { - currentMdpState = getCurrentNumberOfMdpStates(); - extraBottomState = currentMdpState; - mdpStateToBeliefIdMap.push_back(beliefManager->noId()); - probabilityEstimation.push_back(storm::utility::zero()); - insertValueHints(extraBottomStateValue.value(), extraBottomStateValue.value()); - - internalAddTransition(getStartOfCurrentRowGroup(), extraBottomState.value(), storm::utility::one()); - mdpStateToChoiceLabelsMap[getStartOfCurrentRowGroup()][0] = "loop"; - internalAddRowGroupIndex(); - ++nextId; - } else { - extraBottomState = std::nullopt; - } - if (extraTargetStateValue) { - currentMdpState = getCurrentNumberOfMdpStates(); - extraTargetState = currentMdpState; - mdpStateToBeliefIdMap.push_back(beliefManager->noId()); - probabilityEstimation.push_back(storm::utility::zero()); - insertValueHints(extraTargetStateValue.value(), extraTargetStateValue.value()); - - internalAddTransition(getStartOfCurrentRowGroup(), extraTargetState.value(), storm::utility::one()); - mdpStateToChoiceLabelsMap[getStartOfCurrentRowGroup()][0] = "loop"; - internalAddRowGroupIndex(); - - targetStates.grow(getCurrentNumberOfMdpStates(), false); - targetStates.set(extraTargetState.value(), true); - ++nextId; - } else { - extraTargetState = std::nullopt; - } - currentMdpState = noState(); - - // Set up the initial state. - initialMdpState = getOrAddMdpState(beliefManager->getInitialBelief()); -} - -template -void BeliefMdpExplorer::restartExploration() { - STORM_LOG_ASSERT(status == Status::ModelChecked || status == Status::ModelFinished, "Method call is invalid in current status."); - status = Status::Exploring; - // We will not erase old states during the exploration phase, so most state-based data (like mappings between MDP and Belief states) remain valid. - prio = storm::utility::zero(); - stateRemapping.clear(); - exploredBeliefIds.clear(); - exploredBeliefIds.grow(beliefManager->getNumberOfBeliefIds(), false); - exploredMdpTransitions.clear(); - exploredMdpTransitions.resize(exploredMdp->getNumberOfChoices()); - clippingTransitionRewards.clear(); - previousChoiceIndices = exploredMdp->getNondeterministicChoiceIndices(); - exploredChoiceIndices = exploredMdp->getNondeterministicChoiceIndices(); - mdpActionRewards.clear(); - probabilityEstimation.clear(); - if (exploredMdp->hasRewardModel()) { - // Can be overwritten during exploration - mdpActionRewards = exploredMdp->getUniqueRewardModel().getStateActionRewardVector(); - } - targetStates = storm::storage::BitVector(getCurrentNumberOfMdpStates(), false); - truncatedStates = storm::storage::BitVector(getCurrentNumberOfMdpStates(), false); - clippedStates = storm::storage::BitVector(getCurrentNumberOfMdpStates(), false); - delayedExplorationChoices.clear(); - mdpStatesToExplorePrioState.clear(); - mdpStatesToExploreStatePrio.clear(); - - // The extra states are not changed - if (extraBottomState) { - currentMdpState = extraBottomState.value(); - restoreOldBehaviorAtCurrentState(0); - } - if (extraTargetState) { - currentMdpState = extraTargetState.value(); - restoreOldBehaviorAtCurrentState(0); - targetStates.set(extraTargetState.value(), true); - } - currentMdpState = noState(); - - // Set up the initial state. - initialMdpState = getOrAddMdpState(beliefManager->getInitialBelief()); -} - -template -void BeliefMdpExplorer::storeExplorationState() { - explorationStorage.storedMdpStateToBeliefIdMap = std::vector(mdpStateToBeliefIdMap); - explorationStorage.storedBeliefIdToMdpStateMap = std::map(beliefIdToMdpStateMap); - explorationStorage.storedExploredBeliefIds = storm::storage::BitVector(exploredBeliefIds); - explorationStorage.storedMdpStateToChoiceLabelsMap = std::map>(mdpStateToChoiceLabelsMap); - explorationStorage.storedMdpStatesToExplorePrioState = std::multimap(mdpStatesToExplorePrioState); - explorationStorage.storedMdpStatesToExploreStatePrio = std::map(mdpStatesToExploreStatePrio); - explorationStorage.storedProbabilityEstimation = std::vector(probabilityEstimation); - explorationStorage.storedExploredMdpTransitions = std::vector>(exploredMdpTransitions); - explorationStorage.storedExploredChoiceIndices = std::vector(exploredChoiceIndices); - explorationStorage.storedMdpActionRewards = std::vector(mdpActionRewards); - explorationStorage.storedClippingTransitionRewards = std::map(clippingTransitionRewards); - explorationStorage.storedCurrentMdpState = currentMdpState; - explorationStorage.storedStateRemapping = std::map(stateRemapping); - explorationStorage.storedNextId = nextId; - explorationStorage.storedPrio = ValueType(prio); - explorationStorage.storedLowerValueBounds = std::vector(lowerValueBounds); - explorationStorage.storedUpperValueBounds = std::vector(upperValueBounds); - explorationStorage.storedValues = std::vector(values); - - explorationStorage.storedTargetStates = storm::storage::BitVector(targetStates); -} - -template -void BeliefMdpExplorer::restoreExplorationState() { - mdpStateToBeliefIdMap = std::vector(explorationStorage.storedMdpStateToBeliefIdMap); - beliefIdToMdpStateMap = std::map(explorationStorage.storedBeliefIdToMdpStateMap); - exploredBeliefIds = storm::storage::BitVector(explorationStorage.storedExploredBeliefIds); - mdpStateToChoiceLabelsMap = std::map>(explorationStorage.storedMdpStateToChoiceLabelsMap); - mdpStatesToExplorePrioState = std::multimap(explorationStorage.storedMdpStatesToExplorePrioState); - mdpStatesToExploreStatePrio = std::map(explorationStorage.storedMdpStatesToExploreStatePrio); - probabilityEstimation = std::vector(explorationStorage.storedProbabilityEstimation); - exploredMdpTransitions = std::vector>(explorationStorage.storedExploredMdpTransitions); - exploredChoiceIndices = std::vector(explorationStorage.storedExploredChoiceIndices); - mdpActionRewards = std::vector(explorationStorage.storedMdpActionRewards); - clippingTransitionRewards = std::map(explorationStorage.storedClippingTransitionRewards); - currentMdpState = explorationStorage.storedCurrentMdpState; - stateRemapping = std::map(explorationStorage.storedStateRemapping); - nextId = explorationStorage.storedNextId; - prio = ValueType(explorationStorage.storedPrio); - lowerValueBounds = explorationStorage.storedLowerValueBounds; - upperValueBounds = explorationStorage.storedUpperValueBounds; - values = explorationStorage.storedValues; - status = Status::Exploring; - targetStates = explorationStorage.storedTargetStates; - - truncatedStates.clear(); - clippedStates.clear(); - delayedExplorationChoices.clear(); - optimalChoices = std::nullopt; - optimalChoicesReachableMdpStates = std::nullopt; - exploredMdp = nullptr; - scheduler = nullptr; -} - -template -bool BeliefMdpExplorer::hasUnexploredState() const { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - return !mdpStatesToExploreStatePrio.empty(); -} - -template -std::vector BeliefMdpExplorer::getUnexploredStates() { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - std::vector res; - res.reserve(mdpStatesToExploreStatePrio.size()); - for (auto const &entry : mdpStatesToExploreStatePrio) { - res.push_back(entry.first); - } - return res; -} - -template -typename BeliefMdpExplorer::BeliefId BeliefMdpExplorer::exploreNextState() { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - // Mark the end of the previously explored row group. - if (currentMdpState != noState() && mdpStatesToExplorePrioState.rbegin()->second == exploredChoiceIndices.size()) { - internalAddRowGroupIndex(); - } - - // Pop from the queue. - currentMdpState = mdpStatesToExplorePrioState.rbegin()->second; - auto currprio = mdpStatesToExplorePrioState.rbegin()->first; - auto range = mdpStatesToExplorePrioState.equal_range(currprio); - for (auto i = range.first; i != range.second; ++i) { - if (i->second == currentMdpState) { - mdpStatesToExplorePrioState.erase(i); - break; - } - } - mdpStatesToExploreStatePrio.erase(currentMdpState); - if (currentMdpState != nextId && !currentStateHasOldBehavior()) { - stateRemapping[currentMdpState] = nextId; - STORM_LOG_DEBUG("Explore state " << currentMdpState << " [Bel " << getCurrentBeliefId() << " " << beliefManager->toString(getCurrentBeliefId()) - << "] as state with ID " << nextId << " (Prio: " << storm::utility::to_string(currprio) << ")"); - } else { - STORM_LOG_DEBUG("Explore state " << currentMdpState << " [Bel " << getCurrentBeliefId() << " " << beliefManager->toString(getCurrentBeliefId()) << "]" - << " (Prio: " << storm::utility::to_string(currprio) << ")"); - } - - if (!currentStateHasOldBehavior()) { - ++nextId; - } - if (explHeuristic == ExplorationHeuristic::ProbabilityPrio) { - probabilityEstimation.push_back(currprio); - } - - return mdpStateToBeliefIdMap[currentMdpState]; -} - -template -void BeliefMdpExplorer::addTransitionsToExtraStates(uint64_t const &localActionIndex, ValueType const &targetStateValue, - ValueType const &bottomStateValue) { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - STORM_LOG_ASSERT(!currentStateHasOldBehavior() || localActionIndex < previousChoiceIndices[currentMdpState + 1] - previousChoiceIndices[currentMdpState] || - getCurrentStateWasTruncated(), - "Action index " << localActionIndex << " was not valid at non-truncated state " << currentMdpState << " of the previously explored MDP."); - uint64_t row = getStartOfCurrentRowGroup() + localActionIndex; - if (!storm::utility::isZero(bottomStateValue)) { - STORM_LOG_ASSERT(extraBottomState.has_value(), "Requested a transition to the extra bottom state but there is none."); - internalAddTransition(row, extraBottomState.value(), bottomStateValue); - } - if (!storm::utility::isZero(targetStateValue)) { - STORM_LOG_ASSERT(extraTargetState.has_value(), "Requested a transition to the extra target state but there is none."); - internalAddTransition(row, extraTargetState.value(), targetStateValue); - } -} - -template -void BeliefMdpExplorer::addSelfloopTransition(uint64_t const &localActionIndex, ValueType const &value) { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - STORM_LOG_ASSERT(!currentStateHasOldBehavior() || localActionIndex < previousChoiceIndices[currentMdpState + 1] - previousChoiceIndices[currentMdpState] || - getCurrentStateWasTruncated(), - "Action index " << localActionIndex << " was not valid at non-truncated state " << currentMdpState << " of the previously explored MDP."); - uint64_t row = getStartOfCurrentRowGroup() + localActionIndex; - internalAddTransition(row, getCurrentMdpState(), value); -} - -template -bool BeliefMdpExplorer::addTransitionToBelief(uint64_t const &localActionIndex, BeliefId const &transitionTarget, - ValueType const &value, bool ignoreNewBeliefs) { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - STORM_LOG_ASSERT(!currentStateHasOldBehavior() || localActionIndex < previousChoiceIndices[currentMdpState + 1] - previousChoiceIndices[currentMdpState] || - getCurrentStateWasTruncated(), - "Action index " << localActionIndex << " was not valid at non-truncated state " << currentMdpState << " of the previously explored MDP."); - - MdpStateType column; - if (ignoreNewBeliefs) { - column = getExploredMdpState(transitionTarget); - if (column == noState()) { - return false; - } - } else { - column = getOrAddMdpState(transitionTarget, value); - } - if (getCurrentMdpState() == exploredChoiceIndices.size()) { - internalAddRowGroupIndex(); - } - uint64_t row = getStartOfCurrentRowGroup() + localActionIndex; - internalAddTransition(row, column, value); - return true; -} - -template -void BeliefMdpExplorer::computeRewardAtCurrentState(uint64_t const &localActionIndex, ValueType extraReward) { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - if (getCurrentNumberOfMdpChoices() > mdpActionRewards.size()) { - mdpActionRewards.resize(getCurrentNumberOfMdpChoices(), storm::utility::zero()); - } - uint64_t row = getStartOfCurrentRowGroup() + localActionIndex; - mdpActionRewards[row] = beliefManager->getBeliefActionReward(getCurrentBeliefId(), localActionIndex) + extraReward; -} - -template -void BeliefMdpExplorer::addRewardToCurrentState(uint64_t const &localActionIndex, ValueType rewardValue) { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - if (getCurrentNumberOfMdpChoices() > mdpActionRewards.size()) { - mdpActionRewards.resize(getCurrentNumberOfMdpChoices(), storm::utility::zero()); - } - uint64_t row = getStartOfCurrentRowGroup() + localActionIndex; - mdpActionRewards[row] = rewardValue; -} - -template -void BeliefMdpExplorer::addClippingRewardToCurrentState(uint64_t const &localActionIndex, ValueType rewardValue) { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - uint64_t row = getStartOfCurrentRowGroup() + localActionIndex; - clippingTransitionRewards[row] = rewardValue; -} - -template -void BeliefMdpExplorer::setCurrentStateIsTarget() { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - targetStates.grow(getCurrentNumberOfMdpStates(), false); - targetStates.set(getCurrentMdpState(), true); -} - -template -void BeliefMdpExplorer::setCurrentStateIsTruncated() { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - truncatedStates.grow(getCurrentNumberOfMdpStates(), false); - truncatedStates.set(getCurrentMdpState(), true); -} - -template -void BeliefMdpExplorer::setCurrentStateIsClipped() { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - setCurrentStateIsTruncated(); - clippedStates.grow(getCurrentNumberOfMdpStates(), false); - clippedStates.set(getCurrentMdpState(), true); -} - -template -void BeliefMdpExplorer::setCurrentChoiceIsDelayed(uint64_t const &localActionIndex) { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - delayedExplorationChoices.grow(getCurrentNumberOfMdpChoices(), false); - delayedExplorationChoices.set(getStartOfCurrentRowGroup() + localActionIndex, true); -} - -template -bool BeliefMdpExplorer::currentStateHasOldBehavior() const { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - STORM_LOG_ASSERT(getCurrentMdpState() != noState(), "Method 'currentStateHasOldBehavior' called but there is no current state."); - return exploredMdp && getCurrentMdpState() < exploredMdp->getNumberOfStates(); -} - -template -bool BeliefMdpExplorer::getCurrentStateWasTruncated() const { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - STORM_LOG_ASSERT(getCurrentMdpState() != noState(), "Method 'actionAtCurrentStateWasOptimal' called but there is no current state."); - STORM_LOG_ASSERT(currentStateHasOldBehavior(), "Method 'actionAtCurrentStateWasOptimal' called but current state has no old behavior."); - STORM_LOG_ASSERT(exploredMdp, "No 'old' mdp available."); - return exploredMdp->getStateLabeling().getStateHasLabel("truncated", getCurrentMdpState()); -} - -template -bool BeliefMdpExplorer::getCurrentStateWasClipped() const { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - STORM_LOG_ASSERT(getCurrentMdpState() != noState(), "Method 'actionAtCurrentStateWasOptimal' called but there is no current state."); - STORM_LOG_ASSERT(currentStateHasOldBehavior(), "Method 'actionAtCurrentStateWasOptimal' called but current state has no old behavior."); - STORM_LOG_ASSERT(exploredMdp, "No 'old' mdp available."); - return exploredMdp->getStateLabeling().getStateHasLabel("clipped", getCurrentMdpState()); -} - -template -bool BeliefMdpExplorer::stateIsOptimalSchedulerReachable(MdpStateType mdpState) const { - STORM_LOG_ASSERT(status == Status::ModelChecked, "Method call is invalid in current status."); - STORM_LOG_ASSERT(optimalChoicesReachableMdpStates.has_value(), - "Method 'stateIsOptimalSchedulerReachable' called but 'computeOptimalChoicesAndReachableMdpStates' was not called before."); - return optimalChoicesReachableMdpStates->get(mdpState); -} - -template -bool BeliefMdpExplorer::actionIsOptimal(uint64_t const &globalActionIndex) const { - STORM_LOG_ASSERT(status == Status::ModelChecked, "Method call is invalid in current status."); - STORM_LOG_ASSERT(optimalChoices.has_value(), "Method 'actionIsOptimal' called but 'computeOptimalChoicesAndReachableMdpStates' was not called before."); - return optimalChoices->get(globalActionIndex); -} - -template -bool BeliefMdpExplorer::currentStateIsOptimalSchedulerReachable() const { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - STORM_LOG_ASSERT(getCurrentMdpState() != noState(), "Method 'currentStateIsOptimalSchedulerReachable' called but there is no current state."); - STORM_LOG_ASSERT(currentStateHasOldBehavior(), "Method 'currentStateIsOptimalSchedulerReachable' called but current state has no old behavior."); - STORM_LOG_ASSERT(optimalChoicesReachableMdpStates.has_value(), - "Method 'currentStateIsOptimalSchedulerReachable' called but 'computeOptimalChoicesAndReachableMdpStates' was not called before."); - return optimalChoicesReachableMdpStates->get(getCurrentMdpState()); -} - -template -bool BeliefMdpExplorer::actionAtCurrentStateWasOptimal(uint64_t const &localActionIndex) const { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - STORM_LOG_ASSERT(getCurrentMdpState() != noState(), "Method 'actionAtCurrentStateWasOptimal' called but there is no current state."); - STORM_LOG_ASSERT(currentStateHasOldBehavior(), "Method 'actionAtCurrentStateWasOptimal' called but current state has no old behavior."); - STORM_LOG_ASSERT(optimalChoices.has_value(), - "Method 'currentStateIsOptimalSchedulerReachable' called but 'computeOptimalChoicesAndReachableMdpStates' was not called before."); - uint64_t choice = previousChoiceIndices.at(getCurrentMdpState()) + localActionIndex; - return optimalChoices->get(choice); -} - -template -bool BeliefMdpExplorer::getCurrentStateActionExplorationWasDelayed(uint64_t const &localActionIndex) const { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - STORM_LOG_ASSERT(getCurrentMdpState() != noState(), "Method 'actionAtCurrentStateWasOptimal' called but there is no current state."); - STORM_LOG_ASSERT(currentStateHasOldBehavior(), "Method 'actionAtCurrentStateWasOptimal' called but current state has no old behavior."); - STORM_LOG_ASSERT(exploredMdp, "No 'old' mdp available."); - uint64_t choice = exploredMdp->getNondeterministicChoiceIndices()[getCurrentMdpState()] + localActionIndex; - return exploredMdp->hasChoiceLabeling() && exploredMdp->getChoiceLabeling().getLabels().count("delayed") > 0 && - exploredMdp->getChoiceLabeling().getChoiceHasLabel("delayed", choice); -} - -template -void BeliefMdpExplorer::restoreOldBehaviorAtCurrentState(uint64_t const &localActionIndex) { - STORM_LOG_ASSERT(currentStateHasOldBehavior(), "Cannot restore old behavior as the current state does not have any."); - STORM_LOG_ASSERT(localActionIndex < previousChoiceIndices[currentMdpState + 1] - previousChoiceIndices[currentMdpState], - "Action index " << localActionIndex << " was not valid at state " << currentMdpState << " of the previously explored MDP."); - - if (getCurrentMdpState() == exploredChoiceIndices.size()) { - internalAddRowGroupIndex(); - } - - STORM_LOG_ASSERT(getCurrentMdpState() < previousChoiceIndices.size(), "MDP state out of range for previous choices."); - STORM_LOG_ASSERT(getCurrentMdpState() < exploredChoiceIndices.size(), "MDP state out of range for explored choices."); - uint64_t oldChoiceIndex = previousChoiceIndices.at(getCurrentMdpState()) + localActionIndex; - uint64_t newChoiceIndex = exploredChoiceIndices.at(getCurrentMdpState()) + localActionIndex; - - // Insert the transitions - for (auto const &transition : exploredMdp->getTransitionMatrix().getRow(oldChoiceIndex)) { - internalAddTransition(newChoiceIndex, transition.getColumn(), transition.getValue()); - // Check whether exploration is needed - auto beliefId = getBeliefId(transition.getColumn()); - if (beliefId != beliefManager->noId()) { // Not the extra target or bottom state - if (!exploredBeliefIds.get(beliefId)) { - // This belief needs exploration - exploredBeliefIds.set(beliefId, true); - ValueType currentPrio; - switch (explHeuristic) { - case ExplorationHeuristic::BreadthFirst: - currentPrio = prio; - prio = prio - storm::utility::one(); - break; - case ExplorationHeuristic::LowerBoundPrio: - currentPrio = getLowerValueBoundAtCurrentState(); - break; - case ExplorationHeuristic::UpperBoundPrio: - currentPrio = getUpperValueBoundAtCurrentState(); - break; - case ExplorationHeuristic::GapPrio: - currentPrio = getUpperValueBoundAtCurrentState() - getLowerValueBoundAtCurrentState(); - break; - case ExplorationHeuristic::ProbabilityPrio: - if (getCurrentMdpState() != noState()) { - currentPrio = probabilityEstimation[getCurrentMdpState()] * transition.getValue(); - } else { - currentPrio = storm::utility::one(); - } - break; - default: - STORM_LOG_THROW(false, storm::exceptions::NotImplementedException, "Other heuristics not implemented yet."); - } - mdpStatesToExploreStatePrio[transition.getColumn()] = currentPrio; - mdpStatesToExplorePrioState.emplace(currentPrio, transition.getColumn()); - } - } - } - - // Actually, nothing needs to be done for rewards since we already initialize the vector with the "old" values -} - -template -void BeliefMdpExplorer::finishExploration() { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - STORM_LOG_ASSERT(!hasUnexploredState(), "Finishing exploration not possible if there are still unexplored states."); - - // Complete the exploration - // Finish the last row grouping in case the last explored state was new - if (!currentStateHasOldBehavior() || exploredChoiceIndices.back() < getCurrentNumberOfMdpChoices()) { - internalAddRowGroupIndex(); - } - // Resize state- and choice based vectors to the correct size - targetStates.resize(getCurrentNumberOfMdpStates(), false); - truncatedStates.resize(getCurrentNumberOfMdpStates(), false); - clippedStates.resize(getCurrentNumberOfMdpStates(), false); - if (!mdpActionRewards.empty()) { - mdpActionRewards.resize(getCurrentNumberOfMdpChoices(), storm::utility::zero()); - } - - // We are not exploring anymore - currentMdpState = noState(); - - // If this was a restarted exploration, we might still have unexplored states (which were only reachable and explored in a previous build). - // We get rid of these before rebuilding the model - if (exploredMdp) { - dropUnexploredStates(); - } - - // The potentially computed optimal choices and the set of states that are reachable under these choices are not valid anymore. - optimalChoices = std::nullopt; - optimalChoicesReachableMdpStates = std::nullopt; - - // Apply state remapping to the Belief-State maps - if (!stateRemapping.empty()) { - std::vector remappedStateToBeliefIdMap(mdpStateToBeliefIdMap); - for (auto const &entry : stateRemapping) { - remappedStateToBeliefIdMap[entry.second] = mdpStateToBeliefIdMap[entry.first]; - } - mdpStateToBeliefIdMap = remappedStateToBeliefIdMap; - for (auto const &beliefMdpState : beliefIdToMdpStateMap) { - if (stateRemapping.find(beliefMdpState.second) != stateRemapping.end()) { - beliefIdToMdpStateMap[beliefMdpState.first] = stateRemapping[beliefMdpState.second]; - } - } - if (!mdpStateToChoiceLabelsMap.empty()) { - std::map> temp(mdpStateToChoiceLabelsMap); - for (auto const &entry : stateRemapping) { - temp[entry.second] = mdpStateToChoiceLabelsMap[entry.first]; - } - mdpStateToChoiceLabelsMap = temp; - } - } - - // Create the transition matrix - uint64_t entryCount = 0; - for (auto const &row : exploredMdpTransitions) { - entryCount += row.size(); - } - storm::storage::SparseMatrixBuilder builder(getCurrentNumberOfMdpChoices(), getCurrentNumberOfMdpStates(), entryCount, true, true, - getCurrentNumberOfMdpStates()); - for (uint64_t groupIndex = 0; groupIndex < exploredChoiceIndices.size() - 1; ++groupIndex) { - uint64_t rowIndex = exploredChoiceIndices[groupIndex]; - uint64_t groupEnd = exploredChoiceIndices[groupIndex + 1]; - builder.newRowGroup(rowIndex); - for (; rowIndex < groupEnd; ++rowIndex) { - for (auto const &entry : exploredMdpTransitions[rowIndex]) { - if (stateRemapping.find(entry.first) == stateRemapping.end()) { - builder.addNextValue(rowIndex, entry.first, entry.second); - } else { - builder.addNextValue(rowIndex, stateRemapping[entry.first], entry.second); - } - } - } - } - auto mdpTransitionMatrix = builder.build(); - - // Create a standard labeling - storm::models::sparse::StateLabeling mdpLabeling(getCurrentNumberOfMdpStates()); - mdpLabeling.addLabel("init"); - mdpLabeling.addLabelToState("init", initialMdpState); - targetStates.resize(getCurrentNumberOfMdpStates(), false); - mdpLabeling.addLabel("target", std::move(targetStates)); - truncatedStates.resize(getCurrentNumberOfMdpStates(), false); - mdpLabeling.addLabel("truncated", std::move(truncatedStates)); - clippedStates.resize(getCurrentNumberOfMdpStates(), false); - mdpLabeling.addLabel("clipped", std::move(clippedStates)); - - for (uint64_t state = 0; state < getCurrentNumberOfMdpStates(); ++state) { - if (state == extraBottomState || state == extraTargetState) { - if (!mdpLabeling.containsLabel("__extra")) { - mdpLabeling.addLabel("__extra"); - } - mdpLabeling.addLabelToState("__extra", state); - } else { - STORM_LOG_DEBUG("Observation of MDP state " << state << " : " << beliefManager->getObservationLabel(mdpStateToBeliefIdMap[state]) << "\n"); - std::string obsLabel = beliefManager->getObservationLabel(mdpStateToBeliefIdMap[state]); - uint32_t obsId = beliefManager->getBeliefObservation(mdpStateToBeliefIdMap[state]); - if (!obsLabel.empty()) { - if (!mdpLabeling.containsLabel(obsLabel)) { - mdpLabeling.addLabel(obsLabel); - } - mdpLabeling.addLabelToState(obsLabel, state); - } else if (mdpStateToBeliefIdMap[state] != beliefManager->noId()) { - std::string obsIdLabel = "obs_" + std::to_string(obsId); - if (!mdpLabeling.containsLabel(obsIdLabel)) { - mdpLabeling.addLabel(obsIdLabel); - } - mdpLabeling.addLabelToState(obsIdLabel, state); - } - } - } - - // Create a standard reward model (if rewards are available) - std::unordered_map> mdpRewardModels; - if (!mdpActionRewards.empty()) { - mdpActionRewards.resize(getCurrentNumberOfMdpChoices(), storm::utility::zero()); - if (!clippingTransitionRewards.empty()) { - storm::storage::SparseMatrixBuilder rewardBuilder(getCurrentNumberOfMdpChoices(), getCurrentNumberOfMdpStates(), - clippingTransitionRewards.size(), true, true, getCurrentNumberOfMdpStates()); - for (uint64_t groupIndex = 0; groupIndex < exploredChoiceIndices.size() - 1; ++groupIndex) { - uint64_t rowIndex = exploredChoiceIndices[groupIndex]; - uint64_t groupEnd = exploredChoiceIndices[groupIndex + 1]; - rewardBuilder.newRowGroup(rowIndex); - for (; rowIndex < groupEnd; ++rowIndex) { - if (clippingTransitionRewards.find(rowIndex) != clippingTransitionRewards.end()) { - STORM_LOG_ASSERT(extraTargetState.has_value(), "Requested a transition to the extra target state but there is none."); - rewardBuilder.addNextValue(rowIndex, extraTargetState.value(), clippingTransitionRewards[rowIndex]); - } - } - } - auto transitionRewardMatrix = rewardBuilder.build(); - mdpRewardModels.emplace("default", storm::models::sparse::StandardRewardModel( - std::optional>(), std::move(mdpActionRewards), std::move(transitionRewardMatrix))); - } else { - mdpRewardModels.emplace( - "default", storm::models::sparse::StandardRewardModel(std::optional>(), std::move(mdpActionRewards))); - } - } - - // Create model components - storm::storage::sparse::ModelComponents modelComponents(std::move(mdpTransitionMatrix), std::move(mdpLabeling), std::move(mdpRewardModels)); - - // Potentially create a choice labeling - if (!mdpStateToChoiceLabelsMap.empty()) { - modelComponents.choiceLabeling = storm::models::sparse::ChoiceLabeling(getCurrentNumberOfMdpChoices()); - for (auto const &stateMap : mdpStateToChoiceLabelsMap) { - auto rowGroup = stateMap.first; - for (auto const &actionLabel : stateMap.second) { - if (!modelComponents.choiceLabeling->containsLabel(actionLabel.second)) { - modelComponents.choiceLabeling->addLabel(actionLabel.second); - } - modelComponents.choiceLabeling->addLabelToChoice(actionLabel.second, exploredChoiceIndices.at(rowGroup) + actionLabel.first); - } - } - } - - if (!delayedExplorationChoices.empty()) { - modelComponents.choiceLabeling = storm::models::sparse::ChoiceLabeling(getCurrentNumberOfMdpChoices()); - delayedExplorationChoices.resize(getCurrentNumberOfMdpChoices(), false); - modelComponents.choiceLabeling->addLabel("delayed", std::move(delayedExplorationChoices)); - } - - // Create the final model. - exploredMdp = std::make_shared>(std::move(modelComponents)); - status = Status::ModelFinished; - STORM_LOG_DEBUG("Explored Mdp with " << exploredMdp->getNumberOfStates() << " states (" << clippedStates.getNumberOfSetBits() - << " of which were clipped and " << truncatedStates.getNumberOfSetBits() - clippedStates.getNumberOfSetBits() - << " of which were flagged as truncated)."); -} - -template -void BeliefMdpExplorer::dropUnexploredStates() { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - STORM_LOG_ASSERT(!hasUnexploredState(), "Finishing exploration not possible if there are still unexplored states."); - - STORM_LOG_ASSERT(exploredMdp, "Method called although no 'old' MDP is available."); - // Find the states (and corresponding choices) that were not explored. - // These correspond to "empty" MDP transitions - storm::storage::BitVector relevantMdpStates(getCurrentNumberOfMdpStates(), true), relevantMdpChoices(getCurrentNumberOfMdpChoices(), true); - std::vector toRelevantStateIndexMap(getCurrentNumberOfMdpStates(), noState()); - MdpStateType nextRelevantIndex = 0; - for (uint64_t groupIndex = 0; groupIndex < exploredChoiceIndices.size() - 1; ++groupIndex) { - uint64_t rowIndex = exploredChoiceIndices[groupIndex]; - // Check first row in group - if (exploredMdpTransitions[rowIndex].empty()) { - relevantMdpChoices.set(rowIndex, false); - relevantMdpStates.set(groupIndex, false); - } else { - toRelevantStateIndexMap[groupIndex] = nextRelevantIndex; - ++nextRelevantIndex; - } - uint64_t groupEnd = exploredChoiceIndices[groupIndex + 1]; - // process remaining rows in group - for (++rowIndex; rowIndex < groupEnd; ++rowIndex) { - // Assert that all actions at the current state were consistently explored or unexplored. - STORM_LOG_ASSERT(exploredMdpTransitions[rowIndex].empty() != relevantMdpStates.get(groupIndex), - "Actions at 'old' MDP state " << groupIndex << " were only partly explored."); - if (exploredMdpTransitions[rowIndex].empty()) { - relevantMdpChoices.set(rowIndex, false); - } - } - } - - if (relevantMdpStates.full()) { - // All states are relevant so nothing to do - return; - } - - nextId -= (relevantMdpStates.size() - relevantMdpStates.getNumberOfSetBits()); - - // Translate various components to the "new" MDP state set - storm::utility::vector::filterVectorInPlace(mdpStateToBeliefIdMap, relevantMdpStates); - { // beliefIdToMdpStateMap - for (auto belIdToMdpStateIt = beliefIdToMdpStateMap.begin(); belIdToMdpStateIt != beliefIdToMdpStateMap.end();) { - if (relevantMdpStates.get(belIdToMdpStateIt->second)) { - // Translate current entry and move on to the next one. - belIdToMdpStateIt->second = toRelevantStateIndexMap[belIdToMdpStateIt->second]; - ++belIdToMdpStateIt; - } else { - STORM_LOG_ASSERT(!exploredBeliefIds.get(belIdToMdpStateIt->first), - "Inconsistent exploration information: Unexplored MDPState corresponds to explored beliefId."); - // Delete current entry and move on to the next one. - // This works because std::map::erase does not invalidate other iterators within the map! - beliefIdToMdpStateMap.erase(belIdToMdpStateIt++); - } - } - } - { // exploredMdpTransitions - storm::utility::vector::filterVectorInPlace(exploredMdpTransitions, relevantMdpChoices); - // Adjust column indices. Unfortunately, the fastest way seems to be to "rebuild" the map - // It might pay off to do this when building the matrix. - for (auto &transitions : exploredMdpTransitions) { - std::map newTransitions; - for (auto const &entry : transitions) { - STORM_LOG_ASSERT(relevantMdpStates.get(entry.first), "Relevant state has transition to irrelevant state."); - newTransitions.emplace_hint(newTransitions.end(), toRelevantStateIndexMap[entry.first], entry.second); - } - transitions = std::move(newTransitions); - } - } - { // exploredChoiceIndices - MdpStateType newState = 0; - STORM_LOG_ASSERT(exploredChoiceIndices[0] == 0u, "First explored choice index should be 0."); - // Loop invariant: all indices up to exploredChoiceIndices[newState] consider the new row indices and all other entries are not touched. - for (auto const oldState : relevantMdpStates) { - if (oldState != newState) { - STORM_LOG_ASSERT(oldState > newState, "Expected oldState > newState."); - uint64_t groupSize = getRowGroupSizeOfState(oldState); - exploredChoiceIndices.at(newState + 1) = exploredChoiceIndices.at(newState) + groupSize; - } - ++newState; - } - exploredChoiceIndices.resize(newState + 1); - } - if (!mdpActionRewards.empty()) { - storm::utility::vector::filterVectorInPlace(mdpActionRewards, relevantMdpChoices); - } - if (extraBottomState) { - extraBottomState = toRelevantStateIndexMap[extraBottomState.value()]; - } - if (extraTargetState) { - extraTargetState = toRelevantStateIndexMap[extraTargetState.value()]; - } - targetStates = targetStates % relevantMdpStates; - truncatedStates = truncatedStates % relevantMdpStates; - clippedStates = clippedStates % relevantMdpStates; - initialMdpState = toRelevantStateIndexMap[initialMdpState]; - - storm::utility::vector::filterVectorInPlace(lowerValueBounds, relevantMdpStates); - storm::utility::vector::filterVectorInPlace(upperValueBounds, relevantMdpStates); - storm::utility::vector::filterVectorInPlace(values, relevantMdpStates); - - { // mdpStateToChoiceLabelsMap - if (!mdpStateToChoiceLabelsMap.empty()) { - auto temp = std::map>(); - for (auto const relevantState : relevantMdpStates) { - temp[toRelevantStateIndexMap[relevantState]] = mdpStateToChoiceLabelsMap[relevantState]; - } - mdpStateToChoiceLabelsMap = temp; - } - } -} - -template -std::shared_ptr::ValueType>> -BeliefMdpExplorer::getExploredMdp() const { - STORM_LOG_ASSERT(status == Status::ModelFinished || status == Status::ModelChecked, "Method call is invalid in current status."); - STORM_LOG_ASSERT(exploredMdp, "Tried to get the explored MDP but exploration was not finished yet."); - return exploredMdp; -} - -template -typename BeliefMdpExplorer::MdpStateType BeliefMdpExplorer::getCurrentNumberOfMdpStates() const { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - return mdpStateToBeliefIdMap.size(); -} - -template -typename BeliefMdpExplorer::MdpStateType BeliefMdpExplorer::getCurrentNumberOfMdpChoices() const { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - return exploredMdpTransitions.size(); -} - -template -typename BeliefMdpExplorer::MdpStateType BeliefMdpExplorer::getStartOfCurrentRowGroup() const { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - STORM_LOG_ASSERT(getCurrentMdpState() < exploredChoiceIndices.size(), "MDP state index out of range."); - return exploredChoiceIndices.at(getCurrentMdpState()); -} - -template -uint64_t BeliefMdpExplorer::getSizeOfCurrentRowGroup() const { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - STORM_LOG_ASSERT(getCurrentMdpState() < exploredChoiceIndices.size() - 1, "MDP state index out of range."); - return exploredChoiceIndices.at(getCurrentMdpState() + 1) - exploredChoiceIndices.at(getCurrentMdpState()); -} - -template -uint64_t BeliefMdpExplorer::getRowGroupSizeOfState(uint64_t state) const { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - STORM_LOG_ASSERT(state < exploredChoiceIndices.size(), "State index out of range."); - if (state < exploredChoiceIndices.size() - 1) { - return exploredChoiceIndices.at(state + 1) - exploredChoiceIndices.at(state); - } else if (state == exploredChoiceIndices.size() - 1) { - return exploredMdpTransitions.size() - exploredChoiceIndices.at(state); - } else { - return 0; - } -} - -template -bool BeliefMdpExplorer::needsActionAdjustment(uint64_t numActionsNeeded) { - return (currentStateHasOldBehavior() && getCurrentStateWasTruncated() && getCurrentMdpState() < exploredChoiceIndices.size() - 1 && - getSizeOfCurrentRowGroup() != numActionsNeeded); -} - -template -typename BeliefMdpExplorer::ValueType BeliefMdpExplorer::getLowerValueBoundAtCurrentState() const { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - return lowerValueBounds[getCurrentMdpState()]; -} - -template -typename BeliefMdpExplorer::ValueType BeliefMdpExplorer::getUpperValueBoundAtCurrentState() const { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - return upperValueBounds[getCurrentMdpState()]; -} - -template -typename BeliefMdpExplorer::ValueType BeliefMdpExplorer::computeLowerValueBoundAtBelief( - BeliefId const &beliefId) const { - STORM_LOG_ASSERT(!pomdpValueBounds.lower.empty(), "Requested lower value bounds but none were available."); - auto it = pomdpValueBounds.lower.begin(); - ValueType result = beliefManager->getWeightedSum(beliefId, *it); - for (++it; it != pomdpValueBounds.lower.end(); ++it) { - result = std::max(result, beliefManager->getWeightedSum(beliefId, *it)); - } - return result; -} - -template -typename BeliefMdpExplorer::ValueType BeliefMdpExplorer::computeUpperValueBoundAtBelief( - BeliefId const &beliefId) const { - STORM_LOG_ASSERT(!pomdpValueBounds.upper.empty(), "Requested upper value bounds but none were available."); - auto it = pomdpValueBounds.upper.begin(); - ValueType result = beliefManager->getWeightedSum(beliefId, *it); - for (++it; it != pomdpValueBounds.upper.end(); ++it) { - result = std::min(result, beliefManager->getWeightedSum(beliefId, *it)); - } - return result; -} - -template -typename BeliefMdpExplorer::ValueType BeliefMdpExplorer::computeLowerValueBoundForScheduler( - BeliefId const &beliefId, uint64_t schedulerId) const { - STORM_LOG_ASSERT(!pomdpValueBounds.lower.empty(), "Requested lower value bounds but none were available."); - STORM_LOG_ASSERT(pomdpValueBounds.lower.size() > schedulerId, "Requested lower value bound for scheduler with ID " << schedulerId << " not available."); - return beliefManager->getWeightedSum(beliefId, pomdpValueBounds.lower[schedulerId]); -} - -template -typename BeliefMdpExplorer::ValueType BeliefMdpExplorer::computeUpperValueBoundForScheduler( - BeliefId const &beliefId, uint64_t schedulerId) const { - STORM_LOG_ASSERT(!pomdpValueBounds.upper.empty(), "Requested upper value bounds but none were available."); - STORM_LOG_ASSERT(pomdpValueBounds.upper.size() > schedulerId, "Requested upper value bound for scheduler with ID " << schedulerId << " not available."); - return beliefManager->getWeightedSum(beliefId, pomdpValueBounds.upper[schedulerId]); -} - -template -std::pair::ValueType> -BeliefMdpExplorer::computeFMSchedulerValueForMemoryNode(BeliefId const &beliefId, uint64_t memoryNode) const { - STORM_LOG_ASSERT(!fmSchedulerValueList.empty(), "Requested finite memory scheduler value bounds but none were available."); - auto obs = beliefManager->getBeliefObservation(beliefId); - STORM_LOG_ASSERT(fmSchedulerValueList.size() > obs, "Requested value bound for observation " << obs << " not available."); - STORM_LOG_ASSERT(fmSchedulerValueList.at(obs).size() > memoryNode, - "Requested value bound for observation " << obs << " and memory node " << memoryNode << " not available."); - return beliefManager->getWeightedSum(beliefId, fmSchedulerValueList.at(obs).at(memoryNode)); -} - -template -void BeliefMdpExplorer::computeValuesOfExploredMdp(storm::Environment const &env, storm::solver::OptimizationDirection const &dir) { - STORM_LOG_ASSERT(status == Status::ModelFinished, "Method call is invalid in current status."); - STORM_LOG_ASSERT(exploredMdp, "Tried to compute values but the MDP is not explored."); - auto property = createStandardProperty(dir, exploredMdp->hasRewardModel()); - auto task = createStandardCheckTask(property); - - std::unique_ptr res(storm::api::verifyWithSparseEngine(env, exploredMdp, task)); - if (res) { - values = std::move(res->asExplicitQuantitativeCheckResult().getValueVector()); - scheduler = std::make_shared>(res->asExplicitQuantitativeCheckResult().getScheduler()); - STORM_LOG_WARN_COND_DEBUG(storm::utility::vector::compareElementWise(lowerValueBounds, values, std::less_equal()), - "Computed values are smaller than the lower bound."); - STORM_LOG_WARN_COND_DEBUG(storm::utility::vector::compareElementWise(upperValueBounds, values, std::greater_equal()), - "Computed values are larger than the upper bound."); - } else { - STORM_LOG_ASSERT(storm::utility::resources::isTerminate(), "Empty check result!"); - STORM_LOG_ERROR("No result obtained while checking."); - } - status = Status::ModelChecked; -} - -template -bool BeliefMdpExplorer::hasComputedValues() const { - return status == Status::ModelChecked; -} - -template -std::vector::ValueType> const &BeliefMdpExplorer::getValuesOfExploredMdp() - const { - STORM_LOG_ASSERT(status == Status::ModelChecked, "Method call is invalid in current status."); - return values; -} - -template -const std::shared_ptr::ValueType>> & -BeliefMdpExplorer::getSchedulerForExploredMdp() const { - STORM_LOG_ASSERT(status == Status::ModelChecked, "Method call is invalid in current status."); - return scheduler; -} - -template -typename BeliefMdpExplorer::ValueType const &BeliefMdpExplorer::getComputedValueAtInitialState() const { - STORM_LOG_ASSERT(status == Status::ModelChecked, "Method call is invalid in current status."); - STORM_LOG_ASSERT(exploredMdp, "Tried to get a value but no MDP was explored."); - return getValuesOfExploredMdp()[exploredMdp->getInitialStates().getNextSetIndex(0)]; -} - -template -typename BeliefMdpExplorer::MdpStateType BeliefMdpExplorer::getBeliefId( - MdpStateType exploredMdpState) const { - STORM_LOG_ASSERT(status != Status::Uninitialized, "Method call is invalid in current status."); - return mdpStateToBeliefIdMap[exploredMdpState]; -} - -template -void BeliefMdpExplorer::gatherSuccessorObservationInformationAtCurrentState( - uint64_t localActionIndex, std::map &gatheredSuccessorObservations) { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - STORM_LOG_ASSERT(currentStateHasOldBehavior(), "Method call is invalid since the current state has no old behavior."); - uint64_t mdpChoice = getStartOfCurrentRowGroup() + localActionIndex; - gatherSuccessorObservationInformationAtMdpChoice(mdpChoice, gatheredSuccessorObservations); -} - -template -void BeliefMdpExplorer::gatherSuccessorObservationInformationAtMdpChoice( - uint64_t mdpChoice, std::map &gatheredSuccessorObservations) { - STORM_LOG_ASSERT(exploredMdp, "Method call is invalid if no MDP has been explored before."); - for (auto const &entry : exploredMdp->getTransitionMatrix().getRow(mdpChoice)) { - auto const &beliefId = getBeliefId(entry.getColumn()); - if (beliefId != beliefManager->noId()) { - auto const &obs = beliefManager->getBeliefObservation(beliefId); - SuccessorObservationInformation info(entry.getValue(), entry.getValue(), 1); - auto obsInsertion = gatheredSuccessorObservations.emplace(obs, info); - if (!obsInsertion.second) { - // There already is an entry for this observation, so join the two information constructs - obsInsertion.first->second.join(info); - } - beliefManager->joinSupport(beliefId, obsInsertion.first->second.support); - } - } -} - -template -bool BeliefMdpExplorer::currentStateHasSuccessorObservationInObservationSet(uint64_t localActionIndex, - storm::storage::BitVector const &observationSet) { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - STORM_LOG_ASSERT(currentStateHasOldBehavior(), "Method call is invalid since the current state has no old behavior."); - uint64_t mdpChoice = previousChoiceIndices.at(getCurrentMdpState()) + localActionIndex; - return std::any_of(exploredMdp->getTransitionMatrix().getRow(mdpChoice).begin(), exploredMdp->getTransitionMatrix().getRow(mdpChoice).end(), - [this, &observationSet](typename storm::storage::MatrixEntry i) { - return observationSet.get(beliefManager->getBeliefObservation(getBeliefId(i.getColumn()))); - }); -} - -template -void BeliefMdpExplorer::takeCurrentValuesAsUpperBounds() { - STORM_LOG_ASSERT(status == Status::ModelChecked, "Method call is invalid in current status."); - upperValueBounds = values; -} - -template -void BeliefMdpExplorer::takeCurrentValuesAsLowerBounds() { - STORM_LOG_ASSERT(status == Status::ModelChecked, "Method call is invalid in current status."); - lowerValueBounds = values; -} - -template -void BeliefMdpExplorer::computeOptimalChoicesAndReachableMdpStates(ValueType const &ancillaryChoicesEpsilon, - bool relativeDifference) { - STORM_LOG_ASSERT(status == Status::ModelChecked, "Method call is invalid in current status."); - STORM_LOG_ASSERT(exploredMdp, "Method call is invalid in if no MDP is available."); - STORM_LOG_ASSERT(!optimalChoices.has_value(), "Tried to compute optimal scheduler but this has already been done before."); - STORM_LOG_ASSERT(!optimalChoicesReachableMdpStates.has_value(), - "Tried to compute states that are reachable under an optimal scheduler but this has already been done before."); - - // First find the choices that are optimal - optimalChoices = storm::storage::BitVector(exploredMdp->getNumberOfChoices(), false); - auto const &choiceIndices = exploredMdp->getNondeterministicChoiceIndices(); - auto const &transitions = exploredMdp->getTransitionMatrix(); - auto const &targetStatesExploredMDP = exploredMdp->getStates("target"); - for (uint64_t mdpState = 0; mdpState < exploredMdp->getNumberOfStates(); ++mdpState) { - if (targetStatesExploredMDP.get(mdpState)) { - // Target states can be skipped. - continue; - } else { - auto const &stateValue = values[mdpState]; - for (uint64_t globalChoice = choiceIndices[mdpState]; globalChoice < choiceIndices[mdpState + 1]; ++globalChoice) { - ValueType choiceValue = transitions.multiplyRowWithVector(globalChoice, values); - if (exploredMdp->hasRewardModel()) { - choiceValue += exploredMdp->getUniqueRewardModel().getStateActionReward(globalChoice); - } - auto absDiff = storm::utility::abs((choiceValue - stateValue)); - if ((relativeDifference && absDiff <= ancillaryChoicesEpsilon * stateValue) || (!relativeDifference && absDiff <= ancillaryChoicesEpsilon)) { - optimalChoices->set(globalChoice, true); - } - } - STORM_LOG_ASSERT(optimalChoices->getNextSetIndex(choiceIndices[mdpState]) < optimalChoices->size(), "Could not find an optimal choice."); - } - } - - // Then, find the states that are reachable via these choices - optimalChoicesReachableMdpStates = storm::utility::graph::getReachableStates(transitions, exploredMdp->getInitialStates(), ~targetStatesExploredMDP, - targetStatesExploredMDP, false, 0, optimalChoices.value()); -} - -template -bool BeliefMdpExplorer::beliefHasMdpState(BeliefId const &beliefId) const { - return getExploredMdpState(beliefId) != noState(); -} - -template -typename BeliefMdpExplorer::MdpStateType BeliefMdpExplorer::noState() const { - return std::numeric_limits::max(); -} - -template -std::shared_ptr BeliefMdpExplorer::createStandardProperty( - storm::solver::OptimizationDirection const &dir, bool computeRewards) { - std::string propertyString = computeRewards ? "R" : "P"; - propertyString += storm::solver::minimize(dir) ? "min" : "max"; - propertyString += "=? [F \"target\"]"; - std::vector propertyVector = storm::api::parseProperties(propertyString); - return storm::api::extractFormulasFromProperties(propertyVector).front(); -} - -template -storm::modelchecker::CheckTask::ValueType> -BeliefMdpExplorer::createStandardCheckTask(std::shared_ptr &property) { - // Note: The property should not run out of scope after calling this because the task only stores the property by reference. - // Therefore, this method needs the property by reference (and not const reference) - auto task = storm::api::createTask(property, false); - auto hint = storm::modelchecker::ExplicitModelCheckerHint(); - hint.setResultHint(values); - auto hintPtr = std::make_shared>(hint); - task.setHint(hintPtr); - task.setProduceSchedulers(); - return task; -} - -template -typename BeliefMdpExplorer::MdpStateType BeliefMdpExplorer::getCurrentMdpState() const { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - if (stateRemapping.find(currentMdpState) != stateRemapping.end()) { - return stateRemapping.at(currentMdpState); - } else { - return currentMdpState; - } -} - -template -typename BeliefMdpExplorer::MdpStateType BeliefMdpExplorer::getCurrentBeliefId() const { - STORM_LOG_ASSERT(status == Status::Exploring, "Method call is invalid in current status."); - return getBeliefId(currentMdpState); -} - -template -void BeliefMdpExplorer::internalAddTransition(uint64_t const &row, MdpStateType const &column, ValueType const &value) { - STORM_LOG_ASSERT(row <= exploredMdpTransitions.size(), "Skipped at least one row."); - if (row == exploredMdpTransitions.size()) { - exploredMdpTransitions.emplace_back(); - } - STORM_LOG_ASSERT(exploredMdpTransitions[row].count(column) == 0, "Trying to insert multiple transitions to the same state."); - exploredMdpTransitions[row][column] = value; -} - -template -void BeliefMdpExplorer::internalAddRowGroupIndex() { - exploredChoiceIndices.push_back(getCurrentNumberOfMdpChoices()); -} - -template -void BeliefMdpExplorer::markAsGridBelief(BeliefId const &beliefId) { - gridBeliefs.insert(beliefId); -} - -template -bool BeliefMdpExplorer::isMarkedAsGridBelief(BeliefId const &beliefId) { - return gridBeliefs.count(beliefId) > 0; -} - -template -typename BeliefMdpExplorer::MdpStateType BeliefMdpExplorer::getExploredMdpState( - BeliefId const &beliefId) const { - if (beliefId < exploredBeliefIds.size() && exploredBeliefIds.get(beliefId)) { - return beliefIdToMdpStateMap.at(beliefId); - } else { - return noState(); - } -} - -template -void BeliefMdpExplorer::insertValueHints(ValueType const &lowerBound, ValueType const &upperBound) { - lowerValueBounds.push_back(lowerBound); - upperValueBounds.push_back(upperBound); - // Take the middle value as a hint - values.push_back((lowerBound + upperBound) / storm::utility::convertNumber(2)); - STORM_LOG_ASSERT(lowerValueBounds.size() == getCurrentNumberOfMdpStates(), "Value vectors have different size then number of available states."); - STORM_LOG_ASSERT(lowerValueBounds.size() == upperValueBounds.size() && values.size() == upperValueBounds.size(), "Value vectors have inconsistent size."); -} - -template -typename BeliefMdpExplorer::MdpStateType BeliefMdpExplorer::getOrAddMdpState( - BeliefId const &beliefId, ValueType const &transitionValue) { - exploredBeliefIds.grow(beliefId + 1, false); - if (exploredBeliefIds.get(beliefId)) { - if (explHeuristic == ExplorationHeuristic::ProbabilityPrio && - mdpStatesToExploreStatePrio.find(beliefIdToMdpStateMap[beliefId]) != mdpStatesToExploreStatePrio.end()) { - // We check if the value is higher than the current priority and update if necessary - auto newPrio = probabilityEstimation[getCurrentMdpState()] * transitionValue; - if (newPrio > mdpStatesToExploreStatePrio[beliefIdToMdpStateMap[beliefId]]) { - // Erase the state from the "queue" map and re-insert it with the new value - auto range = mdpStatesToExplorePrioState.equal_range(mdpStatesToExploreStatePrio[beliefIdToMdpStateMap[beliefId]]); - for (auto i = range.first; i != range.second; ++i) { - if (i->second == beliefIdToMdpStateMap[beliefId]) { - mdpStatesToExplorePrioState.erase(i); - break; - } - } - mdpStatesToExplorePrioState.emplace(newPrio, beliefIdToMdpStateMap[beliefId]); - mdpStatesToExploreStatePrio[beliefIdToMdpStateMap[beliefId]] = newPrio; - } - } - return beliefIdToMdpStateMap[beliefId]; - } else { - // This state needs exploration - exploredBeliefIds.set(beliefId, true); - - // If this is a restart of the exploration, we still might have an MDP state for the belief - if (exploredMdp) { - auto findRes = beliefIdToMdpStateMap.find(beliefId); - if (findRes != beliefIdToMdpStateMap.end()) { - ValueType currentPrio; - switch (explHeuristic) { - case ExplorationHeuristic::BreadthFirst: - currentPrio = prio; - prio = prio - storm::utility::one(); - break; - case ExplorationHeuristic::LowerBoundPrio: - currentPrio = getLowerValueBoundAtCurrentState(); - break; - case ExplorationHeuristic::UpperBoundPrio: - currentPrio = getUpperValueBoundAtCurrentState(); - break; - case ExplorationHeuristic::GapPrio: - currentPrio = getUpperValueBoundAtCurrentState() - getLowerValueBoundAtCurrentState(); - break; - case ExplorationHeuristic::ProbabilityPrio: - if (getCurrentMdpState() != noState()) { - currentPrio = probabilityEstimation[getCurrentMdpState()] * transitionValue; - } else { - currentPrio = storm::utility::one(); - } - break; - default: - STORM_LOG_THROW(false, storm::exceptions::NotImplementedException, "Other heuristics not implemented yet."); - } - mdpStatesToExploreStatePrio[findRes->second] = currentPrio; - mdpStatesToExplorePrioState.emplace(currentPrio, findRes->second); - return findRes->second; - } - } - // At this point we need to add a new MDP state - MdpStateType result = getCurrentNumberOfMdpStates(); - STORM_LOG_ASSERT(getCurrentNumberOfMdpStates() == mdpStateToBeliefIdMap.size(), "MDP state count mismatch with belief map size."); - mdpStateToBeliefIdMap.push_back(beliefId); - beliefIdToMdpStateMap[beliefId] = result; - insertValueHints(computeLowerValueBoundAtBelief(beliefId), computeUpperValueBoundAtBelief(beliefId)); - ValueType currentPrio; - switch (explHeuristic) { - case ExplorationHeuristic::BreadthFirst: - currentPrio = prio; - prio = prio - storm::utility::one(); - break; - case ExplorationHeuristic::LowerBoundPrio: - currentPrio = getLowerValueBoundAtCurrentState(); - break; - case ExplorationHeuristic::UpperBoundPrio: - currentPrio = getUpperValueBoundAtCurrentState(); - break; - case ExplorationHeuristic::GapPrio: - currentPrio = getUpperValueBoundAtCurrentState() - getLowerValueBoundAtCurrentState(); - break; - case ExplorationHeuristic::ProbabilityPrio: - if (getCurrentMdpState() != noState()) { - currentPrio = probabilityEstimation[getCurrentMdpState()] * transitionValue; - } else { - currentPrio = storm::utility::one(); - } - break; - default: - STORM_LOG_THROW(false, storm::exceptions::NotImplementedException, "Other heuristics not implemented yet."); - } - mdpStatesToExploreStatePrio[result] = currentPrio; - mdpStatesToExplorePrioState.emplace(currentPrio, result); - return result; - } -} - -template -void BeliefMdpExplorer::addChoiceLabelToCurrentState(uint64_t const &localActionIndex, std::string const &label) { - mdpStateToChoiceLabelsMap[currentMdpState][localActionIndex] = label; -} - -template -std::vector::BeliefId> BeliefMdpExplorer::getBeliefsInMdp() { - return mdpStateToBeliefIdMap; -} - -template -std::vector::BeliefId> BeliefMdpExplorer::getBeliefsWithObservationInMdp( - uint32_t obs) const { - std::vector res; - for (auto const &belief : mdpStateToBeliefIdMap) { - if (belief != beliefManager->noId()) { - if (beliefManager->getBeliefObservation(belief) == obs) { - res.push_back(belief); - } - } - } - return res; -} - -template -typename BeliefMdpExplorer::ValueType BeliefMdpExplorer::getTrivialUpperBoundAtPOMDPState( - uint64_t const &pomdpState) { - return pomdpValueBounds.getSmallestUpperBound(pomdpState); -} - -template -typename BeliefMdpExplorer::ValueType BeliefMdpExplorer::getTrivialLowerBoundAtPOMDPState( - uint64_t const &pomdpState) { - return pomdpValueBounds.getHighestLowerBound(pomdpState); -} - -template -void BeliefMdpExplorer::setExtremeValueBound(storm::pomdp::storage::ExtremePOMDPValueBound valueBound) { - extremeValueBound = valueBound; -} - -template -void BeliefMdpExplorer::setFMSchedValueList(std::vector>> valueList) { - fmSchedulerValueList = valueList; -} - -template -uint64_t BeliefMdpExplorer::getNrOfMemoryNodesForObservation(uint32_t observation) const { - return fmSchedulerValueList.at(observation).size(); -} - -template -typename BeliefMdpExplorer::ValueType BeliefMdpExplorer::getExtremeValueBoundAtPOMDPState( - const uint64_t &pomdpState) { - return extremeValueBound.getValueForState(pomdpState); -} - -template -storm::storage::BitVector BeliefMdpExplorer::getStateExtremeBoundIsInfinite() { - return extremeValueBound.isInfinite; -} - -template -uint64_t BeliefMdpExplorer::getNrSchedulersForUpperBounds() { - return pomdpValueBounds.upper.size(); -} - -template -uint64_t BeliefMdpExplorer::getNrSchedulersForLowerBounds() { - return pomdpValueBounds.lower.size(); -} - -template -storm::storage::Scheduler::ValueType> -BeliefMdpExplorer::getLowerValueBoundScheduler(uint64_t schedulerId) const { - STORM_LOG_ASSERT(!pomdpValueBounds.lowerSchedulers.empty(), "Requested lower bound scheduler but none were available."); - STORM_LOG_ASSERT(pomdpValueBounds.lowerSchedulers.size() > schedulerId, - "Requested lower value bound scheduler with ID " << schedulerId << " not available."); - return pomdpValueBounds.lowerSchedulers[schedulerId]; -} - -template -storm::storage::Scheduler::ValueType> -BeliefMdpExplorer::getUpperValueBoundScheduler(uint64_t schedulerId) const { - STORM_LOG_ASSERT(!pomdpValueBounds.upperSchedulers.empty(), "Requested upper bound scheduler but none were available."); - STORM_LOG_ASSERT(pomdpValueBounds.upperSchedulers.size() > schedulerId, - "Requested upper value bound scheduler with ID " << schedulerId << " not available."); - return pomdpValueBounds.upperSchedulers[schedulerId]; -} - -template -std::vector::ValueType>> -BeliefMdpExplorer::getLowerValueBoundSchedulers() const { - STORM_LOG_ASSERT(!pomdpValueBounds.lowerSchedulers.empty(), "Requested lower bound schedulers but none were available."); - return pomdpValueBounds.lowerSchedulers; -} - -template -std::vector::ValueType>> -BeliefMdpExplorer::getUpperValueBoundSchedulers() const { - STORM_LOG_ASSERT(!pomdpValueBounds.upperSchedulers.empty(), "Requested upper bound schedulers but none were available."); - return pomdpValueBounds.upperSchedulers; -} - -template - -bool BeliefMdpExplorer::hasFMSchedulerValues() const { - return !fmSchedulerValueList.empty(); -} - -template -std::vector BeliefMdpExplorer::computeProductWithSparseMatrix( - BeliefId const &beliefId, storm::storage::SparseMatrix &matrix) const { - return beliefManager->computeMatrixBeliefProduct(beliefId, matrix); -} - -template -void BeliefMdpExplorer::adjustActions(uint64_t totalNumberOfActions) { - uint64_t currentRowGroupSize = getSizeOfCurrentRowGroup(); - STORM_LOG_ASSERT(totalNumberOfActions != currentRowGroupSize, "Total actions equals current row group size."); - if (totalNumberOfActions > currentRowGroupSize) { - uint64_t numberOfActionsToAdd = totalNumberOfActions - currentRowGroupSize; - exploredMdpTransitions.insert(exploredMdpTransitions.begin() + (exploredChoiceIndices[getCurrentMdpState() + 1]), numberOfActionsToAdd, - std::map()); - for (uint64_t i = getCurrentMdpState() + 1; i < exploredChoiceIndices.size(); i++) { - exploredChoiceIndices[i] += numberOfActionsToAdd; - } - return; - } - if (totalNumberOfActions < currentRowGroupSize) { - uint64_t numberOfActionsToRemove = currentRowGroupSize - totalNumberOfActions; - exploredMdpTransitions.erase(exploredMdpTransitions.begin() + (exploredChoiceIndices[getCurrentMdpState() + 1]) - numberOfActionsToRemove, - exploredMdpTransitions.begin() + (exploredChoiceIndices[getCurrentMdpState() + 1])); - for (uint64_t i = getCurrentMdpState() + 1; i < exploredChoiceIndices.size(); i++) { - exploredChoiceIndices[i] -= numberOfActionsToRemove; - } - } -} - -template class BeliefMdpExplorer>; - -template class BeliefMdpExplorer, storm::RationalNumber>; - -template class BeliefMdpExplorer, double>; - -template class BeliefMdpExplorer>; -} // namespace builder -} // namespace storm diff --git a/src/storm-pomdp/builder/BeliefMdpExplorer.h b/src/storm-pomdp/builder/BeliefMdpExplorer.h deleted file mode 100644 index 27d20ce10a..0000000000 --- a/src/storm-pomdp/builder/BeliefMdpExplorer.h +++ /dev/null @@ -1,351 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "storm-pomdp/storage/BeliefExplorationBounds.h" -#include "storm-pomdp/storage/BeliefManager.h" -#include "storm/models/sparse/Mdp.h" -#include "storm/storage/BitVector.h" - -namespace storm { -class Environment; - -namespace modelchecker { -template -class CheckTask; -class CheckResult; -} // namespace modelchecker -namespace builder { -enum class ExplorationHeuristic { BreadthFirst, LowerBoundPrio, UpperBoundPrio, GapPrio, ProbabilityPrio }; - -template -class BeliefMdpExplorer { - public: - typedef typename PomdpType::ValueType ValueType; - typedef storm::storage::BeliefManager BeliefManagerType; - typedef typename BeliefManagerType::BeliefId BeliefId; - typedef uint64_t MdpStateType; - - struct SuccessorObservationInformation { - SuccessorObservationInformation(ValueType const &obsProb, ValueType const &maxProb, uint64_t const &count); - void join(SuccessorObservationInformation other); - ValueType observationProbability; /// The probability we move to the corresponding observation. - ValueType maxProbabilityToSuccessorWithObs; /// The maximal probability to move to a successor with the corresponding observation. - uint64_t successorWithObsCount; /// The number of successor beliefstates with this observation - typename BeliefManagerType::BeliefSupportType support; - }; - - enum class Status { Uninitialized, Exploring, ModelFinished, ModelChecked }; - - BeliefMdpExplorer(std::shared_ptr beliefManager, storm::pomdp::storage::PreprocessingPomdpValueBounds const &pomdpValueBounds, - ExplorationHeuristic explorationHeuristic = ExplorationHeuristic::BreadthFirst); - - BeliefMdpExplorer(BeliefMdpExplorer &&other) = default; - - BeliefManagerType const &getBeliefManager() const; - - void startNewExploration(std::optional extraTargetStateValue = boost::none, std::optional extraBottomStateValue = std::nullopt); - - /*! - * Restarts the exploration to allow re-exploring each state. - * After calling this, the "currently explored" MDP has the same number of states and choices as the "old" one, but the choices are still empty - * This method inserts the initial state of the MDP in the exploration queue. - * While re-exploring, the reference to the old MDP remains valid. - */ - void restartExploration(); - - bool hasUnexploredState() const; - - std::vector getUnexploredStates(); - - BeliefId exploreNextState(); - - void addChoiceLabelToCurrentState(uint64_t const &localActionIndex, std::string const &label); - - void addTransitionsToExtraStates(uint64_t const &localActionIndex, ValueType const &targetStateValue = storm::utility::zero(), - ValueType const &bottomStateValue = storm::utility::zero()); - - void addSelfloopTransition(uint64_t const &localActionIndex = 0, ValueType const &value = storm::utility::one()); - - /*! - * Adds the next transition to the given successor belief - * @param localActionIndex - * @param transitionTarget - * @param value - * @param ignoreNewBeliefs If true, beliefs that were not found before are not inserted, i.e. we might not insert the transition. - * @return true iff a transition was actually inserted. False can only happen if ignoreNewBeliefs is true. - */ - bool addTransitionToBelief(uint64_t const &localActionIndex, BeliefId const &transitionTarget, ValueType const &value, bool ignoreNewBeliefs); - - void computeRewardAtCurrentState(uint64_t const &localActionIndex, ValueType extraReward = storm::utility::zero()); - - /*! - * Adds the provided reward value to the given action of the current state - * - * @param localActionIndex - * @param rewardValue - */ - void addRewardToCurrentState(uint64_t const &localActionIndex, ValueType rewardValue); - - void setCurrentStateIsTarget(); - - void setCurrentStateIsTruncated(); - - void setCurrentStateIsClipped(); - - void setCurrentChoiceIsDelayed(uint64_t const &localActionIndex); - - bool currentStateHasOldBehavior() const; - - bool getCurrentStateWasTruncated() const; - - bool getCurrentStateWasClipped() const; - - /*! - * Retrieves whether the current state can be reached under an optimal scheduler - * This requires a previous call of computeOptimalChoicesAndReachableMdpStates. - */ - bool stateIsOptimalSchedulerReachable(MdpStateType mdpState) const; - - /*! - * Retrieves whether the given action at the current state was optimal in the most recent check. - * This requires a previous call of computeOptimalChoicesAndReachableMdpStates. - */ - bool actionIsOptimal(uint64_t const &globalActionIndex) const; - - /*! - * Retrieves whether the current state can be reached under a scheduler that was optimal in the most recent check. - * This requires (i) a previous call of computeOptimalChoicesAndReachableMdpStates and (ii) that the current state has old behavior. - */ - bool currentStateIsOptimalSchedulerReachable() const; - - /*! - * Retrieves whether the given action at the current state was optimal in the most recent check. - * This requires (i) a previous call of computeOptimalChoicesAndReachableMdpStates and (ii) that the current state has old behavior. - */ - bool actionAtCurrentStateWasOptimal(uint64_t const &localActionIndex) const; - - bool getCurrentStateActionExplorationWasDelayed(uint64_t const &localActionIndex) const; - - /*! - * Inserts transitions and rewards at the given action as in the MDP of the previous exploration. - * Does NOT set whether the state is truncated and/or target. - * Will add "old" states that have not been considered before into the exploration queue - * @param localActionIndex - */ - void restoreOldBehaviorAtCurrentState(uint64_t const &localActionIndex); - - void finishExploration(); - - void dropUnexploredStates(); - - std::shared_ptr> getExploredMdp() const; - - MdpStateType getCurrentNumberOfMdpStates() const; - - MdpStateType getCurrentNumberOfMdpChoices() const; - - MdpStateType getStartOfCurrentRowGroup() const; - - uint64_t getSizeOfCurrentRowGroup() const; - - uint64_t getRowGroupSizeOfState(uint64_t state) const; - - bool needsActionAdjustment(uint64_t numActionsNeeded); - - ValueType getLowerValueBoundAtCurrentState() const; - - ValueType getUpperValueBoundAtCurrentState() const; - - ValueType computeLowerValueBoundAtBelief(BeliefId const &beliefId) const; - - ValueType computeUpperValueBoundAtBelief(BeliefId const &beliefId) const; - - ValueType computeLowerValueBoundForScheduler(BeliefId const &beliefId, uint64_t schedulerId) const; - - ValueType computeUpperValueBoundForScheduler(BeliefId const &beliefId, uint64_t schedulerId) const; - - std::pair computeFMSchedulerValueForMemoryNode(BeliefId const &beliefId, uint64_t memoryNode) const; - - storm::storage::Scheduler getUpperValueBoundScheduler(uint64_t schedulerId) const; - - storm::storage::Scheduler getLowerValueBoundScheduler(uint64_t schedulerId) const; - - std::vector> getUpperValueBoundSchedulers() const; - - std::vector> getLowerValueBoundSchedulers() const; - - void computeValuesOfExploredMdp(storm::Environment const &env, storm::solver::OptimizationDirection const &dir); - - bool hasComputedValues() const; - - bool hasFMSchedulerValues() const; - - std::vector const &getValuesOfExploredMdp() const; - - ValueType const &getComputedValueAtInitialState() const; - - MdpStateType getBeliefId(MdpStateType exploredMdpState) const; - - void gatherSuccessorObservationInformationAtCurrentState(uint64_t localActionIndex, - std::map &gatheredSuccessorObservations); - - void gatherSuccessorObservationInformationAtMdpChoice(uint64_t mdpChoice, - std::map &gatheredSuccessorObservations); - - bool currentStateHasSuccessorObservationInObservationSet(uint64_t localActionIndex, storm::storage::BitVector const &observationSet); - - void takeCurrentValuesAsUpperBounds(); - - void takeCurrentValuesAsLowerBounds(); - - /*! - * - * Computes the set of states that are reachable via a path that is consistent with an optimal MDP scheduler. - * States that are only reachable via target states will not be in this set. - * @param ancillaryChoicesEpsilon if the difference of a 1-step value of a choice is only epsilon away from the optimal value, the choice will be included. - * @param relative if set, we consider the relative difference to detect ancillaryChoices - */ - void computeOptimalChoicesAndReachableMdpStates(ValueType const &ancillaryChoicesEpsilon, bool relativeDifference); - - std::vector getBeliefsWithObservationInMdp(uint32_t obs) const; - - std::vector getBeliefsInMdp(); - - void addClippingRewardToCurrentState(uint64_t const &localActionIndex, ValueType rewardValue); - - ValueType getTrivialUpperBoundAtPOMDPState(uint64_t const &pomdpState); - - ValueType getTrivialLowerBoundAtPOMDPState(uint64_t const &pomdpState); - - void setExtremeValueBound(storm::pomdp::storage::ExtremePOMDPValueBound valueBound); - - ValueType getExtremeValueBoundAtPOMDPState(uint64_t const &pomdpState); - - MdpStateType getExploredMdpState(BeliefId const &beliefId) const; - - bool beliefHasMdpState(BeliefId const &beliefId) const; - - storm::storage::BitVector getStateExtremeBoundIsInfinite(); - - uint64_t getNrSchedulersForUpperBounds(); - - uint64_t getNrSchedulersForLowerBounds(); - - void markAsGridBelief(BeliefId const &beliefId); - - bool isMarkedAsGridBelief(BeliefId const &beliefId); - - const std::shared_ptr::ValueType>> &getSchedulerForExploredMdp() const; - - void setFMSchedValueList(std::vector>> valueList); - - uint64_t getNrOfMemoryNodesForObservation(uint32_t observation) const; - - void storeExplorationState(); - - void restoreExplorationState(); - - void adjustActions(uint64_t totalNumberOfActions); - - std::vector computeProductWithSparseMatrix(BeliefId const &beliefId, storm::storage::SparseMatrix &matrix) const; - - private: - MdpStateType noState() const; - - std::shared_ptr createStandardProperty(storm::solver::OptimizationDirection const &dir, bool computeRewards); - - storm::modelchecker::CheckTask createStandardCheckTask(std::shared_ptr &property); - - MdpStateType getCurrentMdpState() const; - - MdpStateType getCurrentBeliefId() const; - - void internalAddTransition(uint64_t const &row, MdpStateType const &column, ValueType const &value); - - void internalAddRowGroupIndex(); - - void insertValueHints(ValueType const &lowerBound, ValueType const &upperBound); - - MdpStateType getOrAddMdpState(BeliefId const &beliefId, ValueType const &transitionValue = storm::utility::zero()); - - // Belief state related information - std::shared_ptr beliefManager; - std::vector mdpStateToBeliefIdMap; - std::map beliefIdToMdpStateMap; - storm::storage::BitVector exploredBeliefIds; - std::map> mdpStateToChoiceLabelsMap; - - // Exploration information - std::multimap mdpStatesToExplorePrioState; - std::map mdpStatesToExploreStatePrio; - std::vector probabilityEstimation; - std::vector> exploredMdpTransitions; - std::vector exploredChoiceIndices; - std::vector previousChoiceIndices; - std::vector mdpActionRewards; - std::map clippingTransitionRewards; - uint64_t currentMdpState; - std::map stateRemapping; - uint64_t nextId; - ValueType prio; - - // Special states and choices during exploration - std::optional extraTargetState; - std::optional extraBottomState; - storm::storage::BitVector targetStates; - storm::storage::BitVector truncatedStates; - storm::storage::BitVector clippedStates; - MdpStateType initialMdpState; - storm::storage::BitVector delayedExplorationChoices; - std::unordered_set gridBeliefs; - - // Final Mdp - std::shared_ptr> exploredMdp; - - // Value and scheduler related information - storm::pomdp::storage::PreprocessingPomdpValueBounds pomdpValueBounds; - storm::pomdp::storage::ExtremePOMDPValueBound extremeValueBound; - std::vector>> fmSchedulerValueList; - std::vector lowerValueBounds; - std::vector upperValueBounds; - std::vector values; // Contains an estimate during building and the actual result after a check has performed - std::optional optimalChoices; - std::optional optimalChoicesReachableMdpStates; - std::shared_ptr> scheduler; - - // The current status of this explorer - ExplorationHeuristic explHeuristic; - Status status; - - struct ExplorationStorage { - std::vector storedMdpStateToBeliefIdMap; - std::map storedBeliefIdToMdpStateMap; - storm::storage::BitVector storedExploredBeliefIds; - std::map> storedMdpStateToChoiceLabelsMap; - std::multimap storedMdpStatesToExplorePrioState; - std::map storedMdpStatesToExploreStatePrio; - std::vector storedProbabilityEstimation; - std::vector> storedExploredMdpTransitions; - std::vector storedExploredChoiceIndices; - std::vector storedMdpActionRewards; - std::map storedClippingTransitionRewards; - uint64_t storedCurrentMdpState; - std::map storedStateRemapping; - uint64_t storedNextId; - ValueType storedPrio; - std::vector storedLowerValueBounds; - std::vector storedUpperValueBounds; - std::vector storedValues; - storm::storage::BitVector storedTargetStates; - }; - - ExplorationStorage explorationStorage; -}; -} // namespace builder -} // namespace storm \ No newline at end of file diff --git a/src/storm-pomdp/modelchecker/BeliefExplorationPomdpModelChecker.cpp b/src/storm-pomdp/modelchecker/BeliefExplorationPomdpModelChecker.cpp deleted file mode 100644 index 85d561658f..0000000000 --- a/src/storm-pomdp/modelchecker/BeliefExplorationPomdpModelChecker.cpp +++ /dev/null @@ -1,1565 +0,0 @@ -#include "BeliefExplorationPomdpModelChecker.h" - -#include - -#include "storm-pomdp/analysis/FiniteBeliefMdpDetection.h" -#include "storm-pomdp/analysis/FormulaInformation.h" -#include "storm-pomdp/transformer/MakeStateSetObservationClosed.h" - -#include "storm/logic/Formulas.h" -#include "storm/utility/ConstantsComparator.h" -#include "storm/utility/NumberTraits.h" - -#include "storm-pomdp/builder/BeliefMdpExplorer.h" -#include "storm-pomdp/modelchecker/PreprocessingPomdpValueBoundsModelChecker.h" -#include "storm/utility/vector.h" - -#include "storm/environment/Environment.h" -#include "storm/exceptions/NotSupportedException.h" -#include "storm/utility/SignalHandler.h" -#include "storm/utility/graph.h" -#include "storm/utility/macros.h" - -namespace storm { -namespace pomdp { -namespace modelchecker { - -/* Struct Functions */ - -template -BeliefExplorationPomdpModelChecker::Result::Result(ValueType lower, ValueType upper) - : lowerBound(lower), upperBound(upper) { - // Intentionally left empty -} - -template -typename BeliefExplorationPomdpModelChecker::ValueType -BeliefExplorationPomdpModelChecker::Result::diff(bool relative) const { - ValueType diff = upperBound - lowerBound; - if (diff < storm::utility::zero()) { - STORM_LOG_WARN_COND(diff >= storm::utility::convertNumber(1e-6), - "Upper bound '" << upperBound << "' is smaller than lower bound '" << lowerBound << "': Difference is " << diff << "."); - diff = storm::utility::zero(); - } - if (relative && !storm::utility::isZero(upperBound)) { - diff /= upperBound; - } - return diff; -} - -template -bool BeliefExplorationPomdpModelChecker::Result::updateLowerBound(ValueType const& value) { - if (value > lowerBound) { - lowerBound = value; - return true; - } - return false; -} - -template -bool BeliefExplorationPomdpModelChecker::Result::updateUpperBound(ValueType const& value) { - if (value < upperBound) { - upperBound = value; - return true; - } - return false; -} - -template -BeliefExplorationPomdpModelChecker::Statistics::Statistics() - : beliefMdpDetectedToBeFinite(false), - refinementFixpointDetected(false), - overApproximationBuildAborted(false), - underApproximationBuildAborted(false), - aborted(false) { - // intentionally left empty; -} - -template -BeliefExplorationPomdpModelChecker::BeliefExplorationPomdpModelChecker(std::shared_ptr pomdp, - Options options) - : options(options), - inputPomdp(pomdp), - beliefTypeCC(storm::utility::convertNumber(this->options.numericPrecision), false), - valueTypeCC(this->options.numericPrecision, false) { - STORM_LOG_ASSERT(inputPomdp, "The given POMDP is not initialized."); - STORM_LOG_ERROR_COND(inputPomdp->isCanonic(), "Input Pomdp is not known to be canonic. This might lead to unexpected verification results."); -} - -/* Public Functions */ - -template -void BeliefExplorationPomdpModelChecker::precomputeValueBounds(storm::logic::Formula const& formula, - storm::Environment const& preProcEnv) { - auto formulaInfo = storm::pomdp::analysis::getFormulaInformation(pomdp(), formula); - - // Compute some initial bounds on the values for each state of the pomdp - // We work with the Belief MDP value type, so if the POMDP is exact, but the belief MDP is not, we need to convert - auto preProcessingMC = PreprocessingPomdpValueBoundsModelChecker(pomdp()); - auto initialPomdpValueBounds = preProcessingMC.getValueBounds(preProcEnv, formula); - pomdpValueBounds.trivialPomdpValueBounds = initialPomdpValueBounds; - - // If we clip and compute rewards, compute the values necessary for the correction terms - if (options.useClipping && formula.isRewardOperatorFormula()) { - pomdpValueBounds.extremePomdpValueBound = preProcessingMC.getExtremeValueBound(preProcEnv, formula); - } -} - -template -typename BeliefExplorationPomdpModelChecker::Result -BeliefExplorationPomdpModelChecker::check( - storm::Environment const& env, storm::logic::Formula const& formula, - std::vector>> const& additionalUnderApproximationBounds) { - return check(env, formula, env, additionalUnderApproximationBounds); -} - -template -typename BeliefExplorationPomdpModelChecker::Result -BeliefExplorationPomdpModelChecker::check( - storm::logic::Formula const& formula, std::vector>> const& additionalUnderApproximationBounds) { - storm::Environment env; - return check(env, formula, env, additionalUnderApproximationBounds); -} - -template -typename BeliefExplorationPomdpModelChecker::Result -BeliefExplorationPomdpModelChecker::check( - storm::logic::Formula const& formula, storm::Environment const& preProcEnv, - std::vector>> const& additionalUnderApproximationBounds) { - storm::Environment env; - return check(env, formula, preProcEnv, additionalUnderApproximationBounds); -} - -template -typename BeliefExplorationPomdpModelChecker::Result -BeliefExplorationPomdpModelChecker::check( - storm::Environment const& env, storm::logic::Formula const& formula, storm::Environment const& preProcEnv, - std::vector>> const& additionalUnderApproximationBounds) { - STORM_LOG_ASSERT(options.unfold || options.discretize || options.interactiveUnfolding, - "Invoked belief exploration but no task (unfold or discretize) given."); - // Potentially reset preprocessed model from previous call - preprocessedPomdp.reset(); - - // Reset all collected statistics - statistics = Statistics(); - statistics.totalTime.start(); - // Extract the relevant information from the formula - auto formulaInfo = storm::pomdp::analysis::getFormulaInformation(pomdp(), formula); - - precomputeValueBounds(formula, preProcEnv); - if (!additionalUnderApproximationBounds.empty()) { - pomdpValueBounds.fmSchedulerValueList = additionalUnderApproximationBounds; - } - uint64_t initialPomdpState = pomdp().getInitialStates().getNextSetIndex(0); - Result result(pomdpValueBounds.trivialPomdpValueBounds.getHighestLowerBound(initialPomdpState), - pomdpValueBounds.trivialPomdpValueBounds.getSmallestUpperBound(initialPomdpState)); - STORM_LOG_INFO("Initial value bounds are [" << result.lowerBound << ", " << result.upperBound << "]"); - - std::optional rewardModelName; - std::set targetObservations; - if (formulaInfo.isNonNestedReachabilityProbability() || formulaInfo.isNonNestedExpectedRewardFormula()) { - if (formulaInfo.getTargetStates().observationClosed) { - targetObservations = formulaInfo.getTargetStates().observations; - } else { - storm::transformer::MakeStateSetObservationClosed obsCloser(inputPomdp); - std::tie(preprocessedPomdp, targetObservations) = obsCloser.transform(formulaInfo.getTargetStates().states); - } - if (formulaInfo.isNonNestedReachabilityProbability()) { - if (!formulaInfo.getSinkStates().empty()) { - storm::storage::sparse::ModelComponents components; - components.stateLabeling = pomdp().getStateLabeling(); - components.rewardModels = pomdp().getRewardModels(); - auto matrix = pomdp().getTransitionMatrix(); - matrix.makeRowGroupsAbsorbing(formulaInfo.getSinkStates().states); - components.transitionMatrix = matrix; - components.observabilityClasses = pomdp().getObservations(); - if (pomdp().hasChoiceLabeling()) { - components.choiceLabeling = pomdp().getChoiceLabeling(); - } - if (pomdp().hasObservationValuations()) { - components.observationValuations = pomdp().getObservationValuations(); - } - preprocessedPomdp = std::make_shared>(std::move(components), true); - auto reachableFromSinkStates = storm::utility::graph::getReachableStates( - pomdp().getTransitionMatrix(), formulaInfo.getSinkStates().states, formulaInfo.getSinkStates().states, ~formulaInfo.getSinkStates().states); - reachableFromSinkStates &= ~formulaInfo.getSinkStates().states; - STORM_LOG_THROW(reachableFromSinkStates.empty(), storm::exceptions::NotSupportedException, - "There are sink states that can reach non-sink states. This is currently not supported."); - } - } else { - // Expected reward formula! - rewardModelName = formulaInfo.getRewardModelName(); - } - } else { - STORM_LOG_THROW(false, storm::exceptions::NotSupportedException, "Unsupported formula '" << formula << "'."); - } - if (storm::pomdp::detectFiniteBeliefMdp(pomdp(), formulaInfo.getTargetStates().states)) { - STORM_LOG_INFO("Detected that the belief MDP is finite."); - statistics.beliefMdpDetectedToBeFinite = true; - } - if (options.interactiveUnfolding) { - unfoldInteractively(env, targetObservations, formulaInfo.minimize(), rewardModelName, pomdpValueBounds, result); - } else { - refineReachability(env, targetObservations, formulaInfo.minimize(), rewardModelName, pomdpValueBounds, result); - } - // "clear" results in case they were actually not requested (this will make the output a bit more clear) - if ((formulaInfo.minimize() && !options.discretize) || (formulaInfo.maximize() && !options.unfold)) { - result.lowerBound = -storm::utility::infinity(); - } - if ((formulaInfo.maximize() && !options.discretize) || (formulaInfo.minimize() && !options.unfold)) { - result.upperBound = storm::utility::infinity(); - } - - if (storm::utility::resources::isTerminate()) { - statistics.aborted = true; - } - statistics.totalTime.stop(); - return result; -} - -template -void BeliefExplorationPomdpModelChecker::printStatisticsToStream(std::ostream& stream) const { - stream << "##### POMDP Approximation Statistics ######\n"; - stream << "# Input model: \n"; - pomdp().printModelInformationToStream(stream); - stream << "# Max. Number of states with same observation: " << pomdp().getMaxNrStatesWithSameObservation() << '\n'; - if (statistics.beliefMdpDetectedToBeFinite) { - stream << "# Pre-computations detected that the belief MDP is finite.\n"; - } - if (statistics.aborted) { - stream << "# Computation aborted early\n"; - } - - stream << "# Total check time: " << statistics.totalTime << '\n'; - // Refinement information: - if (statistics.refinementSteps) { - stream << "# Number of refinement steps: " << statistics.refinementSteps.value() << '\n'; - } - if (statistics.refinementFixpointDetected) { - stream << "# Detected a refinement fixpoint.\n"; - } - - // The overapproximation MDP: - if (statistics.overApproximationStates) { - stream << "# Number of states in the "; - if (options.refine) { - stream << "final "; - } - stream << "grid MDP for the over-approximation: "; - if (statistics.overApproximationBuildAborted) { - stream << ">="; - } - stream << statistics.overApproximationStates.value() << '\n'; - stream << "# Maximal resolution for over-approximation: " << statistics.overApproximationMaxResolution.value() << '\n'; - stream << "# Time spend for building the over-approx grid MDP(s): " << statistics.overApproximationBuildTime << '\n'; - stream << "# Time spend for checking the over-approx grid MDP(s): " << statistics.overApproximationCheckTime << '\n'; - } - - // The underapproximation MDP: - if (statistics.underApproximationStates) { - stream << "# Number of states in the "; - if (options.refine) { - stream << "final "; - } - stream << "belief MDP for the under-approximation: "; - if (statistics.underApproximationBuildAborted) { - stream << ">="; - } - stream << statistics.underApproximationStates.value() << '\n'; - if (statistics.nrClippingAttempts) { - stream << "# Clipping attempts (clipped states) for the under-approximation: "; - if (statistics.underApproximationBuildAborted) { - stream << ">="; - } - stream << statistics.nrClippingAttempts.value() << " (" << statistics.nrClippedStates.value() << ")\n"; - stream << "# Total clipping preprocessing time: " << statistics.clippingPreTime << "\n"; - stream << "# Total clipping time: " << statistics.clipWatch << "\n"; - } else if (statistics.nrTruncatedStates) { - stream << "# Truncated states for the under-approximation: "; - if (statistics.underApproximationBuildAborted) { - stream << ">="; - } - stream << statistics.nrTruncatedStates.value() << "\n"; - } - if (statistics.underApproximationStateLimit) { - stream << "# Exploration state limit for under-approximation: " << statistics.underApproximationStateLimit.value() << '\n'; - } - stream << "# Time spend for building the under-approx grid MDP(s): " << statistics.underApproximationBuildTime << '\n'; - stream << "# Time spend for checking the under-approx grid MDP(s): " << statistics.underApproximationCheckTime << '\n'; - } - - stream << "##########################################\n"; -} - -/* Private Functions */ - -template -PomdpModelType const& BeliefExplorationPomdpModelChecker::pomdp() const { - if (preprocessedPomdp) { - return *preprocessedPomdp; - } else { - return *inputPomdp; - } -} - -template -void BeliefExplorationPomdpModelChecker::refineReachability( - storm::Environment const& env, std::set const& targetObservations, bool min, std::optional rewardModelName, - storm::pomdp::modelchecker::POMDPValueBounds const& valueBounds, Result& result) { - statistics.refinementSteps = 0; - auto trivialPOMDPBounds = valueBounds.trivialPomdpValueBounds; - // Set up exploration data - std::vector observationResolutionVector; - std::shared_ptr overApproxBeliefManager; - std::shared_ptr overApproximation; - HeuristicParameters overApproxHeuristicPar{}; - if (options.discretize) { // Setup and build first OverApproximation - observationResolutionVector = - std::vector(pomdp().getNrObservations(), storm::utility::convertNumber(options.resolutionInit)); - overApproxBeliefManager = std::make_shared( - pomdp(), storm::utility::convertNumber(options.numericPrecision), - options.dynamicTriangulation ? BeliefManagerType::TriangulationMode::Dynamic : BeliefManagerType::TriangulationMode::Static); - if (rewardModelName) { - overApproxBeliefManager->setRewardModel(rewardModelName); - } - overApproximation = std::make_shared(overApproxBeliefManager, trivialPOMDPBounds, storm::builder::ExplorationHeuristic::BreadthFirst); - overApproxHeuristicPar.gapThreshold = options.gapThresholdInit; - overApproxHeuristicPar.observationThreshold = options.obsThresholdInit; - overApproxHeuristicPar.sizeThreshold = options.sizeThresholdInit == 0 ? std::numeric_limits::max() : options.sizeThresholdInit; - overApproxHeuristicPar.optimalChoiceValueEpsilon = options.optimalChoiceValueThresholdInit; - - buildOverApproximation(env, targetObservations, min, rewardModelName.has_value(), false, overApproxHeuristicPar, observationResolutionVector, - overApproxBeliefManager, overApproximation); - if (!overApproximation->hasComputedValues() || storm::utility::resources::isTerminate()) { - return; - } - ValueType const& newValue = overApproximation->getComputedValueAtInitialState(); - bool betterBound = min ? result.updateLowerBound(newValue) : result.updateUpperBound(newValue); - if (betterBound) { - STORM_LOG_INFO("Initial Over-approx result obtained after " << statistics.totalTime << ". Value is '" << newValue << "'.\n"); - } - } - - std::shared_ptr underApproxBeliefManager; - std::shared_ptr underApproximation; - HeuristicParameters underApproxHeuristicPar{}; - if (options.unfold) { // Setup and build first UnderApproximation - underApproxBeliefManager = std::make_shared( - pomdp(), storm::utility::convertNumber(options.numericPrecision), - options.dynamicTriangulation ? BeliefManagerType::TriangulationMode::Dynamic : BeliefManagerType::TriangulationMode::Static); - if (rewardModelName) { - underApproxBeliefManager->setRewardModel(rewardModelName); - } - underApproximation = std::make_shared(underApproxBeliefManager, trivialPOMDPBounds, options.explorationHeuristic); - underApproxHeuristicPar.gapThreshold = options.gapThresholdInit; - underApproxHeuristicPar.optimalChoiceValueEpsilon = options.optimalChoiceValueThresholdInit; - underApproxHeuristicPar.sizeThreshold = options.sizeThresholdInit; - if (underApproxHeuristicPar.sizeThreshold == 0) { - if (!options.refine && options.explorationTimeLimit != 0) { - underApproxHeuristicPar.sizeThreshold = std::numeric_limits::max(); - } else { - underApproxHeuristicPar.sizeThreshold = pomdp().getNumberOfStates() * pomdp().getMaxNrStatesWithSameObservation(); - STORM_LOG_INFO("Heuristically selected an under-approximation MDP size threshold of " << underApproxHeuristicPar.sizeThreshold << ".\n"); - } - } - - if (options.useClipping && rewardModelName.has_value()) { - underApproximation->setExtremeValueBound(valueBounds.extremePomdpValueBound); - } - if (!valueBounds.fmSchedulerValueList.empty()) { - underApproximation->setFMSchedValueList(valueBounds.fmSchedulerValueList); - } - buildUnderApproximation(env, targetObservations, min, rewardModelName.has_value(), false, underApproxHeuristicPar, underApproxBeliefManager, - underApproximation, false); - if (!underApproximation->hasComputedValues() || storm::utility::resources::isTerminate()) { - return; - } - ValueType const& newValue = underApproximation->getComputedValueAtInitialState(); - bool betterBound = min ? result.updateUpperBound(newValue) : result.updateLowerBound(newValue); - if (betterBound) { - STORM_LOG_INFO("Initial Under-approx result obtained after " << statistics.totalTime << ". Value is '" << newValue << "'.\n"); - } - } - - // Do some output - STORM_LOG_INFO("Completed (initial) computation. Current checktime is " << statistics.totalTime << "."); - bool computingLowerBound = false; - bool computingUpperBound = false; - if (options.discretize) { - STORM_LOG_INFO("\tOver-approx MDP has size " << overApproximation->getExploredMdp()->getNumberOfStates() << "."); - (min ? computingLowerBound : computingUpperBound) = true; - } - if (options.unfold) { - STORM_LOG_INFO("\tUnder-approx MDP has size " << underApproximation->getExploredMdp()->getNumberOfStates() << "."); - (min ? computingUpperBound : computingLowerBound) = true; - } - if (computingLowerBound && computingUpperBound) { - STORM_LOG_INFO("\tObtained result is [" << result.lowerBound << ", " << result.upperBound << "]."); - } else if (computingLowerBound) { - STORM_LOG_INFO("\tObtained result is ≥" << result.lowerBound << "."); - } else if (computingUpperBound) { - STORM_LOG_INFO("\tObtained result is ≤" << result.upperBound << "."); - } - - // Start refinement - if (options.refine) { - STORM_LOG_WARN_COND(options.refineStepLimit != 0 || !storm::utility::isZero(options.refinePrecision), - "No termination criterion for refinement given. Consider to specify a steplimit, a non-zero precisionlimit, or a timeout"); - STORM_LOG_WARN_COND(storm::utility::isZero(options.refinePrecision) || (options.unfold && options.discretize), - "Refinement goal precision is given, but only one bound is going to be refined."); - while ((options.refineStepLimit == 0 || statistics.refinementSteps.value() < options.refineStepLimit) && result.diff() > options.refinePrecision) { - bool overApproxFixPoint = true; - bool underApproxFixPoint = true; - if (options.discretize) { - // Refine over-approximation - if (min) { - overApproximation->takeCurrentValuesAsLowerBounds(); - } else { - overApproximation->takeCurrentValuesAsUpperBounds(); - } - overApproxHeuristicPar.gapThreshold *= options.gapThresholdFactor; - overApproxHeuristicPar.sizeThreshold = storm::utility::convertNumber( - storm::utility::convertNumber(overApproximation->getExploredMdp()->getNumberOfStates()) * options.sizeThresholdFactor); - overApproxHeuristicPar.observationThreshold += - options.obsThresholdIncrementFactor * (storm::utility::one() - overApproxHeuristicPar.observationThreshold); - overApproxHeuristicPar.optimalChoiceValueEpsilon *= options.optimalChoiceValueThresholdFactor; - overApproxFixPoint = buildOverApproximation(env, targetObservations, min, rewardModelName.has_value(), true, overApproxHeuristicPar, - observationResolutionVector, overApproxBeliefManager, overApproximation); - if (overApproximation->hasComputedValues() && !storm::utility::resources::isTerminate()) { - ValueType const& newValue = overApproximation->getComputedValueAtInitialState(); - bool betterBound = min ? result.updateLowerBound(newValue) : result.updateUpperBound(newValue); - if (betterBound) { - STORM_LOG_INFO("Over-approx result for refinement improved after " << statistics.totalTime << " in refinement step #" - << (statistics.refinementSteps.value() + 1) << ". New value is '" - << newValue << "'."); - } - } else { - break; - } - } - - if (options.unfold && result.diff() > options.refinePrecision) { - // Refine under-approximation - underApproxHeuristicPar.gapThreshold *= options.gapThresholdFactor; - underApproxHeuristicPar.sizeThreshold = storm::utility::convertNumber( - storm::utility::convertNumber(underApproximation->getExploredMdp()->getNumberOfStates()) * - options.sizeThresholdFactor); - underApproxHeuristicPar.optimalChoiceValueEpsilon *= options.optimalChoiceValueThresholdFactor; - underApproxFixPoint = buildUnderApproximation(env, targetObservations, min, rewardModelName.has_value(), true, underApproxHeuristicPar, - underApproxBeliefManager, underApproximation, true); - if (underApproximation->hasComputedValues() && !storm::utility::resources::isTerminate()) { - ValueType const& newValue = underApproximation->getComputedValueAtInitialState(); - bool betterBound = min ? result.updateUpperBound(newValue) : result.updateLowerBound(newValue); - if (betterBound) { - STORM_LOG_INFO("Under-approx result for refinement improved after " << statistics.totalTime << " in refinement step #" - << (statistics.refinementSteps.value() + 1) << ". New value is '" - << newValue << "'."); - } - } else { - break; - } - } - - if (storm::utility::resources::isTerminate()) { - break; - } else { - ++statistics.refinementSteps.value(); - // Don't make too many outputs (to avoid logfile clutter) - if (statistics.refinementSteps.value() <= 1000) { - STORM_LOG_INFO("Completed iteration #" << statistics.refinementSteps.value() << ". Current checktime is " << statistics.totalTime << "."); - computingLowerBound = false; - computingUpperBound = false; - if (options.discretize) { - STORM_LOG_INFO("\tOver-approx MDP has size " << overApproximation->getExploredMdp()->getNumberOfStates() << "."); - (min ? computingLowerBound : computingUpperBound) = true; - } - if (options.unfold) { - STORM_LOG_INFO("\tUnder-approx MDP has size " << underApproximation->getExploredMdp()->getNumberOfStates() << "."); - (min ? computingUpperBound : computingLowerBound) = true; - } - if (computingLowerBound && computingUpperBound) { - STORM_LOG_INFO("\tCurrent result is [" << result.lowerBound << ", " << result.upperBound << "]."); - } else if (computingLowerBound) { - STORM_LOG_INFO("\tCurrent result is ≥" << result.lowerBound << "."); - } else if (computingUpperBound) { - STORM_LOG_INFO("\tCurrent result is ≤" << result.upperBound << "."); - } - STORM_LOG_WARN_COND(statistics.refinementSteps.value() < 1000, "Refinement requires more than 1000 iterations."); - } - } - if (overApproxFixPoint && underApproxFixPoint) { - STORM_LOG_INFO("Refinement fixpoint reached after " << statistics.refinementSteps.value() << " iterations.\n"); - statistics.refinementFixpointDetected = true; - break; - } - } - } - // Print model information of final over- / under-approximation MDP - if (options.discretize && overApproximation->hasComputedValues()) { - auto printOverInfo = [&overApproximation]() { - std::stringstream str; - str << "Explored and checked Over-Approximation MDP:\n"; - overApproximation->getExploredMdp()->printModelInformationToStream(str); - return str.str(); - }; - STORM_LOG_INFO(printOverInfo()); - } - if (options.unfold && underApproximation->hasComputedValues()) { - auto printUnderInfo = [&underApproximation]() { - std::stringstream str; - str << "Explored and checked Under-Approximation MDP:\n"; - underApproximation->getExploredMdp()->printModelInformationToStream(str); - return str.str(); - }; - STORM_LOG_INFO(printUnderInfo()); - std::shared_ptr> scheduledModel = underApproximation->getExploredMdp(); - if (!options.useStateEliminationCutoff) { - storm::models::sparse::StateLabeling newLabeling(scheduledModel->getStateLabeling()); - auto nrPreprocessingScheds = min ? underApproximation->getNrSchedulersForUpperBounds() : underApproximation->getNrSchedulersForLowerBounds(); - for (uint64_t i = 0; i < nrPreprocessingScheds; ++i) { - newLabeling.addLabel("sched_" + std::to_string(i)); - } - newLabeling.addLabel("cutoff"); - newLabeling.addLabel("clipping"); - newLabeling.addLabel("finite_mem"); - - auto transMatrix = scheduledModel->getTransitionMatrix(); - for (uint64_t i = 0; i < scheduledModel->getNumberOfStates(); ++i) { - if (newLabeling.getStateHasLabel("truncated", i)) { - uint64_t localChosenActionIndex = underApproximation->getSchedulerForExploredMdp()->getChoice(i).getDeterministicChoice(); - auto rowIndex = scheduledModel->getTransitionMatrix().getRowGroupIndices()[i]; - if (scheduledModel->getChoiceLabeling().getLabelsOfChoice(rowIndex + localChosenActionIndex).size() > 0) { - auto label = *(scheduledModel->getChoiceLabeling().getLabelsOfChoice(rowIndex + localChosenActionIndex).begin()); - if (label.rfind("clip", 0) == 0) { - newLabeling.addLabelToState("clipping", i); - auto chosenRow = transMatrix.getRow(i, 0); - auto candidateIndex = (chosenRow.end() - 1)->getColumn(); - transMatrix.makeRowDirac(transMatrix.getRowGroupIndices()[i], candidateIndex); - } else if (label.rfind("mem_node", 0) == 0) { - if (!newLabeling.containsLabel("finite_mem_" + label.substr(9, 1))) { - newLabeling.addLabel("finite_mem_" + label.substr(9, 1)); - } - newLabeling.addLabelToState("finite_mem_" + label.substr(9, 1), i); - newLabeling.addLabelToState("cutoff", i); - } else { - newLabeling.addLabelToState(label, i); - newLabeling.addLabelToState("cutoff", i); - } - } - } - } - newLabeling.removeLabel("truncated"); - - transMatrix.dropZeroEntries(); - storm::storage::sparse::ModelComponents modelComponents(transMatrix, newLabeling); - if (scheduledModel->hasChoiceLabeling()) { - modelComponents.choiceLabeling = scheduledModel->getChoiceLabeling(); - } - storm::models::sparse::Mdp newMDP(modelComponents); - auto inducedMC = newMDP.applyScheduler(*(underApproximation->getSchedulerForExploredMdp()), true); - scheduledModel = std::static_pointer_cast>(inducedMC); - } else { - auto inducedMC = underApproximation->getExploredMdp()->applyScheduler(*(underApproximation->getSchedulerForExploredMdp()), true); - scheduledModel = std::static_pointer_cast>(inducedMC); - } - result.schedulerAsMarkovChain = scheduledModel; - if (min) { - result.cutoffSchedulers = underApproximation->getUpperValueBoundSchedulers(); - } else { - result.cutoffSchedulers = underApproximation->getLowerValueBoundSchedulers(); - } - } -} - -template -void BeliefExplorationPomdpModelChecker::unfoldInteractively( - storm::Environment const& env, std::set const& targetObservations, bool min, std::optional rewardModelName, - storm::pomdp::modelchecker::POMDPValueBounds const& valueBounds, Result& result) { - statistics.refinementSteps = 0; - interactiveResult = result; - unfoldingStatus = Status::Uninitialized; - unfoldingControl = UnfoldingControl::Run; - auto trivialPOMDPBounds = valueBounds.trivialPomdpValueBounds; - // Set up exploration data - std::shared_ptr underApproxBeliefManager; - HeuristicParameters underApproxHeuristicPar{}; - bool firstIteration = true; - // Set up belief manager - underApproxBeliefManager = std::make_shared( - pomdp(), storm::utility::convertNumber(options.numericPrecision), - options.dynamicTriangulation ? BeliefManagerType::TriangulationMode::Dynamic : BeliefManagerType::TriangulationMode::Static); - if (rewardModelName) { - underApproxBeliefManager->setRewardModel(rewardModelName); - } - - // set up belief MDP explorer - interactiveUnderApproximationExplorer = std::make_shared(underApproxBeliefManager, trivialPOMDPBounds, options.explorationHeuristic); - underApproxHeuristicPar.gapThreshold = options.gapThresholdInit; - underApproxHeuristicPar.optimalChoiceValueEpsilon = options.optimalChoiceValueThresholdInit; - underApproxHeuristicPar.sizeThreshold = std::numeric_limits::max() - 1; // we don't set a size threshold - - if (options.useClipping && rewardModelName.has_value()) { - interactiveUnderApproximationExplorer->setExtremeValueBound(valueBounds.extremePomdpValueBound); - } - - if (!valueBounds.fmSchedulerValueList.empty()) { - interactiveUnderApproximationExplorer->setFMSchedValueList(valueBounds.fmSchedulerValueList); - } - - // Start iteration - while (!(unfoldingControl == - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker::UnfoldingControl::Terminate)) { - bool underApproxFixPoint = true; - bool hasTruncatedStates = false; - if (unfoldingStatus != Status::Converged) { - // Continue unfolding underapproximation - underApproxFixPoint = buildUnderApproximation(env, targetObservations, min, rewardModelName.has_value(), false, underApproxHeuristicPar, - underApproxBeliefManager, interactiveUnderApproximationExplorer, firstIteration); - if (interactiveUnderApproximationExplorer->hasComputedValues() && !storm::utility::resources::isTerminate()) { - ValueType const& newValue = interactiveUnderApproximationExplorer->getComputedValueAtInitialState(); - bool betterBound = min ? interactiveResult.updateUpperBound(newValue) : interactiveResult.updateLowerBound(newValue); - if (betterBound) { - STORM_LOG_INFO("Under-approximation result improved after " << statistics.totalTime << " in step #" - << (statistics.refinementSteps.value() + 1) << ". New value is '" << newValue - << "'."); - } - std::shared_ptr> scheduledModel = interactiveUnderApproximationExplorer->getExploredMdp(); - if (!options.useStateEliminationCutoff) { - storm::models::sparse::StateLabeling newLabeling(scheduledModel->getStateLabeling()); - auto nrPreprocessingScheds = min ? interactiveUnderApproximationExplorer->getNrSchedulersForUpperBounds() - : interactiveUnderApproximationExplorer->getNrSchedulersForLowerBounds(); - for (uint64_t i = 0; i < nrPreprocessingScheds; ++i) { - newLabeling.addLabel("sched_" + std::to_string(i)); - } - newLabeling.addLabel("cutoff"); - newLabeling.addLabel("clipping"); - newLabeling.addLabel("finite_mem"); - - auto transMatrix = scheduledModel->getTransitionMatrix(); - for (uint64_t i = 0; i < scheduledModel->getNumberOfStates(); ++i) { - if (newLabeling.getStateHasLabel("truncated", i)) { - hasTruncatedStates = true; - uint64_t localChosenActionIndex = - interactiveUnderApproximationExplorer->getSchedulerForExploredMdp()->getChoice(i).getDeterministicChoice(); - auto rowIndex = scheduledModel->getTransitionMatrix().getRowGroupIndices()[i]; - if (scheduledModel->getChoiceLabeling().getLabelsOfChoice(rowIndex + localChosenActionIndex).size() > 0) { - auto label = *(scheduledModel->getChoiceLabeling().getLabelsOfChoice(rowIndex + localChosenActionIndex).begin()); - if (label.rfind("clip", 0) == 0) { - newLabeling.addLabelToState("clipping", i); - auto chosenRow = transMatrix.getRow(i, 0); - auto candidateIndex = (chosenRow.end() - 1)->getColumn(); - transMatrix.makeRowDirac(transMatrix.getRowGroupIndices()[i], candidateIndex); - } else if (label.rfind("mem_node", 0) == 0) { - if (!newLabeling.containsLabel("finite_mem_" + label.substr(9, 1))) { - newLabeling.addLabel("finite_mem_" + label.substr(9, 1)); - } - newLabeling.addLabelToState("finite_mem_" + label.substr(9, 1), i); - newLabeling.addLabelToState("cutoff", i); - } else { - newLabeling.addLabelToState(label, i); - newLabeling.addLabelToState("cutoff", i); - } - } - } - } - newLabeling.removeLabel("truncated"); - - transMatrix.dropZeroEntries(); - storm::storage::sparse::ModelComponents modelComponents(transMatrix, newLabeling); - if (scheduledModel->hasChoiceLabeling()) { - modelComponents.choiceLabeling = scheduledModel->getChoiceLabeling(); - } - storm::models::sparse::Mdp newMDP(modelComponents); - auto inducedMC = newMDP.applyScheduler(*(interactiveUnderApproximationExplorer->getSchedulerForExploredMdp()), true); - scheduledModel = std::static_pointer_cast>(inducedMC); - } - interactiveResult.schedulerAsMarkovChain = scheduledModel; - if (min) { - interactiveResult.cutoffSchedulers = interactiveUnderApproximationExplorer->getUpperValueBoundSchedulers(); - } else { - interactiveResult.cutoffSchedulers = interactiveUnderApproximationExplorer->getLowerValueBoundSchedulers(); - } - if (firstIteration) { - firstIteration = false; - } - unfoldingStatus = Status::ResultAvailable; - } else { - break; - } - - if (storm::utility::resources::isTerminate()) { - break; - } else { - ++statistics.refinementSteps.value(); - // Don't make too many outputs (to avoid logfile clutter) - if (statistics.refinementSteps.value() <= 1000) { - STORM_LOG_INFO("Completed iteration #" << statistics.refinementSteps.value() << ". Current checktime is " << statistics.totalTime << "."); - bool computingLowerBound = false; - bool computingUpperBound = false; - if (options.unfold) { - STORM_LOG_INFO("\tUnder-approx MDP has size " << interactiveUnderApproximationExplorer->getExploredMdp()->getNumberOfStates() << "."); - (min ? computingUpperBound : computingLowerBound) = true; - } - if (computingLowerBound && computingUpperBound) { - STORM_LOG_INFO("\tCurrent result is [" << interactiveResult.lowerBound << ", " << interactiveResult.upperBound << "]."); - } else if (computingLowerBound) { - STORM_LOG_INFO("\tCurrent result is ≥" << interactiveResult.lowerBound << "."); - } else if (computingUpperBound) { - STORM_LOG_INFO("\tCurrent result is ≤" << interactiveResult.upperBound << "."); - } - } - } - if (underApproxFixPoint) { - STORM_LOG_INFO("Fixpoint reached after " << statistics.refinementSteps.value() << " iterations.\n"); - statistics.refinementFixpointDetected = true; - unfoldingStatus = Status::Converged; - unfoldingControl = UnfoldingControl::Pause; - } - if (!hasTruncatedStates) { - STORM_LOG_INFO("No states have been truncated, so continued iteration does not yield new results.\n"); - unfoldingStatus = Status::Converged; - unfoldingControl = UnfoldingControl::Pause; - } - } - // While we tell the procedure to be paused, idle - while (unfoldingControl == - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker::UnfoldingControl::Pause && - !storm::utility::resources::isTerminate()) { - // Intentionally left empty - } - } - STORM_LOG_INFO("\tInteractive Unfolding terminated.\n"); -} - -template -void BeliefExplorationPomdpModelChecker::unfoldInteractively( - std::set const& targetObservations, bool min, std::optional rewardModelName, - storm::pomdp::modelchecker::POMDPValueBounds const& valueBounds, Result& result) { - storm::Environment env; - unfoldInteractively(env, targetObservations, min, rewardModelName, valueBounds, result); -} - -template -typename BeliefExplorationPomdpModelChecker::Result -BeliefExplorationPomdpModelChecker::getInteractiveResult() { - return interactiveResult; -} - -template -int64_t BeliefExplorationPomdpModelChecker::getStatus() { - if (unfoldingStatus == Status::Uninitialized) - return 0; - if (unfoldingStatus == Status::Exploring) - return 1; - if (unfoldingStatus == Status::ModelExplorationFinished) - return 2; - if (unfoldingStatus == Status::ResultAvailable) - return 3; - if (unfoldingStatus == Status::Terminated) - return 4; - - return -1; -} - -template -ValueType getGap(ValueType const& l, ValueType const& u) { - STORM_LOG_ASSERT(l >= storm::utility::zero() && u >= storm::utility::zero(), - "Gap computation currently does not handle negative values."); - if (storm::utility::isInfinity(u)) { - if (storm::utility::isInfinity(l)) { - return storm::utility::zero(); - } else { - return u; - } - } else if (storm::utility::isZero(u)) { - STORM_LOG_ASSERT(storm::utility::isZero(l), "Upper bound is zero but lower bound is " << l << "."); - return u; - } else { - STORM_LOG_ASSERT(!storm::utility::isInfinity(l), "Lower bound is infinity, but upper bound is " << u << "."); - // get the relative gap - return storm::utility::abs(u - l) * storm::utility::convertNumber(2) / (l + u); - } -} - -template -bool BeliefExplorationPomdpModelChecker::buildOverApproximation( - storm::Environment const& env, std::set const& targetObservations, bool min, bool computeRewards, bool refine, - HeuristicParameters const& heuristicParameters, std::vector& observationResolutionVector, - std::shared_ptr& beliefManager, std::shared_ptr& overApproximation) { - // Detect whether the refinement reached a fixpoint. - bool fixPoint = true; - - statistics.overApproximationBuildTime.start(); - storm::storage::BitVector refinedObservations; - if (!refine) { - // If we build the model from scratch, we first have to set up the explorer for the overApproximation. - if (computeRewards) { - overApproximation->startNewExploration(storm::utility::zero()); - } else { - overApproximation->startNewExploration(storm::utility::one(), storm::utility::zero()); - } - } else { - // If we refine the existing overApproximation, our heuristic also wants to know which states are reachable under an optimal policy - overApproximation->computeOptimalChoicesAndReachableMdpStates(heuristicParameters.optimalChoiceValueEpsilon, true); - // We also need to find out which observation resolutions needs refinement. - // current maximal resolution (needed for refinement heuristic) - auto obsRatings = getObservationRatings(overApproximation, observationResolutionVector); - // If there is a score < 1, we have not reached a fixpoint, yet - auto numericPrecision = storm::utility::convertNumber(options.numericPrecision); - if (std::any_of(obsRatings.begin(), obsRatings.end(), - [&numericPrecision](BeliefValueType const& value) { return value + numericPrecision < storm::utility::one(); })) { - STORM_LOG_INFO_COND(!fixPoint, "Not reaching a refinement fixpoint because there are still observations to refine."); - fixPoint = false; - } - refinedObservations = storm::utility::vector::filter(obsRatings, [&heuristicParameters](BeliefValueType const& r) { - return r <= storm::utility::convertNumber(heuristicParameters.observationThreshold); - }); - STORM_LOG_DEBUG("Refining the resolution of " << refinedObservations.getNumberOfSetBits() << "/" << refinedObservations.size() << " observations."); - for (auto const obs : refinedObservations) { - // Increment the resolution at the refined observations. - // Use storm's rational number to detect overflows properly. - storm::RationalNumber newObsResolutionAsRational = storm::utility::convertNumber(observationResolutionVector[obs]) * - storm::utility::convertNumber(options.resolutionFactor); - static_assert(storm::NumberTraits::IsExact || std::is_same::value, "Unhandled belief value type"); - if (!storm::NumberTraits::IsExact && - newObsResolutionAsRational > storm::utility::convertNumber(std::numeric_limits::max())) { - observationResolutionVector[obs] = storm::utility::convertNumber(std::numeric_limits::max()); - } else { - observationResolutionVector[obs] = storm::utility::convertNumber(newObsResolutionAsRational); - } - } - overApproximation->restartExploration(); - } - statistics.overApproximationMaxResolution = storm::utility::ceil(*std::max_element(observationResolutionVector.begin(), observationResolutionVector.end())); - - // Start exploration - storm::utility::Stopwatch explorationTime; - if (options.explorationTimeLimit != 0) { - explorationTime.start(); - } - bool timeLimitExceeded = false; - std::map gatheredSuccessorObservations; // Declare here to avoid reallocations - uint64_t numRewiredOrExploredStates = 0; - while (overApproximation->hasUnexploredState()) { - if (!timeLimitExceeded && options.explorationTimeLimit != 0 && - static_cast(explorationTime.getTimeInSeconds()) > options.explorationTimeLimit) { - STORM_LOG_INFO("Exploration time limit exceeded."); - timeLimitExceeded = true; - STORM_LOG_INFO_COND(!fixPoint, "Not reaching a refinement fixpoint because the exploration time limit is exceeded."); - fixPoint = false; - } - - uint64_t currId = overApproximation->exploreNextState(); - bool hasOldBehavior = refine && overApproximation->currentStateHasOldBehavior(); - if (!hasOldBehavior) { - STORM_LOG_INFO_COND(!fixPoint, "Not reaching a refinement fixpoint because a new state is explored"); - fixPoint = false; // Exploring a new state! - } - uint32_t currObservation = beliefManager->getBeliefObservation(currId); - if (targetObservations.count(currObservation) != 0) { - overApproximation->setCurrentStateIsTarget(); - overApproximation->addSelfloopTransition(); - } else { - // We need to decide how to treat this state (and each individual enabled action). There are the following cases: - // 1 The state has no old behavior and - // 1.1 we explore all actions or - // 1.2 we truncate all actions - // 2 The state has old behavior and was truncated in the last iteration and - // 2.1 we explore all actions or - // 2.2 we truncate all actions (essentially restoring old behavior, but we do the truncation step again to benefit from updated bounds) - // 3 The state has old behavior and was not truncated in the last iteration and the current action - // 3.1 should be rewired or - // 3.2 should get the old behavior but either - // 3.2.1 none of the successor observation has been refined since the last rewiring or exploration of this action - // 3.2.2 rewiring is only delayed as it could still have an effect in a later refinement step - - // Find out in which case we are - bool exploreAllActions = false; - bool truncateAllActions = false; - bool restoreAllActions = false; - bool checkRewireForAllActions = false; - // Get the relative gap - ValueType gap = getGap(overApproximation->getLowerValueBoundAtCurrentState(), overApproximation->getUpperValueBoundAtCurrentState()); - if (!hasOldBehavior) { - // Case 1 - // If we explore this state and if it has no old behavior, it is clear that an "old" optimal scheduler can be extended to a scheduler that - // reaches this state - if (!timeLimitExceeded && gap >= heuristicParameters.gapThreshold && numRewiredOrExploredStates < heuristicParameters.sizeThreshold) { - exploreAllActions = true; // Case 1.1 - } else { - truncateAllActions = true; // Case 1.2 - overApproximation->setCurrentStateIsTruncated(); - } - } else if (overApproximation->getCurrentStateWasTruncated()) { - // Case 2 - if (!timeLimitExceeded && overApproximation->currentStateIsOptimalSchedulerReachable() && gap > heuristicParameters.gapThreshold && - numRewiredOrExploredStates < heuristicParameters.sizeThreshold) { - exploreAllActions = true; // Case 2.1 - STORM_LOG_INFO_COND(!fixPoint, "Not reaching a refinement fixpoint because a previously truncated state is now explored."); - fixPoint = false; - } else { - truncateAllActions = true; // Case 2.2 - overApproximation->setCurrentStateIsTruncated(); - if (fixPoint) { - // Properly check whether this can still be a fixpoint - if (overApproximation->currentStateIsOptimalSchedulerReachable() && !storm::utility::isZero(gap)) { - STORM_LOG_INFO_COND(!fixPoint, "Not reaching a refinement fixpoint because we truncate a state with non-zero gap " - << gap << " that is reachable via an optimal sched."); - fixPoint = false; - } - // else {} - // In this case we truncated a state that is not reachable under optimal schedulers. - // If no other state is explored (i.e. fixPoint remains true), these states should still not be reachable in subsequent iterations - } - } - } else { - // Case 3 - // The decision for rewiring also depends on the corresponding action, but we have some criteria that lead to case 3.2 (independent of the - // action) - if (!timeLimitExceeded && overApproximation->currentStateIsOptimalSchedulerReachable() && gap > heuristicParameters.gapThreshold && - numRewiredOrExploredStates < heuristicParameters.sizeThreshold) { - checkRewireForAllActions = true; // Case 3.1 or Case 3.2 - } else { - restoreAllActions = true; // Definitely Case 3.2 - // We still need to check for each action whether rewiring makes sense later - checkRewireForAllActions = true; - } - } - bool expandedAtLeastOneAction = false; - for (uint64_t action = 0, numActions = beliefManager->getBeliefNumberOfChoices(currId); action < numActions; ++action) { - bool expandCurrentAction = exploreAllActions || truncateAllActions; - if (checkRewireForAllActions) { - STORM_LOG_ASSERT(refine, "Expected refine to be true."); - // In this case, we still need to check whether this action needs to be expanded - STORM_LOG_ASSERT(!expandCurrentAction, "Action should not be expanded."); - // Check the action dependent conditions for rewiring - // First, check whether this action has been rewired since the last refinement of one of the successor observations (i.e. whether rewiring - // would actually change the successor states) - STORM_LOG_ASSERT(overApproximation->currentStateHasOldBehavior(), "Expected old behavior."); - if (overApproximation->getCurrentStateActionExplorationWasDelayed(action) || - overApproximation->currentStateHasSuccessorObservationInObservationSet(action, refinedObservations)) { - // Then, check whether the other criteria for rewiring are satisfied - if (!restoreAllActions && overApproximation->actionAtCurrentStateWasOptimal(action)) { - // Do the rewiring now! (Case 3.1) - expandCurrentAction = true; - STORM_LOG_INFO_COND(!fixPoint, "Not reaching a refinement fixpoint because we rewire a state."); - fixPoint = false; - } else { - // Delay the rewiring (Case 3.2.2) - overApproximation->setCurrentChoiceIsDelayed(action); - if (fixPoint) { - // Check whether this delay means that a fixpoint has not been reached - if (!overApproximation->getCurrentStateActionExplorationWasDelayed(action) || - (overApproximation->currentStateIsOptimalSchedulerReachable() && - overApproximation->actionAtCurrentStateWasOptimal(action) && !storm::utility::isZero(gap))) { - STORM_LOG_INFO_COND(!fixPoint, - "Not reaching a refinement fixpoint because we delay a rewiring of a state with non-zero gap " - << gap << " that is reachable via an optimal scheduler."); - fixPoint = false; - } - } - } - } // else { Case 3.2.1 } - } - - if (expandCurrentAction) { - expandedAtLeastOneAction = true; - if (!truncateAllActions) { - // Cases 1.1, 2.1, or 3.1 - auto successorGridPoints = beliefManager->expandAndTriangulate(env, currId, action, observationResolutionVector); - for (auto const& successor : successorGridPoints) { - overApproximation->addTransitionToBelief(action, successor.first, successor.second, false); - } - if (computeRewards) { - overApproximation->computeRewardAtCurrentState(action); - } - } else { - // Cases 1.2 or 2.2 - auto truncationProbability = storm::utility::zero(); - auto truncationValueBound = storm::utility::zero(); - auto successorGridPoints = beliefManager->expandAndTriangulate(env, currId, action, observationResolutionVector); - for (auto const& successor : successorGridPoints) { - bool added = overApproximation->addTransitionToBelief(action, successor.first, successor.second, true); - if (!added) { - // We did not explore this successor state. Get a bound on the "missing" value - truncationProbability += successor.second; - truncationValueBound += successor.second * (min ? overApproximation->computeLowerValueBoundAtBelief(successor.first) - : overApproximation->computeUpperValueBoundAtBelief(successor.first)); - } - } - if (computeRewards) { - // The truncationValueBound will be added on top of the reward introduced by the current belief state. - overApproximation->addTransitionsToExtraStates(action, truncationProbability); - overApproximation->computeRewardAtCurrentState(action, truncationValueBound); - } else { - overApproximation->addTransitionsToExtraStates(action, truncationValueBound, truncationProbability - truncationValueBound); - } - } - } else { - // Case 3.2 - overApproximation->restoreOldBehaviorAtCurrentState(action); - } - } - if (expandedAtLeastOneAction) { - ++numRewiredOrExploredStates; - } - } - - for (uint64_t action = 0, numActions = beliefManager->getBeliefNumberOfChoices(currId); action < numActions; ++action) { - if (pomdp().hasChoiceLabeling()) { - auto rowIndex = pomdp().getTransitionMatrix().getRowGroupIndices()[beliefManager->getRepresentativeState(currId)]; - if (pomdp().getChoiceLabeling().getLabelsOfChoice(rowIndex + action).size() > 0) { - overApproximation->addChoiceLabelToCurrentState(action, *(pomdp().getChoiceLabeling().getLabelsOfChoice(rowIndex + action).begin())); - } - } - } - - if (storm::utility::resources::isTerminate()) { - break; - } - } - - if (storm::utility::resources::isTerminate()) { - // don't overwrite statistics of a previous, successful computation - if (!statistics.overApproximationStates) { - statistics.overApproximationBuildAborted = true; - statistics.overApproximationStates = overApproximation->getCurrentNumberOfMdpStates(); - } - statistics.overApproximationBuildTime.stop(); - return false; - } - - overApproximation->finishExploration(); - statistics.overApproximationBuildTime.stop(); - - statistics.overApproximationCheckTime.start(); - overApproximation->computeValuesOfExploredMdp(env, min ? storm::solver::OptimizationDirection::Minimize : storm::solver::OptimizationDirection::Maximize); - statistics.overApproximationCheckTime.stop(); - - // don't overwrite statistics of a previous, successful computation - if (!storm::utility::resources::isTerminate() || !statistics.overApproximationStates) { - statistics.overApproximationStates = overApproximation->getExploredMdp()->getNumberOfStates(); - } - return fixPoint; -} - -template -bool BeliefExplorationPomdpModelChecker::buildUnderApproximation( - storm::Environment const& env, std::set const& targetObservations, bool min, bool computeRewards, bool refine, - HeuristicParameters const& heuristicParameters, std::shared_ptr& beliefManager, std::shared_ptr& underApproximation, - bool firstIteration) { - statistics.underApproximationBuildTime.start(); - - unfoldingStatus = Status::Exploring; - if (options.useClipping) { - STORM_LOG_INFO("Use Belief Clipping with grid beliefs \n"); - statistics.nrClippingAttempts = 0; - statistics.nrClippedStates = 0; - } - - uint64_t nrCutoffStrategies = min ? underApproximation->getNrSchedulersForUpperBounds() : underApproximation->getNrSchedulersForLowerBounds(); - - bool fixPoint = true; - if (heuristicParameters.sizeThreshold != std::numeric_limits::max()) { - statistics.underApproximationStateLimit = heuristicParameters.sizeThreshold; - } - if (!refine) { - if (options.interactiveUnfolding && !firstIteration) { - underApproximation->restoreExplorationState(); - } else if (computeRewards) { // Build a new under approximation - // We use the sink state for infinite cut-off/clipping values - underApproximation->startNewExploration(storm::utility::zero(), storm::utility::infinity()); - } else { - underApproximation->startNewExploration(storm::utility::one(), storm::utility::zero()); - } - } else { - // Restart the building process - underApproximation->restartExploration(); - } - - // Expand the beliefs - storm::utility::Stopwatch explorationTime; - storm::utility::Stopwatch printUpdateStopwatch; - printUpdateStopwatch.start(); - if (options.explorationTimeLimit != 0) { - explorationTime.start(); - } - bool timeLimitExceeded = false; - bool stateStored = false; - while (underApproximation->hasUnexploredState()) { - if (!timeLimitExceeded && options.explorationTimeLimit != 0 && - static_cast(explorationTime.getTimeInSeconds()) > options.explorationTimeLimit) { - STORM_LOG_INFO("Exploration time limit exceeded."); - timeLimitExceeded = true; - } - if (printUpdateStopwatch.getTimeInSeconds() >= 60) { - printUpdateStopwatch.restart(); - STORM_LOG_INFO("### " << underApproximation->getCurrentNumberOfMdpStates() << " beliefs in underapproximation MDP" - << " ##### " << underApproximation->getUnexploredStates().size() << " beliefs queued\n"); - if (underApproximation->getCurrentNumberOfMdpStates() > heuristicParameters.sizeThreshold && options.useClipping) { - STORM_LOG_INFO("##### Clipping Attempts: " << statistics.nrClippingAttempts.value() << " ##### " - << "Clipped States: " << statistics.nrClippedStates.value() << "\n"); - } - } - if (unfoldingControl == UnfoldingControl::Pause && !stateStored) { - underApproximation->storeExplorationState(); - stateStored = true; - } - uint64_t currId = underApproximation->exploreNextState(); - uint32_t currObservation = beliefManager->getBeliefObservation(currId); - uint64_t addedActions = 0; - bool stateAlreadyExplored = refine && underApproximation->currentStateHasOldBehavior() && !underApproximation->getCurrentStateWasTruncated(); - if (!stateAlreadyExplored || timeLimitExceeded) { - fixPoint = false; - } - if (targetObservations.count(beliefManager->getBeliefObservation(currId)) != 0) { - underApproximation->setCurrentStateIsTarget(); - underApproximation->addSelfloopTransition(); - underApproximation->addChoiceLabelToCurrentState(0, "loop"); - } else { - bool stopExploration = false; - bool clipBelief = false; - if (timeLimitExceeded || (options.interactiveUnfolding && unfoldingControl != UnfoldingControl::Run)) { - clipBelief = options.useClipping; - stopExploration = !underApproximation->isMarkedAsGridBelief(currId); - } else if (!stateAlreadyExplored) { - // Check whether we want to explore the state now! - ValueType gap = getGap(underApproximation->getLowerValueBoundAtCurrentState(), underApproximation->getUpperValueBoundAtCurrentState()); - if ((gap < heuristicParameters.gapThreshold) || (gap == 0 && options.cutZeroGap)) { - stopExploration = true; - } else if (underApproximation->getCurrentNumberOfMdpStates() >= - heuristicParameters.sizeThreshold /*&& !statistics.beliefMdpDetectedToBeFinite*/) { - clipBelief = options.useClipping; - stopExploration = !underApproximation->isMarkedAsGridBelief(currId); - } - } - - if (clipBelief && !underApproximation->isMarkedAsGridBelief(currId)) { - // Use a belief grid as clipping candidates - if (!options.useStateEliminationCutoff) { - bool successfulClip = clipToGridExplicitly(env, currId, computeRewards, beliefManager, underApproximation, 0); - // Set again as the current belief might have been detected to be a grid belief - stopExploration = !underApproximation->isMarkedAsGridBelief(currId); - if (successfulClip) { - addedActions += 1; - } - } else { - clipToGrid(env, currId, computeRewards, min, beliefManager, underApproximation); - addedActions += beliefManager->getBeliefNumberOfChoices(currId); - } - } // end Clipping Procedure - - if (stopExploration) { - underApproximation->setCurrentStateIsTruncated(); - } - if (options.useStateEliminationCutoff || !stopExploration) { - // Add successor transitions or cut-off transitions when exploration is stopped - uint64_t numActions = beliefManager->getBeliefNumberOfChoices(currId); - if (underApproximation->needsActionAdjustment(numActions)) { - underApproximation->adjustActions(numActions); - } - for (uint64_t action = 0; action < numActions; ++action) { - // Always restore old behavior if available - if (pomdp().hasChoiceLabeling()) { - auto rowIndex = pomdp().getTransitionMatrix().getRowGroupIndices()[beliefManager->getRepresentativeState(currId)]; - if (pomdp().getChoiceLabeling().getLabelsOfChoice(rowIndex + action).size() > 0) { - underApproximation->addChoiceLabelToCurrentState(addedActions + action, - *(pomdp().getChoiceLabeling().getLabelsOfChoice(rowIndex + action).begin())); - } - } - if (stateAlreadyExplored) { - underApproximation->restoreOldBehaviorAtCurrentState(action); - } else { - auto truncationProbability = storm::utility::zero(); - auto truncationValueBound = storm::utility::zero(); - auto successors = beliefManager->expand(env, currId, action); - for (auto const& successor : successors) { - bool added = underApproximation->addTransitionToBelief(addedActions + action, successor.first, successor.second, stopExploration); - if (!added) { - STORM_LOG_ASSERT(stopExploration, "Didn't add a transition although exploration shouldn't be stopped."); - // We did not explore this successor state. Get a bound on the "missing" value - truncationProbability += successor.second; - // Some care has to be taken here: Essentially, we are triangulating a value for the under-approximation out of - // other under-approximation values. In general, this does not yield a sound underapproximation anymore as the - // values might be achieved by different schedulers. However, in our case this is still the case as the - // under-approximation values are based on a single memory-less scheduler. - truncationValueBound += successor.second * (min ? underApproximation->computeUpperValueBoundAtBelief(successor.first) - : underApproximation->computeLowerValueBoundAtBelief(successor.first)); - } - } - if (stopExploration) { - if (computeRewards) { - if (truncationValueBound == storm::utility::infinity()) { - underApproximation->addTransitionsToExtraStates(addedActions + action, storm::utility::zero(), - truncationProbability); - } else { - underApproximation->addTransitionsToExtraStates(addedActions + action, truncationProbability); - } - } else { - underApproximation->addTransitionsToExtraStates(addedActions + action, truncationValueBound, - truncationProbability - truncationValueBound); - } - } - if (computeRewards) { - // The truncationValueBound will be added on top of the reward introduced by the current belief state. - if (truncationValueBound != storm::utility::infinity()) { - if (!clipBelief) { - underApproximation->computeRewardAtCurrentState(action, truncationValueBound); - } else { - underApproximation->addRewardToCurrentState(addedActions + action, - beliefManager->getBeliefActionReward(currId, action) + truncationValueBound); - } - } - } - } - } - } else { - for (uint64_t i = 0; i < nrCutoffStrategies && !options.skipHeuristicSchedulers; ++i) { - auto cutOffValue = min ? underApproximation->computeUpperValueBoundForScheduler(currId, i) - : underApproximation->computeLowerValueBoundForScheduler(currId, i); - if (computeRewards) { - if (cutOffValue != storm::utility::infinity()) { - underApproximation->addTransitionsToExtraStates(addedActions, storm::utility::one()); - underApproximation->addRewardToCurrentState(addedActions, cutOffValue); - } else { - underApproximation->addTransitionsToExtraStates(addedActions, storm::utility::zero(), storm::utility::one()); - } - } else { - underApproximation->addTransitionsToExtraStates(addedActions, cutOffValue, storm::utility::one() - cutOffValue); - } - if (pomdp().hasChoiceLabeling()) { - underApproximation->addChoiceLabelToCurrentState(addedActions, "sched_" + std::to_string(i)); - } - addedActions++; - } - if (underApproximation->hasFMSchedulerValues()) { - uint64_t transitionNr = 0; - for (uint64_t i = 0; i < underApproximation->getNrOfMemoryNodesForObservation(currObservation); ++i) { - auto resPair = underApproximation->computeFMSchedulerValueForMemoryNode(currId, i); - ValueType cutOffValue; - if (resPair.first) { - cutOffValue = resPair.second; - } else { - STORM_LOG_DEBUG("Skipped cut-off of belief with ID " << currId << " with finite memory scheduler in memory node " << i - << ". Missing values."); - continue; - } - if (computeRewards) { - if (cutOffValue != storm::utility::infinity()) { - underApproximation->addTransitionsToExtraStates(addedActions + transitionNr, storm::utility::one()); - underApproximation->addRewardToCurrentState(addedActions + transitionNr, cutOffValue); - } else { - underApproximation->addTransitionsToExtraStates(addedActions + transitionNr, storm::utility::zero(), - storm::utility::one()); - } - } else { - underApproximation->addTransitionsToExtraStates(addedActions + transitionNr, cutOffValue, - storm::utility::one() - cutOffValue); - } - if (pomdp().hasChoiceLabeling()) { - underApproximation->addChoiceLabelToCurrentState(addedActions + transitionNr, "mem_node_" + std::to_string(i)); - } - ++transitionNr; - } - } - } - } - if (storm::utility::resources::isTerminate()) { - break; - } - } - - if (storm::utility::resources::isTerminate()) { - // don't overwrite statistics of a previous, successful computation - if (!statistics.underApproximationStates) { - statistics.underApproximationBuildAborted = true; - statistics.underApproximationStates = underApproximation->getCurrentNumberOfMdpStates(); - } - statistics.underApproximationBuildTime.stop(); - return false; - } - - underApproximation->finishExploration(); - statistics.underApproximationBuildTime.stop(); - printUpdateStopwatch.stop(); - STORM_LOG_INFO("Finished exploring under-approximation MDP.\nStart analysis...\n"); - unfoldingStatus = Status::ModelExplorationFinished; - statistics.underApproximationCheckTime.start(); - underApproximation->computeValuesOfExploredMdp(env, min ? storm::solver::OptimizationDirection::Minimize : storm::solver::OptimizationDirection::Maximize); - statistics.underApproximationCheckTime.stop(); - if (underApproximation->getExploredMdp()->getStateLabeling().getStates("truncated").getNumberOfSetBits() > 0) { - statistics.nrTruncatedStates = underApproximation->getExploredMdp()->getStateLabeling().getStates("truncated").getNumberOfSetBits(); - } - // don't overwrite statistics of a previous, successful computation - if (!storm::utility::resources::isTerminate() || !statistics.underApproximationStates) { - statistics.underApproximationStates = underApproximation->getExploredMdp()->getNumberOfStates(); - } - return fixPoint; -} - -template -void BeliefExplorationPomdpModelChecker::clipToGrid(storm::Environment const& env, uint64_t clippingStateId, - bool computeRewards, bool min, - std::shared_ptr& beliefManager, - std::shared_ptr& beliefExplorer) { - // Add all transitions to states which are already in the MDP, clip all others to a grid - // To make the resulting MDP smaller, we eliminate intermediate successor states when clipping is applied - for (uint64_t action = 0, numActions = beliefManager->getBeliefNumberOfChoices(clippingStateId); action < numActions; ++action) { - auto rewardBound = utility::zero(); - auto successors = beliefManager->expand(env, clippingStateId, action); - auto absDelta = utility::zero(); - for (auto const& successor : successors) { - // Add transition if successor is in explored space. - // We can directly add the transitions as there is at most one successor for each observation - // Therefore no belief can be clipped to an already added successor - bool added = beliefExplorer->addTransitionToBelief(action, successor.first, successor.second, true); - if (!added) { - // The successor is not in the explored space. Clip it - statistics.nrClippingAttempts = statistics.nrClippingAttempts.value() + 1; - auto clipping = - beliefManager->clipBeliefToGrid(env, successor.first, options.clippingGridRes, - computeRewards ? beliefExplorer->getStateExtremeBoundIsInfinite() : storm::storage::BitVector()); - if (clipping.isClippable) { - // The belief is not on the grid and there is a candidate with finite reward - statistics.nrClippedStates = statistics.nrClippedStates.value() + 1; - // Transition probability to candidate is (probability to successor) * (clipping transition probability) - BeliefValueType transitionProb = - (utility::one() - clipping.delta) * utility::convertNumber(successor.second); - beliefExplorer->addTransitionToBelief(action, clipping.targetBelief, utility::convertNumber(transitionProb), false); - // Collect weighted clipping values - absDelta += clipping.delta * utility::convertNumber(successor.second); - if (computeRewards) { - // collect cumulative reward bounds - auto localRew = utility::zero(); - for (auto const& deltaValue : clipping.deltaValues) { - localRew += deltaValue.second * - utility::convertNumber((beliefExplorer->getExtremeValueBoundAtPOMDPState(deltaValue.first))); - } - if (localRew == utility::infinity()) { - STORM_LOG_WARN("Infinite reward in clipping!"); - } - rewardBound += localRew * utility::convertNumber(successor.second); - } - } else if (clipping.onGrid) { - // If the belief is not clippable, but on the grid, it may need to be explored, too - beliefExplorer->addTransitionToBelief(action, successor.first, successor.second, false); - } else { - // Otherwise, the reward for all candidates is infinite, clipping does not make sense. Cut it off instead - absDelta += utility::convertNumber(successor.second); - rewardBound += utility::convertNumber(successor.second) * - utility::convertNumber(min ? beliefExplorer->computeUpperValueBoundAtBelief(successor.first) - : beliefExplorer->computeLowerValueBoundAtBelief(successor.first)); - } - } - } - // Add the collected clipping transition if necessary - if (absDelta != utility::zero()) { - if (computeRewards) { - if (rewardBound == utility::infinity()) { - // If the reward is infinite, add a transition to the sink state to collect infinite reward - beliefExplorer->addTransitionsToExtraStates(action, utility::zero(), utility::convertNumber(absDelta)); - } else { - beliefExplorer->addTransitionsToExtraStates(action, utility::convertNumber(absDelta)); - BeliefValueType totalRewardVal = rewardBound / absDelta; - beliefExplorer->addClippingRewardToCurrentState(action, utility::convertNumber(totalRewardVal)); - } - } else { - beliefExplorer->addTransitionsToExtraStates(action, utility::zero(), utility::convertNumber(absDelta)); - } - } - if (computeRewards) { - beliefExplorer->computeRewardAtCurrentState(action); - } - } -} - -template -bool BeliefExplorationPomdpModelChecker::clipToGridExplicitly(storm::Environment const& env, - uint64_t clippingStateId, bool computeRewards, - std::shared_ptr& beliefManager, - std::shared_ptr& beliefExplorer, - uint64_t localActionIndex) { - statistics.nrClippingAttempts = statistics.nrClippingAttempts.value() + 1; - auto clipping = beliefManager->clipBeliefToGrid(env, clippingStateId, options.clippingGridRes, - computeRewards ? beliefExplorer->getStateExtremeBoundIsInfinite() : storm::storage::BitVector()); - if (clipping.isClippable) { - // The belief is not on the grid and there is a candidate with finite reward - statistics.nrClippedStates = statistics.nrClippedStates.value() + 1; - // Transition probability to candidate is clipping value - BeliefValueType transitionProb = (utility::one() - clipping.delta); - beliefExplorer->addTransitionToBelief(localActionIndex, clipping.targetBelief, utility::convertNumber(transitionProb), false); - beliefExplorer->markAsGridBelief(clipping.targetBelief); - if (computeRewards) { - // collect cumulative reward bounds - auto reward = utility::zero(); - for (auto const& deltaValue : clipping.deltaValues) { - reward += deltaValue.second * utility::convertNumber((beliefExplorer->getExtremeValueBoundAtPOMDPState(deltaValue.first))); - } - if (reward == utility::infinity()) { - STORM_LOG_WARN("Infinite reward in clipping!"); - // If the reward is infinite, add a transition to the sink state to collect infinite reward in our semantics - beliefExplorer->addTransitionsToExtraStates(localActionIndex, utility::zero(), - utility::convertNumber(clipping.delta)); - } else { - beliefExplorer->addTransitionsToExtraStates(localActionIndex, utility::convertNumber(clipping.delta)); - BeliefValueType totalRewardVal = reward / clipping.delta; - beliefExplorer->addClippingRewardToCurrentState(localActionIndex, utility::convertNumber(totalRewardVal)); - } - } else { - beliefExplorer->addTransitionsToExtraStates(localActionIndex, utility::zero(), - utility::convertNumber(clipping.delta)); - } - beliefExplorer->addChoiceLabelToCurrentState(localActionIndex, "clip"); - return true; - } else { - if (clipping.onGrid) { - // If the belief is not clippable, but on the grid, it may need to be explored, too - beliefExplorer->markAsGridBelief(clippingStateId); - } - } - return false; -} - -template -void BeliefExplorationPomdpModelChecker::setUnfoldingControl( - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker::UnfoldingControl newUnfoldingControl) { - unfoldingControl = newUnfoldingControl; -} - -template -void BeliefExplorationPomdpModelChecker::pauseUnfolding() { - STORM_LOG_TRACE("PAUSE COMMAND ISSUED"); - setUnfoldingControl(UnfoldingControl::Pause); -} - -template -void BeliefExplorationPomdpModelChecker::continueUnfolding() { - STORM_LOG_TRACE("CONTINUATION COMMAND ISSUED"); - setUnfoldingControl(UnfoldingControl::Run); -} - -template -void BeliefExplorationPomdpModelChecker::terminateUnfolding() { - STORM_LOG_TRACE("TERMINATION COMMAND ISSUED"); - setUnfoldingControl(UnfoldingControl::Terminate); -} - -template -bool BeliefExplorationPomdpModelChecker::isResultReady() { - return unfoldingStatus == Status::ResultAvailable || unfoldingStatus == Status::Converged; -} - -template -bool BeliefExplorationPomdpModelChecker::hasConverged() { - return unfoldingStatus == Status::Converged; -} - -template -bool BeliefExplorationPomdpModelChecker::isExploring() { - return unfoldingStatus == Status::Exploring; -} - -template -std::shared_ptr> -BeliefExplorationPomdpModelChecker::getInteractiveBeliefExplorer() { - return interactiveUnderApproximationExplorer; -} - -template -void BeliefExplorationPomdpModelChecker::setFMSchedValueList( - std::vector>> valueList) { - interactiveUnderApproximationExplorer->setFMSchedValueList(valueList); -} - -template -BeliefValueType BeliefExplorationPomdpModelChecker::rateObservation( - typename ExplorerType::SuccessorObservationInformation const& info, BeliefValueType const& observationResolution, BeliefValueType const& maxResolution) { - auto n = storm::utility::convertNumber(info.support.size()); - auto one = storm::utility::one(); - if (storm::utility::isOne(n)) { - // If the belief is Dirac, it has to be approximated precisely. - // In this case, we return the best possible rating - return one; - } else { - // Create the rating for this observation at this choice from the given info - auto obsChoiceRating = storm::utility::convertNumber(info.maxProbabilityToSuccessorWithObs / info.observationProbability); - // At this point, obsRating is the largest triangulation weight (which ranges from 1/n to 1) - // Normalize the rating so that it ranges from 0 to 1, where - // 0 means that the actual belief lies in the middle of the triangulating simplex (i.e. a "bad" approximation) and 1 means that the belief is precisely - // approximated. - obsChoiceRating = (obsChoiceRating * n - one) / (n - one); - // Scale the ratings with the resolutions, so that low resolutions get a lower rating (and are thus more likely to be refined) - obsChoiceRating *= observationResolution / maxResolution; - return obsChoiceRating; - } -} - -template -std::vector BeliefExplorationPomdpModelChecker::getObservationRatings( - std::shared_ptr const& overApproximation, std::vector const& observationResolutionVector) { - uint64_t numMdpStates = overApproximation->getExploredMdp()->getNumberOfStates(); - auto const& choiceIndices = overApproximation->getExploredMdp()->getNondeterministicChoiceIndices(); - BeliefValueType maxResolution = *std::max_element(observationResolutionVector.begin(), observationResolutionVector.end()); - - std::vector resultingRatings(pomdp().getNrObservations(), storm::utility::one()); - - std::map gatheredSuccessorObservations; // Declare here to avoid reallocations - for (uint64_t mdpState = 0; mdpState < numMdpStates; ++mdpState) { - // Check whether this state is reached under an optimal scheduler. - // The heuristic assumes that the remaining states are not relevant for the observation score. - if (overApproximation->stateIsOptimalSchedulerReachable(mdpState)) { - for (uint64_t mdpChoice = choiceIndices[mdpState]; mdpChoice < choiceIndices[mdpState + 1]; ++mdpChoice) { - // Similarly, only optimal actions are relevant - if (overApproximation->actionIsOptimal(mdpChoice)) { - // score the observations for this choice - gatheredSuccessorObservations.clear(); - overApproximation->gatherSuccessorObservationInformationAtMdpChoice(mdpChoice, gatheredSuccessorObservations); - for (auto const& obsInfo : gatheredSuccessorObservations) { - auto const& obs = obsInfo.first; - BeliefValueType obsChoiceRating = rateObservation(obsInfo.second, observationResolutionVector[obs], maxResolution); - - // The rating of the observation will be the minimum over all choice-based observation ratings - resultingRatings[obs] = std::min(resultingRatings[obs], obsChoiceRating); - } - } - } - } - } - return resultingRatings; -} - -template -typename PomdpModelType::ValueType BeliefExplorationPomdpModelChecker::getGap( - typename PomdpModelType::ValueType const& l, typename PomdpModelType::ValueType const& u) { - STORM_LOG_ASSERT(l >= storm::utility::zero() && u >= storm::utility::zero(), - "Gap computation currently does not handle negative values."); - if (storm::utility::isInfinity(u)) { - if (storm::utility::isInfinity(l)) { - return storm::utility::zero(); - } else { - return u; - } - } else if (storm::utility::isZero(u)) { - STORM_LOG_ASSERT(storm::utility::isZero(l), "Upper bound is zero but lower bound is " << l << "."); - return u; - } else { - STORM_LOG_ASSERT(!storm::utility::isInfinity(l), "Lower bound is infinity, but upper bound is " << u << "."); - // get the relative gap - return storm::utility::abs(u - l) * storm::utility::convertNumber(2) / - (l + u); - } -} - -/* Template Instantiations */ - -template class BeliefExplorationPomdpModelChecker>; - -template class BeliefExplorationPomdpModelChecker, storm::RationalNumber>; - -template class BeliefExplorationPomdpModelChecker, double>; - -template class BeliefExplorationPomdpModelChecker>; - -} // namespace modelchecker -} // namespace pomdp -} // namespace storm diff --git a/src/storm-pomdp/modelchecker/BeliefExplorationPomdpModelChecker.h b/src/storm-pomdp/modelchecker/BeliefExplorationPomdpModelChecker.h deleted file mode 100644 index abd076511f..0000000000 --- a/src/storm-pomdp/modelchecker/BeliefExplorationPomdpModelChecker.h +++ /dev/null @@ -1,375 +0,0 @@ -#pragma once - -#include "storm-pomdp/builder/BeliefMdpExplorer.h" -#include "storm-pomdp/modelchecker/BeliefExplorationPomdpModelCheckerOptions.h" -#include "storm-pomdp/storage/BeliefManager.h" -#include "storm/storage/jani/Property.h" -#include "storm/utility/Stopwatch.h" - -namespace storm { -class Environment; - -namespace models { -namespace sparse { -template -class Pomdp; -} -} // namespace models -namespace logic { -class Formula; -} - -namespace pomdp { -namespace modelchecker { - -/** - * Structure for storing values on the POMDP used for cut-offs and clipping. - * trivialPomdpValueBounds is supposed to store - * extremePomdpValueBound stores the values - * - * @tparam ValueType - */ -template -struct POMDPValueBounds { - // values generated by memoryless schedulers during pre-processing - storm::pomdp::storage::PreprocessingPomdpValueBounds trivialPomdpValueBounds; - // values for clipping compensation - storm::pomdp::storage::ExtremePOMDPValueBound extremePomdpValueBound; - // values generated by a finite memory schedulers. Each scheduler is represented by a vector of maps representing (memory node x state) -> value - std::vector>> fmSchedulerValueList; -}; -/** - * Model checker for checking reachability queries on POMDPs using approximations based on exploration of the belief MDP - * @tparam PomdpModelType model type of the POMDP - * @tparam BeliefValueType type used for the state probabilities in beliefs - * @tparam BeliefMDPType number type used for the MDP structure of the belief MDP. - * BeliefMDPType can differ from BeliefValueType as we might want to have exact values for probabilities in beliefs, but are okay with possible imprecision in - * the MDP itself. - */ -template -class BeliefExplorationPomdpModelChecker { - public: - typedef BeliefMDPType ValueType; - typedef typename PomdpModelType::RewardModelType RewardModelType; - typedef storm::storage::BeliefManager BeliefManagerType; - typedef storm::builder::BeliefMdpExplorer ExplorerType; - typedef BeliefExplorationPomdpModelCheckerOptions Options; - - /* Struct Definition(s) */ - /** - * Statuses used for the interactive exploration - */ - enum class Status { - Uninitialized, - Exploring, - ModelExplorationFinished, - ResultAvailable, - Terminated, - Converged, - }; - - /** - * Struct used to store the results of the model checker - */ - struct Result { - Result(ValueType lower, ValueType upper); - ValueType lowerBound; - ValueType upperBound; - ValueType diff(bool relative = false) const; - bool updateLowerBound(ValueType const& value); - bool updateUpperBound(ValueType const& value); - std::shared_ptr> schedulerAsMarkovChain; - std::vector> cutoffSchedulers; - }; - - /* Functions */ - - /** - * Constructor - * @param pomdp pointer to the POMDP to be checked - * @param options object containing the options for the model checker - */ - explicit BeliefExplorationPomdpModelChecker(std::shared_ptr pomdp, Options options = Options()); - - /** - * Performs model checking of the given POMDP with regards to a formula using the previously specified options - * @param formula the formula to check - * @param preProcEnv environment used for solving the pre-processisng - * @param additionalUnderApproximationBounds additional bounds that can be used for cut-offs in the under-approximation. Each element of the outer vector - * represents a scheduler. Each scheduler is represented by a vector of maps representing (memory node x state) -> value - * @return result of the model checking - */ - Result check(storm::Environment const& env, storm::logic::Formula const& formula, storm::Environment const& preProcEnv, - std::vector>> const& additionalUnderApproximationBounds = - std::vector>>()); - Result check(storm::logic::Formula const& formula, storm::Environment const& preProcEnv, - std::vector>> const& additionalUnderApproximationBounds = - std::vector>>()); - Result check(storm::logic::Formula const& formula, - std::vector>> const& additionalUnderApproximationBounds = - std::vector>>()); - Result check(storm::Environment const& env, storm::logic::Formula const& formula, - std::vector>> const& additionalUnderApproximationBounds = - std::vector>>()); - - /** - * Prints statistics of the process to a given output stream - * @param stream the output stream - */ - void printStatisticsToStream(std::ostream& stream) const; - - /** - * Uses model checking on the underlying MDP to generate values used for cut-offs and for clipping compensation if necessary - * @param formula the formula to check - * @param preProcEnv environment used for solving the pre-processisng - */ - void precomputeValueBounds(const logic::Formula& formula, storm::Environment const& preProcEnv); - - /** - * Allows to generate an under-approximation using a controllable unfolding. This provides a method for outside tools to control the unfolding for an - * under-approximation themselves. The unfolding runs until a pausing command is issued. If the unfolding is paused, cut-offs and optionally clipping are - * applied to obtain an abstraction MDP. This MDP is then checked and the result is saved. The unfolding can then be continued from the state before - * cut-offs were applied. - * @param targetObservations the target observations of the objective - * @param min true if the objective is to minimise the value - * @param rewardModelName name of the reward model to be used if one is specified - * @param valueBounds values used for cut-offs and clipping - * @param result the struct to store results - */ - void unfoldInteractively(storm::Environment const& env, std::set const& targetObservations, bool min, std::optional rewardModelName, - storm::pomdp::modelchecker::POMDPValueBounds const& valueBounds, Result& result); - void unfoldInteractively(std::set const& targetObservations, bool min, std::optional rewardModelName, - storm::pomdp::modelchecker::POMDPValueBounds const& valueBounds, Result& result); - - /** - * Pauses a running interactive unfolding - */ - void pauseUnfolding(); - - /** - * Continues a previously paused interactive unfolding. Only works if the checking process has already finished and a result is ready. - */ - void continueUnfolding(); - - /** - * Terminates a running interactive unfolding. Results are computed one last time, then the interactive unfolding is terminated and cannot be continued. - */ - void terminateUnfolding(); - - /** - * Indicates whether there is a result after an interactive unfolding was paused. - * @return True, if the model checking process of the current approximation has finished. - */ - bool isResultReady(); - - /** - * Indicates whether the interactive unfolding is currently in the process of exploring the belief MDP. - * @return True, if the exploration is currently in progress - */ - bool isExploring(); - - /** - * Indicates whether the interactive unfolding has coonverged, i.e. it has completely explored a finite belief MDP - * @return True if the entire belief MDP has been explored - */ - bool hasConverged(); - - /** - * Get the latest saved result obtained by the interactive unfolding - * @return - */ - Result getInteractiveResult(); - - /** - * Get a pointer to the belief explorer used in the interactive unfolding - * @return pointer to the belief explorer - */ - std::shared_ptr getInteractiveBeliefExplorer(); - - void setFMSchedValueList(std::vector>> valueList); - - /** - * Get the current status of the interactive unfolding - * @return the interactive unfolding - */ - int64_t getStatus(); - - private: - /* Struct Definition(s) */ - - /** - * Control parameters for the interactive unfolding - */ - enum class UnfoldingControl { Run, Pause, Terminate }; - - /** - * Struct containing statistics for the belief exploration process - */ - struct Statistics { - Statistics(); - std::optional refinementSteps; - storm::utility::Stopwatch totalTime; - - bool beliefMdpDetectedToBeFinite; - bool refinementFixpointDetected; - - std::optional overApproximationStates; - bool overApproximationBuildAborted; - storm::utility::Stopwatch overApproximationBuildTime; - storm::utility::Stopwatch overApproximationCheckTime; - std::optional overApproximationMaxResolution; - - std::optional underApproximationStates; - bool underApproximationBuildAborted; - storm::utility::Stopwatch underApproximationBuildTime; - storm::utility::Stopwatch underApproximationCheckTime; - std::optional underApproximationStateLimit; - std::optional nrClippingAttempts; - std::optional nrClippedStates; - std::optional nrTruncatedStates; - storm::utility::Stopwatch clipWatch; - storm::utility::Stopwatch clippingPreTime; - - bool aborted; - }; - - /** - * Parameters used for guiding the exploration and abstraction-refinement - */ - struct HeuristicParameters { - ValueType gapThreshold; - ValueType observationThreshold; - uint64_t sizeThreshold; - ValueType optimalChoiceValueEpsilon; - }; - - /* Functions */ - - /** - * Returns the pomdp that is to be analyzed - */ - PomdpModelType const& pomdp() const; - - /** - * Compute the reachability probability of given target observations on a POMDP using the automatic refinement loop - * - * @param targetObservations the set of observations to be reached - * @param min true if minimum probability is to be computed - * @return A struct containing the final over-approximation (overApproxValue) and under-approximation (underApproxValue) values - */ - void refineReachability(storm::Environment const& env, std::set const& targetObservations, bool min, std::optional rewardModelName, - storm::pomdp::modelchecker::POMDPValueBounds const& valueBounds, Result& result); - - /** - * Builds and checks an MDP that over-approximates the POMDP behavior, i.e. provides an upper bound for maximizing and a lower bound for minimizing - * properties - * @param targetObservations targetObservations the target observations of the objective - * @param min true if the objective is to minimise the value - * @param computeRewards true if the objective is to compute a reward value - * @param refine true if the method is called as part of the abstraction-refinement process - * @param heuristicParameters parameters used for guiding the exploration - * @param observationResolutionVector vector of resolutions for each observation used to discretise beliefs - * @param beliefManager the belief manager to be used - * @param overApproximation the belief explorer to be used - * @return True if a fixpoint for the refinement has been detected (i.e. if further refinement steps would not change the MDP) - */ - bool buildOverApproximation(storm::Environment const& env, std::set const& targetObservations, bool min, bool computeRewards, bool refine, - HeuristicParameters const& heuristicParameters, std::vector& observationResolutionVector, - std::shared_ptr& beliefManager, std::shared_ptr& overApproximation); - - /** - * Builds and checks an MDP that under-approximates the POMDP behavior, i.e. provides a lower bound for maximizing and an upper bound for minimizing - * properties - * @param targetObservations targetObservations the target observations of the objective - * @param min true if the objective is to minimise the value - * @param computeRewards true if the objective is to compute a reward value - * @param refine true if the method is called as part of the abstraction-refinement process - * @param heuristicParameters parameters used for guiding the exploration - * @param beliefManager the belief manager to be used - * @param underApproximation the belief explorer to be used - * @param interactive true if the underapproximation is built as part of an interactive unfolding - * @return True if a fixpoint for the refinement has been detected (i.e. if further refinement steps would not change the MDP) - */ - bool buildUnderApproximation(storm::Environment const& env, std::set const& targetObservations, bool min, bool computeRewards, bool refine, - HeuristicParameters const& heuristicParameters, std::shared_ptr& beliefManager, - std::shared_ptr& underApproximation, bool interactive); - - /** - * Clips the belief with the given state ID to a belief grid by clipping its direct successor ("grid clipping") - * Transitions to explored successors and successors on the grid are added, otherwise successors are not generated - * @param env Environment - * @param clippingStateId the state ID of the clipping belief - * @param computeRewards true, if rewards are computed - * @param min true, if objective is to minimise - * @param beliefManager the belief manager used - * @param beliefExplorer the belief MDP explorer used - */ - void clipToGrid(storm::Environment const& env, uint64_t clippingStateId, bool computeRewards, bool min, std::shared_ptr& beliefManager, - std::shared_ptr& beliefExplorer); - - /** - * Clips the belief with the given state ID to a belief grid. - * If a new candidate is added to the belief space, it is expanded. If necessary, its direct successors are added to the exploration queue to be - * handled by the main exploration routine. - * @param env Environment - * @param clippingStateId the state ID of the clipping belief - * @param computeRewards true, if rewards are computed - * @param beliefManager the belief manager used - * @param beliefExplorer the belief MDP explorer used - */ - bool clipToGridExplicitly(storm::Environment const& env, uint64_t clippingStateId, bool computeRewards, std::shared_ptr& beliefManager, - std::shared_ptr& beliefExplorer, uint64_t localActionIndex); - - /** - * Heuristically rates the quality of the approximation described by the given successor observation info. - * Here, 0 means a bad approximation and 1 means a good approximation. - */ - BeliefValueType rateObservation(typename ExplorerType::SuccessorObservationInformation const& info, BeliefValueType const& observationResolution, - BeliefValueType const& maxResolution); - - /** - * Obtains the quality ratings for all observations - * @param overApproximation pointer to the over-approximation belief explorer - * @param observationResolutionVector vector containing the resolutions used in the over-approximation for each observation - * @return vector of ratings - */ - std::vector getObservationRatings(std::shared_ptr const& overApproximation, - std::vector const& observationResolutionVector); - - /** - * Obtains the difference between the given lower and upper bounds - * @param l the lower bound - * @param u the upper bound - * @return the difference - */ - typename PomdpModelType::ValueType getGap(typename PomdpModelType::ValueType const& l, typename PomdpModelType::ValueType const& u); - - /** - * Sets the command for the interactive belief unfolding - * @param newUnfoldingControl the new command - */ - void setUnfoldingControl(UnfoldingControl newUnfoldingControl); - - /* Variables */ - - Statistics statistics; - Options options; - - std::shared_ptr inputPomdp; - std::shared_ptr preprocessedPomdp; - - storm::utility::ConstantsComparator beliefTypeCC; - storm::utility::ConstantsComparator valueTypeCC; - - storm::pomdp::modelchecker::POMDPValueBounds pomdpValueBounds; - - std::shared_ptr interactiveUnderApproximationExplorer; - - Status unfoldingStatus; - UnfoldingControl unfoldingControl; - Result interactiveResult = Result(-storm::utility::infinity(), storm::utility::infinity()); -}; - -} // namespace modelchecker -} // namespace pomdp -} // namespace storm \ No newline at end of file diff --git a/src/storm-pomdp/modelchecker/BeliefExplorationPomdpModelCheckerOptions.h b/src/storm-pomdp/modelchecker/BeliefExplorationPomdpModelCheckerOptions.h deleted file mode 100644 index d4791b3ad0..0000000000 --- a/src/storm-pomdp/modelchecker/BeliefExplorationPomdpModelCheckerOptions.h +++ /dev/null @@ -1,63 +0,0 @@ -#pragma once - -#include "storm-pomdp/builder/BeliefMdpExplorer.h" -#include "storm/utility/NumberTraits.h" -#include "storm/utility/constants.h" - -namespace storm { -namespace builder { -template -class BeliefMdpExplorer; -} -namespace pomdp { -namespace modelchecker { -template -struct BeliefExplorationPomdpModelCheckerOptions { - BeliefExplorationPomdpModelCheckerOptions(bool discretize, bool unfold) : discretize(discretize), unfold(unfold) { - // Intentionally left empty - } - - // TODO documentation? - bool discretize; - bool unfold; - - bool useClipping = false; - bool interactiveUnfolding = false; - bool refine = false; - bool cutZeroGap = false; - bool useStateEliminationCutoff = false; - uint64_t refineStepLimit = 0; - ValueType refinePrecision = storm::utility::convertNumber(1e-4); - uint64_t explorationTimeLimit = 0; - - // Control parameters for the refinement heuristic - // Discretization Resolution - uint64_t resolutionInit = 2; - ValueType resolutionFactor = storm::utility::convertNumber(2); - // The maximal number of newly expanded MDP states in a refinement step - uint64_t sizeThresholdInit = 0; - ValueType sizeThresholdFactor = storm::utility::convertNumber(4); - // Controls how large the gap between known lower- and upper bounds at a belief state needs to be in order to explore - ValueType gapThresholdInit = storm::utility::convertNumber(0.1); - ValueType gapThresholdFactor = storm::utility::convertNumber(0.25); - // Controls whether "almost optimal" choices will be considered optimal - ValueType optimalChoiceValueThresholdInit = storm::utility::convertNumber(1e-3); - ValueType optimalChoiceValueThresholdFactor = storm::utility::one(); - // Controls which observations are refined. - ValueType obsThresholdInit = storm::utility::convertNumber(0.1); - ValueType obsThresholdIncrementFactor = storm::utility::convertNumber(0.1); - - uint64_t clippingGridRes = 2; - - bool skipHeuristicSchedulers = false; - - ValueType numericPrecision = storm::NumberTraits::IsExact - ? storm::utility::zero() - : storm::utility::convertNumber(1e-9); /// Used to decide whether two beliefs are equal - bool dynamicTriangulation = true; // Sets whether the triangulation is done in a dynamic way (yielding more precise triangulations) - - storm::builder::ExplorationHeuristic explorationHeuristic = storm::builder::ExplorationHeuristic::BreadthFirst; -}; -} // namespace modelchecker -} // namespace pomdp -} // namespace storm diff --git a/src/storm-pomdp/modelchecker/PreprocessingPomdpValueBoundsModelChecker.cpp b/src/storm-pomdp/modelchecker/PreprocessingPomdpValueBoundsModelChecker.cpp index 43d372d28a..4799a657c4 100644 --- a/src/storm-pomdp/modelchecker/PreprocessingPomdpValueBoundsModelChecker.cpp +++ b/src/storm-pomdp/modelchecker/PreprocessingPomdpValueBoundsModelChecker.cpp @@ -13,52 +13,53 @@ #include "storm/utility/macros.h" #include "storm/utility/vector.h" -namespace storm { -namespace pomdp { -namespace modelchecker { -template -PreprocessingPomdpValueBoundsModelChecker::PreprocessingPomdpValueBoundsModelChecker(storm::models::sparse::Pomdp const& pomdp) +namespace storm::pomdp::modelchecker { +template +PreprocessingPomdpValueBoundsModelChecker::PreprocessingPomdpValueBoundsModelChecker(PomdpType const& pomdp) : pomdp(pomdp) { /* Intentionally left empty */ } -template -typename PreprocessingPomdpValueBoundsModelChecker::ValueBounds PreprocessingPomdpValueBoundsModelChecker::getValueBounds( +template +PreprocessingPomdpValueBoundsModelChecker::ValueBounds PreprocessingPomdpValueBoundsModelChecker::getValueBounds( storm::Environment const& env, storm::logic::Formula const& formula) { return getValueBounds(env, formula, storm::pomdp::analysis::getFormulaInformation(pomdp, formula)); } -template -typename PreprocessingPomdpValueBoundsModelChecker::ValueBounds PreprocessingPomdpValueBoundsModelChecker::getValueBounds( +template +PreprocessingPomdpValueBoundsModelChecker::ValueBounds PreprocessingPomdpValueBoundsModelChecker::getValueBounds( storm::logic::Formula const& formula) { storm::Environment env; return getValueBounds(env, formula, storm::pomdp::analysis::getFormulaInformation(pomdp, formula)); } -template -std::vector PreprocessingPomdpValueBoundsModelChecker::getChoiceValues(std::vector const& stateValues, - std::vector* actionBasedRewards) { - std::vector choiceValues((pomdp.getNumberOfChoices())); +template +std::vector::PomdpValueType> +PreprocessingPomdpValueBoundsModelChecker::getChoiceValues(std::vector const& stateValues, + std::vector* actionBasedRewards) { + std::vector choiceValues((pomdp.getNumberOfChoices())); pomdp.getTransitionMatrix().multiplyWithVector(stateValues, choiceValues, actionBasedRewards); return choiceValues; } -template -std::pair, storm::storage::Scheduler> PreprocessingPomdpValueBoundsModelChecker::computeValuesForGuessedScheduler( - storm::Environment const& env, std::vector const& stateValues, std::vector* actionBasedRewards, storm::logic::Formula const& formula, - storm::pomdp::analysis::FormulaInformation const& info, std::shared_ptr> underlyingMdp, - ValueType const& scoreThreshold, bool relativeScore) { +template +std::pair::PomdpValueType>, + storm::storage::Scheduler::PomdpValueType>> +PreprocessingPomdpValueBoundsModelChecker::computeValuesForGuessedScheduler( + storm::Environment const& env, std::vector const& stateValues, std::vector* actionBasedRewards, + storm::logic::Formula const& formula, storm::pomdp::analysis::FormulaInformation const& info, + std::shared_ptr> underlyingMdp, PomdpValueType const& scoreThreshold, bool relativeScore) { // Create some positional scheduler for the POMDP - storm::storage::Scheduler pomdpScheduler(pomdp.getNumberOfStates()); + storm::storage::Scheduler pomdpScheduler(pomdp.getNumberOfStates()); // For each state, we heuristically find a good distribution over output actions. auto choiceValues = getChoiceValues(stateValues, actionBasedRewards); auto const& choiceIndices = pomdp.getTransitionMatrix().getRowGroupIndices(); - std::vector> choiceDistributions(pomdp.getNrObservations()); + std::vector> choiceDistributions(pomdp.getNrObservations()); for (uint64_t state = 0; state < pomdp.getNumberOfStates(); ++state) { auto& choiceDistribution = choiceDistributions[pomdp.getObservation(state)]; - ValueType const& stateValue = stateValues[state]; - STORM_LOG_ASSERT(stateValue >= storm::utility::zero(), "State value expected non-negative."); + PomdpValueType const& stateValue = stateValues[state]; + STORM_LOG_ASSERT(stateValue >= storm::utility::zero(), "State value expected non-negative."); for (auto choice = choiceIndices[state]; choice < choiceIndices[state + 1]; ++choice) { - ValueType const& choiceValue = choiceValues[choice]; - STORM_LOG_ASSERT(choiceValue >= storm::utility::zero(), "Choice value expected non-negative."); + PomdpValueType const& choiceValue = choiceValues[choice]; + STORM_LOG_ASSERT(choiceValue >= storm::utility::zero(), "Choice value expected non-negative."); // Rate this choice by considering the relative difference between the choice value and the (optimal) state value // A high score shall mean that the choice is "good" if (storm::utility::isInfinity(stateValue)) { @@ -66,14 +67,14 @@ std::pair, storm::storage::Scheduler> Preproce // This case could be handled a bit more sensible choiceDistribution.addProbability(choice - choiceIndices[state], scoreThreshold); } else { - ValueType choiceScore = info.minimize() ? (choiceValue - stateValue) : (stateValue - choiceValue); + PomdpValueType choiceScore = info.minimize() ? (choiceValue - stateValue) : (stateValue - choiceValue); if (relativeScore) { - ValueType avg = (stateValue + choiceValue) / storm::utility::convertNumber(2); + PomdpValueType avg = (stateValue + choiceValue) / storm::utility::convertNumber(2); if (!storm::utility::isZero(avg)) { choiceScore /= avg; } } - choiceScore = storm::utility::one() - choiceScore; + choiceScore = storm::utility::one() - choiceScore; if (choiceScore >= scoreThreshold) { choiceDistribution.addProbability(choice - choiceIndices[state], choiceScore); } @@ -98,22 +99,26 @@ std::pair, storm::storage::Scheduler> Preproce STORM_LOG_ASSERT(!pomdpScheduler.isPartialScheduler(), "Expected a fully defined scheduler."); auto scheduledModel = underlyingMdp->applyScheduler(pomdpScheduler, false); - auto resultPtr = storm::api::verifyWithSparseEngine(env, scheduledModel, storm::api::createTask(formula.asSharedPointer(), false)); + auto resultPtr = + storm::api::verifyWithSparseEngine(env, scheduledModel, storm::api::createTask(formula.asSharedPointer(), false)); STORM_LOG_THROW(resultPtr, storm::exceptions::UnexpectedException, "No check result obtained."); STORM_LOG_THROW(resultPtr->isExplicitQuantitativeCheckResult(), storm::exceptions::UnexpectedException, "Unexpected Check result Type."); - std::vector pomdpSchedulerResult = std::move(resultPtr->template asExplicitQuantitativeCheckResult().getValueVector()); + std::vector pomdpSchedulerResult = std::move(resultPtr->template asExplicitQuantitativeCheckResult().getValueVector()); return std::make_pair(pomdpSchedulerResult, pomdpScheduler); } -template -std::pair, storm::storage::Scheduler> PreprocessingPomdpValueBoundsModelChecker::computeValuesForRandomFMPolicy( - storm::Environment const& env, storm::logic::Formula const& formula, storm::pomdp::analysis::FormulaInformation const& info, uint64_t memoryBound) { +template +std::pair::PomdpValueType>, + storm::storage::Scheduler::PomdpValueType>> +PreprocessingPomdpValueBoundsModelChecker::computeValuesForRandomFMPolicy(storm::Environment const& env, storm::logic::Formula const& formula, + storm::pomdp::analysis::FormulaInformation const& info, + uint64_t memoryBound) { // Consider memoryless policy on memory-unfolded POMDP - storm::storage::Scheduler pomdpScheduler(pomdp.getNumberOfStates() * memoryBound); + storm::storage::Scheduler pomdpScheduler(pomdp.getNumberOfStates() * memoryBound); STORM_LOG_DEBUG("Computing the unfolding for memory bound " << memoryBound); storm::storage::PomdpMemory memory = storm::storage::PomdpMemoryBuilder().build(storm::storage::PomdpMemoryPattern::Full, memoryBound); - storm::transformer::PomdpMemoryUnfolder memoryUnfolder(pomdp, memory); + storm::transformer::PomdpMemoryUnfolder memoryUnfolder(pomdp, memory); // We keep unreachable states to not mess with the state ordering and capture potential better choices auto memPomdp = memoryUnfolder.transform(false); @@ -131,16 +136,18 @@ std::pair, storm::storage::Scheduler> Preproce } // Model check the DTMC resulting from the policy - auto underlyingMdp = - std::make_shared>(memPomdp->getTransitionMatrix(), memPomdp->getStateLabeling(), memPomdp->getRewardModels()); + auto underlyingMdp = std::make_shared>(memPomdp->getTransitionMatrix(), memPomdp->getStateLabeling(), + memPomdp->getRewardModels()); auto scheduledModel = underlyingMdp->applyScheduler(pomdpScheduler, false); - auto resultPtr = storm::api::verifyWithSparseEngine(env, scheduledModel, storm::api::createTask(formula.asSharedPointer(), false)); + auto resultPtr = + storm::api::verifyWithSparseEngine(env, scheduledModel, storm::api::createTask(formula.asSharedPointer(), false)); STORM_LOG_THROW(resultPtr, storm::exceptions::UnexpectedException, "No check result obtained."); STORM_LOG_THROW(resultPtr->isExplicitQuantitativeCheckResult(), storm::exceptions::UnexpectedException, "Unexpected Check result Type."); - std::vector pomdpSchedulerResult = std::move(resultPtr->template asExplicitQuantitativeCheckResult().getValueVector()); + std::vector pomdpSchedulerResult = std::move(resultPtr->template asExplicitQuantitativeCheckResult().getValueVector()); // Take the optimal value in ANY of the unfolded states for a POMDP state as the resulting state value - std::vector res(pomdp.getNumberOfStates(), info.minimize() ? storm::utility::infinity() : -storm::utility::infinity()); + std::vector res(pomdp.getNumberOfStates(), + info.minimize() ? storm::utility::infinity() : -storm::utility::infinity()); for (uint64_t memPomdpState = 0; memPomdpState < pomdpSchedulerResult.size(); ++memPomdpState) { uint64_t modelState = memPomdpState / memoryBound; if ((info.minimize() && pomdpSchedulerResult[memPomdpState] < res[modelState]) || @@ -151,12 +158,13 @@ std::pair, storm::storage::Scheduler> Preproce return std::make_pair(res, pomdpScheduler); } -template -[[maybe_unused]] std::pair, storm::storage::Scheduler> -PreprocessingPomdpValueBoundsModelChecker::computeValuesForRandomMemorylessPolicy( +template +[[maybe_unused]] std::pair::PomdpValueType>, + storm::storage::Scheduler::PomdpValueType>> +PreprocessingPomdpValueBoundsModelChecker::computeValuesForRandomMemorylessPolicy( storm::Environment const& env, storm::logic::Formula const& formula, storm::pomdp::analysis::FormulaInformation const& info, - std::shared_ptr> underlyingMdp) { - storm::storage::Scheduler pomdpScheduler(pomdp.getNumberOfStates()); + std::shared_ptr> underlyingMdp) { + storm::storage::Scheduler pomdpScheduler(pomdp.getNumberOfStates()); std::vector obsChoiceVector(pomdp.getNrObservations()); std::random_device rd; @@ -174,18 +182,19 @@ PreprocessingPomdpValueBoundsModelChecker::computeValuesForRandomMemo auto scheduledModel = underlyingMdp->applyScheduler(pomdpScheduler, false); - auto resultPtr = storm::api::verifyWithSparseEngine(env, scheduledModel, storm::api::createTask(formula.asSharedPointer(), false)); + auto resultPtr = + storm::api::verifyWithSparseEngine(env, scheduledModel, storm::api::createTask(formula.asSharedPointer(), false)); STORM_LOG_THROW(resultPtr, storm::exceptions::UnexpectedException, "No check result obtained."); STORM_LOG_THROW(resultPtr->isExplicitQuantitativeCheckResult(), storm::exceptions::UnexpectedException, "Unexpected Check result Type."); - std::vector pomdpSchedulerResult = std::move(resultPtr->template asExplicitQuantitativeCheckResult().getValueVector()); + std::vector pomdpSchedulerResult = std::move(resultPtr->template asExplicitQuantitativeCheckResult().getValueVector()); STORM_LOG_DEBUG("Initial Value for guessed Policy: " << pomdpSchedulerResult[pomdp.getInitialStates().getNextSetIndex(0)]); return std::make_pair(pomdpSchedulerResult, pomdpScheduler); } -template -typename PreprocessingPomdpValueBoundsModelChecker::ValueBounds PreprocessingPomdpValueBoundsModelChecker::getValueBounds( +template +PreprocessingPomdpValueBoundsModelChecker::ValueBounds PreprocessingPomdpValueBoundsModelChecker::getValueBounds( storm::Environment const& env, storm::logic::Formula const& formula, storm::pomdp::analysis::FormulaInformation const& info) { STORM_LOG_THROW(info.isNonNestedReachabilityProbability() || info.isNonNestedExpectedRewardFormula(), storm::exceptions::NotSupportedException, "The property type is not supported for this analysis."); @@ -194,53 +203,56 @@ typename PreprocessingPomdpValueBoundsModelChecker::ValueBounds Prepr // We need an actual MDP so that we can apply schedulers below. // Also, the api call in the next line will require a copy anyway. auto underlyingMdp = - std::make_shared>(pomdp.getTransitionMatrix(), pomdp.getStateLabeling(), pomdp.getRewardModels()); - auto resultPtr = storm::api::verifyWithSparseEngine(env, underlyingMdp, storm::api::createTask(formula.asSharedPointer(), false)); + std::make_shared>(pomdp.getTransitionMatrix(), pomdp.getStateLabeling(), pomdp.getRewardModels()); + auto resultPtr = + storm::api::verifyWithSparseEngine(env, underlyingMdp, storm::api::createTask(formula.asSharedPointer(), false)); STORM_LOG_THROW(resultPtr, storm::exceptions::UnexpectedException, "No check result obtained."); STORM_LOG_THROW(resultPtr->isExplicitQuantitativeCheckResult(), storm::exceptions::UnexpectedException, "Unexpected Check result Type."); - std::vector fullyObservableResult = std::move(resultPtr->template asExplicitQuantitativeCheckResult().getValueVector()); + std::vector fullyObservableResult = std::move(resultPtr->template asExplicitQuantitativeCheckResult().getValueVector()); - std::vector actionBasedRewards; - std::vector* actionBasedRewardsPtr = nullptr; + std::vector actionBasedRewards; + std::vector* actionBasedRewardsPtr = nullptr; if (info.isNonNestedExpectedRewardFormula()) { actionBasedRewards = pomdp.getRewardModel(info.getRewardModelName()).getTotalRewardVector(pomdp.getTransitionMatrix()); actionBasedRewardsPtr = &actionBasedRewards; } - std::vector> guessedSchedulerValues; - std::vector> guessedSchedulers; - std::shared_ptr, storm::storage::Scheduler>> guessedSchedulerPair; + std::vector> guessedSchedulerValues; + std::vector> guessedSchedulers; + std::shared_ptr, storm::storage::Scheduler>> guessedSchedulerPair; std::vector> guessParameters({{0.875, false}, {0.875, true}, {0.75, false}, {0.75, true}}); for (auto const& pars : guessParameters) { - guessedSchedulerPair = std::make_shared, storm::storage::Scheduler>>( + guessedSchedulerPair = std::make_shared, storm::storage::Scheduler>>( computeValuesForGuessedScheduler(env, fullyObservableResult, actionBasedRewardsPtr, formula, info, underlyingMdp, - storm::utility::convertNumber(pars.first), pars.second)); + storm::utility::convertNumber(pars.first), pars.second)); guessedSchedulerValues.push_back(guessedSchedulerPair->first); guessedSchedulers.push_back(guessedSchedulerPair->second); } // compute the 'best' guess and do a few iterations on it uint64_t bestGuess = 0; - ValueType bestGuessSum = std::accumulate(guessedSchedulerValues.front().begin(), guessedSchedulerValues.front().end(), storm::utility::zero()); + PomdpValueType bestGuessSum = + std::accumulate(guessedSchedulerValues.front().begin(), guessedSchedulerValues.front().end(), storm::utility::zero()); for (uint64_t guess = 1; guess < guessedSchedulerValues.size(); ++guess) { - ValueType guessSum = std::accumulate(guessedSchedulerValues[guess].begin(), guessedSchedulerValues[guess].end(), storm::utility::zero()); + PomdpValueType guessSum = + std::accumulate(guessedSchedulerValues[guess].begin(), guessedSchedulerValues[guess].end(), storm::utility::zero()); if ((info.minimize() && guessSum < bestGuessSum) || (info.maximize() && guessSum > bestGuessSum)) { bestGuess = guess; bestGuessSum = guessSum; } } - guessedSchedulerPair = std::make_shared, storm::storage::Scheduler>>( + guessedSchedulerPair = std::make_shared, storm::storage::Scheduler>>( computeValuesForGuessedScheduler(env, guessedSchedulerValues[bestGuess], actionBasedRewardsPtr, formula, info, underlyingMdp, - storm::utility::convertNumber(guessParameters[bestGuess].first), guessParameters[bestGuess].second)); + storm::utility::convertNumber(guessParameters[bestGuess].first), guessParameters[bestGuess].second)); guessedSchedulerValues.push_back(guessedSchedulerPair->first); guessedSchedulers.push_back(guessedSchedulerPair->second); - guessedSchedulerPair = std::make_shared, storm::storage::Scheduler>>( + guessedSchedulerPair = std::make_shared, storm::storage::Scheduler>>( computeValuesForGuessedScheduler(env, guessedSchedulerValues.back(), actionBasedRewardsPtr, formula, info, underlyingMdp, - storm::utility::convertNumber(guessParameters[bestGuess].first), guessParameters[bestGuess].second)); + storm::utility::convertNumber(guessParameters[bestGuess].first), guessParameters[bestGuess].second)); guessedSchedulerValues.push_back(guessedSchedulerPair->first); guessedSchedulers.push_back(guessedSchedulerPair->second); - guessedSchedulerPair = std::make_shared, storm::storage::Scheduler>>( + guessedSchedulerPair = std::make_shared, storm::storage::Scheduler>>( computeValuesForGuessedScheduler(env, guessedSchedulerValues.back(), actionBasedRewardsPtr, formula, info, underlyingMdp, - storm::utility::convertNumber(guessParameters[bestGuess].first), guessParameters[bestGuess].second)); + storm::utility::convertNumber(guessParameters[bestGuess].first), guessParameters[bestGuess].second)); guessedSchedulerValues.push_back(guessedSchedulerPair->first); guessedSchedulers.push_back(guessedSchedulerPair->second); @@ -255,7 +267,7 @@ typename PreprocessingPomdpValueBoundsModelChecker::ValueBounds Prepr if (!keptGuesses.get(j)) { continue; } - if (storm::utility::vector::compareElementWise(guessedSchedulerValues[i], guessedSchedulerValues[j], std::less_equal())) { + if (storm::utility::vector::compareElementWise(guessedSchedulerValues[i], guessedSchedulerValues[j], std::less_equal())) { if (info.minimize()) { // In this case we are guessing upper bounds (and smaller upper bounds are better) keptGuesses.set(j, false); @@ -264,7 +276,7 @@ typename PreprocessingPomdpValueBoundsModelChecker::ValueBounds Prepr keptGuesses.set(i, false); break; } - } else if (storm::utility::vector::compareElementWise(guessedSchedulerValues[j], guessedSchedulerValues[i], std::less_equal())) { + } else if (storm::utility::vector::compareElementWise(guessedSchedulerValues[j], guessedSchedulerValues[i], std::less_equal())) { if (info.minimize()) { keptGuesses.set(i, false); break; @@ -276,7 +288,7 @@ typename PreprocessingPomdpValueBoundsModelChecker::ValueBounds Prepr } STORM_LOG_INFO("Keeping scheduler guesses " << keptGuesses); storm::utility::vector::filterVectorInPlace(guessedSchedulerValues, keptGuesses); - std::vector> filteredSchedulers; + std::vector> filteredSchedulers; for (uint64_t i = 0; i < guessedSchedulers.size(); ++i) { if (keptGuesses[i]) { filteredSchedulers.push_back(guessedSchedulers[i]); @@ -294,26 +306,44 @@ typename PreprocessingPomdpValueBoundsModelChecker::ValueBounds Prepr result.upper.push_back(std::move(fullyObservableResult)); result.lowerSchedulers = filteredSchedulers; } - STORM_LOG_WARN_COND_DEBUG(storm::utility::vector::compareElementWise(result.lower.front(), result.upper.front(), std::less_equal()), - "Lower bound is larger than upper bound"); +#ifndef NDEBUG + bool boundsValid = true; + auto maxDifference = storm::utility::zero(); + for (auto const& lower : result.lower) { + for (auto const& upper : result.upper) { + for (uint64_t state = 0; state < pomdp.getNumberOfStates(); ++state) { + if (storm::utility::min(upper.at(state), lower.at(state)) == upper.at(state) && upper.at(state) != lower.at(state)) { + boundsValid = false; + maxDifference = storm::utility::max(maxDifference, lower.at(state) - upper.at(state)); + STORM_LOG_TRACE("Lower bound " << lower.at(state) << " at state " << state << " is larger than upper bound " << upper.at(state)); + } + } + } + } + + STORM_LOG_WARN_COND_DEBUG(boundsValid, "At least one lower bound is not smaller than an upper bound (max. difference: " + << maxDifference + << "). This might be due to floating point imprecisions. Enable TRACE output for more details."); +#endif + return result; } -template -typename PreprocessingPomdpValueBoundsModelChecker::ExtremeValueBound PreprocessingPomdpValueBoundsModelChecker::getExtremeValueBound( +template +PreprocessingPomdpValueBoundsModelChecker::ExtremeValueBound PreprocessingPomdpValueBoundsModelChecker::getExtremeValueBound( storm::logic::Formula const& formula) { storm::Environment env; return getExtremeValueBound(env, formula); } -template -typename PreprocessingPomdpValueBoundsModelChecker::ExtremeValueBound PreprocessingPomdpValueBoundsModelChecker::getExtremeValueBound( +template +PreprocessingPomdpValueBoundsModelChecker::ExtremeValueBound PreprocessingPomdpValueBoundsModelChecker::getExtremeValueBound( storm::Environment const& env, storm::logic::Formula const& formula) { return getExtremeValueBound(env, formula, storm::pomdp::analysis::getFormulaInformation(pomdp, formula)); } -template -typename PreprocessingPomdpValueBoundsModelChecker::ExtremeValueBound PreprocessingPomdpValueBoundsModelChecker::getExtremeValueBound( +template +PreprocessingPomdpValueBoundsModelChecker::ExtremeValueBound PreprocessingPomdpValueBoundsModelChecker::getExtremeValueBound( storm::Environment const& env, storm::logic::Formula const& formula, storm::pomdp::analysis::FormulaInformation const& info) { STORM_LOG_THROW(info.isNonNestedExpectedRewardFormula(), storm::exceptions::NotSupportedException, "The property type is not supported for this analysis."); @@ -328,11 +358,11 @@ typename PreprocessingPomdpValueBoundsModelChecker::ExtremeValueBound } auto formulaPtr = std::make_shared(newFormula); auto underlyingMdp = - std::make_shared>(pomdp.getTransitionMatrix(), pomdp.getStateLabeling(), pomdp.getRewardModels()); - auto resultPtr = storm::api::verifyWithSparseEngine(env, underlyingMdp, storm::api::createTask(formulaPtr, false)); + std::make_shared>(pomdp.getTransitionMatrix(), pomdp.getStateLabeling(), pomdp.getRewardModels()); + auto resultPtr = storm::api::verifyWithSparseEngine(env, underlyingMdp, storm::api::createTask(formulaPtr, false)); STORM_LOG_THROW(resultPtr, storm::exceptions::UnexpectedException, "No check result obtained."); STORM_LOG_THROW(resultPtr->isExplicitQuantitativeCheckResult(), storm::exceptions::UnexpectedException, "Unexpected Check result Type."); - std::vector resultVec = std::move(resultPtr->template asExplicitQuantitativeCheckResult().getValueVector()); + std::vector resultVec = std::move(resultPtr->template asExplicitQuantitativeCheckResult().getValueVector()); ExtremeValueBound res; if (info.minimize()) { res.min = false; @@ -344,9 +374,7 @@ typename PreprocessingPomdpValueBoundsModelChecker::ExtremeValueBound return res; } -template class PreprocessingPomdpValueBoundsModelChecker; +template class PreprocessingPomdpValueBoundsModelChecker>; -template class PreprocessingPomdpValueBoundsModelChecker; -} // namespace modelchecker -} // namespace pomdp -} // namespace storm \ No newline at end of file +template class PreprocessingPomdpValueBoundsModelChecker>; +} // namespace storm::pomdp::modelchecker \ No newline at end of file diff --git a/src/storm-pomdp/modelchecker/PreprocessingPomdpValueBoundsModelChecker.h b/src/storm-pomdp/modelchecker/PreprocessingPomdpValueBoundsModelChecker.h index ade0b950e8..97259a52b1 100644 --- a/src/storm-pomdp/modelchecker/PreprocessingPomdpValueBoundsModelChecker.h +++ b/src/storm-pomdp/modelchecker/PreprocessingPomdpValueBoundsModelChecker.h @@ -3,8 +3,6 @@ #include "storm-pomdp/analysis/FormulaInformation.h" #include "storm-pomdp/storage/BeliefExplorationBounds.h" #include "storm/api/verification.h" -#include "storm/models/sparse/Pomdp.h" -#include "storm/solver/SolverSelectionOptions.h" namespace storm { class Environment; @@ -16,15 +14,15 @@ class CheckResult; namespace logic { class Formula; } -namespace pomdp { -namespace modelchecker { -template +namespace pomdp::modelchecker { +template class PreprocessingPomdpValueBoundsModelChecker { public: - typedef pomdp::storage::PreprocessingPomdpValueBounds ValueBounds; - typedef pomdp::storage::ExtremePOMDPValueBound ExtremeValueBound; + using PomdpValueType = PomdpType::ValueType; + typedef pomdp::storage::PreprocessingPomdpValueBounds ValueBounds; + typedef pomdp::storage::ExtremePOMDPValueBound ExtremeValueBound; - PreprocessingPomdpValueBoundsModelChecker(storm::models::sparse::Pomdp const& pomdp); + explicit PreprocessingPomdpValueBoundsModelChecker(PomdpType const& pomdp); ValueBounds getValueBounds(storm::logic::Formula const& formula); @@ -40,22 +38,21 @@ class PreprocessingPomdpValueBoundsModelChecker { storm::pomdp::analysis::FormulaInformation const& info); private: - storm::models::sparse::Pomdp const& pomdp; + PomdpType const& pomdp; - std::vector getChoiceValues(std::vector const& stateValues, std::vector* actionBasedRewards); + std::vector getChoiceValues(std::vector const& stateValues, std::vector* actionBasedRewards); - std::pair, storm::storage::Scheduler> computeValuesForGuessedScheduler( - storm::Environment const& env, std::vector const& stateValues, std::vector* actionBasedRewards, + std::pair, storm::storage::Scheduler> computeValuesForGuessedScheduler( + storm::Environment const& env, std::vector const& stateValues, std::vector* actionBasedRewards, storm::logic::Formula const& formula, storm::pomdp::analysis::FormulaInformation const& info, - std::shared_ptr> underlyingMdp, ValueType const& scoreThreshold, bool relativeScore); + std::shared_ptr> underlyingMdp, PomdpValueType const& scoreThreshold, bool relativeScore); - std::pair, storm::storage::Scheduler> computeValuesForRandomFMPolicy( + std::pair, storm::storage::Scheduler> computeValuesForRandomFMPolicy( storm::Environment const& env, storm::logic::Formula const& formula, storm::pomdp::analysis::FormulaInformation const& info, uint64_t memoryBound); - [[maybe_unused]] std::pair, storm::storage::Scheduler> computeValuesForRandomMemorylessPolicy( + [[maybe_unused]] std::pair, storm::storage::Scheduler> computeValuesForRandomMemorylessPolicy( storm::Environment const& env, storm::logic::Formula const& formula, storm::pomdp::analysis::FormulaInformation const& info, - std::shared_ptr> underlyingMdp); + std::shared_ptr> underlyingMdp); }; -} // namespace modelchecker -} // namespace pomdp +} // namespace pomdp::modelchecker } // namespace storm \ No newline at end of file diff --git a/src/storm-pomdp/storage/BeliefExplorationBounds.cpp b/src/storm-pomdp/storage/BeliefExplorationBounds.cpp deleted file mode 100644 index 4819652143..0000000000 --- a/src/storm-pomdp/storage/BeliefExplorationBounds.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include "BeliefExplorationBounds.h" - -#include "storm/adapters/RationalNumberAdapter.h" - -namespace storm { -namespace pomdp { -namespace storage { - -template -ValueType PreprocessingPomdpValueBounds::getLowerBound(uint64_t scheduler_id, uint64_t const& state) { - STORM_LOG_ASSERT(!lower.empty(), "Requested a lower bound but none were available."); - return lower[scheduler_id][state]; -} - -template -ValueType PreprocessingPomdpValueBounds::getUpperBound(uint64_t scheduler_id, uint64_t const& state) { - STORM_LOG_ASSERT(!upper.empty(), "Requested an upper bound but none were available."); - return upper[scheduler_id][state]; -} - -template -ValueType PreprocessingPomdpValueBounds::getHighestLowerBound(uint64_t const& state) { - STORM_LOG_ASSERT(!lower.empty(), "Requested a lower bound but none were available."); - auto it = lower.begin(); - ValueType result = (*it)[state]; - for (++it; it != lower.end(); ++it) { - result = std::max(result, (*it)[state]); - } - return result; -} - -template -ValueType PreprocessingPomdpValueBounds::getSmallestUpperBound(uint64_t const& state) { - STORM_LOG_ASSERT(!upper.empty(), "Requested an upper bound but none were available."); - auto it = upper.begin(); - ValueType result = (*it)[state]; - for (++it; it != upper.end(); ++it) { - result = std::min(result, (*it)[state]); - } - return result; -} - -template -ValueType ExtremePOMDPValueBound::getValueForState(uint64_t const& state) { - STORM_LOG_ASSERT(!values.empty(), "Requested an extreme bound but none were available."); - return values[state]; -} - -template struct PreprocessingPomdpValueBounds; -template struct PreprocessingPomdpValueBounds; - -template struct ExtremePOMDPValueBound; -template struct ExtremePOMDPValueBound; -} // namespace storage -} // namespace pomdp -} // namespace storm \ No newline at end of file diff --git a/src/storm-pomdp/storage/BeliefExplorationBounds.h b/src/storm-pomdp/storage/BeliefExplorationBounds.h index cde4853f7d..a95fa8e493 100644 --- a/src/storm-pomdp/storage/BeliefExplorationBounds.h +++ b/src/storm-pomdp/storage/BeliefExplorationBounds.h @@ -1,21 +1,28 @@ #pragma once +#include +#include +#include #include + +#include "storm/storage/BitVector.h" #include "storm/storage/Scheduler.h" +#include "storm/utility/constants.h" +#include "storm/utility/macros.h" + namespace storm { namespace pomdp { namespace storage { - /** * Struct for storing precomputed values bounding the actual values on the POMDP */ -template +template struct PreprocessingPomdpValueBounds { // Vectors containing upper and lower bound values for the POMDP states - std::vector> lower; - std::vector> upper; - std::vector> lowerSchedulers; - std::vector> upperSchedulers; + std::vector> lower; + std::vector> upper; + std::vector> lowerSchedulers; + std::vector> upperSchedulers; /** * Picks the precomputed lower bound for a given scheduler index and state of the POMDP @@ -23,44 +30,139 @@ struct PreprocessingPomdpValueBounds { * @param state the state ID * @return the lower bound value */ - ValueType getLowerBound(uint64_t scheduler_id, uint64_t const& state); + template + OutputValueType getLowerBound(uint64_t scheduler_id, uint64_t const& state) { + STORM_LOG_ASSERT(!lower.empty(), "requested a lower bound but none were available"); + return storm::utility::convertNumber(lower[scheduler_id][state]); + } /** * Picks the precomputed upper bound for a given scheduler index and state of the POMDP * @param scheduler_id the scheduler ID * @param state the state ID * @return the smallest upper bound value */ - ValueType getUpperBound(uint64_t scheduler_id, uint64_t const& state); + template + OutputValueType getUpperBound(uint64_t scheduler_id, uint64_t const& state) { + STORM_LOG_ASSERT(!upper.empty(), "requested an upper bound but none were available"); + return storm::utility::convertNumber(upper[scheduler_id][state]); + } /** * Picks the largest precomputed lower bound for a given state of the POMDP * @param state the state ID * @return the largest lower bound value */ - ValueType getHighestLowerBound(uint64_t const& state); + template + OutputValueType getHighestLowerBound(uint64_t const& state) { + STORM_LOG_ASSERT(!lower.empty(), "requested a lower bound but none were available"); + auto it = lower.begin(); + POMDPValueType result = (*it)[state]; + for (++it; it != lower.end(); ++it) { + result = std::max(result, (*it)[state]); + } + return storm::utility::convertNumber(result); + } /** * Picks the smallest precomputed upper bound for a given state of the POMDP * @param state the state ID * @return the smallest upper bound value */ - ValueType getSmallestUpperBound(uint64_t const& state); + template + OutputValueType getSmallestUpperBound(uint64_t const& state) { + STORM_LOG_ASSERT(!upper.empty(), "requested an upper bound but none were available"); + auto it = upper.begin(); + POMDPValueType result = (*it)[state]; + for (++it; it != upper.end(); ++it) { + result = std::min(result, (*it)[state]); + } + return storm::utility::convertNumber(result); + } + + template + PreprocessingPomdpValueBounds toValueType() { + PreprocessingPomdpValueBounds convertedBounds; + for (auto const& vec : lower) { + std::vector resultVector; + resultVector.reserve(vec.size()); + for (auto const& oldValue : vec) { + resultVector.push_back(storm::utility::convertNumber(oldValue)); + } + convertedBounds.lower.push_back(resultVector); + } + for (auto const& vec : upper) { + std::vector resultVector; + resultVector.reserve(vec.size()); + for (auto const& oldValue : vec) { + resultVector.push_back(storm::utility::convertNumber(oldValue)); + } + convertedBounds.upper.push_back(resultVector); + } + for (auto const& sched : lowerSchedulers) { + convertedBounds.lowerSchedulers.push_back(sched.template toValueType()); + } + for (auto const& sched : upperSchedulers) { + convertedBounds.upperSchedulers.push_back(sched.template toValueType()); + } + return convertedBounds; + } }; /** * Struct to store the extreme bound values needed for the reward correction values when clipping is used */ -template +template struct ExtremePOMDPValueBound { bool min; - std::vector values; + std::vector values; storm::storage::BitVector isInfinite; /** * Get the extreme bound value for a given state * @param state the state ID * @return the bound value */ - ValueType getValueForState(uint64_t const& state); + template + OutputValueType getValueForState(uint64_t const& state) { + STORM_LOG_ASSERT(!values.empty(), "requested an extreme bound but none were available"); + return storm::utility::convertNumber(values[state]); + } + + std::vector copyValues() const { + std::vector resultVector(values); + return resultVector; + } + + template + std::vector copyValues() const { + std::vector resultVector; + resultVector.reserve(values.size()); + for (auto const& oldValue : values) { + resultVector.push_back(storm::utility::convertNumber(oldValue)); + } + return resultVector; + } + + template + ExtremePOMDPValueBound toValueType() { + ExtremePOMDPValueBound convertedBounds; + convertedBounds.values.reserve(values.size()); + for (auto const& oldValue : values) { + convertedBounds.values.push_back(storm::utility::convertNumber(oldValue)); + } + convertedBounds.min = min; + convertedBounds.isInfinite = isInfinite; + return convertedBounds; + } }; + +/** + * Struct for storing precomputed values bounding the actual values on the POMDP + */ +template +struct BeliefExplorationBounds { + std::optional> preprocessingBounds = std::nullopt; + std::optional> extremeBounds = std::nullopt; +}; + } // namespace storage } // namespace pomdp -} // namespace storm \ No newline at end of file +} // namespace storm diff --git a/src/storm-pomdp/storage/BeliefExplorationResult.h b/src/storm-pomdp/storage/BeliefExplorationResult.h new file mode 100644 index 0000000000..9494531e71 --- /dev/null +++ b/src/storm-pomdp/storage/BeliefExplorationResult.h @@ -0,0 +1,58 @@ +#pragma once + +#include + +#include "storm/utility/constants.h" +#include "storm/utility/macros.h" + +namespace storm::pomdp::storage { +/** + * Struct used to store the results of the model checker + */ +template +struct BeliefExplorationResult { + BeliefExplorationResult(ValueType lower, ValueType upper) : lowerBound(lower), upperBound(upper) {}; + ValueType diff(bool relative = false) const { + if (!(upperBound.has_value() && lowerBound.has_value())) { + STORM_LOG_WARN("Either the upper or the lower bound is not set. Difference is undefined."); + return storm::utility::infinity(); + } + ValueType diff = *upperBound - *lowerBound; + if (diff < storm::utility::zero()) { + STORM_LOG_WARN_COND(diff >= storm::utility::convertNumber(1e-6), + "Upper bound '" << *upperBound << "' is smaller than lower bound '" << *lowerBound << "': Difference is " << diff << "."); + diff = storm::utility::zero(); + } + if (relative && !storm::utility::isZero(*upperBound)) { + diff /= *upperBound; + } + return diff; + }; + bool updateLowerBound(ValueType const& value) { + if (value > lowerBound) { + lowerBound = value; + return true; + } + return false; + }; + + bool updateUpperBound(ValueType const& value) { + if (value < upperBound) { + upperBound = value; + return true; + } + return false; + }; + + void removeLowerBound() { + lowerBound = std::nullopt; + }; + + void removeUpperBound() { + upperBound = std::nullopt; + }; + + std::optional lowerBound = std::nullopt; + std::optional upperBound = std::nullopt; +}; +} // namespace storm::pomdp::storage diff --git a/src/storm-pomdp/storage/BeliefManager.cpp b/src/storm-pomdp/storage/BeliefManager.cpp deleted file mode 100644 index 3929b0a16d..0000000000 --- a/src/storm-pomdp/storage/BeliefManager.cpp +++ /dev/null @@ -1,826 +0,0 @@ -#include "storm-pomdp/storage/BeliefManager.h" - -#include "storm/adapters/RationalNumberAdapter.h" -#include "storm/models/sparse/Pomdp.h" -#include "storm/solver/GlpkLpSolver.h" -#include "storm/storage/expressions/Expression.h" -#include "storm/storage/expressions/ExpressionManager.h" -#include "storm/utility/macros.h" - -namespace storm { -namespace storage { - -template -uint64_t BeliefManager::Triangulation::size() const { - return weights.size(); -} - -template -BeliefManager::FreudenthalDiff::FreudenthalDiff(StateType const &dimension, BeliefValueType diff) - : dimension(dimension), diff(std::move(diff)) { - // Intentionally left empty -} - -template -bool BeliefManager::FreudenthalDiff::operator>(FreudenthalDiff const &other) const { - if (diff != other.diff) { - return diff > other.diff; - } else { - return dimension < other.dimension; - } -} - -template -bool BeliefManager::Belief_equal_to::operator()(const BeliefType &lhBelief, const BeliefType &rhBelief) const { - return lhBelief == rhBelief; -} - -template<> -bool BeliefManager, double, uint64_t>::Belief_equal_to::operator()(const BeliefType &lhBelief, - const BeliefType &rhBelief) const { - // If the sizes are different, we don't have to look inside the belief - if (lhBelief.size() != rhBelief.size()) { - return false; - } - // Assumes that beliefs are ordered - auto lhIt = lhBelief.begin(); - auto rhIt = rhBelief.begin(); - while (lhIt != lhBelief.end() || rhIt != rhBelief.end()) { - // Iterate over the entries simultaneously, beliefs not equal if they contain either different states or different values for the same state - if (lhIt->first != rhIt->first || std::fabs(lhIt->second - rhIt->second) > 1e-15) { - return false; - } - ++lhIt; - ++rhIt; - } - return lhIt == lhBelief.end() && rhIt == rhBelief.end(); -} - -template -std::size_t BeliefManager::BeliefHash::operator()(const BeliefType &belief) const { - std::size_t seed = 0; - // Assumes that beliefs are ordered - for (auto const &entry : belief) { - boost::hash_combine(seed, entry.first); - boost::hash_combine(seed, entry.second); - } - return seed; -} - -template<> -std::size_t BeliefManager, double, uint64_t>::BeliefHash::operator()(const BeliefType &belief) const { - std::size_t seed = 0; - // Assumes that beliefs are ordered - for (auto const &entry : belief) { - boost::hash_combine(seed, entry.first); - boost::hash_combine(seed, round(storm::utility::convertNumber(entry.second) * 1e15)); - } - return seed; -} - -template -BeliefManager::BeliefManager(PomdpType const &pomdp, BeliefValueType const &precision, - TriangulationMode const &triangulationMode) - : pomdp(pomdp), cc(precision, false), triangulationMode(triangulationMode) { - beliefToIdMap.resize(pomdp.getNrObservations()); - initialBeliefId = computeInitialBelief(); -} - -template -void BeliefManager::setRewardModel(std::optional rewardModelName) { - if (rewardModelName) { - auto const &rewardModel = pomdp.getRewardModel(rewardModelName.value()); - pomdpActionRewardVector = rewardModel.getTotalRewardVector(pomdp.getTransitionMatrix()); - } else { - setRewardModel(pomdp.getUniqueRewardModelName()); - } -} - -template -void BeliefManager::unsetRewardModel() { - pomdpActionRewardVector.clear(); -} - -template -typename BeliefManager::BeliefId BeliefManager::noId() const { - return std::numeric_limits::max(); -} - -template -bool BeliefManager::isEqual(BeliefId const &first, BeliefId const &second) const { - return isEqual(getBelief(first), getBelief(second)); -} - -template -std::string BeliefManager::toString(BeliefId const &beliefId) const { - return toString(getBelief(beliefId)); -} - -template -std::string BeliefManager::toString(Triangulation const &t) const { - std::stringstream str; - str << "(\n"; - for (uint64_t i = 0; i < t.size(); ++i) { - str << "\t" << t.weights[i] << " * \t" << toString(getBelief(t.gridPoints[i])) << "\n"; - } - str << ")\n"; - return str.str(); -} - -template -typename BeliefManager::ValueType BeliefManager::getWeightedSum( - BeliefId const &beliefId, std::vector const &summands) { - auto result = storm::utility::zero(); - for (auto const &entry : getBelief(beliefId)) { - result += storm::utility::convertNumber(entry.second) * storm::utility::convertNumber(summands.at(entry.first)); - } - return result; -} - -template -std::pair::ValueType> BeliefManager::getWeightedSum( - BeliefId const &beliefId, std::unordered_map const &summands) { - bool successful = true; - auto result = storm::utility::zero(); - for (auto const &entry : getBelief(beliefId)) { - auto probIter = summands.find(entry.first); - if (probIter != summands.end()) { - result += storm::utility::convertNumber(entry.second) * storm::utility::convertNumber(summands.at(entry.first)); - } else { - successful = false; - break; - } - } - return {successful, result}; -} - -template -typename BeliefManager::BeliefId const &BeliefManager::getInitialBelief() const { - return initialBeliefId; -} - -template -typename BeliefManager::ValueType BeliefManager::getBeliefActionReward( - BeliefId const &beliefId, uint64_t const &localActionIndex) const { - auto const &belief = getBelief(beliefId); - STORM_LOG_ASSERT(!pomdpActionRewardVector.empty(), "Requested a reward although no reward model was specified."); - auto result = storm::utility::zero(); - auto const &choiceIndices = pomdp.getTransitionMatrix().getRowGroupIndices(); - for (auto const &entry : belief) { - uint64_t choiceIndex = choiceIndices[entry.first] + localActionIndex; - STORM_LOG_ASSERT(choiceIndex < choiceIndices[entry.first + 1], "Invalid local action index."); - STORM_LOG_ASSERT(choiceIndex < pomdpActionRewardVector.size(), "Invalid choice index."); - result += storm::utility::convertNumber(entry.second) * pomdpActionRewardVector[choiceIndex]; - } - return result; -} - -template -uint32_t BeliefManager::getBeliefObservation(BeliefId beliefId) { - return getBeliefObservation(getBelief(beliefId)); -} - -template -uint64_t BeliefManager::getBeliefNumberOfChoices(BeliefId beliefId) { - auto const &belief = getBelief(beliefId); - return pomdp.getNumberOfChoices(belief.begin()->first); -} - -template -typename BeliefManager::Triangulation BeliefManager::triangulateBelief( - BeliefId beliefId, BeliefValueType resolution) { - return triangulateBelief(getBelief(beliefId), resolution); -} - -template -template -void BeliefManager::addToDistribution(DistributionType &distr, StateType const &state, BeliefValueType const &value) { - auto insertionRes = distr.emplace(state, value); - if (!insertionRes.second) { - insertionRes.first->second += value; - } -} - -template -template -void BeliefManager::adjustDistribution(DistributionType &distr) { - if (distr.size() == 1 && cc.isEqual(distr.begin()->second, storm::utility::one())) { - // If the distribution consists of only one entry and its value is sufficiently close to 1, make it exactly 1 to avoid numerical problems - distr.begin()->second = storm::utility::one(); - } -} - -template -void BeliefManager::joinSupport(BeliefId const &beliefId, BeliefSupportType &support) { - auto const &belief = getBelief(beliefId); - for (auto const &entry : belief) { - support.insert(entry.first); - } -} - -template -typename BeliefManager::BeliefId BeliefManager::getNumberOfBeliefIds() const { - return beliefs.size(); -} - -template -std::vector::BeliefId, - typename BeliefManager::ValueType>> -BeliefManager::expandAndTriangulate(storm::Environment const &env, BeliefId const &beliefId, uint64_t actionIndex, - std::vector const &observationResolutions) { - return expandInternal(env, beliefId, actionIndex, observationResolutions); -} - -template -std::vector::BeliefId, - typename BeliefManager::ValueType>> -BeliefManager::expandAndClip(storm::Environment const &env, BeliefId const &beliefId, uint64_t actionIndex, - std::vector const &observationResolutions) { - return expandInternal(env, beliefId, actionIndex, std::nullopt, observationResolutions); -} - -template -std::vector::BeliefId, - typename BeliefManager::ValueType>> -BeliefManager::expand(storm::Environment const &env, BeliefId const &beliefId, uint64_t actionIndex) { - return expandInternal(env, beliefId, actionIndex); -} - -template -typename BeliefManager::BeliefType const &BeliefManager::getBelief( - BeliefId const &id) const { - STORM_LOG_ASSERT(id != noId(), "Tried to get a non-existent belief."); - STORM_LOG_ASSERT(id < getNumberOfBeliefIds(), "Belief index " << id << " is out of range."); - return beliefs[id]; -} - -template -typename BeliefManager::BeliefId BeliefManager::getId( - BeliefType const &belief) const { - uint32_t obs = getBeliefObservation(belief); - STORM_LOG_ASSERT(obs < beliefToIdMap.size(), "Belief has unknown observation."); - auto idIt = beliefToIdMap[obs].find(belief); - STORM_LOG_ASSERT(idIt != beliefToIdMap[obs].end(), "Unknown Belief."); - return idIt->second; -} - -template -std::string BeliefManager::toString(BeliefType const &belief) const { - std::stringstream str; - str << "{ "; - bool first = true; - for (auto const &entry : belief) { - if (first) { - first = false; - } else { - str << ", "; - } - str << entry.first << ": " << std::setprecision(std::numeric_limits::max_digits10 + 1) << entry.second; - } - str << " }"; - return str.str(); -} - -template -bool BeliefManager::isEqual(BeliefType const &first, BeliefType const &second) const { - if (first.size() != second.size()) { - return false; - } - auto secondIt = second.begin(); - for (auto const &firstEntry : first) { - if (firstEntry.first != secondIt->first) { - return false; - } - if (!cc.isEqual(firstEntry.second, secondIt->second)) { - return false; - } - ++secondIt; - } - return true; -} - -template -bool BeliefManager::assertBelief(BeliefType const &belief) const { - auto sum = storm::utility::zero(); - std::optional observation; - for (auto const &entry : belief) { - if (entry.first >= pomdp.getNumberOfStates()) { - STORM_LOG_ERROR("Belief does refer to non-existing pomdp state " << entry.first << "."); - return false; - } - uint64_t entryObservation = pomdp.getObservation(entry.first); - if (observation) { - if (observation.value() != entryObservation) { - STORM_LOG_ERROR("Beliefsupport contains different observations."); - return false; - } - } else { - observation = entryObservation; - } - // Don't use cc for these checks, because computations with zero are usually fine - if (storm::utility::isZero(entry.second)) { - // We assume that beliefs only consider their support. - STORM_LOG_ERROR("Zero belief probability."); - return false; - } - if (entry.second < storm::utility::zero()) { - STORM_LOG_ERROR("Negative belief probability."); - return false; - } - if (cc.isLess(storm::utility::one(), entry.second)) { - STORM_LOG_ERROR("Belief probability greater than one."); - return false; - } - sum += entry.second; - } - if (!cc.isOne(sum)) { - STORM_LOG_ERROR("Belief does not sum up to one. (" << sum << " instead)."); - return false; - } - return true; -} - -template -bool BeliefManager::assertTriangulation(BeliefType const &belief, Triangulation const &triangulation) const { - if (triangulation.weights.size() != triangulation.gridPoints.size()) { - STORM_LOG_ERROR("Number of weights and points in triangulation does not match."); - return false; - } - if (triangulation.size() == 0) { - STORM_LOG_ERROR("Empty triangulation."); - return false; - } - BeliefType triangulatedBelief; - auto weightSum = storm::utility::zero(); - for (uint64_t i = 0; i < triangulation.weights.size(); ++i) { - if (cc.isZero(triangulation.weights[i])) { - STORM_LOG_ERROR("Zero weight in triangulation."); - return false; - } - if (cc.isLess(triangulation.weights[i], storm::utility::zero())) { - STORM_LOG_ERROR("Negative weight in triangulation."); - return false; - } - if (cc.isLess(storm::utility::one(), triangulation.weights[i])) { - STORM_LOG_ERROR("Weight greater than one in triangulation."); - } - weightSum += triangulation.weights[i]; - BeliefType const &gridPoint = getBelief(triangulation.gridPoints[i]); - for (auto const &pointEntry : gridPoint) { - BeliefValueType &triangulatedValue = triangulatedBelief.emplace(pointEntry.first, storm::utility::zero()).first->second; - triangulatedValue += triangulation.weights[i] * pointEntry.second; - } - } - if (!cc.isOne(weightSum)) { - STORM_LOG_ERROR("Triangulation weights do not sum up to one."); - return false; - } - if (!assertBelief(triangulatedBelief)) { - STORM_LOG_ERROR("Triangulated belief is not a belief."); - } - if (!isEqual(belief, triangulatedBelief)) { - STORM_LOG_ERROR("Belief:\n\t" << toString(belief) << "\ndoes not match triangulated belief:\n\t" << toString(triangulatedBelief) << "."); - return false; - } - return true; -} - -template -uint32_t BeliefManager::getBeliefObservation(BeliefType belief) const { - STORM_LOG_ASSERT(assertBelief(belief), "Invalid belief."); - return pomdp.getObservation(belief.begin()->first); -} - -template -void BeliefManager::triangulateBeliefFreudenthal(BeliefType const &belief, BeliefValueType const &resolution, - Triangulation &result) { - STORM_LOG_ASSERT(resolution != 0, "Invalid resolution: 0."); - STORM_LOG_ASSERT(storm::utility::isInteger(resolution), "Expected an integer resolution."); - StateType numEntries = belief.size(); - // This is the Freudenthal Triangulation as described in Lovejoy (a whole lotta math) - // Probabilities will be triangulated to values in 0/N, 1/N, 2/N, ..., N/N - // Variable names are mostly based on the paper - // However, we speed this up a little by exploiting that belief states usually have sparse support (i.e. numEntries is much smaller than - // pomdp.getNumberOfStates()). Initialize diffs and the first row of the 'qs' matrix (aka v) - std::set> sorted_diffs; // d (and p?) in the paper - std::vector qsRow; // Row of the 'qs' matrix from the paper (initially corresponds to v - qsRow.reserve(numEntries); - std::vector toOriginalIndicesMap; // Maps 'local' indices to the original pomdp state indices - toOriginalIndicesMap.reserve(numEntries); - BeliefValueType x = resolution; - for (auto const &entry : belief) { - qsRow.push_back(storm::utility::floor(x)); // v - sorted_diffs.emplace(toOriginalIndicesMap.size(), x - qsRow.back()); // x-v - toOriginalIndicesMap.push_back(entry.first); - x -= entry.second * resolution; - } - // Insert a dummy 0 column in the qs matrix so the loops below are a bit simpler - qsRow.push_back(storm::utility::zero()); - - result.weights.reserve(numEntries); - result.gridPoints.reserve(numEntries); - auto currentSortedDiff = sorted_diffs.begin(); - auto previousSortedDiff = sorted_diffs.end(); - --previousSortedDiff; - for (StateType i = 0; i < numEntries; ++i) { - // Compute the weight for the grid points - BeliefValueType weight = previousSortedDiff->diff - currentSortedDiff->diff; - if (i == 0) { - // The first weight is a bit different - weight += storm::utility::one(); - } else { - // 'compute' the next row of the qs matrix - qsRow[previousSortedDiff->dimension] += storm::utility::one(); - } - if (!cc.isZero(weight)) { - result.weights.push_back(weight); - // Compute the grid point - BeliefType gridPoint; - for (StateType j = 0; j < numEntries; ++j) { - BeliefValueType gridPointEntry = qsRow[j] - qsRow[j + 1]; - if (!cc.isZero(gridPointEntry)) { - gridPoint[toOriginalIndicesMap[j]] = gridPointEntry / resolution; - } - } - result.gridPoints.push_back(getOrAddBeliefId(gridPoint)); - } - previousSortedDiff = currentSortedDiff++; - } -} - -template -void BeliefManager::triangulateBeliefDynamic(BeliefType const &belief, BeliefValueType const &resolution, - Triangulation &result) { - // Find the best resolution for this belief, i.e., N such that the largest distance between one of the belief values to a value in {i/N | 0 ≤ i ≤ N} is - // minimal - STORM_LOG_ASSERT(storm::utility::isInteger(resolution), "Expected an integer resolution."); - BeliefValueType finalResolution = resolution; - uint64_t finalResolutionMisses = belief.size() + 1; - // We don't need to check resolutions that are smaller than the maximal resolution divided by 2 (as we already checked multiples of these) - for (BeliefValueType currResolution = resolution; currResolution > resolution / 2; --currResolution) { - uint64_t currResMisses = 0; - bool continueWithNextResolution = false; - for (auto const &belEntry : belief) { - BeliefValueType product = belEntry.second * currResolution; - if (!cc.isZero(product - storm::utility::round(product))) { - ++currResMisses; - if (currResMisses >= finalResolutionMisses) { - // This resolution is not better than a previous resolution - continueWithNextResolution = true; - break; - } - } - } - if (!continueWithNextResolution) { - STORM_LOG_ASSERT(currResMisses < finalResolutionMisses, "Distance for this resolution should not be larger than a previously checked one."); - finalResolution = currResolution; - finalResolutionMisses = currResMisses; - if (currResMisses == 0) { - break; - } - } - } - - STORM_LOG_TRACE("Picking resolution " << finalResolution << " for belief " << toString(belief)); - - // do standard freudenthal with the found resolution - triangulateBeliefFreudenthal(belief, finalResolution, result); -} - -template -typename BeliefManager::Triangulation BeliefManager::triangulateBelief( - BeliefType const &belief, BeliefValueType const &resolution) { - STORM_LOG_ASSERT(assertBelief(belief), "Input belief for triangulation is not valid."); - Triangulation result; - // Quickly triangulate Dirac beliefs - if (belief.size() == 1u) { - result.weights.push_back(storm::utility::one()); - result.gridPoints.push_back(getOrAddBeliefId(belief)); - } else { - auto ceiledResolution = storm::utility::ceil(resolution); - switch (triangulationMode) { - case TriangulationMode::Static: - triangulateBeliefFreudenthal(belief, ceiledResolution, result); - break; - case TriangulationMode::Dynamic: - triangulateBeliefDynamic(belief, ceiledResolution, result); - break; - default: - STORM_LOG_ASSERT(false, "Invalid triangulation mode."); - } - } - STORM_LOG_ASSERT(assertTriangulation(belief, result), "Incorrect triangulation: " << toString(result)); - return result; -} - -template -std::vector::BeliefId, - typename BeliefManager::ValueType>> -BeliefManager::expandInternal(storm::Environment const &env, BeliefId const &beliefId, uint64_t actionIndex, - std::optional> const &observationTriangulationResolutions, - std::optional> const &observationGridClippingResolutions) { - std::vector> destinations; - - BeliefType belief = getBelief(beliefId); - - // Find the probability we go to each observation - BeliefType successorObs; // This is actually not a belief but has the same type - for (auto const &pointEntry : belief) { - uint64_t state = pointEntry.first; - for (auto const &pomdpTransition : pomdp.getTransitionMatrix().getRow(state, actionIndex)) { - if (!storm::utility::isZero(pomdpTransition.getValue())) { - auto obs = pomdp.getObservation(pomdpTransition.getColumn()); - addToDistribution(successorObs, obs, pointEntry.second * storm::utility::convertNumber(pomdpTransition.getValue())); - } - } - } - adjustDistribution(successorObs); - - // Now for each successor observation we find and potentially triangulate the successor belief - for (auto const &successor : successorObs) { - BeliefType successorBelief; - for (auto const &pointEntry : belief) { - uint64_t state = pointEntry.first; - for (auto const &pomdpTransition : pomdp.getTransitionMatrix().getRow(state, actionIndex)) { - if (pomdp.getObservation(pomdpTransition.getColumn()) == successor.first) { - BeliefValueType prob = pointEntry.second * storm::utility::convertNumber(pomdpTransition.getValue()) / successor.second; - addToDistribution(successorBelief, pomdpTransition.getColumn(), prob); - } - } - } - adjustDistribution(successorBelief); - STORM_LOG_ASSERT(assertBelief(successorBelief), "Invalid successor belief."); - - // Insert the destination. We know that destinations have to be disjoint since they have different observations - if (observationTriangulationResolutions) { - Triangulation triangulation = triangulateBelief(successorBelief, observationTriangulationResolutions.value()[successor.first]); - for (size_t j = 0; j < triangulation.size(); ++j) { - // Here we additionally assume that triangulation.gridPoints does not contain the same point multiple times - BeliefValueType a = triangulation.weights[j] * successor.second; - destinations.emplace_back(triangulation.gridPoints[j], storm::utility::convertNumber(a)); - } - } else if (observationGridClippingResolutions) { - BeliefClipping clipping = clipBeliefToGrid(env, successorBelief, observationGridClippingResolutions.value()[successor.first], - storm::storage::BitVector(pomdp.getNumberOfStates())); - if (clipping.isClippable) { - BeliefValueType a = (storm::utility::one() - clipping.delta) * successor.second; - destinations.emplace_back(clipping.targetBelief, storm::utility::convertNumber(a)); - } else { - // Belief on Grid - destinations.emplace_back(getOrAddBeliefId(successorBelief), storm::utility::convertNumber(successor.second)); - } - } else { - destinations.emplace_back(getOrAddBeliefId(successorBelief), storm::utility::convertNumber(successor.second)); - } - } - - return destinations; -} - -template -typename BeliefManager::BeliefClipping BeliefManager::clipBeliefToGrid( - storm::Environment const &env, BeliefId const &beliefId, uint64_t resolution, storm::storage::BitVector isInfinite) { - auto res = clipBeliefToGrid(env, getBelief(beliefId), resolution, isInfinite); - res.startingBelief = beliefId; - return res; -} - -template -typename BeliefManager::BeliefClipping BeliefManager::clipBeliefToGrid( - storm::Environment const &env, BeliefType const &belief, uint64_t resolution, const storm::storage::BitVector &isInfinite) { - [[maybe_unused]] uint32_t obs = getBeliefObservation(belief); - STORM_LOG_ASSERT(obs < beliefToIdMap.size(), "Belief has unknown observation."); - if (!lpSolver) { - lpSolver = storm::utility::solver::getLpSolver(env, "POMDP LP Solver"); - } else { - lpSolver->pop(); - } - lpSolver->push(); - - std::vector helper(belief.size(), storm::utility::zero()); - helper[0] = storm::utility::convertNumber(resolution); - bool done = false; - // Set-up Variables - std::vector decisionVariables; - // Add variable for the clipping value, it is to be minimized - auto bigDelta = lpSolver->addBoundedContinuousVariable("D", storm::utility::zero(), storm::utility::one(), - storm::utility::one()); - // State clipping values - std::vector deltas; - uint64_t i = 0; - for (auto const &state : belief) { - // This is a quite dirty fix to enable GLPK for the TACAS '22 implementation without substantially changing the implementation for Gurobi. - if (typeid(*lpSolver) == typeid(storm::solver::GlpkLpSolver) && !isInfinite.empty()) { - if (isInfinite[state.first]) { - auto localDelta = lpSolver->addBoundedContinuousVariable("d_" + std::to_string(i), storm::utility::zero(), state.second); - auto deltaExpr = storm::expressions::Expression(localDelta); - deltas.push_back(deltaExpr); - lpSolver->addConstraint("state_val_inf_" + std::to_string(i), deltaExpr == lpSolver->getConstant(storm::utility::zero())); - } - } else { - BeliefValueType bound = state.second; - if (!isInfinite.empty()) { - bound = isInfinite[state.first] ? storm::utility::zero() : state.second; - } - auto localDelta = lpSolver->addBoundedContinuousVariable("d_" + std::to_string(i), storm::utility::zero(), bound); - deltas.push_back(storm::expressions::Expression(localDelta)); - } - ++i; - } - lpSolver->update(); - std::vector gridCandidates; - while (!done) { - BeliefType candidate; - auto belIter = belief.begin(); - for (uint64_t j = 0; j < belief.size() - 1; ++j) { - if (!cc.isEqual(helper[j] - helper[j + 1], storm::utility::zero())) { - candidate[belIter->first] = (helper[j] - helper[j + 1]) / storm::utility::convertNumber(resolution); - } - belIter++; - } - if (!cc.isEqual(helper[belief.size() - 1], storm::utility::zero())) { - candidate[belIter->first] = helper[belief.size() - 1] / storm::utility::convertNumber(resolution); - } - if (isEqual(candidate, belief)) { - // TODO Improve handling of successors which are already on the grid - return BeliefClipping{false, noId(), noId(), storm::utility::zero(), {}, true}; - } else { - gridCandidates.push_back(candidate); - - // Add variables a_j - auto decisionVar = lpSolver->addBinaryVariable("a_" + std::to_string(gridCandidates.size() - 1)); - decisionVariables.push_back(storm::expressions::Expression(decisionVar)); - lpSolver->update(); - - i = 0; - for (auto const &state : belief) { - // Add the constraint to describe the transformation between the state values in the beliefs - // d_i - storm::expressions::Expression leftSide = deltas[i]; - storm::expressions::Expression targetValue = lpSolver->getConstant(candidate[i]); - if (candidate.find(state.first) != candidate.end()) { - targetValue = lpSolver->getConstant(candidate.at(state.first)); - } else { - targetValue = lpSolver->getConstant(storm::utility::zero()); - } - - // b(s_i) - b_j(s_i) + D * b_j(s_i) - 1 + a_j - storm::expressions::Expression rightSide = - lpSolver->getConstant(state.second) - targetValue + storm::expressions::Expression(bigDelta) * targetValue - - lpSolver->getConstant(storm::utility::one()) + storm::expressions::Expression(decisionVar); - - // Add left >= right - lpSolver->addConstraint("state_eq_" + std::to_string(i) + "_" + std::to_string(gridCandidates.size() - 1), leftSide >= rightSide); - ++i; - lpSolver->update(); - } - } - if (helper.back() == storm::utility::convertNumber(resolution)) { - // If the last entry of helper is the gridResolution, we have enumerated all necessary distributions - done = true; - } else { - // Update helper by finding the index to increment - auto helperIt = helper.end() - 1; - while (*helperIt == *(helperIt - 1)) { - --helperIt; - } - STORM_LOG_ASSERT(helperIt != helper.begin(), "Error in grid clipping - index wrong."); - // Increment the value at the index - *helperIt += 1; - // Reset all indices greater than the changed one to 0 - ++helperIt; - while (helperIt != helper.end()) { - *helperIt = 0; - ++helperIt; - } - } - } - - // Only one target belief should be chosen - lpSolver->addConstraint("choice", storm::expressions::sum(decisionVariables) == lpSolver->getConstant(storm::utility::one())); - // Link D and d_i - lpSolver->addConstraint("delta", storm::expressions::Expression(bigDelta) == storm::expressions::sum(deltas)); - // Exclude D = 0 (self-loop) - lpSolver->addConstraint("not_zero", storm::expressions::Expression(bigDelta) > lpSolver->getConstant(storm::utility::zero())); - - lpSolver->update(); - - lpSolver->optimize(); - // Get the optimal belief for clipping - BeliefId targetBelief = noId(); - // Not a belief but has the same type - BeliefType deltaValues; - auto optDelta = storm::utility::zero(); - auto deltaSum = storm::utility::zero(); - if (lpSolver->isOptimal()) { - optDelta = lpSolver->getObjectiveValue(); - for (uint64_t dist = 0; dist < gridCandidates.size(); ++dist) { - if (lpSolver->getBinaryValue(lpSolver->getManager().getVariable("a_" + std::to_string(dist)))) { - targetBelief = getOrAddBeliefId(gridCandidates[dist]); - break; - } - } - i = 0; - for (auto const &state : belief) { - auto val = lpSolver->getContinuousValue(lpSolver->getManager().getVariable("d_" + std::to_string(i))); - if (cc.isLess(storm::utility::zero(), val)) { - deltaValues.emplace(state.first, val); - deltaSum += val; - } - ++i; - } - - if (cc.isEqual(optDelta, storm::utility::zero())) { - // If we get an optimal value of 0, the LP solver considers two beliefs to be equal, possibly due to numerical instability - // For a sound result, we consider the state to not be clippable - STORM_LOG_WARN("LP solver returned an optimal value of 0. This should definitely not happen when using a grid"); - STORM_LOG_WARN("Origin" << toString(belief)); - STORM_LOG_WARN("Target [Bel " << targetBelief << "] " << toString(targetBelief)); - return BeliefClipping{false, noId(), noId(), storm::utility::zero(), {}, false}; - } - - if (optDelta == storm::utility::one()) { - STORM_LOG_WARN("LP solver returned an optimal value of 1. Sum of state clipping values is " << deltaSum); - // If we get an optimal value of 1, we cannot clip the belief as by definition this would correspond to a division by 0. - STORM_LOG_DEBUG("Origin" << toString(belief)); - STORM_LOG_DEBUG("Target [Bel " << targetBelief << "] " << toString(targetBelief)); - - if (deltaSum == storm::utility::one()) { - return BeliefClipping{false, noId(), noId(), storm::utility::zero(), {}, false}; - } - optDelta = deltaSum; - } - } - return BeliefClipping{lpSolver->isOptimal(), noId(), targetBelief, optDelta, deltaValues, false}; -} - -template -typename BeliefManager::BeliefId BeliefManager::computeInitialBelief() { - STORM_LOG_ASSERT(pomdp.getInitialStates().getNumberOfSetBits() < 2, "POMDP contains more than one initial state."); - STORM_LOG_ASSERT(pomdp.getInitialStates().getNumberOfSetBits() == 1, "POMDP does not contain an initial state."); - BeliefType belief; - belief[*pomdp.getInitialStates().begin()] = storm::utility::one(); - - STORM_LOG_ASSERT(assertBelief(belief), "Invalid initial belief."); - return getOrAddBeliefId(belief); -} - -template -typename BeliefManager::BeliefId BeliefManager::getOrAddBeliefId( - BeliefType const &belief) { - uint32_t obs = getBeliefObservation(belief); - STORM_LOG_ASSERT(obs < beliefToIdMap.size(), "Belief has unknown observation."); - auto insertioRes = beliefToIdMap[obs].emplace(belief, beliefs.size()); - if (insertioRes.second) { - // There actually was an insertion, so add the new belief - STORM_LOG_TRACE("Add Belief " << beliefs.size() << " " << toString(belief)); - beliefs.push_back(belief); - } - // Return the id - return insertioRes.first->second; -} -template -uint64_t BeliefManager::getRepresentativeState(BeliefId const &beliefId) { - return getBelief(beliefId).begin()->first; -} - -template -std::string BeliefManager::getObservationLabel(BeliefId const &beliefId) { - if (pomdp.hasObservationValuations()) { - return pomdp.getObservationValuations().toString(getBeliefObservation(beliefId)); - } else { - STORM_LOG_TRACE("Cannot get observation labels as no observation valuation has been defined for the POMDP. Return empty label instead."); - return ""; - } -} - -template -std::vector BeliefManager::getBeliefAsVector(BeliefId const &beliefId) { - return getBeliefAsVector(getBelief(beliefId)); -} - -template -std::vector BeliefManager::getBeliefAsVector(const BeliefType &belief) { - std::vector res(pomdp.getNumberOfStates(), storm::utility::zero()); - for (auto const &stateprob : belief) { - res[stateprob.first] = stateprob.second; - } - return res; -} - -template -std::vector BeliefManager::computeMatrixBeliefProduct( - const BeliefId &beliefId, storm::storage::SparseMatrix &matrix) { - std::vector beliefAsVector = getBeliefAsVector(beliefId); - std::vector res(matrix.getRowCount()); - matrix.multiplyWithVector(beliefAsVector, res); - return res; -} - -template class BeliefManager>; -template class BeliefManager, storm::RationalNumber>; - -template class BeliefManager, double>; -template class BeliefManager>; -} // namespace storage -} // namespace storm diff --git a/src/storm-pomdp/storage/BeliefManager.h b/src/storm-pomdp/storage/BeliefManager.h deleted file mode 100644 index 8f6b63a84f..0000000000 --- a/src/storm-pomdp/storage/BeliefManager.h +++ /dev/null @@ -1,172 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -#include "storm/solver/LpSolver.h" -#include "storm/storage/BitVector.h" -#include "storm/utility/ConstantsComparator.h" -#include "storm/utility/constants.h" -#include "storm/utility/solver.h" - -namespace storm { -namespace storage { -// Forward declaration -template -class SparseMatrix; - -template -class BeliefManager { - public: - typedef typename PomdpType::ValueType ValueType; - typedef boost::container::flat_map BeliefType; // iterating over this shall be ordered (for correct hash computation) - typedef boost::container::flat_set BeliefSupportType; - typedef uint64_t BeliefId; - - enum class TriangulationMode { Static, Dynamic }; - - BeliefManager(PomdpType const &pomdp, BeliefValueType const &precision, TriangulationMode const &triangulationMode); - - void setRewardModel(std::optional rewardModelName = std::nullopt); - - void unsetRewardModel(); - - struct Triangulation { - std::vector gridPoints; - std::vector weights; - uint64_t size() const; - }; - - struct BeliefClipping { - bool isClippable; - BeliefId startingBelief; - BeliefId targetBelief; - BeliefValueType delta; - BeliefType deltaValues; - bool onGrid = false; - }; - - BeliefId noId() const; - - bool isEqual(BeliefId const &first, BeliefId const &second) const; - - std::string toString(BeliefId const &beliefId) const; - - std::string toString(Triangulation const &t) const; - - ValueType getWeightedSum(BeliefId const &beliefId, std::vector const &summands); - - std::pair getWeightedSum(BeliefId const &beliefId, std::unordered_map const &summands); - - BeliefId const &getInitialBelief() const; - - ValueType getBeliefActionReward(BeliefId const &beliefId, uint64_t const &localActionIndex) const; - - uint32_t getBeliefObservation(BeliefId beliefId); - - uint64_t getBeliefNumberOfChoices(BeliefId beliefId); - - /** - * Returns the first state in the belief as a representative - * @param beliefId - * @return - */ - uint64_t getRepresentativeState(BeliefId const &beliefId); - - Triangulation triangulateBelief(BeliefId beliefId, BeliefValueType resolution); - - template - void addToDistribution(DistributionType &distr, StateType const &state, BeliefValueType const &value); - - void joinSupport(BeliefId const &beliefId, BeliefSupportType &support); - - BeliefId getNumberOfBeliefIds() const; - - std::vector> expandAndTriangulate(storm::Environment const &env, BeliefId const &beliefId, uint64_t actionIndex, - std::vector const &observationResolutions); - - std::vector> expandAndClip(storm::Environment const &env, BeliefId const &beliefId, uint64_t actionIndex, - std::vector const &observationResolutions); - - std::vector> expand(storm::Environment const &env, BeliefId const &beliefId, uint64_t actionIndex); - - BeliefClipping clipBeliefToGrid(storm::Environment const &env, BeliefId const &beliefId, uint64_t resolution, - storm::storage::BitVector isInfinite = storm::storage::BitVector()); - - std::string getObservationLabel(BeliefId const &beliefId); - - std::vector computeMatrixBeliefProduct(BeliefId const &beliefId, storm::storage::SparseMatrix &matrix); - - private: - std::vector getBeliefAsVector(BeliefId const &beliefId); - - std::vector getBeliefAsVector(const BeliefType &belief); - - BeliefClipping clipBeliefToGrid(storm::Environment const &env, BeliefType const &belief, uint64_t resolution, const storm::storage::BitVector &isInfinite); - - template - void adjustDistribution(DistributionType &distr); - - struct BeliefHash { - std::size_t operator()(const BeliefType &belief) const; - }; - - struct Belief_equal_to { - bool operator()(const BeliefType &lhBelief, const BeliefType &rhBelief) const; - }; - - struct FreudenthalDiff { - FreudenthalDiff(StateType const &dimension, BeliefValueType diff); - - StateType dimension; // i - BeliefValueType diff; // d[i] - bool operator>(FreudenthalDiff const &other) const; - }; - - BeliefType const &getBelief(BeliefId const &id) const; - - BeliefId getId(BeliefType const &belief) const; - - std::string toString(BeliefType const &belief) const; - - bool isEqual(BeliefType const &first, BeliefType const &second) const; - - bool assertBelief(BeliefType const &belief) const; - - bool assertTriangulation(BeliefType const &belief, Triangulation const &triangulation) const; - - uint32_t getBeliefObservation(BeliefType belief) const; - - void triangulateBeliefFreudenthal(BeliefType const &belief, BeliefValueType const &resolution, Triangulation &result); - - void triangulateBeliefDynamic(BeliefType const &belief, BeliefValueType const &resolution, Triangulation &result); - - Triangulation triangulateBelief(BeliefType const &belief, BeliefValueType const &resolution); - - std::vector> expandInternal( - storm::Environment const &env, BeliefId const &beliefId, uint64_t actionIndex, - std::optional> const &observationTriangulationResolutions = std::nullopt, - std::optional> const &observationGridClippingResolutions = std::nullopt); - - BeliefId computeInitialBelief(); - - BeliefId getOrAddBeliefId(BeliefType const &belief); - - PomdpType const &pomdp; - std::vector pomdpActionRewardVector; - - std::vector beliefs; - std::vector> beliefToIdMap; - BeliefId initialBeliefId; - - storm::utility::ConstantsComparator cc; - - std::shared_ptr> lpSolver; - - TriangulationMode triangulationMode; -}; -} // namespace storage -} // namespace storm diff --git a/src/storm-pomdp/transformer/RewardBoundUnfolder.cpp b/src/storm-pomdp/transformer/RewardBoundUnfolder.cpp new file mode 100644 index 0000000000..c3d9ed50d1 --- /dev/null +++ b/src/storm-pomdp/transformer/RewardBoundUnfolder.cpp @@ -0,0 +1,491 @@ +#include "storm-pomdp/transformer/RewardBoundUnfolder.h" + +#include + +#include "storm/adapters/RationalNumberAdapter.h" +#include "storm/logic/AtomicLabelFormula.h" +#include "storm/logic/BinaryBooleanStateFormula.h" +#include "storm/logic/BoundedUntilFormula.h" +#include "storm/logic/FragmentSpecification.h" +#include "storm/logic/ProbabilityOperatorFormula.h" +#include "storm/logic/UntilFormula.h" +#include "storm/models/sparse/Pomdp.h" +#include "storm/models/sparse/StandardRewardModel.h" +#include "storm/storage/expressions/ExpressionManager.h" +#include "storm/storage/sparse/ModelComponents.h" +#include "storm/utility/builder.h" +#include "storm/utility/macros.h" + +#include "storm/exceptions/InvalidPropertyException.h" +#include "storm/exceptions/NotSupportedException.h" + +namespace storm::pomdp::transformer { +namespace detail { +// Typedefs for readability +using StateIdType = uint64_t; +using ChoiceIdType = uint64_t; // global choices, i.e., each (state-action) pair has a unique id +using EpochType = std::vector; +using StateEpochPair = std::pair; + +/*! + * Contains information for a single dimension of the unfolding + */ +template +struct Dimension { + enum class Relation { greater, lessEqual } const relation{Relation::lessEqual}; + int64_t const threshold; + int64_t const levelWidth; + std::string const freshLevelRewardOrActiveLabelName; // A name for the newly introduced level reward or active label + uint64_t const originalFormulaDimension; + storm::models::sparse::StandardRewardModel const& rewardModel; +}; + +/*! + * Extracts the dimension information from the input formula. + * Also performs various sanity/compatibility checks. + */ +template +std::vector> extractDimensions(storm::models::sparse::Model const& model, + storm::logic::BoundedUntilFormula const& boundedUntilFormula, std::vector const& levelWidths) { + STORM_LOG_THROW(boundedUntilFormula.getRightSubformula().isInFragment(storm::logic::propositional()), storm::exceptions::NotSupportedException, + "Only propositional right subformulas are supported."); // Temporal sub-formulas are potentially not preserved by the construction + STORM_LOG_THROW(boundedUntilFormula.getLeftSubformula().isInFragment(storm::logic::propositional()), storm::exceptions::NotSupportedException, + "Only propositional left subformulas are supported."); // Temporal sub-formulas are potentially not preserved by the construction + std::vector> dimensions; + for (uint64_t formulaDim = 0; formulaDim < boundedUntilFormula.getDimension(); ++formulaDim) { + int64_t const levelWidth = formulaDim < levelWidths.size() ? levelWidths[formulaDim] : 0; + auto const& tbr = boundedUntilFormula.getTimeBoundReference(formulaDim); + STORM_LOG_THROW(tbr.hasRewardModelName(), storm::exceptions::NotSupportedException, + "The reward model for bound reference " << formulaDim << " has no name."); + STORM_LOG_THROW( + !tbr.hasRewardAccumulation(), storm::exceptions::NotSupportedException, + "The reward model for bound reference " << formulaDim << " has non-trivial reward accumulation which is not supported in this context."); + auto const& rewardModel = model.getRewardModel(tbr.getRewardModelName()); + // Note: computation of successor epoch is slightly more involved with transition rewards which is why we do not support them for now + STORM_LOG_THROW(!rewardModel.hasTransitionRewards(), storm::exceptions::NotSupportedException, + "The reward model for bound reference " << formulaDim << " uses transition rewards. These are currently unsupported."); + // All assigned rewards need to be integer (might support rational via scaling, but its unclear how to scale levelWidth) + STORM_LOG_THROW(!rewardModel.hasStateRewards() || std::all_of(rewardModel.getStateRewardVector().begin(), rewardModel.getStateRewardVector().end(), + storm::utility::isInteger), + storm::exceptions::NotSupportedException, "State rewards in reward model " << tbr.getRewardModelName() << " are not integers."); + STORM_LOG_THROW( + !rewardModel.hasStateActionRewards() || std::all_of(rewardModel.getStateActionRewardVector().begin(), + rewardModel.getStateActionRewardVector().end(), storm::utility::isInteger), + storm::exceptions::NotSupportedException, "State action rewards in reward model " << tbr.getRewardModelName() << " are not integers."); + // The finite epoch abstraction assumes that accumulated rewards never decrease. In particular, a negative-reward cycle would + // increase an epoch indefinitely and therefore make the unfolding infinite. + STORM_LOG_THROW(!rewardModel.hasNegativeRewards(), storm::exceptions::NotSupportedException, + "Reward model " << tbr.getRewardModelName() << " contains negative rewards. These are currently unsupported."); + + // Helper function to generate fresh identifiers (either for level reward or active label) + auto getFreshIdentifier = [&]() { + std::string prefix = "dim" + std::to_string(dimensions.size()) + (levelWidth == 0 ? "_active" : "_levelReward"); + auto identifier = prefix; + for (uint64_t i = 0; (levelWidth == 0 ? model.hasLabel(identifier) : model.hasRewardModel(identifier)); ++i) { + identifier = prefix + "_" + std::to_string(i); + } + return identifier; + }; + + if (boundedUntilFormula.hasUpperBound(formulaDim)) { + STORM_LOG_THROW(boundedUntilFormula.hasIntegerUpperBound(formulaDim), storm::exceptions::NotSupportedException, + "Bound " << formulaDim << " is not an integer"); // might support rational via scaling (how to scale levelWidth?) + int64_t const threshold = + boundedUntilFormula.getUpperBound(formulaDim).evaluateAsInt() - (boundedUntilFormula.isUpperBoundStrict(formulaDim) ? 1ul : 0ul); + STORM_LOG_THROW(threshold >= 0, storm::exceptions::NotSupportedException, + "Upper reward bound in dimension " << formulaDim << " is not satisfiable."); + dimensions.push_back( + Dimension{Dimension::Relation::lessEqual, threshold, levelWidth, getFreshIdentifier(), formulaDim, rewardModel}); + } + if (boundedUntilFormula.hasLowerBound(formulaDim)) { + STORM_LOG_THROW(boundedUntilFormula.hasIntegerLowerBound(formulaDim), storm::exceptions::NotSupportedException, + "Bound " << formulaDim << " is not an integer"); // might support rational via scaling (how to scale levelWidth?) + int64_t const threshold = + boundedUntilFormula.getLowerBound(formulaDim).evaluateAsInt() - (boundedUntilFormula.isLowerBoundStrict(formulaDim) ? 0ul : 1ul); + STORM_LOG_THROW(threshold >= 0, storm::exceptions::NotSupportedException, + "Lower reward bound in dimension " << formulaDim << " is not satisfiable."); + dimensions.push_back( + Dimension{Dimension::Relation::greater, threshold, levelWidth, getFreshIdentifier(), formulaDim, rewardModel}); + } + } + return dimensions; +} + +/*! + * Helper function to compute the modulos + * @note operator% is not the modulos for negative numerators. E.g. -1 % 3 = -1, but mod(-1, 3) = 2 + */ +int64_t mod(int64_t a, int64_t b) { + return (a % b + b) % b; +} + +template +EpochType computeInitialEpoch(std::vector> const& dimensions) { + EpochType epoch; + epoch.reserve(dimensions.size()); + for (auto const& dim : dimensions) { + if (dim.levelWidth == 0) { + epoch.push_back(dim.threshold); + } else { + epoch.push_back(mod(dim.threshold, dim.levelWidth)); + } + } + return epoch; +} + +template +EpochType computeSuccessorEpoch(StateIdType currentState, EpochType const& currentEpoch, ChoiceIdType choice, + std::vector> const& dimensions) { + EpochType successorEpoch = currentEpoch; + for (auto eIt = successorEpoch.begin(); auto const& dim : dimensions) { + auto const& rew = dim.rewardModel; + ValueType const reward = (rew.hasStateRewards() ? rew.getStateReward(currentState) : storm::utility::zero()) + + (rew.hasStateActionRewards() ? rew.getStateActionReward(choice) : storm::utility::zero()); + *eIt -= storm::utility::convertNumber(reward); + ++eIt; + } + return successorEpoch; +} + +template +void computeLevelReward(EpochType const& successorEpoch, std::vector> const& dimensions, LevelRewardSetterType const& setLevelReward) { + for (uint64_t dimIndex = 0; dimIndex < dimensions.size(); ++dimIndex) { + auto const lvlWidth = dimensions[dimIndex].levelWidth; + if (lvlWidth != 0) { + auto const reward = storm::utility::ceil(storm::utility::convertNumber(-successorEpoch[dimIndex]) / + storm::utility::convertNumber(lvlWidth)); + STORM_LOG_ASSERT(reward >= storm::utility::zero(), "Expected non-negative level reward, got " + << reward << ". Succ epoch is " << successorEpoch[dimIndex] << " and lvlWidth is " + << lvlWidth); + // The level reward is the smallest number of times we can add lvlWidth to the epoch entry to get a non-negative value + setLevelReward(dimIndex, reward); + } + } +} + +template +void applyEpochAbstraction(EpochType& epoch, std::vector> const& dimensions) { + for (auto eIt = epoch.begin(); auto const& dim : dimensions) { + if (*eIt < 0) { + if (dim.levelWidth == 0) { + // bottom epoch. If this is an upper bound, we can set all epoch entries to -1 + if (dim.relation == Dimension::Relation::lessEqual) { + epoch.assign(epoch.size(), -1); + return; + } + *eIt = -1; // bottom epoch + } else { + *eIt = mod(*eIt, dim.levelWidth); // level abstraction + } + } + ++eIt; + } +} + +template +void computeActiveDimensions(EpochType const& epoch, std::vector> const& dimensions, ActiveDimensionSetterType const& setActiveDimension) { + for (uint64_t dimIndex = 0; dimIndex < dimensions.size(); ++dimIndex) { + auto const& dim = dimensions[dimIndex]; + if (dim.levelWidth == 0) { + bool const inBottomEpoch = epoch[dimIndex] < 0; + if ((dim.relation == Dimension::Relation::greater) ? inBottomEpoch : !inBottomEpoch) { + setActiveDimension(dimIndex); + } + } + } +} + +struct StateEpochCollector { + std::pair getOrAddStateEpoch(StateIdType state, EpochType epoch) { + auto [idIt, isNewEntry] = stateEpochPairToId.try_emplace(std::make_pair(state, std::forward(epoch)), idToStateEpochPair.size()); + if (isNewEntry) { + idToStateEpochPair.push_back(idIt->first); + } + return {idIt->second, isNewEntry}; + } + std::map stateEpochPairToId; + std::vector idToStateEpochPair; +}; + +template +struct ExplorationResult { + storm::storage::SparseMatrix matrix; + storm::storage::BitVector initialStates; + std::vector> levelRewardsForDimensions; + std::vector activeDimensions; + std::vector stateIdToOgStateEpochPair; +}; + +template +ExplorationResult exploreUnfolding(storm::models::sparse::Model const& model, std::vector> const& dimensions) { + auto const& ogMatrix = model.getTransitionMatrix(); + bool const hasRowGroups = !ogMatrix.hasTrivialRowGrouping(); + + // Initialize data objects that we are going to fill + storm::storage::SparseMatrixBuilder matrixBuilder(0, 0, 0, 0, hasRowGroups); + storm::storage::BitVector initialStateIds; + std::vector> levelRewardsForDimensions(dimensions.size()); + std::vector activeDimensions(dimensions.size()); + + // Exploration data + std::queue queue; + StateEpochCollector stateEpochCollector; + + // Auxiliary function that is called whenever a (state-epoch) pair is found + auto processNewStateEpoch = [&queue, &stateEpochCollector](StateIdType stateId, EpochType epoch) { + auto [newId, isNewEntry] = stateEpochCollector.getOrAddStateEpoch(stateId, epoch); + if (isNewEntry) { + queue.push(newId); + } + return newId; + }; + + // Fill queue with initial state(s) + auto initEpoch = computeInitialEpoch(dimensions); + for (auto const& initState : model.getInitialStates()) { + auto initId = processNewStateEpoch(initState, initEpoch); + initialStateIds.grow(initId + 1); + initialStateIds.set(initId); + } + + // Start BFS exploration + uint64_t numChoicesInUnfolding = 0; + while (!queue.empty()) { + // Pop from queue + auto const currentStateEpochId = queue.front(); + queue.pop(); + auto [currentState, currentEpoch] = stateEpochCollector.idToStateEpochPair[currentStateEpochId]; + // Can't take currentEpoch by reference because it might be invalidated when finding new epochs + + // Compute active dimensions + computeActiveDimensions(currentEpoch, dimensions, [&activeDimensions, ¤tStateEpochId](uint64_t dimIndex) { + activeDimensions[dimIndex].grow(currentStateEpochId + 1); + activeDimensions[dimIndex].set(currentStateEpochId); + }); + + // Explore successors + if (hasRowGroups) { + matrixBuilder.newRowGroup(numChoicesInUnfolding); + } + for (auto const choice : ogMatrix.getRowGroupIndices(currentState)) { + // Since we (for now) assume models without transition branch rewards, the successor epoch does not depend on the state that we reach + auto successorEpoch = computeSuccessorEpoch(currentState, currentEpoch, choice, dimensions); + // Compute the level rewards for the dimensions with level widths (needs to be done before abstracting away that information!) + computeLevelReward(successorEpoch, dimensions, [&levelRewardsForDimensions](uint64_t dimIndex, ValueType levelReward) { + levelRewardsForDimensions[dimIndex].push_back(levelReward); + }); + // Abstract the successor epoch. Ensures that we only reach a finite set of epochs. + applyEpochAbstraction(successorEpoch, dimensions); + for (auto const& entry : ogMatrix.getRow(choice)) { + auto successorInUnfolding = processNewStateEpoch(entry.getColumn(), successorEpoch); + STORM_LOG_ASSERT(entry.getValue() > storm::utility::zero(), "Transition probabilities must be positive."); + matrixBuilder.addNextValue(numChoicesInUnfolding, successorInUnfolding, entry.getValue()); + } + ++numChoicesInUnfolding; + } + } + auto matrix = matrixBuilder.build(numChoicesInUnfolding, stateEpochCollector.idToStateEpochPair.size(), stateEpochCollector.idToStateEpochPair.size()); + STORM_LOG_ASSERT(matrix.isProbabilistic(storm::utility::zero()), "Resulting transition matrix is not a probability matrix."); + STORM_LOG_ASSERT(matrix.hasOnlyPositiveEntries(), "Resulting transition matrix has non-positive entries."); + + return ExplorationResult{std::move(matrix), std::move(initialStateIds), std::move(levelRewardsForDimensions), std::move(activeDimensions), + std::move(stateEpochCollector.idToStateEpochPair)}; +} + +template +storm::storage::sparse::ModelComponents constructComponents(storm::models::sparse::Model const& originalModel, + std::vector> const& dimensions, + std::set const& preservedRewardModels, + ExplorationResult&& explorationResult) { + uint64_t const numStates = explorationResult.matrix.getColumnCount(); + uint64_t const numChoices = explorationResult.matrix.getRowCount(); + + storm::storage::sparse::ModelComponents components(std::move(explorationResult.matrix), storm::models::sparse::StateLabeling(numStates)); + + // Helper functions to iterate over all states/choices. Unfolding states/choices are called in ascending order of their ids. + auto forEachState = [&explorationResult, &numStates](auto const& f) { + for (uint64_t unfoldingState = 0; unfoldingState < numStates; ++unfoldingState) { + f(unfoldingState, explorationResult.stateIdToOgStateEpochPair[unfoldingState].first); + } + }; + auto forEachChoice = [&originalModel, &components, &explorationResult, &numStates](auto const& f) { + for (uint64_t unfoldingState = 0; unfoldingState < numStates; ++unfoldingState) { + uint64_t const originalState = explorationResult.stateIdToOgStateEpochPair[unfoldingState].first; + STORM_LOG_ASSERT(components.transitionMatrix.getRowGroupSize(unfoldingState) == originalModel.getTransitionMatrix().getRowGroupSize(originalState), + "Number of choices in unfolding and original model differ."); + auto origChoice = originalModel.getTransitionMatrix().getRowGroupIndices()[originalState]; + for (auto unfoldingChoice : components.transitionMatrix.getRowGroupIndices(unfoldingState)) { + f(unfoldingChoice, origChoice); + ++origChoice; + } + } + }; + + // Create the state labeling + explorationResult.initialStates.resize(numStates); + components.stateLabeling.addLabel("init", std::move(explorationResult.initialStates)); + for (auto label : originalModel.getStateLabeling().getLabels()) { + if (label == "init") { + continue; + } + auto const& originalLabel = originalModel.getStateLabeling().getStates(label); + storm::storage::BitVector newLabel(numStates, false); + forEachState([&newLabel, &originalLabel](uint64_t unfoldingState, StateIdType originalState) { + if (originalLabel.get(originalState)) { + newLabel.set(unfoldingState); + } + }); + components.stateLabeling.addLabel(label, std::move(newLabel)); + } + for (uint64_t dimIndex = 0; dimIndex < dimensions.size(); ++dimIndex) { + if (dimensions[dimIndex].levelWidth == 0) { + explorationResult.activeDimensions[dimIndex].resize(numStates); + components.stateLabeling.addLabel(dimensions[dimIndex].freshLevelRewardOrActiveLabelName, std::move(explorationResult.activeDimensions[dimIndex])); + } + } + + // Create the (optional) choice labeling + if (originalModel.hasChoiceLabeling()) { + components.choiceLabeling.emplace(numChoices); + for (auto label : originalModel.getChoiceLabeling().getLabels()) { + auto const& originalLabel = originalModel.getChoiceLabeling().getChoices(label); + storm::storage::BitVector newLabel(numChoices, false); + forEachChoice([&newLabel, &originalLabel](uint64_t unfoldingChoice, uint64_t originalChoice) { + if (originalLabel.get(originalChoice)) { + newLabel.set(unfoldingChoice); + } + }); + components.choiceLabeling->addLabel(label, std::move(newLabel)); + } + } + + // Create the reward models + for (auto const& [name, rewmodel] : originalModel.getRewardModels()) { + if (!preservedRewardModels.contains(name)) { + continue; + } + STORM_LOG_THROW(!rewmodel.hasTransitionRewards(), storm::exceptions::NotSupportedException, + "Transition rewards are currently not supported in this context."); + std::optional> stateRewards, stateActionRewards; + if (rewmodel.hasStateRewards()) { + stateRewards.emplace(); + stateRewards->reserve(numStates); + forEachState( + [&stateRewards, &rewmodel](uint64_t _, StateIdType originalState) { stateRewards->push_back(rewmodel.getStateReward(originalState)); }); + } + if (rewmodel.hasStateActionRewards()) { + stateActionRewards.emplace(); + stateActionRewards->reserve(numChoices); + forEachChoice([&stateActionRewards, &rewmodel](auto _, StateIdType originalChoice) { + stateActionRewards->push_back(rewmodel.getStateActionReward(originalChoice)); + }); + } + components.rewardModels.emplace(name, storm::models::sparse::StandardRewardModel{std::move(stateRewards), std::move(stateActionRewards)}); + } + for (uint64_t dimIndex = 0; dimIndex < dimensions.size(); ++dimIndex) { + if (dimensions[dimIndex].levelWidth != 0) { + explorationResult.levelRewardsForDimensions[dimIndex].shrink_to_fit(); + components.rewardModels.emplace( + dimensions[dimIndex].freshLevelRewardOrActiveLabelName, + storm::models::sparse::StandardRewardModel{std::nullopt, std::move(explorationResult.levelRewardsForDimensions[dimIndex])}); + } + } + + // Create POMDP-specific components + if (originalModel.isOfType(storm::models::ModelType::Pomdp)) { + auto const& pomdp = static_cast const&>(originalModel); + std::vector unfoldingStateObservations; + unfoldingStateObservations.reserve(numStates); + forEachState([&unfoldingStateObservations, &pomdp](auto _, StateIdType originalState) { + unfoldingStateObservations.push_back(pomdp.getObservation(originalState)); + }); + components.observabilityClasses = std::move(unfoldingStateObservations); + STORM_LOG_WARN_COND(!pomdp.hasObservationValuations(), "Observation valuations are dropped."); + + } else { + STORM_LOG_THROW(originalModel.isOfType(storm::models::ModelType::Mdp), storm::exceptions::NotSupportedException, + "Unfolding is only supported POMDP and MDP models right now."); // DTMCs might work, too? + } + return components; +} + +template +std::shared_ptr constructFormula(storm::logic::BoundedUntilFormula boundedUntilFormula, + std::vector> const& dimensions) { + // Construct a new (bounded or unbounded) until formula + auto lhs = boundedUntilFormula.getLeftSubformula().clone(); + auto rhs = boundedUntilFormula.getRightSubformula().clone(); + std::vector> lowerBounds, upperBounds; + std::vector timeBoundReferences; + STORM_LOG_ASSERT(boundedUntilFormula.getDimension() > 0, "did not expect a 0-dimensional formula."); + auto const& exprManager = + boundedUntilFormula.hasLowerBound(0) ? boundedUntilFormula.getLowerBound(0).getManager() : boundedUntilFormula.getUpperBound(0).getManager(); + for (auto const& dim : dimensions) { + if (dim.levelWidth == 0) { + // Dimension is unfolded up to the threshold. Inject the label where the bound is active + auto activeFormula = std::make_shared(dim.freshLevelRewardOrActiveLabelName); + rhs = std::make_shared(storm::logic::BinaryBooleanOperatorType::And, std::move(rhs), + std::move(activeFormula)); + if (dim.relation == Dimension::Relation::lessEqual) { + auto activeFormula = std::make_shared(dim.freshLevelRewardOrActiveLabelName); + lhs = std::make_shared(storm::logic::BinaryBooleanOperatorType::And, std::move(lhs), + std::move(activeFormula)); + } + } else { + // Dimension is unfolded up to level width. Add a reward bound addressing the level jump reward + auto const lvlthreshold = exprManager.integer(storm::utility::convertNumber(storm::utility::floor( + storm::utility::convertNumber(dim.threshold) / storm::utility::convertNumber(dim.levelWidth)))); + if (dim.relation == Dimension::Relation::lessEqual) { + upperBounds.push_back(storm::logic::TimeBound{false, lvlthreshold}); + lowerBounds.emplace_back(); + } else { + upperBounds.emplace_back(); + lowerBounds.push_back(storm::logic::TimeBound{true, lvlthreshold}); + } + timeBoundReferences.emplace_back(dim.freshLevelRewardOrActiveLabelName); + } + } + + // Construct the new formula + + if (timeBoundReferences.empty()) { + return std::make_shared(lhs, rhs); + } else { + return std::make_shared(lhs, rhs, std::move(lowerBounds), std::move(upperBounds), std::move(timeBoundReferences)); + } +} + +} // namespace detail + +template +RewardBoundUnfolder::ReturnType RewardBoundUnfolder::transform(storm::models::sparse::Model const& model, + storm::logic::Formula const& formula, UnfoldingOptions const& options) { + if (formula.isProbabilityOperatorFormula()) { + // Recursive call with subformula + auto const& opFormula = formula.asProbabilityOperatorFormula(); + auto result = transform(model, opFormula.getSubformula(), options); + result.formula = std::make_shared(std::move(result.formula), opFormula.getOperatorInformation()); + return result; + } + + // Process with boundedUntilFormula + STORM_LOG_THROW(formula.isBoundedUntilFormula(), storm::exceptions::InvalidPropertyException, "Unexpected formula type." << formula << "."); + auto const& boundedUntilFormula = formula.asBoundedUntilFormula(); + auto const dimensions = detail::extractDimensions(model, boundedUntilFormula, options.levelWidths); + auto explorationResult = detail::exploreUnfolding(model, dimensions); + auto components = detail::constructComponents(model, dimensions, options.preservedRewardModels, std::move(explorationResult)); + auto unfoldedModel = storm::utility::builder::buildModelFromComponents(model.getType(), std::move(components)); + + if (unfoldedModel->isOfType(storm::models::ModelType::Pomdp)) { + auto& unfoldedPomdp = static_cast&>(*unfoldedModel); + auto const& originalPomdp = static_cast const&>(model); + unfoldedPomdp.setIsCanonic(originalPomdp.isCanonic()); + } + + return {std::move(unfoldedModel), std::move(detail::constructFormula(boundedUntilFormula, dimensions))}; +} + +template class RewardBoundUnfolder; +template class RewardBoundUnfolder; + +} // namespace storm::pomdp::transformer diff --git a/src/storm-pomdp/transformer/RewardBoundUnfolder.h b/src/storm-pomdp/transformer/RewardBoundUnfolder.h new file mode 100644 index 0000000000..631f866bb1 --- /dev/null +++ b/src/storm-pomdp/transformer/RewardBoundUnfolder.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "storm/logic/Formula.h" +#include "storm/models/sparse/Pomdp.h" + +namespace storm::pomdp::transformer { + +template +class RewardBoundUnfolder { + public: + struct ReturnType { + std::shared_ptr> model; + std::shared_ptr formula; + }; + + struct UnfoldingOptions { + /// Allows to define a levelWidth for each dimension. + /// If a non-zero levelWidth is given for a dimension, the unfolding in that dimension is performed according to the given width + /// A fresh reward assignment will be introduced that indicates how often a transition exceeds the given levelWidth + /// The returned formula will then have a rewardBound in terms of the new level reward. + /// Assume a dimension i. Let c=levelWidth[i]!=0 and let t be the reward bound threshold (i.e. either <=t or >t). + /// A transition from epoch a=e[i] with reward b=r[i] leads to epoch (a-b) mod c and yields level reward ceil((b-a)/c). + /// The initial epoch is -t mod c (Note: in C++ this is (-t)%c+c because % is not the modulos for negative numerators). + /// The output formula will have reward bound threshold ceil(t/c), where t is the original threshold. + /// Note that if c=1, no unfolding will be performed. + /// The case levelWidth[i]=0 is special: this means that the dimension is unfolded until past the threshold. + /// No level reward is introduced in this case and the bound dimension is removed from the formula. + /// levelWidth.size() <= i is treated equivalently to levelWidth[i]=0. + std::vector levelWidths{}; + + /// The reward models that will be preserved in the unfolding. + std::set preservedRewardModels{}; + }; + + /** + * Unfolds reward-bounded dimensions into the model state space and returns an equivalent unbounded formula. + * + * Zero level widths unfold a dimension completely and remove its bound. Positive level widths retain a compact + * epoch representation and introduce a corresponding level reward model. + */ + static ReturnType transform(storm::models::sparse::Model const& model, storm::logic::Formula const& formula, + UnfoldingOptions const& options = {}); +}; + +} // namespace storm::pomdp::transformer diff --git a/src/storm-pomdp/transformer/ToStateBasedObservationTransformer.cpp b/src/storm-pomdp/transformer/ToStateBasedObservationTransformer.cpp new file mode 100644 index 0000000000..7e8e2f8fa9 --- /dev/null +++ b/src/storm-pomdp/transformer/ToStateBasedObservationTransformer.cpp @@ -0,0 +1,231 @@ +#include "storm-pomdp/transformer/ToStateBasedObservationTransformer.h" + +#include +#include +#include +#include + +#include "storm/adapters/RationalNumberAdapter.h" +#include "storm/exceptions/InvalidModelException.h" +#include "storm/exceptions/NotSupportedException.h" +#include "storm/utility/builder.h" +#include "storm/utility/macros.h" + +namespace storm::pomdp::transformer { + +template +std::shared_ptr> ToStateBasedObservationTransformer::transform( + storm::models::sparse::Mdp const& mdp, TransitionObservationFunction const& transitionObservationFunction, ObservationType initialObservation) { + auto const& transitionMatrix = mdp.getTransitionMatrix(); + + // Create a vector that for each state contains the set of observations with which we may enter that state. + std::vector> stateObservations(mdp.getNumberOfStates()); + // Start with the initial observations + for (auto const initState : mdp.getInitialStates()) { + stateObservations[initState].push_back(initialObservation); + } + // Now run over all transitions. + // Also gather all transition observations so that we do not have to query them twice + std::vector transitionTargetObservations; + transitionTargetObservations.reserve(transitionMatrix.getEntryCount()); + for (uint64_t state = 0; state < transitionMatrix.getRowGroupCount(); ++state) { + for (auto choice : transitionMatrix.getRowGroupIndices(state)) { + for (auto const& entry : transitionMatrix.getRow(choice)) { + auto const obs = transitionObservationFunction(state, choice, entry.getColumn()); + transitionTargetObservations.push_back(obs); + auto& obsSet = stateObservations[entry.getColumn()]; + if (std::find(obsSet.begin(), obsSet.end(), obs) == obsSet.end()) { + obsSet.push_back(obs); + } + } + } + } + + // Create state offsets. The entry for a given input model state is the id of the first copy of that state. + std::vector stateOffsets; + stateOffsets.reserve(mdp.getNumberOfStates() + 1); + stateOffsets.push_back(0); + for (uint64_t offset = 0; auto const& obsSet : stateObservations) { + STORM_LOG_THROW(!obsSet.empty(), storm::exceptions::InvalidModelException, + "There are states that are neither initial nor have an incoming transition."); + offset += obsSet.size(); + stateOffsets.push_back(offset); + } + + // Create the transition matrix of the resulting model. + storm::storage::SparseMatrixBuilder matrixBuilder(0, stateOffsets.back(), 0, true, true, stateOffsets.back()); + auto transTargetObsIt = transitionTargetObservations.begin(); + uint64_t rowInResultMatrix = 0; + for (uint64_t state = 0; state < transitionMatrix.getRowGroupCount(); ++state) { + auto const transTargetObsBegin = transTargetObsIt; + for (auto _ [[maybe_unused]] : stateObservations[state]) { + transTargetObsIt = transTargetObsBegin; + matrixBuilder.newRowGroup(rowInResultMatrix); + for (auto choice : transitionMatrix.getRowGroupIndices(state)) { + for (auto const& entry : transitionMatrix.getRow(choice)) { + auto const targetObs = *transTargetObsIt; + auto const& targetObsSet = stateObservations[entry.getColumn()]; + auto const findIt = std::find(targetObsSet.begin(), targetObsSet.end(), targetObs); + STORM_LOG_ASSERT(findIt != targetObsSet.end(), "Transition target observation not found in target state."); + auto transitionTarget = stateOffsets[entry.getColumn()] + std::distance(targetObsSet.begin(), findIt); + matrixBuilder.addNextValue(rowInResultMatrix, transitionTarget, entry.getValue()); + ++transTargetObsIt; + } + ++rowInResultMatrix; + } + } + } + + uint64_t const numResultStates = stateOffsets.back(); + uint64_t const numResultChoices = rowInResultMatrix; + + storm::storage::sparse::ModelComponents components(matrixBuilder.build(numResultChoices, numResultStates, numResultStates), + storm::models::sparse::StateLabeling(numResultStates)); + + // Helper functions to iterate over all states/choices. Unfolding states/choices are called in ascending order of their ids. + auto forEachState = [&stateOffsets](auto const& f) { + for (uint64_t ogState = 0; ogState < stateOffsets.size() - 1; ++ogState) { + for (uint64_t resultState = stateOffsets[ogState]; resultState < stateOffsets[ogState + 1]; ++resultState) { + f(resultState, ogState); + } + } + }; + auto forEachChoice = [&stateOffsets, &transitionMatrix, &components](auto const& f) { + for (uint64_t ogState = 0; ogState < stateOffsets.size() - 1; ++ogState) { + for (uint64_t resultState = stateOffsets[ogState]; resultState < stateOffsets[ogState + 1]; ++resultState) { + STORM_LOG_ASSERT(components.transitionMatrix.getRowGroupSize(resultState) == transitionMatrix.getRowGroupSize(ogState), + "Number of choices in unfolding and original model differ."); + auto ogChoice = transitionMatrix.getRowGroupIndices()[ogState]; + for (auto resultChoice : components.transitionMatrix.getRowGroupIndices(resultState)) { + f(resultChoice, ogChoice); + ++ogChoice; + } + } + } + }; + + // Create the state labeling + storm::storage::BitVector initialStates(numResultStates, false); + for (auto ogInitState : mdp.getInitialStates()) { + auto const& obsSet = stateObservations[ogInitState]; + auto const findIt = std::find(obsSet.begin(), obsSet.end(), initialObservation); + STORM_LOG_ASSERT(findIt != obsSet.end(), "Initial observation not found in initial state."); + auto const resultInitState = stateOffsets[ogInitState] + std::distance(obsSet.begin(), findIt); + initialStates.set(resultInitState); + } + components.stateLabeling.addLabel("init", std::move(initialStates)); + for (auto label : mdp.getStateLabeling().getLabels()) { + if (label == "init") { + continue; + } + auto const& originalLabel = mdp.getStateLabeling().getStates(label); + storm::storage::BitVector newLabel(numResultStates, false); + forEachState([&newLabel, &originalLabel](uint64_t resultState, StateIdType ogState) { + if (originalLabel.get(ogState)) { + newLabel.set(resultState); + } + }); + components.stateLabeling.addLabel(label, std::move(newLabel)); + } + + // Create the (optional) choice labeling + if (mdp.hasChoiceLabeling()) { + components.choiceLabeling.emplace(numResultChoices); + for (auto label : mdp.getChoiceLabeling().getLabels()) { + auto const& originalLabel = mdp.getChoiceLabeling().getChoices(label); + storm::storage::BitVector newLabel(numResultChoices, false); + forEachChoice([&newLabel, &originalLabel](uint64_t unfoldingChoice, uint64_t originalChoice) { + if (originalLabel.get(originalChoice)) { + newLabel.set(unfoldingChoice); + } + }); + components.choiceLabeling->addLabel(label, std::move(newLabel)); + } + } + + // Create the reward models + for (auto const& [name, rewmodel] : mdp.getRewardModels()) { + STORM_LOG_THROW(!rewmodel.hasTransitionRewards(), storm::exceptions::NotSupportedException, + "Transition rewards are currently not supported in this context."); + std::optional> stateRewards, stateActionRewards; + if (rewmodel.hasStateRewards()) { + stateRewards.emplace(); + stateRewards->reserve(numResultStates); + forEachState( + [&stateRewards, &rewmodel](uint64_t _, StateIdType originalState) { stateRewards->push_back(rewmodel.getStateReward(originalState)); }); + } + if (rewmodel.hasStateActionRewards()) { + stateActionRewards.emplace(); + stateActionRewards->reserve(numResultChoices); + forEachChoice([&stateActionRewards, &rewmodel](auto _, StateIdType originalChoice) { + stateActionRewards->push_back(rewmodel.getStateActionReward(originalChoice)); + }); + } + components.rewardModels.emplace(name, storm::models::sparse::StandardRewardModel{std::move(stateRewards), std::move(stateActionRewards)}); + } + + // Create POMDP-specific components + std::vector flatStateObservations; + flatStateObservations.reserve(numResultStates); + for (auto const& obsSet : stateObservations) { + flatStateObservations.insert(flatStateObservations.end(), obsSet.begin(), obsSet.end()); + } + components.observabilityClasses = std::move(flatStateObservations); + + return storm::utility::builder::buildModelFromComponents(storm::models::ModelType::Pomdp, std::move(components)) + ->template as>(); +} + +template +std::shared_ptr> ToStateBasedObservationTransformer::transformRewardAware( + storm::models::sparse::Pomdp const& pomdp, std::set const& observableRewardModels) { + STORM_LOG_THROW(pomdp.getInitialStates().getNumberOfSetBits() == 1, storm::exceptions::NotSupportedException, + "The model must have exactly one initial state."); + auto const initialObservation = pomdp.getObservation(pomdp.getInitialStates().getNextSetIndex(0)); + + struct TransitionObservation { + ObservationType targetStateObservation; + std::vector rewards; + bool operator<(TransitionObservation const& other) const { + if (targetStateObservation != other.targetStateObservation) { + return targetStateObservation < other.targetStateObservation; + } + return rewards < other.rewards; + } + }; + std::map observationIndexStorage; + // Generated observations must not collide with the original observations retained by initial-state copies. + ObservationType freshObservation = pomdp.getNrObservations(); + auto getOrAddObservationIndex = [&observationIndexStorage, &freshObservation](TransitionObservation const& obs) { + auto [it, inserted] = observationIndexStorage.try_emplace(obs, freshObservation); + if (inserted) { + ++freshObservation; + } + return it->second; + }; + auto result = transform( + pomdp, + [&pomdp, &getOrAddObservationIndex, &observableRewardModels](StateIdType srcState, ActionIdType action, StateIdType targetState) { + TransitionObservation obs{pomdp.getObservation(targetState), {}}; + for (auto const& rewName : observableRewardModels) { + auto const& rewModel = pomdp.getRewardModel(rewName); + obs.rewards.push_back(storm::utility::zero()); + if (rewModel.hasStateRewards()) { + obs.rewards.back() += rewModel.getStateReward(srcState); + } + if (rewModel.hasStateActionRewards()) { + obs.rewards.back() += rewModel.getStateActionReward(action); + } + STORM_LOG_THROW(!rewModel.hasTransitionRewards(), storm::exceptions::NotSupportedException, + "Transition rewards are currently not supported in this context."); + } + return getOrAddObservationIndex(obs); + }, + initialObservation); + result->setIsCanonic(pomdp.isCanonic()); + return result; +} + +template class ToStateBasedObservationTransformer; +template class ToStateBasedObservationTransformer; +} // namespace storm::pomdp::transformer diff --git a/src/storm-pomdp/transformer/ToStateBasedObservationTransformer.h b/src/storm-pomdp/transformer/ToStateBasedObservationTransformer.h new file mode 100644 index 0000000000..799d1a0e2e --- /dev/null +++ b/src/storm-pomdp/transformer/ToStateBasedObservationTransformer.h @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include +#include + +#include "storm/models/sparse/Pomdp.h" +#include "storm/models/sparse/StandardRewardModel.h" + +namespace storm::pomdp::transformer { + +template +class ToStateBasedObservationTransformer { + public: + using ObservationType = uint32_t; + using StateIdType = uint64_t; + using ActionIdType = uint64_t; // Action indices are local, i.e. 0 is always the first action of any state + using TransitionObservationFunction = std::function; + + /*! + * Creates a POMDP (with state-based observations) out of an MDP and a transition-based observation function. + * @param mdp The input MDP. If this is actually a POMDP, its observation function will be ignored. + * @param transitionObservationFunction An observation function mapping transitions to observations O : S x Act x S -> Z + * @param initialObservation The observation used for the initial state(s) of the POMDP + * @return A POMDP with state-based observations that is equivalent to the input MDP with the given observation function. + * Specifically, the observation assigned to the state is equal to the transition-observation with which the state has been entered. + * This might require to create copies of states that can be entered with different observations. + * Initial states retain @p initialObservation. + */ + static std::shared_ptr> transform(storm::models::sparse::Mdp const& mdp, + TransitionObservationFunction const& transitionObservationFunction, + ObservationType initialObservation); + + /** + * Makes rewards observable by extending observations with the state and state-action reward values of the selected models. + * + * The input POMDP must have exactly one initial state and no transition rewards in the selected reward models. + */ + static std::shared_ptr> transformRewardAware(storm::models::sparse::Pomdp const& pomdp, + std::set const& observableRewardModels); +}; + +} // namespace storm::pomdp::transformer diff --git a/src/storm/logic/BoundedUntilFormula.cpp b/src/storm/logic/BoundedUntilFormula.cpp index d97b4341d7..ab8389d182 100644 --- a/src/storm/logic/BoundedUntilFormula.cpp +++ b/src/storm/logic/BoundedUntilFormula.cpp @@ -251,6 +251,22 @@ storm::expressions::Expression const& BoundedUntilFormula::getUpperBound(unsigne return upperBound.at(i).get().getBound(); } +std::optional BoundedUntilFormula::getLowerBoundAsOptionalTimeBound(unsigned i) const { + if (hasLowerBound(i)) { + return lowerBound.at(i).get(); + } else { + return std::nullopt; + } +} + +std::optional BoundedUntilFormula::getUpperBoundAsOptionalTimeBound(unsigned i) const { + if (hasUpperBound(i)) { + return upperBound.at(i).get(); + } else { + return std::nullopt; + } +} + template<> double BoundedUntilFormula::getLowerBound(unsigned i) const { if (!hasLowerBound(i)) { diff --git a/src/storm/logic/BoundedUntilFormula.h b/src/storm/logic/BoundedUntilFormula.h index 1e82ff4926..e70ce26172 100644 --- a/src/storm/logic/BoundedUntilFormula.h +++ b/src/storm/logic/BoundedUntilFormula.h @@ -59,6 +59,9 @@ class BoundedUntilFormula : public PathFormula { storm::expressions::Expression const& getLowerBound(unsigned i = 0) const; storm::expressions::Expression const& getUpperBound(unsigned i = 0) const; + std::optional getLowerBoundAsOptionalTimeBound(unsigned i = 0) const; + std::optional getUpperBoundAsOptionalTimeBound(unsigned i = 0) const; + template ValueType getLowerBound(unsigned i = 0) const; diff --git a/src/storm/transformer/SparseModelValueTypeTransformer.cpp b/src/storm/transformer/SparseModelValueTypeTransformer.cpp new file mode 100644 index 0000000000..eb4c9ec3eb --- /dev/null +++ b/src/storm/transformer/SparseModelValueTypeTransformer.cpp @@ -0,0 +1,114 @@ +#include "SparseModelValueTypeTransformer.h" + +#include "storm/exceptions/IllegalArgumentTypeException.h" +#include "storm/models/sparse/Ctmc.h" +#include "storm/models/sparse/Dtmc.h" +#include "storm/models/sparse/MarkovAutomaton.h" +#include "storm/models/sparse/Mdp.h" +#include "storm/models/sparse/Pomdp.h" +#include "storm/models/sparse/Smg.h" +#include "storm/models/sparse/StochasticTwoPlayerGame.h" +#include "storm/storage/sparse/ModelComponents.h" +#include "storm/utility/macros.h" +#include "storm/utility/vector.h" + +namespace storm::transformer { +template +std::shared_ptr> SparseModelValueTypeTransformer::transformModel( + std::shared_ptr> const& inputModel) { + STORM_LOG_THROW(inputModel, storm::exceptions::IllegalArgumentTypeException, "Cannot transform a null model."); + storm::storage::sparse::ModelComponents convertedComponents; + convertedComponents.transitionMatrix = inputModel->getTransitionMatrix().template toValueType(); + convertedComponents.choiceLabeling = inputModel->getOptionalChoiceLabeling(); + convertedComponents.stateLabeling = inputModel->getStateLabeling(); + convertedComponents.stateValuations = inputModel->getOptionalStateValuations(); + convertedComponents.choiceOrigins = inputModel->getOptionalChoiceOrigins(); + for (auto const& [rewardModelName, rewardModel] : inputModel->getRewardModels()) { + // Transform reward models + std::optional> optionalStateRewardVector = std::nullopt; + std::optional> optionalStateActionRewardVector = std::nullopt; + std::optional> optionalTransitionRewardMatrix = std::nullopt; + if (rewardModel.hasStateRewards()) { + std::vector resultVector; + resultVector.reserve(rewardModel.getStateRewardVector().size()); + for (auto const& oldValue : rewardModel.getStateRewardVector()) { + resultVector.push_back(storm::utility::convertNumber(oldValue)); + } + optionalStateRewardVector = resultVector; + } + if (rewardModel.hasStateActionRewards()) { + std::vector resultVector; + resultVector.reserve(rewardModel.getStateActionRewardVector().size()); + for (auto const& oldValue : rewardModel.getStateActionRewardVector()) { + resultVector.push_back(storm::utility::convertNumber(oldValue)); + } + optionalStateActionRewardVector = resultVector; + } + if (rewardModel.hasTransitionRewards()) { + optionalTransitionRewardMatrix = rewardModel.getTransitionRewardMatrix().template toValueType(); + } + convertedComponents.rewardModels.emplace( + rewardModelName, storm::models::sparse::StandardRewardModel( + std::move(optionalStateRewardVector), std::move(optionalStateActionRewardVector), std::move(optionalTransitionRewardMatrix))); + } + switch (inputModel->getType()) { + case storm::models::ModelType::Dtmc: + return std::make_shared>(storm::models::sparse::Dtmc(convertedComponents)); + case storm::models::ModelType::Mdp: + return std::make_shared>(storm::models::sparse::Mdp(convertedComponents)); + case storm::models::ModelType::Ctmc: { + auto ctmc = inputModel->template as>(); + std::vector resultVector; + resultVector.reserve(ctmc->getExitRateVector().size()); + for (auto const& oldValue : ctmc->getExitRateVector()) { + resultVector.push_back(storm::utility::convertNumber(oldValue)); + } + convertedComponents.exitRates = resultVector; + // Markov automata store probabilities in their transition matrix and rates separately in exitRates. + convertedComponents.rateTransitions = false; + return std::make_shared>(storm::models::sparse::Ctmc(convertedComponents)); + } + case storm::models::ModelType::MarkovAutomaton: { + auto ma = inputModel->template as>(); + std::vector resultVector; + resultVector.reserve(ma->getExitRates().size()); + for (auto const& oldValue : ma->getExitRates()) { + resultVector.push_back(storm::utility::convertNumber(oldValue)); + } + convertedComponents.exitRates = resultVector; + convertedComponents.rateTransitions = true; + convertedComponents.markovianStates = ma->getMarkovianStates(); + return std::make_shared>( + storm::models::sparse::MarkovAutomaton(convertedComponents)); + } + case storm::models::ModelType::Pomdp: { + auto pomdp = inputModel->template as>(); + convertedComponents.observabilityClasses = pomdp->getObservations(); + convertedComponents.observationValuations = pomdp->getOptionalObservationValuations(); + return std::make_shared>(models::sparse::Pomdp(convertedComponents, pomdp->isCanonic())); + } + case storm::models::ModelType::Smg: { + auto smg = inputModel->template as>(); + convertedComponents.statePlayerIndications = smg->getStatePlayerIndications(); + convertedComponents.playerNameToIndexMap = smg->getPlayerNamesToIndex(); + return std::make_shared>(models::sparse::Smg(convertedComponents)); + } + case storm::models::ModelType::S2pg: { + auto s2pg = inputModel->template as>(); + convertedComponents.player1Matrix = s2pg->getPlayer1Matrix(); + return std::make_shared>( + models::sparse::StochasticTwoPlayerGame(convertedComponents)); + } + default: + STORM_LOG_THROW(false, storm::exceptions::IllegalArgumentTypeException, + "Value type transformation is not supported for models of type " << inputModel->getType() << "."); + } + return nullptr; +} + +template class SparseModelValueTypeTransformer; +template class SparseModelValueTypeTransformer; +template class SparseModelValueTypeTransformer; +template class SparseModelValueTypeTransformer; + +} // namespace storm::transformer diff --git a/src/storm/transformer/SparseModelValueTypeTransformer.h b/src/storm/transformer/SparseModelValueTypeTransformer.h new file mode 100644 index 0000000000..f6f02ca7e0 --- /dev/null +++ b/src/storm/transformer/SparseModelValueTypeTransformer.h @@ -0,0 +1,23 @@ +#pragma once + +#include + +#include "storm/models/sparse/Model.h" + +namespace storm::transformer { + +template +/** Converts the numeric values of a sparse model while preserving its model-specific metadata. */ +class SparseModelValueTypeTransformer { + public: + explicit SparseModelValueTypeTransformer() = default; + + /** + * Returns a model equivalent to @p inputModel with all probabilities, rates, and rewards converted to OutputValueType. + * + * @pre inputModel is not null. + */ + std::shared_ptr> transformModel(std::shared_ptr> const& inputModel); +}; + +} // namespace storm::transformer diff --git a/src/storm/transformer/TransitionToActionRewardTransformer.cpp b/src/storm/transformer/TransitionToActionRewardTransformer.cpp new file mode 100644 index 0000000000..eb84521130 --- /dev/null +++ b/src/storm/transformer/TransitionToActionRewardTransformer.cpp @@ -0,0 +1,232 @@ +#include "storm/transformer/TransitionToActionRewardTransformer.h" + +#include "storm/adapters/RationalFunctionAdapter.h" +#include "storm/adapters/RationalNumberAdapter.h" +#include "storm/exceptions/UnexpectedException.h" +#include "storm/models/sparse/MarkovAutomaton.h" +#include "storm/models/sparse/StandardRewardModel.h" +#include "storm/storage/SparseMatrix.h" +#include "storm/storage/sparse/ModelComponents.h" +#include "storm/utility/OptionalRef.h" +#include "storm/utility/builder.h" +#include "storm/utility/macros.h" +#include "storm/utility/vector.h" + +namespace storm::transformer { + +namespace detail { +template +using MultiRewardVector = std::vector; + +template +class RewardTransitionIterator { + public: + RewardTransitionIterator(storm::storage::SparseMatrix const& m) : transitionMatrix(m) {} + + void addRewardModel(storm::models::sparse::StandardRewardModel const& rewardModel) { + if (rewardModel.hasTransitionRewards()) { + transitionRewards.emplace_back(rewardModel.getTransitionRewardMatrix()); + STORM_LOG_ASSERT(transitionRewards.back()->isSubmatrixOf(transitionMatrix), "Invalid reward matrix."); + } else { + transitionRewards.emplace_back(); + } + } + + template + void forEachRowEntry(uint64_t rowIndex, bool skip0RewardEntries, CallBackType&& callBack) { + // Set-up iterators + std::vector::const_iterator> rewardIterators; + std::vector::const_iterator> rewardIteratorsEnd; + for (auto const& rewardMatrix : transitionRewards) { + if (rewardMatrix) { + rewardIterators.push_back(rewardMatrix->begin(rowIndex)); + rewardIteratorsEnd.push_back(rewardMatrix->end(rowIndex)); + } else { + rewardIterators.emplace_back(); + rewardIteratorsEnd.emplace_back(); + } + } + + std::vector rewards(transitionRewards.size()); + for (auto const& entry : transitionMatrix.getRow(rowIndex)) { + // Fill in rewards for this entry + bool skipEntry = skip0RewardEntries; + for (uint64_t i = 0; i < transitionRewards.size(); ++i) { + if (rewardIterators[i] != rewardIteratorsEnd[i] && rewardIterators[i]->getColumn() == entry.getColumn()) { + rewards[i] = rewardIterators[i]->getValue(); + ++rewardIterators[i]; + skipEntry = skipEntry && storm::utility::isZero(rewards[i]); + } else { + rewards[i] = storm::utility::zero(); + } + } + if (!skipEntry) { + callBack(entry.getColumn(), entry.getValue(), rewards); + } + } + } + + private: + storm::storage::SparseMatrix const& transitionMatrix; + std::vector const>> transitionRewards; +}; + +} // namespace detail + +template +TransitionToActionRewardTransformerReturnType transformTransitionToActionRewards( + std::shared_ptr> originalModel, std::vector const& relevantRewardModelNames) { + STORM_LOG_ASSERT(originalModel, "Model must not be null."); + detail::RewardTransitionIterator rewardTransitionIterator(originalModel->getTransitionMatrix()); + bool hasTransitionRewards = false; + for (auto const& rewardModelName : relevantRewardModelNames) { + auto const& rewardModel = originalModel->getRewardModel(rewardModelName); + if (rewardModel.hasTransitionRewards()) { + hasTransitionRewards = true; + } + rewardTransitionIterator.addRewardModel(rewardModel); + } + if (!hasTransitionRewards) { + return {originalModel->template as>(), + {storm::utility::vector::buildVectorForRange(0, originalModel->getNumberOfStates())}}; + } + + // Make a pass to find the different rewards with which a state is entered + std::vector>> incomingRewards(originalModel->getNumberOfStates()); + auto const& transitions = originalModel->getTransitionMatrix(); + for (uint64_t row = 0; row < transitions.getRowCount(); ++row) { + rewardTransitionIterator.forEachRowEntry( + row, true, + [&incomingRewards](uint64_t column, ValueType, detail::MultiRewardVector const& rewards) { incomingRewards[column].insert(rewards); }); + } + + // Create a mapping from original to new indices + std::vector originalToNewIndex; + uint64_t numStates = 0; + for (auto const& incRewardsSet : incomingRewards) { + numStates += incRewardsSet.size(); + originalToNewIndex.push_back(numStates); + ++numStates; + } + + // Populate the new transition matrix and (action) rewards for intermediate states + uint64_t const numIntermediateStates = numStates - originalModel->getNumberOfStates(); + bool const useGroups = !transitions.hasTrivialRowGrouping(); + storm::storage::SparseMatrixBuilder newTransitionsBuilder(transitions.getRowCount() + numIntermediateStates, numStates, + transitions.getEntryCount() + numIntermediateStates, true, useGroups, + useGroups ? numStates : 0ull); + std::vector> newActionRewards(relevantRewardModelNames.size(), + std::vector(transitions.getRowCount() + numIntermediateStates)); + uint64_t currNewRow = 0; + for (uint64_t currOrigState = 0; currOrigState < originalModel->getNumberOfStates(); ++currOrigState) { + uint64_t const currNewState = originalToNewIndex[currOrigState]; + // First add the transitions and rewards for the intermediate states + for (auto const& incRewardsSet : incomingRewards[currOrigState]) { + if (useGroups) { + newTransitionsBuilder.newRowGroup(currNewRow); + } + newTransitionsBuilder.addNextValue(currNewRow, currNewState, storm::utility::one()); + auto newRewIt = newActionRewards.begin(); + for (auto const& rew : incRewardsSet) { + (*newRewIt)[currNewRow] = rew; + ++newRewIt; + } + ++currNewRow; + } + // Add the transitions and rewards for the original state + if (useGroups) { + newTransitionsBuilder.newRowGroup(currNewRow); + } + for (auto origRowIndex : transitions.getRowGroupIndices(currOrigState)) { + rewardTransitionIterator.forEachRowEntry( + origRowIndex, false, + [&newTransitionsBuilder, &originalToNewIndex, &incomingRewards, &currNewRow](uint64_t column, ValueType prob, + detail::MultiRewardVector const& rewards) { + if (std::all_of(rewards.begin(), rewards.end(), [](ValueType const& r) { return storm::utility::isZero(r); })) { + // No transition reward collected so use originial state + newTransitionsBuilder.addNextValue(currNewRow, originalToNewIndex[column], prob); + } else { + // Redirect to intermediate state + auto incomingRewardsIt = incomingRewards[column].find(rewards); + STORM_LOG_ASSERT(incomingRewardsIt != incomingRewards[column].end(), "Invalid incoming rewards."); + uint64_t const intermediateStateIndex = + originalToNewIndex[column] - incomingRewards[column].size() + std::distance(incomingRewards[column].begin(), incomingRewardsIt); + newTransitionsBuilder.addNextValue(currNewRow, intermediateStateIndex, prob); + } + }); + ++currNewRow; + } + } + + // create new state labels and init components + storm::models::sparse::StateLabeling newLabeling(numStates); + for (auto const& l : originalModel->getStateLabeling().getLabels()) { + newLabeling.addLabel(l); + for (auto origIndex : originalModel->getStateLabeling().getStates(l)) { + newLabeling.addLabelToState(l, originalToNewIndex[origIndex]); + } + } + storm::storage::sparse::ModelComponents components(newTransitionsBuilder.build(), std::move(newLabeling)); + + // create new reward models + uint64_t rewardIndex = 0; + for (auto const& rewardModelName : relevantRewardModelNames) { + auto& newActionRewardVector = newActionRewards[rewardIndex++]; + auto const& oldRewardModel = originalModel->getRewardModel(rewardModelName); + for (uint64_t oldState = 0; oldState < originalModel->getNumberOfStates(); ++oldState) { + uint64_t const oldStartRow = transitions.getRowGroupIndices()[oldState]; + uint64_t const newState = originalToNewIndex[oldState]; + uint64_t const newStartRow = useGroups ? components.transitionMatrix.getRowGroupIndices()[newState] : newState; + uint64_t const numRowsInGroup = useGroups ? transitions.getRowGroupSize(oldState) : 1ull; + for (uint64_t groupOffset = 0; groupOffset < numRowsInGroup; ++groupOffset) { + auto& rewValue = newActionRewardVector[newStartRow + groupOffset]; + if (oldRewardModel.hasStateRewards()) { + rewValue += oldRewardModel.getStateReward(oldState); + } + if (oldRewardModel.hasStateActionRewards()) { + rewValue += oldRewardModel.getStateActionReward(oldStartRow + groupOffset); + } + } + } + storm::models::sparse::StandardRewardModel newRewardModel(std::nullopt, std::move(newActionRewardVector)); + components.rewardModels.emplace(rewardModelName, std::move(newRewardModel)); + } + + STORM_LOG_WARN_COND(!originalModel->hasChoiceLabeling(), "Choice labellings will be dropped as the transformation is currently not implemented."); + STORM_LOG_WARN_COND(!originalModel->hasStateValuations(), "State valuations will be dropped as the transformation is currently not implemented."); + STORM_LOG_WARN_COND(!originalModel->hasChoiceOrigins(), "Choice origins will be dropped as the transformation is currently not implemented."); + + // Model type specific components + if (originalModel->isOfType(storm::models::ModelType::MarkovAutomaton)) { + auto const& ma = *originalModel->template as>(); + components.markovianStates = storm::storage::BitVector(numStates); + components.exitRates = std::vector(numStates, storm::utility::zero()); + for (uint64_t origState = 0; origState < originalModel->getNumberOfStates(); ++origState) { + uint64_t const newState = originalToNewIndex[origState]; + if (ma.isMarkovianState(origState)) { + components.markovianStates->set(newState, true); + components.exitRates->at(newState) = ma.getExitRate(origState); + } + } + components.rateTransitions = false; // Note that originalModel->getTransitionMatrix() contains probabilities + } else if (originalModel->isOfType(storm::models::ModelType::Ctmc)) { + components.rateTransitions = true; + } else { + STORM_LOG_THROW(originalModel->isOfType(storm::models::ModelType::Dtmc) || originalModel->isOfType(storm::models::ModelType::Mdp), + storm::exceptions::UnexpectedException, "Unhandled model type."); + } + return {storm::utility::builder::buildModelFromComponents(originalModel->getType(), std::move(components)), std::move(originalToNewIndex)}; +} + +template struct TransitionToActionRewardTransformerReturnType; +template struct TransitionToActionRewardTransformerReturnType; +template struct TransitionToActionRewardTransformerReturnType; + +template TransitionToActionRewardTransformerReturnType transformTransitionToActionRewards( + std::shared_ptr> originalModel, std::vector const& relevantRewardModelNames); +template TransitionToActionRewardTransformerReturnType transformTransitionToActionRewards( + std::shared_ptr> originalModel, std::vector const& relevantRewardModelNames); +template TransitionToActionRewardTransformerReturnType transformTransitionToActionRewards( + std::shared_ptr> originalModel, std::vector const& relevantRewardModelNames); + +} // namespace storm::transformer diff --git a/src/storm/transformer/TransitionToActionRewardTransformer.h b/src/storm/transformer/TransitionToActionRewardTransformer.h new file mode 100644 index 0000000000..3af168f198 --- /dev/null +++ b/src/storm/transformer/TransitionToActionRewardTransformer.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include + +#include "storm/models/sparse/Model.h" +#include "storm/storage/BitVector.h" + +namespace storm::transformer { + +template +struct TransitionToActionRewardTransformerReturnType { + std::shared_ptr> model; + std::vector originalToNewStateIndices; +}; +/*! + * + * Replaces transition branch rewards from all given reward models and replaces them by equivalent state-action based rewards. + * This is done by potentially adding intermediate states at which the corresponding reward is collected and which have a Dirac transition to the original + * state. + * Notes: + * - this construction potentially invalidates step-based properties, e.g., step-bounded reachability or discrete-time LRA properties. + * Also Until formulas with non-trivial left-hand-side will likely be invalidated + * - originalToNewStateIndices maps states of the original model to their positions within the transformed model. Those states are kept in the same order. + * - the introduced intermediate states that lead to original state 's' are located directly in front of 's'. + * - the number of intermediate states is kept small, e.g., if two distinct states 's_1' and 's_2' transition to 's' with the same transition reward + * (w.r.t. *all* reward models), only one intermediate state is introduced. + * - for Markov automata, the intermediate states are probabilistic (instantaneous). For CTMCs, the intermediate states have rate 1. + * - intermediate states do not get any label. All labels from the original model are preserved at the original states + * + * possible improvement: Preprocessing: move transition rewards to action if it is the same for all successor states + * + * @param originalModel The original model. + * @param relevantRewardModelNames The names of the reward models that should be transformed. Error if the model does not contain a reward model with this name. + * @return The transformed model and the positions of the original model states in the new (larger) transformed model. + */ +template +TransitionToActionRewardTransformerReturnType transformTransitionToActionRewards( + std::shared_ptr> originalModel, std::vector const& relevantRewardModelNames); + +} // namespace storm::transformer diff --git a/src/test/storm-pomdp/api/BeliefExplorationAPITest.cpp b/src/test/storm-pomdp/api/BeliefExplorationAPITest.cpp deleted file mode 100644 index f168ab2a4e..0000000000 --- a/src/test/storm-pomdp/api/BeliefExplorationAPITest.cpp +++ /dev/null @@ -1,316 +0,0 @@ -#include "storm-config.h" -#include "test/storm_gtest.h" - -#include "storm-parsers/api/storm-parsers.h" -#include "storm-pomdp/analysis/QualitativeAnalysisOnGraphs.h" -#include "storm-pomdp/api/verification.h" -#include "storm-pomdp/transformer/GlobalPOMDPSelfLoopEliminator.h" -#include "storm-pomdp/transformer/KnownProbabilityTransformer.h" -#include "storm/api/storm.h" -#include "storm/environment/solver/MinMaxSolverEnvironment.h" -#include "storm/transformer/MakePOMDPCanonic.h" - -class DefaultDoubleVIEnvironment { - public: - typedef double ValueType; - static storm::Environment createEnvironment() { - storm::Environment env; - env.solver().minMax().setMethod(storm::solver::MinMaxMethod::ValueIteration); - env.solver().minMax().setPrecision(storm::utility::convertNumber(1e-6)); - return env; - } - static bool const isExactModelChecking = false; - static ValueType precision() { - return storm::utility::convertNumber(0.12); - } // there actually aren't any precision guarantees, but we still want to detect if results are weird. - static void adaptOptions(storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions&) { /* intentionally left empty */ } -}; - -template -class BeliefExplorationAPITest : public ::testing::Test { - public: - typedef typename TestType::ValueType ValueType; - BeliefExplorationAPITest() : _environment(TestType::createEnvironment()) {} - - void SetUp() override { -#ifndef STORM_HAVE_Z3 - GTEST_SKIP() << "Z3 not available."; -#endif - } - - storm::Environment const& env() const { - return _environment; - } - - ValueType parseNumber(std::string const& str) { - return storm::utility::convertNumber(str); - } - struct Input { - std::shared_ptr> model; - std::shared_ptr formula; - }; - Input buildPrism(std::string const& programFile, std::string const& formulaAsString, std::string const& constantsAsString = "") const { - // Parse and build input - storm::prism::Program program = storm::api::parseProgram(programFile); - program = program.preprocess(constantsAsString); - Input input; - input.formula = storm::api::parsePropertiesForPrismProgram(formulaAsString, program).front().getRawFormula(); - input.model = storm::api::buildSparseModel(program, {input.formula})->template as>(); - - // Preprocess - storm::transformer::MakePOMDPCanonic makeCanonic(*input.model); - input.model = makeCanonic.transform(); - EXPECT_TRUE(input.model->isCanonic()); - return input; - } - ValueType precision() const { - return TestType::precision(); - } - ValueType modelcheckingPrecision() const { - if (TestType::isExactModelChecking) - return storm::utility::zero(); - else - return storm::utility::convertNumber(1e-6); - } - - private: - storm::Environment _environment; -}; - -typedef ::testing::Types TestingTypes; - -TYPED_TEST_SUITE(BeliefExplorationAPITest, TestingTypes, ); - -TYPED_TEST(BeliefExplorationAPITest, simple_Pmax) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmax=? [F \"goal\" ]", "slippery=0"); - auto task = storm::api::createTask(data.formula, false); - auto result = storm::pomdp::api::underapproximateWithCutoffs(data.model, task, 100); - - ValueType expected = this->parseNumber("7/10"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - - EXPECT_EQ(1ul, storm::pomdp::api::getNumberOfPreprocessingSchedulers(result)); - EXPECT_NO_THROW(storm::pomdp::api::extractSchedulerAsMarkovChain(result)); - EXPECT_NO_THROW(storm::pomdp::api::getCutoffScheduler(result, 0)); - EXPECT_THROW(storm::pomdp::api::getCutoffScheduler(result, 1), std::out_of_range); -} - -TYPED_TEST(BeliefExplorationAPITest, simple_Pmin) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmin=? [F \"goal\" ]", "slippery=0"); - auto task = storm::api::createTask(data.formula, false); - auto result = storm::pomdp::api::underapproximateWithCutoffs(data.model, task, 100); - - ValueType expected = this->parseNumber("3/10"); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - - EXPECT_EQ(1ul, storm::pomdp::api::getNumberOfPreprocessingSchedulers(result)); - EXPECT_NO_THROW(storm::pomdp::api::extractSchedulerAsMarkovChain(result)); - EXPECT_NO_THROW(storm::pomdp::api::getCutoffScheduler(result, 0)); - EXPECT_THROW(storm::pomdp::api::getCutoffScheduler(result, 1), std::out_of_range); -} - -TYPED_TEST(BeliefExplorationAPITest, simple_slippery_Pmax) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmax=? [F \"goal\" ]", "slippery=0.4"); - auto task = storm::api::createTask(data.formula, false); - auto result = storm::pomdp::api::underapproximateWithCutoffs(data.model, task, 100); - - ValueType expected = this->parseNumber("7/10"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - - EXPECT_EQ(1ul, storm::pomdp::api::getNumberOfPreprocessingSchedulers(result)); - EXPECT_NO_THROW(storm::pomdp::api::extractSchedulerAsMarkovChain(result)); - EXPECT_NO_THROW(storm::pomdp::api::getCutoffScheduler(result, 0)); - EXPECT_THROW(storm::pomdp::api::getCutoffScheduler(result, 1), std::out_of_range); -} - -TYPED_TEST(BeliefExplorationAPITest, simple_slippery_Pmin) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmin=? [F \"goal\" ]", "slippery=0.4"); - auto task = storm::api::createTask(data.formula, false); - auto result = storm::pomdp::api::underapproximateWithCutoffs(data.model, task, 100); - - ValueType expected = this->parseNumber("3/10"); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - - EXPECT_EQ(1ul, storm::pomdp::api::getNumberOfPreprocessingSchedulers(result)); - - EXPECT_NO_THROW(storm::pomdp::api::extractSchedulerAsMarkovChain(result)); - EXPECT_NO_THROW(storm::pomdp::api::getCutoffScheduler(result, 0)); - EXPECT_THROW(storm::pomdp::api::getCutoffScheduler(result, 1), std::out_of_range); -} - -TYPED_TEST(BeliefExplorationAPITest, simple_Rmax) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmax=? [F s>4 ]", "slippery=0"); - auto task = storm::api::createTask(data.formula, false); - auto result = storm::pomdp::api::underapproximateWithCutoffs(data.model, task, 100); - - ValueType expected = this->parseNumber("29/50"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - - EXPECT_EQ(1ul, storm::pomdp::api::getNumberOfPreprocessingSchedulers(result)); - EXPECT_NO_THROW(storm::pomdp::api::extractSchedulerAsMarkovChain(result)); - EXPECT_NO_THROW(storm::pomdp::api::getCutoffScheduler(result, 0)); - EXPECT_THROW(storm::pomdp::api::getCutoffScheduler(result, 1), std::out_of_range); -} - -TYPED_TEST(BeliefExplorationAPITest, simple_Rmin) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmin=? [F s>4 ]", "slippery=0"); - auto task = storm::api::createTask(data.formula, false); - auto result = storm::pomdp::api::underapproximateWithCutoffs(data.model, task, 100); - - ValueType expected = this->parseNumber("19/50"); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - - EXPECT_EQ(1ul, storm::pomdp::api::getNumberOfPreprocessingSchedulers(result)); - EXPECT_NO_THROW(storm::pomdp::api::extractSchedulerAsMarkovChain(result)); - EXPECT_NO_THROW(storm::pomdp::api::getCutoffScheduler(result, 0)); - EXPECT_THROW(storm::pomdp::api::getCutoffScheduler(result, 1), std::out_of_range); -} - -TYPED_TEST(BeliefExplorationAPITest, simple_slippery_Rmax) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmax=? [F s>4 ]", "slippery=0.4"); - auto task = storm::api::createTask(data.formula, false); - auto result = storm::pomdp::api::underapproximateWithCutoffs(data.model, task, 100); - - ValueType expected = this->parseNumber("29/30"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - - EXPECT_EQ(1ul, storm::pomdp::api::getNumberOfPreprocessingSchedulers(result)); - EXPECT_NO_THROW(storm::pomdp::api::extractSchedulerAsMarkovChain(result)); - EXPECT_NO_THROW(storm::pomdp::api::getCutoffScheduler(result, 0)); - EXPECT_THROW(storm::pomdp::api::getCutoffScheduler(result, 5), std::out_of_range); -} - -TYPED_TEST(BeliefExplorationAPITest, simple_slippery_Rmin) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmin=? [F s>4 ]", "slippery=0.4"); - auto task = storm::api::createTask(data.formula, false); - auto result = storm::pomdp::api::underapproximateWithCutoffs(data.model, task, 100); - - ValueType expected = this->parseNumber("19/30"); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - - EXPECT_EQ(1ul, storm::pomdp::api::getNumberOfPreprocessingSchedulers(result)); - EXPECT_NO_THROW(storm::pomdp::api::extractSchedulerAsMarkovChain(result)); - EXPECT_NO_THROW(storm::pomdp::api::getCutoffScheduler(result, 0)); - EXPECT_THROW(storm::pomdp::api::getCutoffScheduler(result, 5), std::out_of_range); -} - -TYPED_TEST(BeliefExplorationAPITest, maze2_Rmin) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmin=? [F \"goal\"]", "sl=0"); - auto task = storm::api::createTask(data.formula, false); - auto result = storm::pomdp::api::underapproximateWithCutoffs(data.model, task, 100); - - ValueType expected = this->parseNumber("74/91"); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - - EXPECT_EQ(1ul, storm::pomdp::api::getNumberOfPreprocessingSchedulers(result)); - EXPECT_NO_THROW(storm::pomdp::api::extractSchedulerAsMarkovChain(result)); - EXPECT_NO_THROW(storm::pomdp::api::getCutoffScheduler(result, 0)); - EXPECT_THROW(storm::pomdp::api::getCutoffScheduler(result, 1), std::out_of_range); -} - -TYPED_TEST(BeliefExplorationAPITest, maze2_slippery_Rmin) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmin=? [F \"goal\"]", "sl=0.075"); - auto task = storm::api::createTask(data.formula, false); - auto result = storm::pomdp::api::underapproximateWithCutoffs(data.model, task, 100); - - ValueType expected = this->parseNumber("80/91"); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - - EXPECT_EQ(1ul, storm::pomdp::api::getNumberOfPreprocessingSchedulers(result)); - EXPECT_NO_THROW(storm::pomdp::api::extractSchedulerAsMarkovChain(result)); - EXPECT_NO_THROW(storm::pomdp::api::getCutoffScheduler(result, 0)); - EXPECT_THROW(storm::pomdp::api::getCutoffScheduler(result, 1), std::out_of_range); -} - -TYPED_TEST(BeliefExplorationAPITest, refuel_Pmax) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/refuel.prism", "Pmax=?[\"notbad\" U \"goal\"]", "N=4"); - auto task = storm::api::createTask(data.formula, false); - auto result = storm::pomdp::api::underapproximateWithCutoffs(data.model, task, 100); - - ValueType expected = this->parseNumber("38/155"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - - EXPECT_EQ(2ul, storm::pomdp::api::getNumberOfPreprocessingSchedulers(result)); - EXPECT_NO_THROW(storm::pomdp::api::extractSchedulerAsMarkovChain(result)); - EXPECT_NO_THROW(storm::pomdp::api::getCutoffScheduler(result, 0)); - EXPECT_NO_THROW(storm::pomdp::api::getCutoffScheduler(result, 1)); - EXPECT_THROW(storm::pomdp::api::getCutoffScheduler(result, 2), std::out_of_range); -} - -TYPED_TEST(BeliefExplorationAPITest, refuel_Pmin) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/refuel.prism", "Pmin=?[\"notbad\" U \"goal\"]", "N=4"); - auto task = storm::api::createTask(data.formula, false); - auto result = storm::pomdp::api::underapproximateWithCutoffs(data.model, task, 100); - - ValueType expected = this->parseNumber("0"); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - - EXPECT_EQ(1ul, storm::pomdp::api::getNumberOfPreprocessingSchedulers(result)); - EXPECT_NO_THROW(storm::pomdp::api::extractSchedulerAsMarkovChain(result)); - EXPECT_NO_THROW(storm::pomdp::api::getCutoffScheduler(result, 0)); - EXPECT_THROW(storm::pomdp::api::getCutoffScheduler(result, 1), std::out_of_range); -} - -TYPED_TEST(BeliefExplorationAPITest, simple2_Rmax) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple2.prism", "Rmax=?[F \"goal\"]"); - auto task = storm::api::createTask(data.formula, false); - auto result = storm::pomdp::api::underapproximateWithCutoffs(data.model, task, 10); - - ValueType expected = this->parseNumber("59040588757/103747000000"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - - EXPECT_EQ(2ul, storm::pomdp::api::getNumberOfPreprocessingSchedulers(result)); - EXPECT_NO_THROW(storm::pomdp::api::extractSchedulerAsMarkovChain(result)); - EXPECT_NO_THROW(storm::pomdp::api::getCutoffScheduler(result, 0)); - EXPECT_NO_THROW(storm::pomdp::api::getCutoffScheduler(result, 1)); - EXPECT_THROW(storm::pomdp::api::getCutoffScheduler(result, 2), std::out_of_range); - - std::vector> obs0vals{{{0, 0}, {1, 0}}, {{0, 0.7}}, {{0, 1}, {1, 1}}}; - std::vector> obs1vals{{{2, 1}}, {{2, 1}}}; - std::vector>> additionalVals{obs0vals, obs1vals}; - - result = storm::pomdp::api::underapproximateWithCutoffs(data.model, task, 10, additionalVals); - - EXPECT_LE(result.lowerBound, storm::utility::one() + this->modelcheckingPrecision()); -} - -TYPED_TEST(BeliefExplorationAPITest, noHeuristicValues) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple2.prism", "Rmax=?[F \"goal\"]"); - auto task = storm::api::createTask(data.formula, false); - - std::vector> obs0vals{{{0, 0}, {1, 0}}, {{0, 0.7}}, {{0, 1}, {1, 1}}}; - std::vector> obs1vals{{{2, 1}}, {{2, 1}}}; - std::vector>> additionalVals{obs0vals, obs1vals}; - - auto result = storm::pomdp::api::underapproximateWithoutHeuristicValues(data.model, task, 10, additionalVals); - - EXPECT_LE(result.lowerBound, storm::utility::one() + this->modelcheckingPrecision()); -} diff --git a/src/test/storm-pomdp/modelchecker/BeliefBasedModelCheckerTest.cpp b/src/test/storm-pomdp/modelchecker/BeliefBasedModelCheckerTest.cpp new file mode 100644 index 0000000000..5db970ae0b --- /dev/null +++ b/src/test/storm-pomdp/modelchecker/BeliefBasedModelCheckerTest.cpp @@ -0,0 +1,1275 @@ +#include "storm-config.h" +#include "test/storm_gtest.h" + +#include "storm-parsers/api/storm-parsers.h" +#include "storm-pomdp/analysis/FormulaInformation.h" +#include "storm-pomdp/analysis/QualitativeAnalysisOnGraphs.h" +#include "storm-pomdp/beliefs/verification/BeliefBasedModelChecker.h" +#include "storm-pomdp/modelchecker/PreprocessingPomdpValueBoundsModelChecker.h" +#include "storm-pomdp/transformer/GlobalPOMDPSelfLoopEliminator.h" +#include "storm-pomdp/transformer/KnownProbabilityTransformer.h" +#include "storm-pomdp/transformer/MakeStateSetObservationClosed.h" +#include "storm/api/storm.h" +#include "storm/environment/solver/MinMaxSolverEnvironment.h" +#include "storm/transformer/MakePOMDPCanonic.h" +#include "storm/utility/graph.h" + +namespace { +enum class PreprocessingType { None, SelfloopReduction, QualitativeReduction, All }; + +class DefaultDoubleVIEnvironment { + public: + typedef double POMDPValueType; + typedef double BeliefValueType; + typedef double BeliefMDPValueType; + + static storm::Environment createEnvironment() { + storm::Environment env; + env.solver().minMax().setMethod(storm::solver::MinMaxMethod::ValueIteration); + env.solver().minMax().setPrecision(storm::utility::convertNumber(1e-6)); + return env; + } + static bool const isExactModelChecking = false; + static POMDPValueType precision() { + return storm::utility::convertNumber(0.12); + } // there actually aren't any precision guarantees, but we still want to detect if results are weird. + static PreprocessingType const preprocessingType = PreprocessingType::None; + static uint64_t overApproxResolution() { + return 2; + } +}; + +class SelfloopReductionDefaultDoubleVIEnvironment { + public: + typedef double POMDPValueType; + typedef double BeliefValueType; + typedef double BeliefMDPValueType; + static storm::Environment createEnvironment() { + storm::Environment env; + env.solver().minMax().setMethod(storm::solver::MinMaxMethod::ValueIteration); + env.solver().minMax().setPrecision(storm::utility::convertNumber(1e-6)); + return env; + } + static bool const isExactModelChecking = false; + static POMDPValueType precision() { + return storm::utility::convertNumber(0.12); + } // there actually aren't any precision guarantees, but we still want to detect if results are weird. + static PreprocessingType const preprocessingType = PreprocessingType::SelfloopReduction; + static uint64_t overApproxResolution() { + return 2; + } +}; + +class QualitativeReductionDefaultDoubleVIEnvironment { + public: + typedef double POMDPValueType; + typedef double BeliefValueType; + typedef double BeliefMDPValueType; + static storm::Environment createEnvironment() { + storm::Environment env; + env.solver().minMax().setMethod(storm::solver::MinMaxMethod::ValueIteration); + env.solver().minMax().setPrecision(storm::utility::convertNumber(1e-6)); + return env; + } + static bool const isExactModelChecking = false; + static POMDPValueType precision() { + return storm::utility::convertNumber(0.12); + } // there actually aren't any precision guarantees, but we still want to detect if results are weird. + static PreprocessingType const preprocessingType = PreprocessingType::QualitativeReduction; + static uint64_t overApproxResolution() { + return 2; + } +}; + +class PreprocessedDefaultDoubleVIEnvironment { + public: + typedef double POMDPValueType; + typedef double BeliefValueType; + typedef double BeliefMDPValueType; + static storm::Environment createEnvironment() { + storm::Environment env; + env.solver().minMax().setMethod(storm::solver::MinMaxMethod::ValueIteration); + env.solver().minMax().setPrecision(storm::utility::convertNumber(1e-6)); + return env; + } + static bool const isExactModelChecking = false; + static POMDPValueType precision() { + return storm::utility::convertNumber(0.12); + } // there actually aren't any precision guarantees, but we still want to detect if results are weird. + static PreprocessingType const preprocessingType = PreprocessingType::All; + static uint64_t overApproxResolution() { + return 2; + } +}; + +class FineDoubleVIEnvironment { + public: + typedef double POMDPValueType; + typedef double BeliefValueType; + typedef double BeliefMDPValueType; + static storm::Environment createEnvironment() { + storm::Environment env; + env.solver().minMax().setMethod(storm::solver::MinMaxMethod::ValueIteration); + env.solver().minMax().setPrecision(storm::utility::convertNumber(1e-6)); + return env; + } + static bool const isExactModelChecking = false; + static POMDPValueType precision() { + return storm::utility::convertNumber(0.02); + } // there actually aren't any precision guarantees, but we still want to detect if results are weird. + static PreprocessingType const preprocessingType = PreprocessingType::None; + static uint64_t overApproxResolution() { + return 24; + } +}; + +class DefaultDoubleOVIEnvironment { + public: + typedef double POMDPValueType; + typedef double BeliefValueType; + typedef double BeliefMDPValueType; + static storm::Environment createEnvironment() { + storm::Environment env; + env.solver().minMax().setMethod(storm::solver::MinMaxMethod::OptimisticValueIteration); + env.solver().minMax().setPrecision(storm::utility::convertNumber(1e-6)); + env.solver().setForceSoundness(true); + return env; + } + static bool const isExactModelChecking = false; + static POMDPValueType precision() { + return storm::utility::convertNumber(0.12); + } // there actually aren't any precision guarantees, but we still want to detect if results are weird. + static PreprocessingType const preprocessingType = PreprocessingType::None; + static uint64_t overApproxResolution() { + return 2; + } +}; + +class DefaultDoubleSVIEnvironment { + public: + typedef double POMDPValueType; + typedef double BeliefValueType; + typedef double BeliefMDPValueType; + static storm::Environment createEnvironment() { + storm::Environment env; + env.solver().minMax().setMethod(storm::solver::MinMaxMethod::SoundValueIteration); + env.solver().minMax().setPrecision(storm::utility::convertNumber(1e-6)); + env.solver().setForceSoundness(true); + return env; + } + static bool const isExactModelChecking = false; + static POMDPValueType precision() { + return storm::utility::convertNumber(0.12); + } // there actually aren't any precision guarantees, but we still want to detect if results are weird. + static PreprocessingType const preprocessingType = PreprocessingType::None; + static uint64_t overApproxResolution() { + return 2; + } +}; + +class DefaultRationalPIEnvironment { + public: + typedef storm::RationalNumber POMDPValueType; + typedef storm::RationalNumber BeliefValueType; + typedef storm::RationalNumber BeliefMDPValueType; + + static storm::Environment createEnvironment() { + storm::Environment env; + env.solver().minMax().setMethod(storm::solver::MinMaxMethod::PolicyIteration); + env.solver().setForceExact(true); + return env; + } + static bool const isExactModelChecking = true; + static POMDPValueType precision() { + return storm::utility::convertNumber(0.12); + } // there actually aren't any precision guarantees, but we still want to detect if results are weird. + static PreprocessingType const preprocessingType = PreprocessingType::None; + static uint64_t overApproxResolution() { + return 2; + } +}; + +class PreprocessedDefaultRationalPIEnvironment { + public: + typedef storm::RationalNumber POMDPValueType; + typedef storm::RationalNumber BeliefValueType; + typedef storm::RationalNumber BeliefMDPValueType; + + static storm::Environment createEnvironment() { + storm::Environment env; + env.solver().minMax().setMethod(storm::solver::MinMaxMethod::PolicyIteration); + env.solver().setForceExact(true); + return env; + } + static bool const isExactModelChecking = true; + static POMDPValueType precision() { + return storm::utility::convertNumber(0.12); + } // there actually aren't any precision guarantees, but we still want to detect if results are weird. + static PreprocessingType const preprocessingType = PreprocessingType::All; + static uint64_t overApproxResolution() { + return 2; + } +}; + +template +class BeliefBasedModelCheckerTest : public ::testing::Test { + public: + typedef typename TestType::POMDPValueType POMDPValueType; + typedef typename TestType::BeliefValueType BeliefValueType; + typedef typename TestType::BeliefMDPValueType BeliefMDPValueType; + + BeliefBasedModelCheckerTest() : _environment(TestType::createEnvironment()) {} + + void SetUp() override { +#ifndef STORM_HAVE_Z3 + GTEST_SKIP() << "Z3 not available."; +#endif + } + + storm::Environment const& env() const { + return _environment; + } + + template + ValueType parseNumber(std::string const& str) { + return storm::utility::convertNumber(str); + } + struct Input { + std::shared_ptr> model; + std::shared_ptr formula; + std::shared_ptr propertyInfo = std::make_shared(); + }; + Input buildPrism(std::string const& programFile, std::string const& formulaAsString, std::string const& constantsAsString = "") const { + // Parse and build input + storm::prism::Program program = storm::api::parseProgram(programFile); + program = program.preprocess(constantsAsString); + Input input; + input.formula = storm::api::parsePropertiesForPrismProgram(formulaAsString, program).front().getRawFormula(); + input.model = storm::api::buildSparseModel(program, {input.formula})->template as>(); + + // Preprocess + storm::transformer::MakePOMDPCanonic makeCanonic(*input.model); + input.model = makeCanonic.transform(); + EXPECT_TRUE(input.model->isCanonic()); + if (TestType::preprocessingType == PreprocessingType::SelfloopReduction || TestType::preprocessingType == PreprocessingType::All) { + storm::transformer::GlobalPOMDPSelfLoopEliminator selfLoopEliminator(*input.model); + if (selfLoopEliminator.preservesFormula(*input.formula)) { + input.model = selfLoopEliminator.transform(); + } else { + EXPECT_TRUE(input.formula->isOperatorFormula()); + EXPECT_TRUE(input.formula->asOperatorFormula().hasOptimalityType()); + bool maximizing = storm::solver::maximize(input.formula->asOperatorFormula().getOptimalityType()); + // Valid reasons for unpreserved formulas: + EXPECT_TRUE(maximizing || input.formula->isProbabilityOperatorFormula()); + EXPECT_TRUE(!maximizing || input.formula->isRewardOperatorFormula()); + } + } + if (TestType::preprocessingType == PreprocessingType::QualitativeReduction || TestType::preprocessingType == PreprocessingType::All) { + EXPECT_TRUE(input.formula->isOperatorFormula()); + EXPECT_TRUE(input.formula->asOperatorFormula().hasOptimalityType()); + if (input.formula->isProbabilityOperatorFormula() && storm::solver::maximize(input.formula->asOperatorFormula().getOptimalityType())) { + storm::analysis::QualitativeAnalysisOnGraphs qualitativeAnalysis(*input.model); + storm::storage::BitVector prob0States = qualitativeAnalysis.analyseProb0(input.formula->asProbabilityOperatorFormula()); + storm::storage::BitVector prob1States = qualitativeAnalysis.analyseProb1(input.formula->asProbabilityOperatorFormula()); + storm::pomdp::transformer::KnownProbabilityTransformer kpt; + input.model = kpt.transform(*input.model, prob0States, prob1States); + } + } + EXPECT_TRUE(input.model->isCanonic()); + auto formulaInfo = storm::pomdp::analysis::getFormulaInformation(*input.model, *input.formula); + std::optional rewardModelName; + std::set targetObservations; + EXPECT_TRUE(formulaInfo.isNonNestedReachabilityProbability() || formulaInfo.isNonNestedExpectedRewardFormula()); + if (formulaInfo.getTargetStates().observationClosed) { + targetObservations = formulaInfo.getTargetStates().observations; + } else { + storm::transformer::MakeStateSetObservationClosed obsCloser(input.model); + std::tie(input.model, targetObservations) = obsCloser.transform(formulaInfo.getTargetStates().states); + } + if (formulaInfo.isNonNestedReachabilityProbability()) { + if (!formulaInfo.getSinkStates().empty()) { + storm::storage::sparse::ModelComponents components; + components.stateLabeling = input.model->getStateLabeling(); + components.rewardModels = input.model->getRewardModels(); + auto matrix = input.model->getTransitionMatrix(); + matrix.makeRowGroupsAbsorbing(formulaInfo.getSinkStates().states); + components.transitionMatrix = matrix; + components.observabilityClasses = input.model->getObservations(); + if (input.model->hasChoiceLabeling()) { + components.choiceLabeling = input.model->getChoiceLabeling(); + } + if (input.model->hasObservationValuations()) { + components.observationValuations = input.model->getObservationValuations(); + } + input.model = std::make_shared>(std::move(components), true); + auto reachableFromSinkStates = + storm::utility::graph::getReachableStates(input.model->getTransitionMatrix(), formulaInfo.getSinkStates().states, + formulaInfo.getSinkStates().states, ~formulaInfo.getSinkStates().states); + reachableFromSinkStates &= ~formulaInfo.getSinkStates().states; + STORM_LOG_THROW(reachableFromSinkStates.empty(), storm::exceptions::NotSupportedException, + "There are sink states that can reach non-sink states. This is currently not supported"); + } + } else { + // Expected reward formula! + rewardModelName = formulaInfo.getRewardModelName(); + } + + if (rewardModelName) { + input.propertyInfo->kind = storm::pomdp::beliefs::PropertyInformation::Kind::ExpectedTotalReachabilityReward; + input.propertyInfo->rewardModelName = rewardModelName; + } else { + input.propertyInfo->kind = storm::pomdp::beliefs::PropertyInformation::Kind::ReachabilityProbability; + } + input.propertyInfo->dir = formulaInfo.getOptimizationDirection(); + input.propertyInfo->targetObservations = targetObservations; + + return input; + } + POMDPValueType precision() const { + return TestType::precision(); + } + uint64_t overApproxResolution() const { + return TestType::overApproxResolution(); + } + template + ValueType modelcheckingPrecision() const { + if (TestType::isExactModelChecking) { + return storm::utility::zero(); + } else { + return storm::utility::convertNumber(1e-6); + } + } + bool isExact() const { + return TestType::isExactModelChecking; + } + + private: + storm::Environment _environment; +}; + +typedef ::testing::Types + TestingTypes; + +TYPED_TEST_SUITE(BeliefBasedModelCheckerTest, TestingTypes, ); + +TYPED_TEST(BeliefBasedModelCheckerTest, simple_Pmax) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmax=? [F \"goal\" ]", "slippery=0"); + storm::pomdp::beliefs::BeliefBasedModelChecker checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + + BeliefMDPValueType overResultValue; + BeliefMDPValueType underResultValue; + bool completedOverExploration; + bool completedUnderExploration; + auto expected = this->template parseNumber("7/10"); + std::tie(overResultValue, completedOverExploration) = + checker.checkDiscretize(this->env(), *data.propertyInfo, options, this->overApproxResolution(), true, precomputedBeliefBounds); + EXPECT_GE(overResultValue, expected - this->template modelcheckingPrecision()); + + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_LE(underResultValue, expected + this->template modelcheckingPrecision()); + EXPECT_LE(storm::utility::abs(BeliefMDPValueType(overResultValue - underResultValue)), this->precision()) + << "Result [" << underResultValue << ", " << overResultValue + << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; +} + +TYPED_TEST(BeliefBasedModelCheckerTest, simple_Pmin) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmin=? [F \"goal\" ]", "slippery=0"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + + BeliefMDPValueType overResultValue; + BeliefMDPValueType underResultValue; + bool completedOverExploration; + bool completedUnderExploration; + + BeliefMDPValueType expected = this->template parseNumber("3/10"); + std::tie(overResultValue, completedOverExploration) = + checker.checkDiscretize(this->env(), *data.propertyInfo, options, this->overApproxResolution(), true, precomputedBeliefBounds); + EXPECT_LE(overResultValue, expected + this->template modelcheckingPrecision()); + + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_GE(underResultValue, expected - this->template modelcheckingPrecision()); + EXPECT_LE(storm::utility::abs(BeliefMDPValueType(overResultValue - underResultValue)), this->precision()) + << "Result [" << overResultValue << ", " << underResultValue + << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; +} + +TYPED_TEST(BeliefBasedModelCheckerTest, simple_slippery_Pmax) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmax=? [F \"goal\" ]", "slippery=0.4"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + BeliefMDPValueType overResultValue; + BeliefMDPValueType underResultValue; + bool completedOverExploration; + bool completedUnderExploration; + + BeliefMDPValueType expected = this->template parseNumber("7/10"); + std::tie(overResultValue, completedOverExploration) = + checker.checkDiscretize(this->env(), *data.propertyInfo, options, this->overApproxResolution(), true, precomputedBeliefBounds); + EXPECT_GE(overResultValue, expected - this->template modelcheckingPrecision()); + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_LE(underResultValue, expected + this->template modelcheckingPrecision()); + EXPECT_LE(storm::utility::abs(BeliefMDPValueType(overResultValue - underResultValue)), this->precision()) + << "Result [" << underResultValue << ", " << overResultValue + << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; +} + +TYPED_TEST(BeliefBasedModelCheckerTest, simple_slippery_Pmin) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmin=? [F \"goal\" ]", "slippery=0.4"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + BeliefMDPValueType overResultValue; + BeliefMDPValueType underResultValue; + bool completedOverExploration; + bool completedUnderExploration; + + POMDPValueType expected = this->template parseNumber("3/10"); + std::tie(overResultValue, completedOverExploration) = + checker.checkDiscretize(this->env(), *data.propertyInfo, options, this->overApproxResolution(), true, precomputedBeliefBounds); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + if (this->isExact()) { + // This model's value can only be approximated arbitrarily close but never reached + // Exact arithmetics will thus not reach the value with absoulute precision either. + POMDPValueType approxPrecision = storm::utility::convertNumber(1e-5); + EXPECT_GE(underResultValue, expected - approxPrecision); + EXPECT_LE(overResultValue, expected + approxPrecision); + } else { + EXPECT_GE(underResultValue, expected - this->template modelcheckingPrecision()); + EXPECT_LE(overResultValue, expected + this->template modelcheckingPrecision()); + } + EXPECT_LE(storm::utility::abs(BeliefMDPValueType(overResultValue - underResultValue)), this->precision()) + << "Result [" << underResultValue << ", " << overResultValue + << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; +} + +TYPED_TEST(BeliefBasedModelCheckerTest, simple_Rmax) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmax=? [F s>4 ]", "slippery=0"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + + BeliefMDPValueType overResultValue; + BeliefMDPValueType underResultValue; + bool completedOverExploration; + bool completedUnderExploration; + + BeliefMDPValueType expected = this->template parseNumber("29/50"); + std::tie(overResultValue, completedOverExploration) = + checker.checkDiscretize(this->env(), *data.propertyInfo, options, this->overApproxResolution(), true, precomputedBeliefBounds); + EXPECT_GE(overResultValue, expected - this->template modelcheckingPrecision()); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_LE(underResultValue, expected + this->template modelcheckingPrecision()); + EXPECT_LE(storm::utility::abs(BeliefMDPValueType(overResultValue - underResultValue)), this->precision()) + << "Result [" << underResultValue << ", " << overResultValue + << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; +} + +TYPED_TEST(BeliefBasedModelCheckerTest, simple_Rmin) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmin=? [F s>4 ]", "slippery=0"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + + BeliefMDPValueType overResultValue; + BeliefMDPValueType underResultValue; + bool completedOverExploration; + bool completedUnderExploration; + + BeliefMDPValueType expected = this->template parseNumber("19/50"); + std::tie(overResultValue, completedOverExploration) = + checker.checkDiscretize(this->env(), *data.propertyInfo, options, this->overApproxResolution(), true, precomputedBeliefBounds); + EXPECT_LE(overResultValue, expected + this->template modelcheckingPrecision()); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_GE(underResultValue, expected - this->template modelcheckingPrecision()); + EXPECT_LE(storm::utility::abs(BeliefMDPValueType(overResultValue - underResultValue)), this->precision()) + << "Result [" << overResultValue << ", " << underResultValue + << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; +} + +TYPED_TEST(BeliefBasedModelCheckerTest, simple_slippery_Rmax) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmax=? [F s>4 ]", "slippery=0.4"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + + BeliefMDPValueType overResultValue; + BeliefMDPValueType underResultValue; + bool completedOverExploration; + bool completedUnderExploration; + + BeliefMDPValueType expected = this->template parseNumber("29/30"); + std::tie(overResultValue, completedOverExploration) = + checker.checkDiscretize(this->env(), *data.propertyInfo, options, this->overApproxResolution(), true, precomputedBeliefBounds); + EXPECT_GE(overResultValue, expected - this->template modelcheckingPrecision()); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_LE(underResultValue, expected + this->template modelcheckingPrecision()); + EXPECT_LE(storm::utility::abs(BeliefMDPValueType(overResultValue - underResultValue)), this->precision()) + << "Result [" << underResultValue << ", " << overResultValue + << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; +} + +TYPED_TEST(BeliefBasedModelCheckerTest, simple_slippery_Rmin) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmin=? [F s>4 ]", "slippery=0.4"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + + BeliefMDPValueType overResultValue; + BeliefMDPValueType underResultValue; + bool completedOverExploration; + bool completedUnderExploration; + + BeliefMDPValueType expected = this->template parseNumber("19/30"); + std::tie(overResultValue, completedOverExploration) = + checker.checkDiscretize(this->env(), *data.propertyInfo, options, this->overApproxResolution(), true, precomputedBeliefBounds); + EXPECT_LE(overResultValue, expected + this->template modelcheckingPrecision()); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_GE(underResultValue, expected - this->template modelcheckingPrecision()); + EXPECT_LE(storm::utility::abs(BeliefMDPValueType(overResultValue - underResultValue)), this->precision()) + << "Result [" << overResultValue << ", " << underResultValue + << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; +} + +TYPED_TEST(BeliefBasedModelCheckerTest, maze2_Rmin) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmin=? [F \"goal\"]", "sl=0"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + + BeliefMDPValueType overResultValue; + BeliefMDPValueType underResultValue; + bool completedOverExploration; + bool completedUnderExploration; + + BeliefMDPValueType expected = this->template parseNumber("74/91"); + std::tie(overResultValue, completedOverExploration) = + checker.checkDiscretize(this->env(), *data.propertyInfo, options, this->overApproxResolution(), true, precomputedBeliefBounds); + EXPECT_LE(overResultValue, expected + this->template modelcheckingPrecision()); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_GE(underResultValue, expected - this->template modelcheckingPrecision()); + EXPECT_LE(storm::utility::abs(BeliefMDPValueType(overResultValue - underResultValue)), this->precision()) + << "Result [" << overResultValue << ", " << underResultValue + << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; +} + +TYPED_TEST(BeliefBasedModelCheckerTest, maze2_Rmax) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmax=? [F \"goal\"]", "sl=0"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + + BeliefMDPValueType overResultValue; + BeliefMDPValueType underResultValue; + bool completedOverExploration; + bool completedUnderExploration; + + std::tie(overResultValue, completedOverExploration) = + checker.checkDiscretize(this->env(), *data.propertyInfo, options, this->overApproxResolution(), true, precomputedBeliefBounds); + EXPECT_TRUE(storm::utility::isInfinity(overResultValue)); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_TRUE(storm::utility::isInfinity(underResultValue)); +} + +TYPED_TEST(BeliefBasedModelCheckerTest, maze2_slippery_Rmin) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmin=? [F \"goal\"]", "sl=0.075"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + + BeliefMDPValueType overResultValue; + BeliefMDPValueType underResultValue; + bool completedOverExploration; + bool completedUnderExploration; + + BeliefMDPValueType expected = this->template parseNumber("80/91"); + std::tie(overResultValue, completedOverExploration) = + checker.checkDiscretize(this->env(), *data.propertyInfo, options, this->overApproxResolution(), true, precomputedBeliefBounds); + EXPECT_LE(overResultValue, expected + this->template modelcheckingPrecision()); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_GE(underResultValue, expected - this->template modelcheckingPrecision()); + EXPECT_LE(storm::utility::abs(BeliefMDPValueType(overResultValue - underResultValue)), this->precision()) + << "Result [" << overResultValue << ", " << underResultValue + << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; +} + +TYPED_TEST(BeliefBasedModelCheckerTest, maze2_slippery_Rmax) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmax=? [F \"goal\"]", "sl=0.075"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + + BeliefMDPValueType overResultValue; + BeliefMDPValueType underResultValue; + bool completedOverExploration; + bool completedUnderExploration; + + std::tie(overResultValue, completedOverExploration) = + checker.checkDiscretize(this->env(), *data.propertyInfo, options, this->overApproxResolution(), true, precomputedBeliefBounds); + EXPECT_TRUE(storm::utility::isInfinity(overResultValue)); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_TRUE(storm::utility::isInfinity(underResultValue)); +} + +TYPED_TEST(BeliefBasedModelCheckerTest, refuel_Pmax) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/refuel.prism", "Pmax=?[\"notbad\" U \"goal\"]", "N=4"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + + BeliefMDPValueType overResultValue; + BeliefMDPValueType underResultValue; + bool completedOverExploration; + bool completedUnderExploration; + + BeliefMDPValueType expected = this->template parseNumber("38/155"); + std::tie(overResultValue, completedOverExploration) = + checker.checkDiscretize(this->env(), *data.propertyInfo, options, this->overApproxResolution(), true, precomputedBeliefBounds); + EXPECT_GE(overResultValue, expected - this->template modelcheckingPrecision()); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_LE(underResultValue, expected + this->template modelcheckingPrecision()); + EXPECT_LE(storm::utility::abs(BeliefMDPValueType(overResultValue - underResultValue)), this->precision()) + << "Result [" << underResultValue << ", " << overResultValue + << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; +} + +TYPED_TEST(BeliefBasedModelCheckerTest, refuel_Pmin) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/refuel.prism", "Pmin=?[\"notbad\" U \"goal\"]", "N=4"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + + BeliefMDPValueType overResultValue; + BeliefMDPValueType underResultValue; + bool completedOverExploration; + bool completedUnderExploration; + + BeliefMDPValueType expected = this->template parseNumber("0"); + std::tie(overResultValue, completedOverExploration) = + checker.checkDiscretize(this->env(), *data.propertyInfo, options, this->overApproxResolution(), true, precomputedBeliefBounds); + EXPECT_LE(overResultValue, expected + this->template modelcheckingPrecision()); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_GE(underResultValue, expected - this->template modelcheckingPrecision()); + EXPECT_LE(storm::utility::abs(BeliefMDPValueType(overResultValue - underResultValue)), this->precision()) + << "Result [" << overResultValue << ", " << underResultValue + << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; +} + +#if defined STORM_HAVE_LP_SOLVER +TYPED_TEST(BeliefBasedModelCheckerTest, clip_simple_Pmax) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmax=? [F \"goal\" ]", "slippery=0"); + storm::pomdp::beliefs::BeliefBasedModelChecker checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + options.useClipping = true; + options.clippingResolutions = std::vector(data.model->getNrObservations(), 2); + + BeliefMDPValueType overResultValue; + BeliefMDPValueType underResultValue; + bool completedOverExploration; + bool completedUnderExploration; + auto expected = this->template parseNumber("7/10"); + + std::tie(overResultValue, completedOverExploration) = + checker.checkDiscretize(this->env(), *data.propertyInfo, options, this->overApproxResolution(), true, precomputedBeliefBounds); + EXPECT_LE(overResultValue, expected + this->template modelcheckingPrecision()); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_LE(underResultValue, expected + this->template modelcheckingPrecision()); + EXPECT_LE(storm::utility::abs(BeliefMDPValueType(overResultValue - underResultValue)), this->precision()) + << "Result [" << underResultValue << ", " << overResultValue + << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; +} + +TYPED_TEST(BeliefBasedModelCheckerTest, clip_simple_Pmin) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmin=? [F \"goal\" ]", "slippery=0"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + options.useClipping = true; + options.clippingResolutions = std::vector(data.model->getNrObservations(), 2); + + BeliefMDPValueType overResultValue; + BeliefMDPValueType underResultValue; + bool completedOverExploration; + bool completedUnderExploration; + + BeliefMDPValueType expected = this->template parseNumber("3/10"); + + std::tie(overResultValue, completedOverExploration) = + checker.checkDiscretize(this->env(), *data.propertyInfo, options, this->overApproxResolution(), true, precomputedBeliefBounds); + EXPECT_GE(overResultValue, expected - this->template modelcheckingPrecision()); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_GE(underResultValue, expected - this->template modelcheckingPrecision()); + EXPECT_LE(storm::utility::abs(BeliefMDPValueType(overResultValue - underResultValue)), this->precision()) + << "Result [" << overResultValue << ", " << underResultValue + << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; +} + +TYPED_TEST(BeliefBasedModelCheckerTest, clip_simple_slippery_Pmax) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmax=? [F \"goal\" ]", "slippery=0.4"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + options.useClipping = true; + options.clippingResolutions = std::vector(data.model->getNrObservations(), 2); + + BeliefMDPValueType underResultValue; + bool completedUnderExploration; + + BeliefMDPValueType expected = this->template parseNumber("7/10"); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_LE(underResultValue, expected + this->template modelcheckingPrecision()); +} + +TYPED_TEST(BeliefBasedModelCheckerTest, clip_simple_slippery_Pmin) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmin=? [F \"goal\" ]", "slippery=0.4"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + options.useClipping = true; + options.clippingResolutions = std::vector(data.model->getNrObservations(), 2); + + BeliefMDPValueType overResultValue; + BeliefMDPValueType underResultValue; + bool completedOverExploration; + bool completedUnderExploration; + + POMDPValueType expected = this->template parseNumber("3/10"); + std::tie(overResultValue, completedOverExploration) = + checker.checkDiscretize(this->env(), *data.propertyInfo, options, this->overApproxResolution(), true, precomputedBeliefBounds); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + if (this->isExact()) { + // This model's value can only be approximated arbitrarily close but never reached + // Exact arithmetics will thus not reach the value with absoulute precision either. + POMDPValueType approxPrecision = storm::utility::convertNumber(1e-5); + EXPECT_GE(underResultValue, expected - approxPrecision); + EXPECT_LE(overResultValue, expected + approxPrecision); + } else { + EXPECT_GE(underResultValue, expected - this->template modelcheckingPrecision()); + EXPECT_LE(overResultValue, expected + this->template modelcheckingPrecision()); + } + EXPECT_LE(storm::utility::abs(BeliefMDPValueType(overResultValue - underResultValue)), this->precision()) + << "Result [" << underResultValue << ", " << overResultValue + << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; +} + +TYPED_TEST(BeliefBasedModelCheckerTest, clip_simple_Rmax) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmax=? [F s>4 ]", "slippery=0"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + precomputedBeliefBounds.extremeBounds = preprocessChecker.getExtremeValueBound(this->env(), *data.formula); + + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + options.useClipping = true; + options.clippingResolutions = std::vector(data.model->getNrObservations(), 2); + + BeliefMDPValueType underResultValue; + bool completedUnderExploration; + + BeliefMDPValueType expected = this->template parseNumber("29/50"); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_LE(underResultValue, expected + this->template modelcheckingPrecision()); +} + +TYPED_TEST(BeliefBasedModelCheckerTest, clip_simple_Rmin) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmin=? [F s>4 ]", "slippery=0"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + precomputedBeliefBounds.extremeBounds = preprocessChecker.getExtremeValueBound(this->env(), *data.formula); + + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + options.useClipping = true; + options.clippingResolutions = std::vector(data.model->getNrObservations(), 2); + + BeliefMDPValueType underResultValue; + bool completedUnderExploration; + + BeliefMDPValueType expected = this->template parseNumber("19/50"); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_GE(underResultValue, expected - this->template modelcheckingPrecision()); +} + +TYPED_TEST(BeliefBasedModelCheckerTest, clip_simple_slippery_Rmax) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmax=? [F s>4 ]", "slippery=0.4"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + precomputedBeliefBounds.extremeBounds = preprocessChecker.getExtremeValueBound(this->env(), *data.formula); + + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + options.useClipping = true; + options.clippingResolutions = std::vector(data.model->getNrObservations(), 2); + + BeliefMDPValueType underResultValue; + bool completedUnderExploration; + + BeliefMDPValueType expected = this->template parseNumber("29/30"); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_LE(underResultValue, expected + this->template modelcheckingPrecision()); +} + +TYPED_TEST(BeliefBasedModelCheckerTest, clip_simple_slippery_Rmin) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmin=? [F s>4 ]", "slippery=0.4"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + precomputedBeliefBounds.extremeBounds = preprocessChecker.getExtremeValueBound(this->env(), *data.formula); + + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + options.useClipping = true; + options.clippingResolutions = std::vector(data.model->getNrObservations(), 2); + + BeliefMDPValueType underResultValue; + bool completedUnderExploration; + + BeliefMDPValueType expected = this->template parseNumber("19/30"); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_GE(underResultValue, expected - this->template modelcheckingPrecision()); +} + +TYPED_TEST(BeliefBasedModelCheckerTest, clip_maze2_Rmin) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmin=? [F \"goal\"]", "sl=0"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + precomputedBeliefBounds.extremeBounds = preprocessChecker.getExtremeValueBound(this->env(), *data.formula); + + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + options.useClipping = true; + options.clippingResolutions = std::vector(data.model->getNrObservations(), 2); + + BeliefMDPValueType underResultValue; + bool completedUnderExploration; + + BeliefMDPValueType expected = this->template parseNumber("74/91"); + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_GE(underResultValue, expected - this->template modelcheckingPrecision()); +} + +TYPED_TEST(BeliefBasedModelCheckerTest, clip_maze2_Rmax) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmax=? [F \"goal\"]", "sl=0"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + precomputedBeliefBounds.extremeBounds = preprocessChecker.getExtremeValueBound(this->env(), *data.formula); + + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + options.useClipping = true; + options.clippingResolutions = std::vector(data.model->getNrObservations(), 2); + + BeliefMDPValueType underResultValue; + bool completedUnderExploration; + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_TRUE(storm::utility::isInfinity(underResultValue)); +} + +TYPED_TEST(BeliefBasedModelCheckerTest, clip_maze2_slippery_Rmin) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmin=? [F \"goal\"]", "sl=0.075"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + precomputedBeliefBounds.extremeBounds = preprocessChecker.getExtremeValueBound(this->env(), *data.formula); + + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + options.useClipping = true; + options.clippingResolutions = std::vector(data.model->getNrObservations(), 2); + + BeliefMDPValueType underResultValue; + bool completedUnderExploration; + + BeliefMDPValueType expected = this->template parseNumber("80/91"); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_GE(underResultValue, expected - this->template modelcheckingPrecision()); +} + +TYPED_TEST(BeliefBasedModelCheckerTest, clip_maze2_slippery_Rmax) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmax=? [F \"goal\"]", "sl=0.075"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + precomputedBeliefBounds.extremeBounds = preprocessChecker.getExtremeValueBound(this->env(), *data.formula); + + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + options.useClipping = true; + options.clippingResolutions = std::vector(data.model->getNrObservations(), 2); + + BeliefMDPValueType underResultValue; + bool completedUnderExploration; + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_TRUE(storm::utility::isInfinity(underResultValue)); +} + +TYPED_TEST(BeliefBasedModelCheckerTest, clip_refuel_Pmax) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/refuel.prism", "Pmax=?[\"notbad\" U \"goal\"]", "N=4"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + options.useClipping = true; + options.clippingResolutions = std::vector(data.model->getNrObservations(), 2); + + BeliefMDPValueType underResultValue; + bool completedUnderExploration; + + BeliefMDPValueType expected = this->template parseNumber("38/155"); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_LE(underResultValue, expected + this->template modelcheckingPrecision()); +} + +TYPED_TEST(BeliefBasedModelCheckerTest, clip_refuel_Pmin) { + typedef storm::models::sparse::Pomdp POMDPType; + typedef typename TestFixture::POMDPValueType POMDPValueType; + typedef typename TestFixture::BeliefValueType BeliefValueType; + typedef typename TestFixture::BeliefMDPValueType BeliefMDPValueType; + + auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/refuel.prism", "Pmin=?[\"notbad\" U \"goal\"]", "N=4"); + storm::pomdp::beliefs::BeliefBasedModelChecker, BeliefValueType, BeliefMDPValueType> checker(*data.model); + storm::pomdp::modelchecker::PreprocessingPomdpValueBoundsModelChecker preprocessChecker(*data.model); + + storm::pomdp::storage::BeliefExplorationBounds precomputedBeliefBounds; + precomputedBeliefBounds.preprocessingBounds = preprocessChecker.getValueBounds(this->env(), *data.formula); + + storm::pomdp::beliefs::BeliefBasedModelCheckerOptions options; + options.buildChoiceLabeling = false; + options.explorationQueueOrder = storm::pomdp::beliefs::ExplorationQueueOrder::FIFO; + options.useClipping = true; + options.clippingResolutions = std::vector(data.model->getNrObservations(), 2); + + BeliefMDPValueType underResultValue; + bool completedUnderExploration; + + BeliefMDPValueType expected = this->template parseNumber("0"); + + options.maxExplorationSize = data.model->getNumberOfStates() * data.model->getMaxNrStatesWithSameObservation(); + std::tie(underResultValue, completedUnderExploration) = checker.checkUnfold(this->env(), *data.propertyInfo, options, precomputedBeliefBounds); + EXPECT_GE(underResultValue, expected - this->template modelcheckingPrecision()); +} + +#endif // defined STORM_HAVE_LP_SOLVER + +} // namespace diff --git a/src/test/storm-pomdp/modelchecker/BeliefExplorationPomdpModelCheckerTest.cpp b/src/test/storm-pomdp/modelchecker/BeliefExplorationPomdpModelCheckerTest.cpp deleted file mode 100644 index a8468bcf79..0000000000 --- a/src/test/storm-pomdp/modelchecker/BeliefExplorationPomdpModelCheckerTest.cpp +++ /dev/null @@ -1,956 +0,0 @@ -#include "storm-config.h" -#include "test/storm_gtest.h" - -#include "storm-parsers/api/storm-parsers.h" -#include "storm-pomdp/analysis/QualitativeAnalysisOnGraphs.h" -#include "storm-pomdp/modelchecker/BeliefExplorationPomdpModelChecker.h" -#include "storm-pomdp/transformer/GlobalPOMDPSelfLoopEliminator.h" -#include "storm-pomdp/transformer/KnownProbabilityTransformer.h" -#include "storm/api/storm.h" -#include "storm/environment/solver/MinMaxSolverEnvironment.h" -#include "storm/transformer/MakePOMDPCanonic.h" - -namespace { -enum class PreprocessingType { None, SelfloopReduction, QualitativeReduction, All }; - -class DefaultDoubleVIEnvironment { - public: - typedef double ValueType; - static storm::Environment createEnvironment() { - storm::Environment env; - env.solver().minMax().setMethod(storm::solver::MinMaxMethod::ValueIteration); - env.solver().minMax().setPrecision(storm::utility::convertNumber(1e-6)); - return env; - } - static bool const isExactModelChecking = false; - static ValueType precision() { - return storm::utility::convertNumber(0.12); - } // there actually aren't any precision guarantees, but we still want to detect if results are weird. - static void adaptOptions(storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions&) { /* intentionally left empty */ } - static PreprocessingType const preprocessingType = PreprocessingType::None; -}; - -class SelfloopReductionDefaultDoubleVIEnvironment { - public: - typedef double ValueType; - static storm::Environment createEnvironment() { - storm::Environment env; - env.solver().minMax().setMethod(storm::solver::MinMaxMethod::ValueIteration); - env.solver().minMax().setPrecision(storm::utility::convertNumber(1e-6)); - return env; - } - static bool const isExactModelChecking = false; - static ValueType precision() { - return storm::utility::convertNumber(0.12); - } // there actually aren't any precision guarantees, but we still want to detect if results are weird. - static void adaptOptions(storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions&) { /* intentionally left empty */ } - static PreprocessingType const preprocessingType = PreprocessingType::SelfloopReduction; -}; - -class QualitativeReductionDefaultDoubleVIEnvironment { - public: - typedef double ValueType; - static storm::Environment createEnvironment() { - storm::Environment env; - env.solver().minMax().setMethod(storm::solver::MinMaxMethod::ValueIteration); - env.solver().minMax().setPrecision(storm::utility::convertNumber(1e-6)); - return env; - } - static bool const isExactModelChecking = false; - static ValueType precision() { - return storm::utility::convertNumber(0.12); - } // there actually aren't any precision guarantees, but we still want to detect if results are weird. - static void adaptOptions(storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions&) { /* intentionally left empty */ } - static PreprocessingType const preprocessingType = PreprocessingType::QualitativeReduction; -}; - -class PreprocessedDefaultDoubleVIEnvironment { - public: - typedef double ValueType; - static storm::Environment createEnvironment() { - storm::Environment env; - env.solver().minMax().setMethod(storm::solver::MinMaxMethod::ValueIteration); - env.solver().minMax().setPrecision(storm::utility::convertNumber(1e-6)); - return env; - } - static bool const isExactModelChecking = false; - static ValueType precision() { - return storm::utility::convertNumber(0.12); - } // there actually aren't any precision guarantees, but we still want to detect if results are weird. - static void adaptOptions(storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions&) { /* intentionally left empty */ } - static PreprocessingType const preprocessingType = PreprocessingType::All; -}; - -class FineDoubleVIEnvironment { - public: - typedef double ValueType; - static storm::Environment createEnvironment() { - storm::Environment env; - env.solver().minMax().setMethod(storm::solver::MinMaxMethod::ValueIteration); - env.solver().minMax().setPrecision(storm::utility::convertNumber(1e-6)); - return env; - } - static bool const isExactModelChecking = false; - static ValueType precision() { - return storm::utility::convertNumber(0.02); - } // there actually aren't any precision guarantees, but we still want to detect if results are weird. - static void adaptOptions(storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions& options) { - options.resolutionInit = 24; - } - static PreprocessingType const preprocessingType = PreprocessingType::None; -}; - -class RefineDoubleVIEnvironment { - public: - typedef double ValueType; - static storm::Environment createEnvironment() { - storm::Environment env; - env.solver().minMax().setMethod(storm::solver::MinMaxMethod::ValueIteration); - env.solver().minMax().setPrecision(storm::utility::convertNumber(1e-6)); - return env; - } - static bool const isExactModelChecking = false; - static ValueType precision() { - return storm::utility::convertNumber(0.005); - } - static PreprocessingType const preprocessingType = PreprocessingType::None; - static void adaptOptions(storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions& options) { - options.refine = true; - options.refinePrecision = precision(); - } -}; - -class PreprocessedRefineDoubleVIEnvironment { - public: - typedef double ValueType; - static storm::Environment createEnvironment() { - storm::Environment env; - env.solver().minMax().setMethod(storm::solver::MinMaxMethod::ValueIteration); - env.solver().minMax().setPrecision(storm::utility::convertNumber(1e-6)); - return env; - } - static bool const isExactModelChecking = false; - static ValueType precision() { - return storm::utility::convertNumber(0.005); - } - static PreprocessingType const preprocessingType = PreprocessingType::All; - static void adaptOptions(storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions& options) { - options.refine = true; - options.refinePrecision = precision(); - } -}; - -class DefaultDoubleOVIEnvironment { - public: - typedef double ValueType; - static storm::Environment createEnvironment() { - storm::Environment env; - env.solver().minMax().setMethod(storm::solver::MinMaxMethod::SoundValueIteration); - env.solver().minMax().setPrecision(storm::utility::convertNumber(1e-6)); - env.solver().setForceSoundness(true); - return env; - } - static bool const isExactModelChecking = false; - static ValueType precision() { - return storm::utility::convertNumber(0.12); - } // there actually aren't any precision guarantees, but we still want to detect if results are weird. - static void adaptOptions(storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions&) { /* intentionally left empty */ } - static PreprocessingType const preprocessingType = PreprocessingType::None; -}; - -class DefaultRationalPIEnvironment { - public: - typedef storm::RationalNumber ValueType; - static storm::Environment createEnvironment() { - storm::Environment env; - env.solver().minMax().setMethod(storm::solver::MinMaxMethod::PolicyIteration); - env.solver().setForceExact(true); - return env; - } - static bool const isExactModelChecking = true; - static ValueType precision() { - return storm::utility::convertNumber(0.12); - } // there actually aren't any precision guarantees, but we still want to detect if results are weird. - static void adaptOptions(storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions&) { /* intentionally left empty */ } - static PreprocessingType const preprocessingType = PreprocessingType::None; -}; - -class PreprocessedDefaultRationalPIEnvironment { - public: - typedef storm::RationalNumber ValueType; - static storm::Environment createEnvironment() { - storm::Environment env; - env.solver().minMax().setMethod(storm::solver::MinMaxMethod::PolicyIteration); - env.solver().setForceExact(true); - return env; - } - static bool const isExactModelChecking = true; - static ValueType precision() { - return storm::utility::convertNumber(0.12); - } // there actually aren't any precision guarantees, but we still want to detect if results are weird. - static void adaptOptions(storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions&) { /* intentionally left empty */ } - static PreprocessingType const preprocessingType = PreprocessingType::All; -}; - -template -class BeliefExplorationPomdpModelCheckerTest : public ::testing::Test { - public: - typedef typename TestType::ValueType ValueType; - BeliefExplorationPomdpModelCheckerTest() : _environment(TestType::createEnvironment()) {} - - void SetUp() override { -#ifndef STORM_HAVE_Z3 - GTEST_SKIP() << "Z3 not available."; -#endif - } - - storm::Environment const& env() const { - return _environment; - } - storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions options() const { - storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions opt(true, true); // Always compute both bounds (lower and upper) - opt.gapThresholdInit = 0; - TestType::adaptOptions(opt); - return opt; - } - storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions optionsWithStateElimination() const { - storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions opt(true, true); // Always compute both bounds (lower and upper) - opt.gapThresholdInit = 0; - TestType::adaptOptions(opt); - opt.useStateEliminationCutoff = true; - return opt; - } - storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions optionsWithClipping() const { - storm::pomdp::modelchecker::BeliefExplorationPomdpModelCheckerOptions opt(true, true); // Always compute both bounds (lower and upper) - opt.gapThresholdInit = 0; - TestType::adaptOptions(opt); - opt.useClipping = true; - return opt; - } - ValueType parseNumber(std::string const& str) { - return storm::utility::convertNumber(str); - } - struct Input { - std::shared_ptr> model; - std::shared_ptr formula; - }; - Input buildPrism(std::string const& programFile, std::string const& formulaAsString, std::string const& constantsAsString = "") const { - // Parse and build input - storm::prism::Program program = storm::api::parseProgram(programFile); - program = program.preprocess(constantsAsString); - Input input; - input.formula = storm::api::parsePropertiesForPrismProgram(formulaAsString, program).front().getRawFormula(); - input.model = storm::api::buildSparseModel(program, {input.formula})->template as>(); - - // Preprocess - storm::transformer::MakePOMDPCanonic makeCanonic(*input.model); - input.model = makeCanonic.transform(); - EXPECT_TRUE(input.model->isCanonic()); - if (TestType::preprocessingType == PreprocessingType::SelfloopReduction || TestType::preprocessingType == PreprocessingType::All) { - storm::transformer::GlobalPOMDPSelfLoopEliminator selfLoopEliminator(*input.model); - if (selfLoopEliminator.preservesFormula(*input.formula)) { - input.model = selfLoopEliminator.transform(); - } else { - EXPECT_TRUE(input.formula->isOperatorFormula()); - EXPECT_TRUE(input.formula->asOperatorFormula().hasOptimalityType()); - bool maximizing = storm::solver::maximize(input.formula->asOperatorFormula().getOptimalityType()); - // Valid reasons for unpreserved formulas: - EXPECT_TRUE(maximizing || input.formula->isProbabilityOperatorFormula()); - EXPECT_TRUE(!maximizing || input.formula->isRewardOperatorFormula()); - } - } - if (TestType::preprocessingType == PreprocessingType::QualitativeReduction || TestType::preprocessingType == PreprocessingType::All) { - EXPECT_TRUE(input.formula->isOperatorFormula()); - EXPECT_TRUE(input.formula->asOperatorFormula().hasOptimalityType()); - if (input.formula->isProbabilityOperatorFormula() && storm::solver::maximize(input.formula->asOperatorFormula().getOptimalityType())) { - storm::analysis::QualitativeAnalysisOnGraphs qualitativeAnalysis(*input.model); - storm::storage::BitVector prob0States = qualitativeAnalysis.analyseProb0(input.formula->asProbabilityOperatorFormula()); - storm::storage::BitVector prob1States = qualitativeAnalysis.analyseProb1(input.formula->asProbabilityOperatorFormula()); - storm::pomdp::transformer::KnownProbabilityTransformer kpt; - input.model = kpt.transform(*input.model, prob0States, prob1States); - } - } - EXPECT_TRUE(input.model->isCanonic()); - return input; - } - ValueType precision() const { - return TestType::precision(); - } - ValueType modelcheckingPrecision() const { - if (TestType::isExactModelChecking) - return storm::utility::zero(); - else - return storm::utility::convertNumber(1e-6); - } - bool isExact() const { - return TestType::isExactModelChecking; - } - - private: - storm::Environment _environment; -}; - -typedef ::testing::Types - TestingTypes; - -TYPED_TEST_SUITE(BeliefExplorationPomdpModelCheckerTest, TestingTypes, ); - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_Pmax) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmax=? [F \"goal\" ]", "slippery=0"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->options()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("7/10"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_Pmax_SE) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmax=? [F \"goal\" ]", "slippery=0"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, - this->optionsWithStateElimination()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("7/10"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_Pmin) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmin=? [F \"goal\" ]", "slippery=0"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->options()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("3/10"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_Pmin_SE) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmin=? [F \"goal\" ]", "slippery=0"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, - this->optionsWithStateElimination()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("3/10"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_slippery_Pmax) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmax=? [F \"goal\" ]", "slippery=0.4"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->options()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("7/10"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_slippery_Pmax_SE) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmax=? [F \"goal\" ]", "slippery=0.4"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, - this->optionsWithStateElimination()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("7/10"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_slippery_Pmin) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmin=? [F \"goal\" ]", "slippery=0.4"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->options()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("3/10"); - if (this->isExact()) { - // This model's value can only be approximated arbitrarily close but never reached - // Exact arithmetics will thus not reach the value with absoulute precision either. - ValueType approxPrecision = storm::utility::convertNumber(1e-5); - EXPECT_LE(result.lowerBound, expected + approxPrecision); - EXPECT_GE(result.upperBound, expected - approxPrecision); - } else { - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - } - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_slippery_Pmin_SE) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmin=? [F \"goal\" ]", "slippery=0.4"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, - this->optionsWithStateElimination()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("3/10"); - if (this->isExact()) { - // This model's value can only be approximated arbitrarily close but never reached - // Exact arithmetics will thus not reach the value with absoulute precision either. - ValueType approxPrecision = storm::utility::convertNumber(1e-5); - EXPECT_LE(result.lowerBound, expected + approxPrecision); - EXPECT_GE(result.upperBound, expected - approxPrecision); - } else { - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - } - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_Rmax) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmax=? [F s>4 ]", "slippery=0"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->options()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("29/50"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_Rmax_SE) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmax=? [F s>4 ]", "slippery=0"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, - this->optionsWithStateElimination()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("29/50"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_Rmin) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmin=? [F s>4 ]", "slippery=0"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->options()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("19/50"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_Rmin_SE) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmin=? [F s>4 ]", "slippery=0"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, - this->optionsWithStateElimination()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("19/50"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_slippery_Rmax) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmax=? [F s>4 ]", "slippery=0.4"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->options()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("29/30"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_slippery_Rmax_SE) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmax=? [F s>4 ]", "slippery=0.4"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, - this->optionsWithStateElimination()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("29/30"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_slippery_Rmin) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmin=? [F s>4 ]", "slippery=0.4"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->options()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("19/30"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_slippery_Rmin_SE) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmin=? [F s>4 ]", "slippery=0.4"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, - this->optionsWithStateElimination()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("19/30"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, maze2_Rmin) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmin=? [F \"goal\"]", "sl=0"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->options()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("74/91"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - // Use relative difference of bounds for this one - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, maze2_Rmin_SE) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmin=? [F \"goal\"]", "sl=0"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, - this->optionsWithStateElimination()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("74/91"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - // Use relative difference of bounds for this one - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, maze2_Rmax) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmax=? [F \"goal\"]", "sl=0"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->options()); - auto result = checker.check(this->env(), *data.formula); - - EXPECT_TRUE(storm::utility::isInfinity(result.lowerBound)); - EXPECT_TRUE(storm::utility::isInfinity(result.upperBound)); -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, maze2_Rmax_SE) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmax=? [F \"goal\"]", "sl=0"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, - this->optionsWithStateElimination()); - auto result = checker.check(this->env(), *data.formula); - - EXPECT_TRUE(storm::utility::isInfinity(result.lowerBound)); - EXPECT_TRUE(storm::utility::isInfinity(result.upperBound)); -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, maze2_slippery_Rmin) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmin=? [F \"goal\"]", "sl=0.075"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->options()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("80/91"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - // Use relative difference of bounds for this one - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, maze2_slippery_Rmin_SE) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmin=? [F \"goal\"]", "sl=0.075"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, - this->optionsWithStateElimination()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("80/91"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - // Use relative difference of bounds for this one - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, maze2_slippery_Rmax) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmax=? [F \"goal\"]", "sl=0.075"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->options()); - auto result = checker.check(this->env(), *data.formula); - - EXPECT_TRUE(storm::utility::isInfinity(result.lowerBound)); - EXPECT_TRUE(storm::utility::isInfinity(result.upperBound)); -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, maze2_slippery_Rmax_SE) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmax=? [F \"goal\"]", "sl=0.075"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, - this->optionsWithStateElimination()); - auto result = checker.check(this->env(), *data.formula); - - EXPECT_TRUE(storm::utility::isInfinity(result.lowerBound)); - EXPECT_TRUE(storm::utility::isInfinity(result.upperBound)); -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, refuel_Pmax) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/refuel.prism", "Pmax=?[\"notbad\" U \"goal\"]", "N=4"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->options()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("38/155"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - // Use relative difference of bounds for this one - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, refuel_Pmax_SE) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/refuel.prism", "Pmax=?[\"notbad\" U \"goal\"]", "N=4"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, - this->optionsWithStateElimination()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("38/155"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - // Use relative difference of bounds for this one - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, refuel_Pmin) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/refuel.prism", "Pmin=?[\"notbad\" U \"goal\"]", "N=4"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->options()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("0"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - // Use relative difference of bounds for this one - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, refuel_Pmin_SE) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/refuel.prism", "Pmin=?[\"notbad\" U \"goal\"]", "N=4"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, - this->optionsWithStateElimination()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("0"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - // Use relative difference of bounds for this one - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_Pmax_Clip) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmax=? [F \"goal\" ]", "slippery=0"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->optionsWithClipping()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("7/10"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_Pmin_Clip) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmin=? [F \"goal\" ]", "slippery=0"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->optionsWithClipping()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("3/10"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_slippery_Pmax_Clip) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmax=? [F \"goal\" ]", "slippery=0.4"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->optionsWithClipping()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("7/10"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_slippery_Pmin_Clip) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Pmin=? [F \"goal\" ]", "slippery=0.4"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->optionsWithClipping()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("3/10"); - if (this->isExact()) { - // This model's value can only be approximated arbitrarily close but never reached - // Exact arithmetics will thus not reach the value with absoulute precision either. - ValueType approxPrecision = storm::utility::convertNumber(1e-4); - EXPECT_LE(result.lowerBound, expected + approxPrecision); - EXPECT_GE(result.upperBound, expected - approxPrecision); - } else { - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision() * 10); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision() * 10); - } - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_Rmax_Clip) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmax=? [F s>4 ]", "slippery=0"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->optionsWithClipping()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("29/50"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_Rmin_Clip) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmin=? [F s>4 ]", "slippery=0"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->optionsWithClipping()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("19/50"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_slippery_Rmax_Clip) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmax=? [F s>4 ]", "slippery=0.4"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->optionsWithClipping()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("29/30"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, simple_slippery_Rmin_Clip) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/simple.prism", "Rmin=? [F s>4 ]", "slippery=0.4"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->optionsWithClipping()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("19/30"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, maze2_Rmin_Clip) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmin=? [F \"goal\"]", "sl=0"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->optionsWithClipping()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("74/91"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - // Use relative difference of bounds for this one - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, maze2_Rmax_Clip) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmax=? [F \"goal\"]", "sl=0"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->optionsWithClipping()); - auto result = checker.check(this->env(), *data.formula); - - EXPECT_TRUE(storm::utility::isInfinity(result.lowerBound)); - EXPECT_TRUE(storm::utility::isInfinity(result.upperBound)); -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, maze2_slippery_Rmin_Clip) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmin=? [F \"goal\"]", "sl=0.075"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->optionsWithClipping()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("80/91"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - // Use relative difference of bounds for this one - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, maze2_slippery_Rmax_Clip) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/maze2.prism", "Rmax=? [F \"goal\"]", "sl=0.075"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->optionsWithClipping()); - auto result = checker.check(this->env(), *data.formula); - - EXPECT_TRUE(storm::utility::isInfinity(result.lowerBound)); - EXPECT_TRUE(storm::utility::isInfinity(result.upperBound)); -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, refuel_Pmax_Clip) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/refuel.prism", "Pmax=?[\"notbad\" U \"goal\"]", "N=4"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->optionsWithClipping()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("38/155"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - // Use relative difference of bounds for this one - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -TYPED_TEST(BeliefExplorationPomdpModelCheckerTest, refuel_Pmin_Clip) { - typedef typename TestFixture::ValueType ValueType; - - auto data = this->buildPrism(STORM_TEST_RESOURCES_DIR "/pomdp/refuel.prism", "Pmin=?[\"notbad\" U \"goal\"]", "N=4"); - storm::pomdp::modelchecker::BeliefExplorationPomdpModelChecker> checker(data.model, this->optionsWithClipping()); - auto result = checker.check(this->env(), *data.formula); - - ValueType expected = this->parseNumber("0"); - EXPECT_LE(result.lowerBound, expected + this->modelcheckingPrecision()); - EXPECT_GE(result.upperBound, expected - this->modelcheckingPrecision()); - // Use relative difference of bounds for this one - EXPECT_LE(result.diff(), this->precision()) - << "Result [" << result.lowerBound << ", " << result.upperBound - << "] is not precise enough. If (only) this fails, the result bounds are still correct, but they might be unexpectedly imprecise.\n"; -} - -} // namespace