Skip to content

Scope allocation ownership #35

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
stanbot8 wants to merge 1 commit into from
Draft

Scope allocation ownership #35

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 9 additions & 30 deletions src/core/resource_manager.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,13 @@
#include "core/environment/environment.h"
#include "core/simulation.h"
#include "core/util/partition.h"
#include "core/util/plot_memory_layout.h"
#include "core/util/timing.h"

namespace bdm {

ResourceManager::ResourceManager() {
// Must be called prior any other function call to libnuma
if (auto ret = numa_available() == -1) {
Log::Fatal("ResourceManager",
"Call to numa_available failed with return code: ", ret);
}
agents_.resize(numa_num_configured_nodes());
agents_lb_.resize(numa_num_configured_nodes());
agents_.resize(thread_info_->GetNumaNodes());
agents_lb_.resize(thread_info_->GetNumaNodes());

auto* param = Simulation::GetActive()->GetParam();
if (param->export_visualization || param->insitu_visualization) {
Expand Down Expand Up @@ -65,8 +59,6 @@ void ResourceManager::ForEachAgentParallel(
auto nid = thread_info_->GetNumaNode(tid);
auto threads_in_numa = thread_info_->GetThreadsInNumaNode(nid);
auto& numa_agents = agents_[nid];
assert(thread_info_->GetNumaNode(tid) == numa_node_of_cpu(sched_getcpu()));

// use static scheduling for now
auto correction = numa_agents.size() % threads_in_numa == 0 ? 0 : 1;
auto chunk = numa_agents.size() / threads_in_numa + correction;
Expand Down Expand Up @@ -156,8 +148,6 @@ void ResourceManager::ForEachAgentParallel(
auto p_numa_nodes = thread_info_->GetNumaNodes();
auto p_max_threads = omp_get_max_threads();
auto p_chunk = chunk;
assert(thread_info_->GetNumaNode(tid) == numa_node_of_cpu(sched_getcpu()));

// dynamic scheduling
uint64_t start = 0;
uint64_t end = 0;
Expand Down Expand Up @@ -190,7 +180,7 @@ void ResourceManager::ForEachAgentParallel(
old_count = (*(counters[current_tid]))++;
}
} // work stealing loop numa_nodes_
} // work stealing loop threads
} // work stealing loop threads
}

for (auto* counter : counters) {
Expand Down Expand Up @@ -242,10 +232,6 @@ void ResourceManager::LoadBalance() {
// the environment before accessing it again.
MarkEnvironmentOutOfSync();
auto* param = Simulation::GetActive()->GetParam();
if (param->plot_memory_layout) {
PlotNeighborMemoryHistogram(true);
}

// balance agents per numa node according to the number of
// threads associated with each numa domain
auto numa_nodes = thread_info_->GetNumaNodes();
Expand All @@ -268,10 +254,12 @@ void ResourceManager::LoadBalance() {
// using first touch policy - page will be allocated to the numa domain of
// the thread that accesses it first.
// alternative, use numa_alloc_onnode.
int ret = numa_run_on_node(0);
if (ret != 0) {
Log::Fatal("ResourceManager",
"Run on numa node failed. Return code: ", ret);
if (thread_info_->IsNumaAvailable()) {
int ret = numa_run_on_node(0);
if (ret != 0) {
Log::Fatal("ResourceManager",
"Run on numa node failed. Return code: ", ret);
}
}
auto* env = Simulation::GetActive()->GetEnvironment();
auto lbi = env->GetLoadBalanceInfo();
Expand All @@ -295,8 +283,6 @@ void ResourceManager::LoadBalance() {
#pragma omp barrier

auto threads_in_numa = thread_info_->GetThreadsInNumaNode(nid);
assert(thread_info_->GetNumaNode(tid) == numa_node_of_cpu(sched_getcpu()));

// use static scheduling
auto correction = agent_per_numa[nid] % threads_in_numa == 0 ? 0 : 1;
auto chunk = agent_per_numa[nid] / threads_in_numa + correction;
Expand All @@ -321,13 +307,6 @@ void ResourceManager::LoadBalance() {

for (int n = 0; n < numa_nodes; n++) {
agents_[n].swap(agents_lb_[n]);
if (param->plot_memory_layout) {
PlotMemoryLayout(agents_[n], n);
PlotMemoryHistogram(agents_[n], n);
}
}
if (param->plot_memory_layout) {
PlotNeighborMemoryHistogram();
}

if (Simulation::GetActive()->GetParam()->debug_numa) {
Expand Down
70 changes: 14 additions & 56 deletions src/core/resource_manager.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@
#include "core/simulation.h"
#include "core/type_index.h"
#include "core/util/numa.h"
#include "core/util/root.h"
#include "core/util/thread_info.h"
#include "core/util/type.h"

Expand All @@ -52,53 +51,14 @@ namespace bdm {
/// simulation.
class ResourceManager {
public:
explicit ResourceManager(TRootIOCtor* r) {}

ResourceManager();

virtual ~ResourceManager();

ResourceManager& operator=(ResourceManager&& other) noexcept {
if (agents_.size() != other.agents_.size()) {
Log::Fatal(
"Restored ResourceManager has different number of NUMA nodes.");
}
for (auto& el : continuum_models_) {
delete el.second;
}
for (auto& numa_agents : agents_) {
for (auto* agent : numa_agents) {
delete agent;
}
}
agents_ = std::move(other.agents_);
agents_lb_.resize(agents_.size());
continuum_models_ = std::move(other.continuum_models_);

RebuildAgentUidMap();
// restore type_index_
if (type_index_) {
for (auto& numa_agents : agents_) {
for (auto* agent : numa_agents) {
type_index_->Add(agent);
}
}
}
return *this;
}

void RebuildAgentUidMap() {
// rebuild uid_ah_map_
uid_ah_map_.clear();
auto* agent_uid_generator = Simulation::GetActive()->GetAgentUidGenerator();
uid_ah_map_.resize(agent_uid_generator->GetHighestIndex() + 1);
for (AgentHandle::NumaNode_t n = 0; n < agents_.size(); ++n) {
for (AgentHandle::ElementIdx_t i = 0; i < agents_[n].size(); ++i) {
auto* agent = agents_[n][i];
this->uid_ah_map_.Insert(agent->GetUid(), AgentHandle(n, i));
}
}
}
ResourceManager(const ResourceManager&) = delete;
ResourceManager& operator=(const ResourceManager&) = delete;
ResourceManager(ResourceManager&&) = delete;
ResourceManager& operator=(ResourceManager&&) = delete;

Agent* GetAgent(const AgentUid& uid) {
if (!uid_ah_map_.Contains(uid)) {
Expand Down Expand Up @@ -431,20 +391,21 @@ class ResourceManager {
/// not overlap!
virtual void AddAgents(typename AgentHandle::NumaNode_t numa_node,
uint64_t offset,
const std::vector<Agent*>& new_agents) {
std::vector<std::unique_ptr<Agent>>& new_agents) {
uint64_t i = 0;
for (auto* agent : new_agents) {
for (auto& new_agent : new_agents) {
auto* agent = new_agent.get();
auto uid = agent->GetUid();
uid_ah_map_.Insert(
uid, AgentHandle(numa_node,
static_cast<AgentHandle::ElementIdx_t>(offset + i)));
agents_[numa_node][offset + i] = agent;
agents_[numa_node][offset + i] = new_agent.release();
i++;
}
if (type_index_) {
#pragma omp critical
for (auto* agent : new_agents) {
type_index_->Add(agent);
for (uint64_t j = 0; j < new_agents.size(); ++j) {
type_index_->Add(agents_[numa_node][offset + j]);
}
}
#pragma omp single
Expand Down Expand Up @@ -498,13 +459,13 @@ class ResourceManager {
void MarkEnvironmentOutOfSync() const;

/// Maps an AgentUid to its storage location in `agents_` \n
AgentUidMap<AgentHandle> uid_ah_map_ = AgentUidMap<AgentHandle>(100u); //!
AgentUidMap<AgentHandle> uid_ah_map_ = AgentUidMap<AgentHandle>(100u);
/// Pointer container for all agents
std::vector<std::vector<Agent*>> agents_;
/// Container used during load balancing
std::vector<std::vector<Agent*>> agents_lb_; //!
std::vector<std::vector<Agent*>> agents_lb_;

ThreadInfo* thread_info_ = ThreadInfo::GetInstance(); //!
ThreadInfo* thread_info_ = ThreadInfo::GetInstance();

TypeIndex* type_index_ = nullptr;

Expand All @@ -514,16 +475,13 @@ class ResourceManager {
};

/// auxiliary data required for parallel agent removal
ParallelRemovalAuxData parallel_remove_; //!
ParallelRemovalAuxData parallel_remove_;

friend class SimulationBackup;
friend std::ostream& operator<<(std::ostream& os, const ResourceManager& rm);

private:
/// Maps a continuum ID to the pointer to the continuum models
std::unordered_map<uint64_t, Continuum*> continuum_models_;

BDM_CLASS_DEF_NV(ResourceManager, 2);
};

inline std::ostream& operator<<(std::ostream& os, const ResourceManager& rm) {
Expand Down
5 changes: 0 additions & 5 deletions test/unit/core/resource_manager_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
// I/O related code must be in header file
#include "unit/core/resource_manager_test.h"
#include "core/model_initializer.h"
#include "unit/test_util/io_test.h"
#include "unit/test_util/test_agent.h"

namespace bdm {
Expand Down Expand Up @@ -144,10 +143,6 @@ TEST(ResourceManagerTest, ForEachAgentParallel) {
RunForEachAgentParallelTest();
}

#ifdef USE_DICT
TEST(ResourceManagerTest, IO) { RunIOTest(); }
#endif // USE_DICT

TEST(ResourceManagerTest, PushBackAndGetAgentTest) {
RunPushBackAndGetAgentTest();
}
Expand Down
83 changes: 13 additions & 70 deletions test/unit/core/resource_manager_test.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,14 @@
#include "core/diffusion/euler_grid.h"
#include "core/environment/environment.h"
#include "core/resource_manager.h"
#include "core/util/io.h"
#include "core/util/type.h"
#include "unit/test_util/test_agent.h"
#include "unit/test_util/test_util.h"

#define ROOTFILE "bdmFile.root"

namespace bdm {

class A : public TestAgent {
BDM_AGENT_HEADER(A, TestAgent, 1);
BDM_AGENT_HEADER(A, TestAgent);

public:
A() = default;
Expand All @@ -45,7 +42,7 @@ class A : public TestAgent {
};

class B : public TestAgent {
BDM_AGENT_HEADER(B, TestAgent, 1);
BDM_AGENT_HEADER(B, TestAgent);

public:
B() = default;
Expand Down Expand Up @@ -322,8 +319,13 @@ struct CheckForEachAgentFunctor : Functor<void, Agent*> {
if (numa_checks && handle.GetNumaNode() != GetNumaNodeForMemory(agent)) {
numa_memory_errors++;
}
if (numa_checks &&
handle.GetNumaNode() != numa_node_of_cpu(sched_getcpu())) {
unsigned int current_numa_node = 0;
#ifdef USE_NUMA
if (numa_checks) {
EXPECT_EQ(0, getcpu(nullptr, &current_numa_node));
}
#endif
if (numa_checks && handle.GetNumaNode() != current_numa_node) {
numa_thread_errors++;
}

Expand Down Expand Up @@ -385,7 +387,8 @@ inline void RunSortAndForEachAgentParallel(uint64_t num_agent_per_type) {
simulation.GetEnvironment()->Update();
rm->LoadBalance();

CheckForEachAgent(rm, num_agent_per_type, true);
CheckForEachAgent(rm, num_agent_per_type,
ThreadInfo::GetInstance()->IsNumaAvailable());

// check if agent uids still point to the correct object
for (auto& entry : a_x_values) {
Expand Down Expand Up @@ -556,7 +559,8 @@ inline void RunSortAndForEachAgentParallelDynamic(uint64_t num_agent_per_type,
simulation.GetEnvironment()->Update();
rm->LoadBalance();

CheckForEachAgentDynamic(rm, num_agent_per_type, batch_size, true);
CheckForEachAgentDynamic(rm, num_agent_per_type, batch_size,
ThreadInfo::GetInstance()->IsNumaAvailable());

// check if agent uids still point to the correct object
for (auto& entry : a_x_values) {
Expand Down Expand Up @@ -588,67 +592,6 @@ inline void RunSortAndForEachAgentParallelDynamic() {
}
}

inline void RunIOTest() {
const real_t kEpsilon = abs_error<real_t>::value;
Simulation simulation("ResourceManagerTest-RunIOTest");
auto* rm = simulation.GetResourceManager();

auto ref_uid = AgentUid(simulation.GetAgentUidGenerator()->GetHighestIndex());
remove(ROOTFILE);

// setup
rm->AddAgent(new A(12));
rm->AddAgent(new A(34));
rm->AddAgent(new A(42));

rm->AddAgent(new B(3.14));
rm->AddAgent(new B(6.28));

DiffusionGrid* dgrid_1 = new EulerGrid(0, "Kalium", 0.4, 0, 2);
DiffusionGrid* dgrid_2 = new EulerGrid(1, "Natrium", 0.2, 0.1, 1);
rm->AddContinuum(dgrid_1);
rm->AddContinuum(dgrid_2);

// backup
WritePersistentObject(ROOTFILE, "rm", *rm, "new");

rm->ClearAgents();

// restore
ResourceManager* restored_rm = nullptr;
GetPersistentObject(ROOTFILE, "rm", restored_rm);
restored_rm->RebuildAgentUidMap();

// validate
EXPECT_EQ(5u, restored_rm->GetNumAgents());

EXPECT_EQ(12, dynamic_cast<A*>(restored_rm->GetAgent(ref_uid))->GetData());
EXPECT_EQ(34,
dynamic_cast<A*>(restored_rm->GetAgent(ref_uid + 1))->GetData());
EXPECT_EQ(42,
dynamic_cast<A*>(restored_rm->GetAgent(ref_uid + 2))->GetData());

EXPECT_NEAR(3.14,
dynamic_cast<B*>(restored_rm->GetAgent(ref_uid + 3))->GetData(),
kEpsilon);
EXPECT_NEAR(6.28,
dynamic_cast<B*>(restored_rm->GetAgent(ref_uid + 4))->GetData(),
kEpsilon);

EXPECT_EQ(0, restored_rm->GetDiffusionGrid(0)->GetContinuumId());
EXPECT_EQ(1, restored_rm->GetDiffusionGrid(1)->GetContinuumId());
EXPECT_EQ("Kalium", restored_rm->GetDiffusionGrid(0)->GetContinuumName());
EXPECT_EQ("Natrium", restored_rm->GetDiffusionGrid(1)->GetContinuumName());
EXPECT_EQ(real_t(0.6),
restored_rm->GetDiffusionGrid(0)->GetDiffusionCoefficients()[0]);
EXPECT_EQ(real_t(0.8),
restored_rm->GetDiffusionGrid(1)->GetDiffusionCoefficients()[0]);

delete restored_rm;

remove(ROOTFILE);
}

} // namespace bdm

#endif // UNIT_CORE_RESOURCE_MANAGER_TEST_H_
Loading