Commit bce1fa75 authored by Timo Koch's avatar Timo Koch
Browse files

Use Kokkos in assembler

parent 23f3a773
Pipeline #10398 waiting for manual action with stages
......@@ -24,16 +24,7 @@
#ifndef DUMUX_FV_ASSEMBLER_HH
#define DUMUX_FV_ASSEMBLER_HH
#if HAVE_TBB
#include <oneapi/tbb/info.h>
#include <oneapi/tbb/parallel_for.h>
#include <oneapi/tbb/task_arena.h>
#define DUMUX_MULTITHREADING_ASSEMBLY_DEFAULT true
#endif
#ifndef DUMUX_MULTITHREADING_ASSEMBLY_DEFAULT
#define DUMUX_MULTITHREADING_ASSEMBLY_DEFAULT false
#endif
#include <Kokkos_Core.hpp>
#include <type_traits>
......@@ -145,7 +136,7 @@ public:
{
static_assert(isImplicit, "Explicit assembler for stationary problem doesn't make sense!");
enableMultithreading_ = SupportsColoring<GridGeometry::discMethod>::value
&& getParam<bool>("Assembly.Multithreading", DUMUX_MULTITHREADING_ASSEMBLY_DEFAULT);
&& getParam<bool>("Assembly.Multithreading", false);
completeConfiguration_();
}
......@@ -168,7 +159,7 @@ public:
, isStationaryProblem_(!timeLoop)
{
enableMultithreading_ = SupportsColoring<GridGeometry::discMethod>::value
&& getParam<bool>("Assembly.Multithreading", DUMUX_MULTITHREADING_ASSEMBLY_DEFAULT);
&& getParam<bool>("Assembly.Multithreading", false);
completeConfiguration_();
}
......@@ -410,13 +401,6 @@ private:
// some final configuration steps
void completeConfiguration_()
{
#if !HAVE_TBB
if (enableMultithreading_)
DUNE_THROW(Dune::Exception,
"Multithread assembly has been explicitly requested but TBB has not been found! Please install TBB first.");
#else
maxNumThreads_ = getParam<int>("Assembly.MaxNumThreads", oneapi::tbb::info::default_concurrency());
#endif
}
// reset the residual vector to 0.0
......@@ -469,7 +453,6 @@ private:
// try assembling using the local assembly function
try
{
#if HAVE_TBB
// make this element loop run in parallel
// for this we have to color the elements so that we don't get
// race conditions when writing into the global matrix
......@@ -478,27 +461,17 @@ private:
if (enableMultithreading_)
{
// constrict thread pool to maxNumThreads_
oneapi::tbb::task_arena arena(maxNumThreads_);
arena.execute([&]
for (const auto& elements : elementSets_)
{
for (const auto& elements : elementSets_)
{
oneapi::tbb::parallel_for(oneapi::tbb::blocked_range<std::size_t>(0, elements.size()),
[&](const oneapi::tbb::blocked_range<size_t>& r)
{
for (std::size_t i=r.begin(); i<r.end(); ++i)
{
const auto element = gridView().grid().entity(elements[i]);
assembleElement(element);
}
});
}
});
Kokkos::parallel_for(elements.size(), [&](const int i){
const auto element = gridView().grid().entity(elements[i]);
assembleElement(element);
});
}
}
else
#endif
for (const auto& element : elements(gridView()))
assembleElement(element);
for (const auto& element : elements(gridView()))
assembleElement(element);
// if we get here, everything worked well on this process
succeeded = true;
......@@ -569,7 +542,6 @@ private:
//! element sets for parallel assembly
bool enableMultithreading_ = false;
int maxNumThreads_ = 1;
std::deque<std::vector<ElementSeed>> elementSets_;
};
......
......@@ -44,6 +44,8 @@
#include "properties.hh"
#include <Kokkos_Core.hpp>
int main(int argc, char** argv)
{
......@@ -57,6 +59,7 @@ int main(int argc, char** argv)
// initialize MPI, finalize is done automatically on exit
const auto& mpiHelper = Dune::MPIHelper::instance(argc, argv);
Kokkos::initialize(argc, argv);
// print dumux start message
if (mpiHelper.rank() == 0)
......@@ -119,8 +122,8 @@ int main(int argc, char** argv)
auto assembler = std::make_shared<Assembler>(problem, gridGeometry, gridVariables, timeLoop, xOld);
// the linear solver
using LinearSolver = AMGBiCGSTABBackend<LinearSolverTraits<GridGeometry>>;
auto linearSolver = std::make_shared<LinearSolver>(leafGridView, gridGeometry->dofMapper());
using LinearSolver = ILUnBiCGSTABBackend;
auto linearSolver = std::make_shared<LinearSolver>();
// the non-linear solver
using NewtonSolver = NewtonSolver<Assembler, LinearSolver>;
......@@ -165,6 +168,8 @@ int main(int argc, char** argv)
DumuxMessage::print(/*firstCall=*/false);
}
Kokkos::finalize();
return 0;
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment