From fef0c4fe966c5fb70b2bb19cc86828fb8befe8c8 Mon Sep 17 00:00:00 2001 From: hamidrezanorouzi Date: Mon, 2 Jan 2023 14:31:40 +0330 Subject: [PATCH] modification for coupling, data transfer --- DEMSystems/CMakeLists.txt | 1 + DEMSystems/DEMSystem/DEMSystem.cpp | 21 +++- DEMSystems/DEMSystem/DEMSystem.hpp | 46 ++++++-- .../domainDistribute/domainDistribute.cpp | 69 +++++++++--- .../domainDistribute/domainDistribute.hpp | 5 +- .../sphereDEMSystem/sphereDEMSystem.cpp | 104 ++++++++++++------ .../sphereDEMSystem/sphereDEMSystem.hpp | 28 ++++- .../sphereDEMSystem/sphereFluidParticles.cpp | 100 +++++++++++++++++ .../sphereDEMSystem/sphereFluidParticles.hpp | 91 +++++++++++++++ .../sphereFluidParticlesKernels.hpp | 79 +++++++++++++ .../dynamicPointStructure.hpp | 10 ++ src/Particles/particles/particles.hpp | 2 +- .../repository/Time/timeControl.hpp | 4 + .../pointStructure/pointStructure.hpp | 6 + 14 files changed, 499 insertions(+), 67 deletions(-) create mode 100644 DEMSystems/sphereDEMSystem/sphereFluidParticles.cpp create mode 100644 DEMSystems/sphereDEMSystem/sphereFluidParticles.hpp create mode 100644 DEMSystems/sphereDEMSystem/sphereFluidParticlesKernels.hpp diff --git a/DEMSystems/CMakeLists.txt b/DEMSystems/CMakeLists.txt index 243df700..1a1a6f9c 100644 --- a/DEMSystems/CMakeLists.txt +++ b/DEMSystems/CMakeLists.txt @@ -2,6 +2,7 @@ set(SourceFiles DEMSystem/DEMSystem.cpp sphereDEMSystem/sphereDEMSystem.cpp +sphereDEMSystem/sphereFluidParticles.cpp domainDistribute/domainDistribute.cpp ) diff --git a/DEMSystems/DEMSystem/DEMSystem.cpp b/DEMSystems/DEMSystem/DEMSystem.cpp index 7616ff35..85143431 100644 --- a/DEMSystems/DEMSystem/DEMSystem.cpp +++ b/DEMSystems/DEMSystem/DEMSystem.cpp @@ -18,6 +18,9 @@ Licence: -----------------------------------------------------------------------------*/ +// from phasicFlow +#include "KokkosTypes.hpp" + #include "DEMSystem.hpp" @@ -27,10 +30,16 @@ pFlow::DEMSystem::DEMSystem( int argc, char* argv[]) : - ControlDict_(), - domains_(domains) + ControlDict_() { + REPORT(0)<<"Initializing host/device execution spaces . . . \n"; + REPORT(1)<<"Host execution space is "<< greenText(DefaultHostExecutionSpace::name())<( ControlDict_.startTime(), @@ -43,7 +52,13 @@ pFlow::DEMSystem::DEMSystem( } pFlow::DEMSystem::~DEMSystem() -{} +{ + + Control_.reset(); + + output<< "\nFinalizing host/device execution space ...."< diff --git a/DEMSystems/DEMSystem/DEMSystem.hpp b/DEMSystems/DEMSystem/DEMSystem.hpp index 02c62bff..ee3c6ec3 100644 --- a/DEMSystems/DEMSystem/DEMSystem.hpp +++ b/DEMSystems/DEMSystem/DEMSystem.hpp @@ -43,16 +43,8 @@ protected: uniquePtr Control_ = nullptr; - std::vector domains_; - uniquePtr timers_; - // methods - auto& Control() - { - return Control_(); - } - public: @@ -91,6 +83,17 @@ public: return Control_->g(); } + // methods + auto& Control() + { + return Control_(); + } + + const auto& Control()const + { + return Control_(); + } + auto inline constexpr usingDoulle()const { return pFlow::usingDouble__; @@ -101,6 +104,9 @@ public: return Control_->timers(); } + virtual + bool updateParticleDistribution(real extentFraction, const std::vector domains) = 0; + virtual int32 numParInDomain(int32 di)const = 0; @@ -110,15 +116,33 @@ public: virtual span parIndexInDomain(int32 di)const = 0; - virtual - bool changeDomainsSizeUpdateParticles(const std::vector& domains) = 0; + virtual + span parDiameter() = 0; virtual - bool updateParticles() = 0; + span parVelocity() = 0; + + virtual + span parPosition() = 0; + + virtual + span parFluidForce() = 0; + + virtual + span parFluidTorque() = 0; + + virtual + bool sendFluidForceToDEM() = 0; + + virtual + bool sendFluidTorqueToDEM() = 0; virtual real maxBounndingSphereSize()const = 0; + virtual + bool beforeIteration() = 0; + virtual bool iterate(int32 n, real timeToWrite, word timeName) = 0; diff --git a/DEMSystems/domainDistribute/domainDistribute.cpp b/DEMSystems/domainDistribute/domainDistribute.cpp index 5c53ca4d..b888f2d6 100644 --- a/DEMSystems/domainDistribute/domainDistribute.cpp +++ b/DEMSystems/domainDistribute/domainDistribute.cpp @@ -49,6 +49,60 @@ maxBoundingBoxSize_(maxBoundingBox) bool pFlow::domainDistribute::locateParticles( ViewType1D points, includeMask mask) { + + range activeRange = mask.activeRange(); + + for(int32 di=0; di& domains) +{ + domainExtension_ = extentFraction; + maxBoundingBoxSize_ = maxBoundingBoxSize; + + if(domains.size()!= numDomains_) + { + fatalErrorInFunction<<"number of new domians differs"< points, includeMask mask) +{ range active = mask.activeRange(); auto numInDomain = numParInDomain_.deviceVectorAll(); auto numDomains = numDomains_; @@ -102,17 +156,4 @@ bool pFlow::domainDistribute::locateParticles( return true; -} - -bool pFlow::domainDistribute::changeDomainsSize( - const std::vector& domains) -{ - if(domains.size()!= numDomains_) - { - fatalErrorInFunction<<"number of new domians differs"<& domains); + bool changeDomainsSize( + real extentFraction, + real maxBoundingBoxSize, + const std::vector& domains); //template diff --git a/DEMSystems/sphereDEMSystem/sphereDEMSystem.cpp b/DEMSystems/sphereDEMSystem/sphereDEMSystem.cpp index 2f720849..f3836b2c 100644 --- a/DEMSystems/sphereDEMSystem/sphereDEMSystem.cpp +++ b/DEMSystems/sphereDEMSystem/sphereDEMSystem.cpp @@ -28,17 +28,8 @@ pFlow::sphereDEMSystem::sphereDEMSystem( char* argv[]) : DEMSystem(demSystemName, domains, argc, argv) -{ - - REPORT(0)<<"Initializing host/device execution spaces . . . \n"; - REPORT(1)<<"Host execution space is "<< greenText(DefaultHostExecutionSpace::name())<( Control().caseSetup().path()+propertyFile__); @@ -47,7 +38,7 @@ pFlow::sphereDEMSystem::sphereDEMSystem( geometry_ = geometry::create(Control(), Property()); REPORT(0)<<"\nReading sphere particles . . ."<(Control(), Property()); + particles_ = makeUnique(Control(), Property()); REPORT(0)<<"\nCreating interaction model for sphere-sphere contact and sphere-wall contact . . ."< domains) +{ + if(!particleDistribution_->changeDomainsSize( + extentFraction, + maxBounndingSphereSize(), + domains)) + { + fatalErrorInFunction<< + "cannot change the domain size"<locateParticles( + parPosition_, + particles_->pStruct().activePointsMaskH())) + { + fatalErrorInFunction<< + "error in locating particles among sub-domains"<particlesInDomain(di); } -bool pFlow::sphereDEMSystem::changeDomainsSizeUpdateParticles( - const std::vector& domains) -{ - if( !particleDistribution_->changeDomainsSize(domains)) - return false; - - // should update list of particles here - //************************************************************************************************ - notImplementedFunction; - return false; +pFlow::span pFlow::sphereDEMSystem::parDiameter() +{ + return span(parDiameter_.data(), parDiameter_.size()); } -bool pFlow::sphereDEMSystem::updateParticles() +pFlow::span pFlow::sphereDEMSystem::parVelocity() { - notImplementedFunction; - return false; + return span(parVelocity_.data(), parVelocity_.size()); } +pFlow::span pFlow::sphereDEMSystem::parPosition() +{ + return span(parPosition_.data(), parPosition_.size()); +} + +pFlow::span pFlow::sphereDEMSystem::parFluidForce() +{ + auto& hVec = particles_->fluidTorqueHostAll(); + return span(hVec.data(), hVec.size()); +} + +pFlow::span pFlow::sphereDEMSystem::parFluidTorque() +{ + auto& hVec = particles_->fluidForceHostAll(); + return span(hVec.data(), hVec.size()); +} + +bool pFlow::sphereDEMSystem::sendFluidForceToDEM() +{ + particles_->fluidForceHostUpdatedSync(); + return true; +} + +bool pFlow::sphereDEMSystem::sendFluidTorqueToDEM() +{ + particles_->fluidTorqueHostUpdatedSync(); + return true; +} + +bool pFlow::sphereDEMSystem::beforeIteration() +{ + parVelocity_ = particles_->dynPointStruct().velocityHostAll(); + parPosition_ = particles_->dynPointStruct().pointPositionHostAll(); + parDiameter_ = particles_->diameter().hostVectorAll(); + return true; +} + + bool pFlow::sphereDEMSystem::iterate( int32 n, real timeToWrite, diff --git a/DEMSystems/sphereDEMSystem/sphereDEMSystem.hpp b/DEMSystems/sphereDEMSystem/sphereDEMSystem.hpp index abf86742..3272a6e1 100644 --- a/DEMSystems/sphereDEMSystem/sphereDEMSystem.hpp +++ b/DEMSystems/sphereDEMSystem/sphereDEMSystem.hpp @@ -25,7 +25,7 @@ Licence: #include "property.hpp" #include "uniquePtr.hpp" #include "geometry.hpp" -#include "sphereParticles.hpp" +#include "sphereFluidParticles.hpp" #include "interaction.hpp" #include "Insertions.hpp" #include "domainDistribute.hpp" @@ -46,7 +46,7 @@ protected: uniquePtr geometry_ = nullptr; - uniquePtr particles_ = nullptr; + uniquePtr particles_ = nullptr; uniquePtr insertion_ = nullptr; @@ -54,6 +54,13 @@ protected: uniquePtr particleDistribution_=nullptr; + // to be used for CPU communications + ViewType1D parVelocity_; + + ViewType1D parPosition_; + + ViewType1D parDiameter_; + // protected member functions auto& Property() { @@ -97,6 +104,7 @@ public: word); + bool updateParticleDistribution(real extentFraction, const std::vector domains) override; int32 numParInDomain(int32 di)const override; @@ -104,9 +112,21 @@ public: span parIndexInDomain(int32 di)const override; - bool changeDomainsSizeUpdateParticles(const std::vector& domains) override; + span parDiameter() override; - bool updateParticles() override; + span parVelocity() override; + + span parPosition() override; + + span parFluidForce() override; + + span parFluidTorque() override; + + bool sendFluidForceToDEM() override; + + bool sendFluidTorqueToDEM() override; + + bool beforeIteration() override; bool iterate(int32 n, real timeToWrite, word timeName) override; diff --git a/DEMSystems/sphereDEMSystem/sphereFluidParticles.cpp b/DEMSystems/sphereDEMSystem/sphereFluidParticles.cpp new file mode 100644 index 00000000..386ac859 --- /dev/null +++ b/DEMSystems/sphereDEMSystem/sphereFluidParticles.cpp @@ -0,0 +1,100 @@ +/*------------------------------- phasicFlow --------------------------------- + O C enter of + O O E ngineering and + O O M ultiscale modeling of + OOOOOOO F luid flow +------------------------------------------------------------------------------ + Copyright (C): www.cemf.ir + email: hamid.r.norouzi AT gmail.com +------------------------------------------------------------------------------ +Licence: + This file is part of phasicFlow code. It is a free software for simulating + granular and multiphase flows. You can redistribute it and/or modify it under + the terms of GNU General Public License v3 or any other later versions. + + phasicFlow is distributed to help others in their research in the field of + granular and multiphase flows, but WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + +-----------------------------------------------------------------------------*/ + +#include "sphereFluidParticles.hpp" +#include "sphereFluidParticlesKernels.hpp" + + +pFlow::sphereFluidParticles::sphereFluidParticles( + systemControl &control, + const property& prop +) +: + sphereParticles(control, prop), + fluidForce_( + this->time().emplaceObject( + objectFile( + "fluidForce", + "", + objectFile::READ_IF_PRESENT, + objectFile::WRITE_ALWAYS + ), + pStruct(), + zero3 + ) + ), + fluidTorque_( + this->time().emplaceObject( + objectFile( + "fluidTorque", + "", + objectFile::READ_IF_PRESENT, + objectFile::WRITE_ALWAYS + ), + pStruct(), + zero3 + ) + ) +{} + + + +bool pFlow::sphereFluidParticles::iterate() +{ + + accelerationTimer_.start(); + pFlow::sphereFluidParticlesKernels::acceleration( + control().g(), + mass().deviceVectorAll(), + contactForce().deviceVectorAll(), + fluidForce().deviceVectorAll(), + I().deviceVectorAll(), + contactTorque().deviceVectorAll(), + fluidTorque().deviceVectorAll(), + pStruct().activePointsMaskD(), + accelertion().deviceVectorAll(), + rAcceleration().deviceVectorAll() + ); + accelerationTimer_.end(); + + intCorrectTimer_.start(); + + dynPointStruct_.correct(this->dt(), accelertion_); + + rVelIntegration_().correct(this->dt(), rVelocity_, rAcceleration_); + + intCorrectTimer_.end(); + + return true; +} + +void pFlow::sphereFluidParticles::fluidForceHostUpdatedSync() +{ + fluidForce_.modifyOnHost(); + fluidForce_.syncViews(); + return; +} + +void pFlow::sphereFluidParticles::fluidTorqueHostUpdatedSync() +{ + fluidTorque_.modifyOnHost(); + fluidTorque_.syncViews(); + return; +} \ No newline at end of file diff --git a/DEMSystems/sphereDEMSystem/sphereFluidParticles.hpp b/DEMSystems/sphereDEMSystem/sphereFluidParticles.hpp new file mode 100644 index 00000000..48b73f34 --- /dev/null +++ b/DEMSystems/sphereDEMSystem/sphereFluidParticles.hpp @@ -0,0 +1,91 @@ +/*------------------------------- phasicFlow --------------------------------- + O C enter of + O O E ngineering and + O O M ultiscale modeling of + OOOOOOO F luid flow +------------------------------------------------------------------------------ + Copyright (C): www.cemf.ir + email: hamid.r.norouzi AT gmail.com +------------------------------------------------------------------------------ +Licence: + This file is part of phasicFlow code. It is a free software for simulating + granular and multiphase flows. You can redistribute it and/or modify it under + the terms of GNU General Public License v3 or any other later versions. + + phasicFlow is distributed to help others in their research in the field of + granular and multiphase flows, but WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + +-----------------------------------------------------------------------------*/ + +/*! + @class pFlow::sphereParticles + + @brief Class for managing spherical particles + + This is a top-level class that contains the essential components for + defining spherical prticles in a DEM simulation. + */ + +#ifndef __sphereFluidParticles_hpp__ +#define __sphereFluidParticles_hpp__ + +#include "sphereParticles.hpp" + + +namespace pFlow +{ + +class sphereFluidParticles +: + public sphereParticles +{ +protected: + + /// pointField of rotational acceleration of particles on device + realx3PointField_HD& fluidForce_; + + realx3PointField_HD& fluidTorque_; + + +public: + + /// construct from systemControl and property + sphereFluidParticles(systemControl &control, const property& prop); + + + /// iterate particles + bool iterate() override; + + + auto& fluidForce() + { + return fluidForce_; + } + + auto& fluidTorque() + { + return fluidTorque_; + } + + + auto& fluidForceHostAll() + { + return fluidForce_.hostVectorAll(); + } + + auto& fluidTorqueHostAll() + { + return fluidTorque_.hostVectorAll(); + } + + + void fluidForceHostUpdatedSync(); + + void fluidTorqueHostUpdatedSync(); + +}; //sphereFluidParticles + +} // pFlow + +#endif //__sphereFluidParticles_hpp__ diff --git a/DEMSystems/sphereDEMSystem/sphereFluidParticlesKernels.hpp b/DEMSystems/sphereDEMSystem/sphereFluidParticlesKernels.hpp new file mode 100644 index 00000000..a8277894 --- /dev/null +++ b/DEMSystems/sphereDEMSystem/sphereFluidParticlesKernels.hpp @@ -0,0 +1,79 @@ +/*------------------------------- phasicFlow --------------------------------- + O C enter of + O O E ngineering and + O O M ultiscale modeling of + OOOOOOO F luid flow +------------------------------------------------------------------------------ + Copyright (C): www.cemf.ir + email: hamid.r.norouzi AT gmail.com +------------------------------------------------------------------------------ +Licence: + This file is part of phasicFlow code. It is a free software for simulating + granular and multiphase flows. You can redistribute it and/or modify it under + the terms of GNU General Public License v3 or any other later versions. + + phasicFlow is distributed to help others in their research in the field of + granular and multiphase flows, but WITHOUT ANY WARRANTY; without even the + implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + +-----------------------------------------------------------------------------*/ + +#ifndef __sphereFluidParticlesKernels_hpp__ +#define __sphereFluidParticlesKernels_hpp__ + +namespace pFlow::sphereFluidParticlesKernels +{ + +using rpAcceleration = Kokkos::RangePolicy< + DefaultExecutionSpace, + Kokkos::Schedule, + Kokkos::IndexType + >; + +template +void acceleration( + realx3 g, + deviceViewType1D mass, + deviceViewType1D force, + deviceViewType1D fluidForce, + deviceViewType1D I, + deviceViewType1D torque, + deviceViewType1D fluidTorque, + IncludeFunctionType incld, + deviceViewType1D lAcc, + deviceViewType1D rAcc + ) +{ + + auto activeRange = incld.activeRange(); + if(incld.allActive()) + { + Kokkos::parallel_for( + "pFlow::sphereParticlesKernels::acceleration", + rpAcceleration(activeRange.first, activeRange.second), + LAMBDA_HD(int32 i){ + lAcc[i] = (force[i]+fluidForce[i])/mass[i] + g; + rAcc[i] = (torque[i]+fluidTorque[i])/I[i]; + }); + } + else + { + Kokkos::parallel_for( + "pFlow::sphereParticlesKernels::acceleration", + rpAcceleration(activeRange.first, activeRange.second), + LAMBDA_HD(int32 i){ + if(incld(i)) + { + lAcc[i] = (force[i]+fluidForce[i])/mass[i] + g; + rAcc[i] = (torque[i]+fluidTorque[i])/I[i]; + } + }); + + } + + Kokkos::fence(); +} + +} + +#endif diff --git a/src/Particles/dynamicPointStructure/dynamicPointStructure.hpp b/src/Particles/dynamicPointStructure/dynamicPointStructure.hpp index da3ecf85..7e101b31 100644 --- a/src/Particles/dynamicPointStructure/dynamicPointStructure.hpp +++ b/src/Particles/dynamicPointStructure/dynamicPointStructure.hpp @@ -100,6 +100,16 @@ public: return velocity_; } + inline auto velocityHostAll() + { + return velocity_.hostVectorAll(); + } + + inline auto pointPositionHostAll() + { + return pStruct_.pointPositionHostAll(); + } + auto markDeleteOutOfBox(const box& domain) { return pStruct_.markDeleteOutOfBox(domain); diff --git a/src/Particles/particles/particles.hpp b/src/Particles/particles/particles.hpp index d2337c5d..0cfcdf57 100644 --- a/src/Particles/particles/particles.hpp +++ b/src/Particles/particles/particles.hpp @@ -67,7 +67,7 @@ protected: realx3PointField_D& contactForce_; - realx3PointField_D& contactTorque_; + realx3PointField_D& contactTorque_; // - object handling particle id diff --git a/src/phasicFlow/repository/Time/timeControl.hpp b/src/phasicFlow/repository/Time/timeControl.hpp index 9ad2916e..51491ce9 100644 --- a/src/phasicFlow/repository/Time/timeControl.hpp +++ b/src/phasicFlow/repository/Time/timeControl.hpp @@ -108,6 +108,10 @@ public: return tmp; } + real startTime()const + { + return startTime_; + } real currentTime() const { diff --git a/src/phasicFlow/structuredData/pointStructure/pointStructure.hpp b/src/phasicFlow/structuredData/pointStructure/pointStructure.hpp index 70c19be5..4f53f2ce 100644 --- a/src/phasicFlow/structuredData/pointStructure/pointStructure.hpp +++ b/src/phasicFlow/structuredData/pointStructure/pointStructure.hpp @@ -255,6 +255,12 @@ public: FUNCTION_H const int8Field_HD& pointFlag()const; + INLINE_FUNCTION_H + auto pointPositionHostAll() + { + return pointPosition_.hostVectorAll(); + } + // - size of data structure FUNCTION_H label size()const;