From 5a25556b8a15bb3f79bfe0c29b656f8e2b66a4da Mon Sep 17 00:00:00 2001
From: HRN <hamid.r.norouzi@gmail.com>
Date: Fri, 24 May 2024 00:10:15 +0330
Subject: [PATCH] changing blocking-recive to non-blocking (in
 boundaryProcessor) to improve MPI run

---
 .../processorBoundarySphereInteraction.cpp    |  4 +-
 .../boundaries/boundaryProcessor.cpp          | 86 +++++++++++--------
 .../boundaries/boundaryProcessor.hpp          |  6 +-
 3 files changed, 59 insertions(+), 37 deletions(-)

diff --git a/src/Interaction/sphereInteraction/boundaries/processorBoundarySphereInteraction/processorBoundarySphereInteraction.cpp b/src/Interaction/sphereInteraction/boundaries/processorBoundarySphereInteraction/processorBoundarySphereInteraction.cpp
index f0e2a9a1..d55ab2b3 100644
--- a/src/Interaction/sphereInteraction/boundaries/processorBoundarySphereInteraction/processorBoundarySphereInteraction.cpp
+++ b/src/Interaction/sphereInteraction/boundaries/processorBoundarySphereInteraction/processorBoundarySphereInteraction.cpp
@@ -106,7 +106,7 @@ bool pFlow::MPI::processorBoundarySphereInteraction<cFM, gMM>::sphereSphereInter
 			return true;
 		}
 		
-		if(iter % 1000 == 0u)
+		if(iter % 100 == 0u)
 		{
 			pOutput<<"inter "<< inter_.totalTime()<<endl;
 			pOutput<<"send "<< send_.totalTime()<<endl<<endl;;
@@ -144,7 +144,7 @@ bool pFlow::MPI::processorBoundarySphereInteraction<cFM, gMM>::sphereSphereInter
 			return true;
 		}
 		
-		if(iter % 1000 == 0u)
+		if(iter % 100 == 0u)
 		{
 			pOutput<<"recive "<< recv_.totalTime()<<endl;
 			pOutput<<"add "<< add_.totalTime()<<endl<<endl;
diff --git a/src/phasicFlow/MPIParallelization/pointStructure/boundaries/boundaryProcessor.cpp b/src/phasicFlow/MPIParallelization/pointStructure/boundaries/boundaryProcessor.cpp
index 54bed6ad..cf34617d 100644
--- a/src/phasicFlow/MPIParallelization/pointStructure/boundaries/boundaryProcessor.cpp
+++ b/src/phasicFlow/MPIParallelization/pointStructure/boundaries/boundaryProcessor.cpp
@@ -65,43 +65,61 @@ pFlow::MPI::boundaryProcessor::boundaryProcessor(
 }
 
 bool
-pFlow::MPI::boundaryProcessor::beforeIteration(uint32 iterNum, real t, real dt)
+pFlow::MPI::boundaryProcessor::beforeIteration(
+	uint32 step, 
+	uint32 iterNum, 
+	real t, 
+	real dt)
 {
-
-	thisNumPoints_ = size();
-
-	auto req = MPI_REQUEST_NULL;
-	MPI_Isend(
-		&thisNumPoints_,
-		1,
-		MPI_UNSIGNED,
-		neighborProcessorNo(),
-		thisBoundaryIndex(),
-		pFlowProcessors().localCommunicator(),
-		&req);
-
-	MPI_Recv(
-		&neighborProcNumPoints_,
-		1,
-		MPI_UNSIGNED,
-		neighborProcessorNo(),
-		mirrorBoundaryIndex(),
-		pFlowProcessors().localCommunicator(),
-		MPI_STATUS_IGNORE
-	);
-	MPI_Request_free(&req);
-
-	anyList varList;
-	message msg;
-
-	varList.emplaceBack(msg.addAndName(message::BNDR_PROC_SIZE_CHANGED), neighborProcNumPoints_);
-
-	if( !notify(iterNum, t, dt, msg, varList) )
+		if(step == 1 )
 	{
-		fatalErrorInFunction;
-		return false;
+		thisNumPoints_ = size();
+
+		uint32 oldNeighborProcNumPoints = neighborProcNumPoints_;
+
+		MPI_Isend(
+			&thisNumPoints_,
+			1,
+			MPI_UNSIGNED,
+			neighborProcessorNo(),
+			thisBoundaryIndex(),
+			pFlowProcessors().localCommunicator(),
+			&numPointsRequest0_);
+
+		MPI_Irecv(
+			&neighborProcNumPoints_,
+			1,
+			MPI_UNSIGNED,
+			neighborProcessorNo(),
+			mirrorBoundaryIndex(),
+			pFlowProcessors().localCommunicator(),
+			&numPointsRequest_
+		);
+
 	}
-	
+	else if(step == 2 )
+	{
+		if(numPointsRequest_ != RequestNull)
+		{
+			MPI_Wait(&numPointsRequest_, MPI_STATUS_IGNORE);
+			if(numPointsRequest0_!= RequestNull)
+			{
+				MPI_Request_free(&numPointsRequest0_);
+			}
+		}
+
+		anyList varList;
+		message msg;
+
+		varList.emplaceBack(msg.addAndName(message::BNDR_PROC_SIZE_CHANGED), neighborProcNumPoints_);
+
+		if( !notify(iterNum, t, dt, msg, varList) )
+		{
+			fatalErrorInFunction;
+			return false;
+		}
+	}
+
 	return true;
 }
 
diff --git a/src/phasicFlow/MPIParallelization/pointStructure/boundaries/boundaryProcessor.hpp b/src/phasicFlow/MPIParallelization/pointStructure/boundaries/boundaryProcessor.hpp
index 8771869e..4f62c86c 100644
--- a/src/phasicFlow/MPIParallelization/pointStructure/boundaries/boundaryProcessor.hpp
+++ b/src/phasicFlow/MPIParallelization/pointStructure/boundaries/boundaryProcessor.hpp
@@ -42,6 +42,10 @@ namespace pFlow::MPI
 
 		uint32               thisNumPoints_ = 0;
 
+		Request 			 numPointsRequest_ = RequestNull;
+
+		Request 			 numPointsRequest0_ = RequestNull;
+
 		realx3Vector_D       neighborProcPoints_;
 
 		dataSender<realx3>   sender_;
@@ -87,7 +91,7 @@ namespace pFlow::MPI
 			boundaryProcessor,
 			dictionary);
 
-		bool beforeIteration(uint32 iterNum, real t, real dt) override;
+		bool beforeIteration(uint32 step, uint32 iterNum, real t, real dt) override;
 
 		bool iterate(uint32 iterNum, real t, real dt) override;