From bb7dc17c67755b44fc0d3877e003402f858f983b Mon Sep 17 00:00:00 2001
From: Hamidreza Norouzi <hamid.r.norouzi@gmail.com>
Date: Thu, 28 Sep 2023 23:14:12 +0330
Subject: [PATCH] processors is added and error part modified

---
 CMakeLists.txt                           |   1 +
 src/phasicFlow/CMakeLists.txt            |   2 +
 src/phasicFlow/globals/error.cpp         |  29 ++---
 src/phasicFlow/globals/error.hpp         |   4 +-
 src/phasicFlow/processors/processors.cpp | 151 +++++++++++++++++++++++
 src/phasicFlow/processors/processors.hpp | 136 ++++++++++++++++++++
 6 files changed, 300 insertions(+), 23 deletions(-)
 create mode 100644 src/phasicFlow/processors/processors.cpp
 create mode 100644 src/phasicFlow/processors/processors.hpp

diff --git a/CMakeLists.txt b/CMakeLists.txt
index aef27fa7..64532366 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -58,6 +58,7 @@ endif()
 
 if(pFlow_Build_MPI)
 	find_package(MPI REQUIRED)
+	link_libraries(MPI::MPI_CXX)
 endif()
 
 
diff --git a/src/phasicFlow/CMakeLists.txt b/src/phasicFlow/CMakeLists.txt
index b9ab1280..155dc7db 100644
--- a/src/phasicFlow/CMakeLists.txt
+++ b/src/phasicFlow/CMakeLists.txt
@@ -4,6 +4,8 @@ types/basicTypes/bTypesFunctions.cpp
 types/basicTypes/Logical.cpp
 types/types.cpp
 
+processors/processors.cpp
+
 globals/error.cpp)
 
 set(link_libs Kokkos::kokkos tbb)
diff --git a/src/phasicFlow/globals/error.cpp b/src/phasicFlow/globals/error.cpp
index 5eb8bf43..d32d771e 100644
--- a/src/phasicFlow/globals/error.cpp
+++ b/src/phasicFlow/globals/error.cpp
@@ -21,13 +21,11 @@ Licence:
 #include <iostream>
 #include <stdlib.h>
 
-#ifdef pFlow_Build_MPI
-	#include <mpi.h>
-#endif
 
 #include <Kokkos_Core.hpp>
 
 #include "error.hpp"
+#include "processors.hpp"
 #include "streams.hpp"
 
 
@@ -93,29 +91,18 @@ pFlow::iOstream& warningMessage(const char* fnName, const char* fileName, int li
 	return errorStream;
 }
 
-pFlow::iOstream& reportAndExit()
+pFlow::iOstream& reportAndExit(int errorCode)
 {
 	errorStream<<"\n>>> phasicFlow is exiting . . ." << pFlow::endl;
-	exit(EXIT_FAILURE);
+	fatalExitPhasicFlow(errorCode);
 	return errorStream;
 }
 
-int exitPhasicFlow()
+int fatalExitPhasicFlow(int errorCode)
 {
-// Kokkos should be finalized first
-Kokkos::finalize();
+	// Kokkos should be finalized first
+	Kokkos::finalize();
 
-#ifdef pFlow_Build_MPI
-	int flag=0;
-	MPI_Initialized(&flag)
-	if(flag == 1)
-	{
-		MPI_Abort(MPI_COMM_WORLD, EXIT_FAILURE);
-		MPI_Finalize();
-    }
-    exit(EXIT_FAILURE);
-#else
-    exit(EXIT_FAILURE);
-#endif
-	return EXIT_FAILURE;
+	pFlow::processors::abort(errorCode);
+	return errorCode;
 }
diff --git a/src/phasicFlow/globals/error.hpp b/src/phasicFlow/globals/error.hpp
index 8ce8c153..a58a30c0 100644
--- a/src/phasicFlow/globals/error.hpp
+++ b/src/phasicFlow/globals/error.hpp
@@ -35,7 +35,7 @@ namespace pFlow
 //- Decleartions 
 
 /// Take actions to fatal exit phasicFlow
-int fatalExitPhasicFlow();
+int fatalExitPhasicFlow(int errorCode = EXIT_FAILURE);
 
 pFlow::iOstream& fatalErrorMessage(const char* fileName, int linNumber );
 pFlow::iOstream& fatalErrorInMessage(const char* fnName, const char* fileName, int linNumber );
@@ -43,7 +43,7 @@ pFlow::iOstream& notImplementedErrorMessage(const char*fnName, const char* fileN
 pFlow::iOstream& ioErrorMessage(const pFlow::word& fileName, int fileLineNumber, const char* fnName, const char* fName, int lNumber);
 pFlow::iOstream& ioErrorMessage(const char* fileName, int fileLineNumber, const char* fnName, const char* fName, int lNumber);
 pFlow::iOstream& warningMessage(const char* fnName, const char* fileName, int linNumber );
-pFlow::iOstream& reportAndExit();
+pFlow::iOstream& reportAndExit(int errorCode = EXIT_FAILURE);
 
 
 /// Report a fatal error and exit the applicaiton 
diff --git a/src/phasicFlow/processors/processors.cpp b/src/phasicFlow/processors/processors.cpp
new file mode 100644
index 00000000..544b01ed
--- /dev/null
+++ b/src/phasicFlow/processors/processors.cpp
@@ -0,0 +1,151 @@
+/*------------------------------- phasicFlow ---------------------------------
+      O        C enter of
+     O O       E ngineering and
+    O   O      M ultiscale modeling of
+   OOOOOOO     F luid flow       
+------------------------------------------------------------------------------
+  Copyright (C): www.cemf.ir
+  email: hamid.r.norouzi AT gmail.com
+------------------------------------------------------------------------------  
+Licence:
+  This file is part of phasicFlow code. It is a free software for simulating 
+  granular and multiphase flows. You can redistribute it and/or modify it under
+  the terms of GNU General Public License v3 or any other later versions. 
+ 
+  phasicFlow is distributed to help others in their research in the field of 
+  granular and multiphase flows, but WITHOUT ANY WARRANTY; without even the
+  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+-----------------------------------------------------------------------------*/
+
+#include "phasicFlowConfig.H"
+
+#ifdef pFlow_Build_MPI
+	#include <mpi.h>
+#endif
+
+// from PhasicFlow
+#include "error.hpp"
+#include "processors.hpp"
+#include "streams.hpp"
+
+
+
+void pFlow::processors::initProcessors(int argc, char *argv[])
+{
+
+#ifdef pFlow_Build_MPI
+	if(!processors::isInitialized())
+	{
+		CheckMPI(MPI_Init(&argc, &argv), true);
+		isSelfInitialized_ = true;
+
+		processors::globalSize_ = MPI::COMM_WORLD.Get_size();
+		processors::globalRank_ = MPI::COMM_WORLD.Get_rank();
+		//CheckMPI(MPI_comm_size( MPI_COMM_WORLD, &processors::globalSize_), true );
+		//CheckMPI(MPI_comm_rank( MPI_COMM_WORLD, &processors::globalRank_), true );
+	}
+#else
+
+#endif
+
+}
+
+void pFlow::processors::finalizeProcessors()
+{
+
+#ifdef pFlow_Build_MPI
+	if(isSelfInitialized_ && !isFinalized())
+	{
+		CheckMPI(MPI_Finalize(), true);
+	}
+#else
+
+#endif
+}
+
+pFlow::processors::processors()
+{
+
+#ifdef pFlow_Build_MPI	
+	if(isParallel() && !isInitialized())
+	{
+		fatalErrorInFunction<<
+		"MPI communication is not initialized yet!"<<endl;
+		processors::abort(MPI_ERR_OP);
+	}
+#endif
+	
+}
+
+pFlow::processors::~processors()
+{}
+
+
+bool pFlow::processors::isInitialized()
+{
+	
+#ifdef pFlow_Build_MPI
+	int res;
+	MPI_Initialized(&res);
+	return res;
+#endif
+	return false;
+
+}
+
+bool pFlow::processors::isFinalized()
+{
+
+#ifdef pFlow_Build_MPI
+	int res;
+	MPI_Finalized(&res);
+	return res;
+#endif
+	return false;
+
+}
+
+void pFlow::processors::abort(int error)
+{
+
+#ifdef pFlow_Build_MPI
+	int flag=0;
+	MPI_Initialized(&flag);
+	if(flag == 1)
+	{
+		MPI_Abort(MPI_COMM_WORLD, error);
+		MPI_Finalize();
+  }
+    exit(error);
+#else
+    exit(error);
+#endif
+
+}
+
+bool pFlow::checkMPI(const char* funcName, int error, bool forceAbort, const char* fileName, int lineNumebr)
+{
+
+#ifdef pFlow_Build_MPI
+	
+	if(error == MPI_SUCCESS) return true;
+	fatalErrorInMessage(funcName, fileName, lineNumebr);
+	
+	if(!forceAbort) return false;
+	reportAndExit(error);
+	
+	return false;
+
+#else
+
+	return true;
+
+#endif
+}
+
+
+
+
+
+
diff --git a/src/phasicFlow/processors/processors.hpp b/src/phasicFlow/processors/processors.hpp
new file mode 100644
index 00000000..49fe3d1f
--- /dev/null
+++ b/src/phasicFlow/processors/processors.hpp
@@ -0,0 +1,136 @@
+/*------------------------------- phasicFlow ---------------------------------
+      O        C enter of
+     O O       E ngineering and
+    O   O      M ultiscale modeling of
+   OOOOOOO     F luid flow       
+------------------------------------------------------------------------------
+  Copyright (C): www.cemf.ir
+  email: hamid.r.norouzi AT gmail.com
+------------------------------------------------------------------------------  
+Licence:
+  This file is part of phasicFlow code. It is a free software for simulating 
+  granular and multiphase flows. You can redistribute it and/or modify it under
+  the terms of GNU General Public License v3 or any other later versions. 
+ 
+  phasicFlow is distributed to help others in their research in the field of 
+  granular and multiphase flows, but WITHOUT ANY WARRANTY; without even the
+  implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+
+-----------------------------------------------------------------------------*/
+
+#ifndef __processors_H__
+#define __processors_H__ 
+
+
+namespace pFlow
+{
+	// - Forward
+	bool checkMPI
+	(
+		const char* funcName,
+		int 				error, 
+		bool 				forceAbort, 
+		const char* fileName, 
+		int 				lineNumebr
+	);
+}
+
+// - Macro for automating line and file logs
+#define CheckMPI(caller, fAbort)\
+   pFlow::checkMPI(#caller, (caller), fAbort, __FILE__, __LINE__);
+
+
+namespace pFlow
+{
+
+/**
+ * This class holds the information about the global execution 
+ * world and number of processors in communication.
+ * 
+ */
+class processors
+{
+protected:
+
+	/// Is the static member initProcessors is called
+	static inline 
+	bool isSelfInitialized_ = false;
+
+	/// Global rank of the current processor
+	static inline
+	int globalRank_ = 0;
+
+	/// The global size of all processors 
+	static inline
+	int globalSize_ = 1;
+
+public:
+
+	/// Initialize MPI processors
+	static 
+	void initProcessors(int argc, char *argv[]);
+
+	/// Finalize MPI processors 
+	static
+	void finalizeProcessors();
+
+	/// Constructor 
+	processors();
+	
+
+	/// Destructor 
+	~processors();
+	
+	/// Master processors number (globaly in MPI).
+	static
+	int masterNo()
+	{
+		return 0;
+	}
+
+	/// Is this a parallel MPI run.
+	static
+	bool isParallel()
+	{
+		return processors::globalSize()>1;
+	}
+	
+	/// Is MPI initialized?
+	static
+	bool isInitialized();
+	
+	/// Is MPI finalized?
+	static
+	bool isFinalized();
+	
+	/// Is this processor the master processor?
+	static
+	bool isMaster()
+	{
+		return processors::globalRank() == processors::masterNo();
+	}
+	
+	/// Global size of processors
+	static
+	int globalSize()
+	{
+		return globalSize_;
+	}
+	
+	/// Rank of the processor in the global MPI
+	static
+	int globalRank()
+	{
+		return globalRank_;
+	}
+
+	/// Abort MPI run or regular run
+	static
+	void abort(int error);
+
+}; //processors
+
+
+} // pFlow
+
+#endif //__processors_H__
\ No newline at end of file