701 {
703
706
707
708
709 {
714 sendReq.fill(MPI_REQUEST_NULL);
715 recvReq.fill(MPI_REQUEST_NULL);
716 sendBufferCnts.fill(0);
717 recvBufferCnts.fill(0);
718 for(
MInt i = 0; i <
fvSolver().noNeighborDomains(); i++) {
719 sendBufferCnts(i) = noCVars *
fvSolver().m_noMaxLevelWindowCells[i];
720 recvBufferCnts(i) = noCVars *
fvSolver().m_noMaxLevelHaloCells[i];
721 MInt sendBufferCounter = 0;
722 for(
MInt j = 0; j <
fvSolver().m_noMaxLevelWindowCells[i]; j++) {
724 for(
MInt v = 0; v < noCVars; v++) {
725 fvSolver().m_sendBuffers[i][sendBufferCounter] =
fvSolver().a_variable(cellId, v);
726 sendBufferCounter++;
727 }
728 }
729 }
733 for(
MInt i = 0; i <
fvSolver().noNeighborDomains(); i++) {
734 if(sendBufferCnts(i) == 0) continue;
735 ASSERT(sendBufferCnts(i) <=
fvSolver().m_noMaxLevelWindowCells[i] *
fvSolver().m_dataBlockSize,
"");
737 fvSolver().mpiComm(), &sendReq[sendCnt], AT_,
"m_sendBuffers[" + std::to_string(i) +
"],0");
738 sendCnt++;
739 }
740 for(
MInt i = 0; i <
fvSolver().noNeighborDomains(); i++) {
741 if(recvBufferCnts(i) == 0) continue;
742 ASSERT(recvBufferCnts(i) <=
fvSolver().m_noMaxLevelHaloCells[i] *
fvSolver().m_dataBlockSize,
"");
744 fvSolver().mpiComm(), &recvReq[recvCnt], AT_,
"m_receiveBuffers[" + std::to_string(i) +
"],0");
745 recvCnt++;
746 }
747 if(recvCnt > 0)
MPI_Waitall(recvCnt, &recvReq[0], MPI_STATUSES_IGNORE, AT_);
748 } else {
749 for(
MInt i = 0; i <
fvSolver().noNeighborDomains(); i++) {
750 if(sendBufferCnts(i) == 0) continue;
751 ASSERT(sendBufferCnts(i) <=
fvSolver().m_noMaxLevelWindowCells[i] *
fvSolver().m_dataBlockSize,
"");
753 fvSolver().mpiComm(), &sendReq[sendCnt], AT_,
"m_sendBuffers[" + std::to_string(i) +
"],0");
754 sendCnt++;
755 }
756 for(
MInt i = 0; i <
fvSolver().noNeighborDomains(); i++) {
757 if(recvBufferCnts(i) == 0) continue;
758 ASSERT(recvBufferCnts(i) <=
fvSolver().m_noMaxLevelHaloCells[i] *
fvSolver().m_dataBlockSize,
"");
760 fvSolver().mpiComm(), MPI_STATUS_IGNORE, AT_,
"m_receiveBuffers[" + std::to_string(i) +
"],0");
761 recvCnt++;
762 }
763 }
764 for(
MInt i = 0; i <
fvSolver().noNeighborDomains(); i++) {
765 MInt recvBufferCounter = 0;
766 for(
MInt j = 0; j <
fvSolver().m_noMaxLevelHaloCells[i]; j++) {
768 for(
MInt v = 0; v < noCVars; v++) {
769 fvSolver().a_variable(cellId, v) =
fvSolver().m_receiveBuffers[i][recvBufferCounter];
770 recvBufferCounter++;
771 }
772 }
773 }
774 if(sendCnt > 0)
MPI_Waitall(sendCnt, &sendReq[0], MPI_STATUSES_IGNORE, AT_);
775 }
776
777
778
779 struct transferCV {
784 };
785 constexpr MInt noTransferCV = 4;
786 std::map<MInt, transferCV>
787 haloCorrections;
789 if(
fvSolver().a_isHalo(cellId))
continue;
790 if(!
fvSolver().c_isLeafCell(cellId))
continue;
791
793
796 const MFloat deltaMass = 0.0 - origMass;
797 const std::vector<MInt> neighbors =
findRedistCells(cellId,
true, 0.0);
798 std::vector<MInt> redistNeighbors = neighbors;
799 MInt noRedistNbs = 0;
800 MFloat massNeedPerCell = 0.0;
801 while(true) {
802 MBool change =
false;
803 noRedistNbs = redistNeighbors.size();
804 massNeedPerCell = deltaMass / noRedistNbs;
805 for(auto it = redistNeighbors.begin(); it != redistNeighbors.end(); ++it) {
806 const MInt candidateId = *it;
808 < massNeedPerCell) {
809 change = true;
810 redistNeighbors.erase(it);
811 break;
812 }
813 }
814 if(change) continue;
815 break;
816 }
817 MFloat momentumTransfer[3] = {0.0, 0.0, 0.0};
818 if(!redistNeighbors.empty()) {
819 for(auto redistId : redistNeighbors) {
820 const MFloat delRho = massNeedPerCell /
fvSolver().a_cellVolume(redistId);
825 for(
MInt dir = 0; dir < 3; dir++) {
826 fvSolver().a_variable(redistId,
fvSolver().CV->A_RHO_VV[dir]) -= delRhoVV[dir];
827 momentumTransfer[dir] += delRhoVV[dir] *
fvSolver().a_cellVolume(redistId);
828 }
829 fvSolver().setPrimitiveVariables(redistId);
830
832 const transferCV delCV = {
833 -delRho,
834 -delRhoVV[0],
835 -delRhoVV[1],
836 -delRhoVV[2],
837 };
838 auto success = haloCorrections.insert(std::pair<MInt, transferCV>(redistId, delCV));
839 if(!success.second) {
840 success.first->second.rhoAlpha += delCV.rhoAlpha;
841 success.first->second.rhoUAlpha += delCV.rhoUAlpha;
842 success.first->second.rhoVAlpha += delCV.rhoVAlpha;
843 success.first->second.rhoWAlpha += delCV.rhoWAlpha;
844 }
845 }
846 }
848 massNeedPerCell * noRedistNbs /
fvSolver().a_cellVolume(cellId);
849 for(
MInt dir = 0; dir < 3; dir++) {
851 momentumTransfer[dir] /
fvSolver().a_cellVolume(cellId);
852 }
853 fvSolver().setPrimitiveVariables(cellId);
854 } else {
855
856 redistNeighbors = neighbors;
857
858 MFloat massTransfer = 0.0;
859 for(auto redistId : redistNeighbors) {
862 const MFloat delRho = massAvail /
fvSolver().a_cellVolume(redistId);
866 massTransfer += massAvail;
868 for(
MInt dir = 0; dir < 3; dir++) {
869 fvSolver().a_variable(redistId,
fvSolver().CV->A_RHO_VV[dir]) -= delRhoVV[dir];
870 momentumTransfer[dir] += delRhoVV[dir] *
fvSolver().a_cellVolume(redistId);
871 }
872 fvSolver().setPrimitiveVariables(redistId);
873
875 const transferCV delCV = {
876 -delRho,
877 -delRhoVV[0],
878 -delRhoVV[1],
879 -delRhoVV[2],
880 };
881 auto success = haloCorrections.insert(std::pair<MInt, transferCV>(redistId, delCV));
882 if(!success.second) {
883 success.first->second.rhoAlpha += delCV.rhoAlpha;
884 success.first->second.rhoUAlpha += delCV.rhoUAlpha;
885 success.first->second.rhoVAlpha += delCV.rhoVAlpha;
886 success.first->second.rhoWAlpha += delCV.rhoWAlpha;
887 }
888 }
889 }
891 for(
MInt dir = 0; dir < 3; dir++) {
893 momentumTransfer[dir] /
fvSolver().a_cellVolume(cellId);
894 }
895 fvSolver().setPrimitiveVariables(cellId);
896 }
902 MBool isOutlet =
false;
903
904 if(
fvSolver().a_bndryId(cellId) >= 0) {
906 for(
MInt srfc = 0; srfc <
fvSolver().m_bndryCells->a[bndryId].m_noSrfcs; srfc++) {
907 const MInt bndrCndId =
fvSolver().m_fvBndryCnd->m_bndryCell[bndryId].m_srfcs[srfc]->m_bndryCndId;
908 if(bndrCndId == 1002) {
909 isOutlet = true;
910 break;
911 }
912 }
913 }
914 if(isOutlet) {
915 for(
MUint dir = 0; dir < 26; dir++) {
916 const MInt neighborId =
fvSolver().grid().neighborList(cellId, dir);
917 if(
fvSolver().a_isBndryGhostCell(neighborId)) {
918 neighbors.push_back(neighborId);
919 }
920 }
921 }
922 std::vector<MInt> redistNeighbors = neighbors;
923
924 MInt noRedistNbs = 0;
925 MFloat massExcessPerCell = 0.0;
926 while(true) {
927 MBool change =
false;
928 noRedistNbs = redistNeighbors.size();
929 massExcessPerCell = -deltaMass / noRedistNbs;
930 for(auto it = redistNeighbors.begin(); it != redistNeighbors.end(); ++it) {
931 const MInt candidateId = *it;
932 if(!
fvSolver().a_isBndryGhostCell(candidateId)
934 + massExcessPerCell /
fvSolver().a_cellVolume(candidateId))
936 change = true;
937 redistNeighbors.erase(it);
938 break;
939 }
940 }
941 if(change) continue;
942 break;
943 }
944 MFloat momentumTransfer[3] = {0.0, 0.0, 0.0};
945 if(!redistNeighbors.empty() || isOutlet) {
946 if(redistNeighbors.empty()) {
947 noRedistNbs = 1;
948 massExcessPerCell = -deltaMass / noRedistNbs;
949 for(
MInt dir = 0; dir < 3; dir++) {
950 momentumTransfer[dir] += massExcessPerCell *
fvSolver().a_pvariable(cellId,
fvSolver().PV->VV[dir]);
951 }
952 }
953 for(auto redistId : redistNeighbors) {
954 if(
fvSolver().a_isBndryGhostCell(redistId))
continue;
955 const MFloat delRho = massExcessPerCell /
fvSolver().a_cellVolume(redistId);
960 for(
MInt dir = 0; dir < 3; dir++) {
961 fvSolver().a_variable(redistId,
fvSolver().CV->A_RHO_VV[dir]) += delRhoVV[dir];
962 momentumTransfer[dir] += delRhoVV[dir] *
fvSolver().a_cellVolume(redistId);
963 }
964 fvSolver().setPrimitiveVariables(redistId);
965
967 const transferCV delCV = {
968 delRho,
969 delRhoVV[0],
970 delRhoVV[1],
971 delRhoVV[2],
972 };
973 auto success = haloCorrections.insert(std::pair<MInt, transferCV>(redistId, delCV));
974 if(!success.second) {
975 success.first->second.rhoAlpha += delCV.rhoAlpha;
976 success.first->second.rhoUAlpha += delCV.rhoUAlpha;
977 success.first->second.rhoVAlpha += delCV.rhoVAlpha;
978 success.first->second.rhoWAlpha += delCV.rhoWAlpha;
979 }
980 }
981 }
983 massExcessPerCell /
fvSolver().a_cellVolume(cellId) * noRedistNbs;
984 for(
MInt dir = 0; dir < 3; dir++) {
986 momentumTransfer[dir] /
fvSolver().a_cellVolume(cellId);
987 }
988 fvSolver().setPrimitiveVariables(cellId);
989 } else {
990
991 redistNeighbors = neighbors;
992 if(redistNeighbors.empty()) {
993 redistNeighbors =
findRedistCells(cellId,
false, std::numeric_limits<MFloat>::max());
994 }
995 MFloat massTransfer = 0.0;
996 for(auto redistId : redistNeighbors) {
999 *
fvSolver().a_cellVolume(redistId);
1000 if(massCapacityAvail < 0.0) continue;
1001 const MFloat delRho = massCapacityAvail /
fvSolver().a_cellVolume(redistId);
1005 massTransfer += massCapacityAvail;
1007 for(
MInt dir = 0; dir < 3; dir++) {
1008 fvSolver().a_variable(redistId,
fvSolver().CV->A_RHO_VV[dir]) += delRhoVV[dir];
1009 momentumTransfer[dir] += delRhoVV[dir] *
fvSolver().a_cellVolume(redistId);
1010 }
1011 fvSolver().setPrimitiveVariables(redistId);
1012
1013 if(
fvSolver().a_isHalo(redistId)) {
1014 const transferCV delCV = {
1015 delRho,
1016 delRhoVV[0],
1017 delRhoVV[1],
1018 delRhoVV[2],
1019 };
1020 auto success = haloCorrections.insert(std::pair<MInt, transferCV>(redistId, delCV));
1021 if(!success.second) {
1022 success.first->second.rhoAlpha += delCV.rhoAlpha;
1023 success.first->second.rhoUAlpha += delCV.rhoUAlpha;
1024 success.first->second.rhoVAlpha += delCV.rhoVAlpha;
1025 success.first->second.rhoWAlpha += delCV.rhoWAlpha;
1026 }
1027 }
1028 }
1030 for(
MInt dir = 0; dir < 3; dir++) {
1032 momentumTransfer[dir] /
fvSolver().a_cellVolume(cellId);
1033 }
1034 fvSolver().setPrimitiveVariables(cellId);
1035 }
1036 }
1037 }
1038
1039
1040
1041 {
1042 if(noTransferCV >
fvSolver().m_dataBlockSize)
1043 mTerm(1, AT_,
"The send- and receive Buffers are too small for this communication!");
1044
1045 for(
MInt i = 0; i <
fvSolver().noNeighborDomains(); i++) {
1046 MInt receiveBufferCounter = 0;
1047 for(
MInt j = 0; j <
fvSolver().m_noMaxLevelHaloCells[i]; j++) {
1048 auto it = haloCorrections.find(
fvSolver().m_maxLevelHaloCells[i][j]);
1049 if(it != haloCorrections.end()) {
1050 fvSolver().m_receiveBuffers[i][receiveBufferCounter +
fvSolver().CV->A_RHO] = it->second.rhoAlpha;
1051 fvSolver().m_receiveBuffers[i][receiveBufferCounter +
fvSolver().CV->A_RHO_U] = it->second.rhoUAlpha;
1052 fvSolver().m_receiveBuffers[i][receiveBufferCounter +
fvSolver().CV->A_RHO_V] = it->second.rhoVAlpha;
1053 fvSolver().m_receiveBuffers[i][receiveBufferCounter +
fvSolver().CV->A_RHO_W] = it->second.rhoWAlpha;
1054 } else {
1055 fvSolver().m_receiveBuffers[i][receiveBufferCounter +
fvSolver().CV->A_RHO] = 0.0;
1056 fvSolver().m_receiveBuffers[i][receiveBufferCounter +
fvSolver().CV->A_RHO_U] = 0.0;
1057 fvSolver().m_receiveBuffers[i][receiveBufferCounter +
fvSolver().CV->A_RHO_V] = 0.0;
1058 fvSolver().m_receiveBuffers[i][receiveBufferCounter +
fvSolver().CV->A_RHO_W] = 0.0;
1059 }
1060 receiveBufferCounter += noTransferCV;
1061 }
1062 }
1063
1064
1066 for(
MInt i = 0; i <
fvSolver().noNeighborDomains(); i++) {
1067 bufSize =
fvSolver().m_noMaxLevelHaloCells[i] * noTransferCV;
1070 "m_receiveBuffers[" + std::to_string(i) + "],1");
1071 }
1072
1073
1074 MPI_Status status;
1075 for(
MInt i = 0; i <
fvSolver().noNeighborDomains(); i++) {
1076 bufSize =
fvSolver().m_noMaxLevelWindowCells[i] * noTransferCV;
1078 &status, AT_, "m_sendBuffers[" + std::to_string(i) + "],1");
1079 }
1080 for(
MInt i = 0; i <
fvSolver().noNeighborDomains(); i++) {
1082 }
1083
1084
1085 MInt sendBufferCounter = 0;
1086 for(
MInt i = 0; i <
fvSolver().noNeighborDomains(); i++) {
1087 sendBufferCounter = 0;
1088 for(
MInt j = 0; j <
fvSolver().m_noMaxLevelWindowCells[i]; j++) {
1091 for(
MInt dir = 0; dir < nDim; dir++) {
1093 fvSolver().m_sendBuffers[i][sendBufferCounter +
fvSolver().CV->A_RHO_VV[dir]];
1094 }
1095 sendBufferCounter += noTransferCV;
1096 }
1097 }
1098 }
1101}
std::vector< MInt > findRedistCells(const MInt cellId, const MBool searchUp, const MFloat limit)
find neighbor Cells to cellId, that are candidates for alpha corrections
MFloat & a_variable(const MInt cellId, const MInt varId, const MInt id=0)
This class is a ScratchSpace.
constexpr T mMax(const T &x, const T &y)
int MPI_Recv(void *buf, int count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Status *status, const MString &name, const MString &varname)
same as MPI_Recv
int MPI_Isend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request, const MString &name, const MString &varname)
same as MPI_Isend
int MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Request *request, const MString &name, const MString &varname)
same as MPI_Irecv
int MPI_Issend(const void *buf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm, MPI_Request *request, const MString &name, const MString &varname)
same as MPI_Issend
int MPI_Wait(MPI_Request *request, MPI_Status *status, const MString &name)
same as MPI_Wait
int MPI_Waitall(int count, MPI_Request *request, MPI_Status *status, const MString &name)
same as MPI_Waitall