501 int new_rows = localDistribution.size();
506 unsigned rows_on_left_process = 0;
507 std::vector<int> node_distr_on_left_process;
509 unsigned num_local_rows = localDistribution.size();
511 MPI_Send(&num_local_rows, 1, MPI_UNSIGNED, proc_right, 123, PETSC_COMM_WORLD);
512 MPI_Recv(&rows_on_left_process, 1, MPI_UNSIGNED, proc_left, 123, PETSC_COMM_WORLD, &status);
514 node_distr_on_left_process.resize(rows_on_left_process > 0 ? rows_on_left_process : 1);
516 MPI_Send(&localDistribution[0], num_local_rows, MPI_INT, proc_right, 123, PETSC_COMM_WORLD);
517 MPI_Recv(&node_distr_on_left_process[0], rows_on_left_process, MPI_INT, proc_left, 123, PETSC_COMM_WORLD, &status);
523 for (
unsigned i=0; i<localDistribution.size(); i++)
525 local_load += localDistribution[i];
527 int load_on_left_proc = 0;
528 for (
unsigned i=0; i<node_distr_on_left_process.size(); i++)
530 load_on_left_proc += node_distr_on_left_process[i];
537 int local_to_left_sq = (local_load - load_on_left_proc) * (local_load - load_on_left_proc);
538 int delta_left = ( (local_load + node_distr_on_left_process[node_distr_on_left_process.size() - 1]) - (load_on_left_proc - node_distr_on_left_process[node_distr_on_left_process.size() - 1]) );
539 delta_left = delta_left*delta_left - local_to_left_sq;
541 int delta_right = ( (local_load - localDistribution[0]) - (load_on_left_proc + localDistribution[0]));
542 delta_right = delta_right*delta_right - local_to_left_sq;
545 int local_change = 0;
546 bool move_left = (!(delta_left > 0) && (node_distr_on_left_process.size() > 1));
552 bool move_right = !(delta_right > 0) && (localDistribution.size() > 2);
558 if (move_left && move_right)
560 local_change = (fabs((
double)delta_right) > fabs((
double)delta_left)) ? -1 : 1;
564 new_rows += local_change;
567 MPI_Send(&local_change, 1, MPI_INT, proc_left, 123, PETSC_COMM_WORLD);
571 int remote_change = 0;
572 MPI_Recv(&remote_change, 1, MPI_INT, proc_right, 123, PETSC_COMM_WORLD, &status);
575 new_rows -= remote_change;
583 if (mAreLocalBoxesSet)
585 EXCEPTION(
"Local Boxes Are Already Set");
597 for (
unsigned global_index = mMinBoxIndex; global_index<mMaxBoxIndex+1; global_index++)
599 std::set<unsigned> local_boxes;
602 local_boxes.insert(global_index);
605 bool right = (global_index==mNumBoxesEachDirection(0)-1);
606 bool left = (global_index == 0);
607 bool proc_left = (global_index == mpDistributedBoxStackFactory->GetLow());
612 local_boxes.insert(global_index+1);
615 if (proc_left && !left)
617 local_boxes.insert(global_index-1);
620 mLocalBoxes.push_back(local_boxes);
630 unsigned j_start = CalculateGridIndices(mMinBoxIndex)(1);
631 unsigned j_end = CalculateGridIndices(mMaxBoxIndex)(1);
633 unsigned nI = mNumBoxesEachDirection(0);
634 unsigned nJ = mNumBoxesEachDirection(1);
636 unsigned bottom_proc = mpDistributedBoxStackFactory->GetLow();
637 unsigned top_proc = mpDistributedBoxStackFactory->GetHigh();
640 for (
unsigned j = j_start; j < (j_end+1); j++ )
643 for (
unsigned i = 0; i < nI; i++ )
645 std::set<unsigned> local_boxes;
652 int dj = -1 * (int)(j == bottom_proc && (j > 0 || (mIsPeriodicInY && top_proc < nJ)) );
655 for (; dj < std::min((
int)nJ-j_mod,(
int)2); dj++ )
659 if ( mIsPeriodicInY && j == 0 && dj < 1 && top_proc < nJ )
661 j_mod_2 = ( dj < 0 ) ? nJ : 0;
666 int boxi = std::max((
int) i-1*std::abs(dj),(
int) 0);
667 for ( ; boxi < std::min((
int)i+2,(
int)nI); boxi++ )
669 local_boxes.insert( (j_mod+dj+j_mod_2)*nI + boxi );
672 if ( i==(nI-1) && mIsPeriodicInX )
674 local_boxes.insert( (j_mod+dj+j_mod_2)*nI );
678 if ( mIsPeriodicInY && j == (nJ-1) && dj == 0 )
684 if ( mIsPeriodicInX && i == 0 )
688 local_boxes.insert( (j+2)*nI - 1 );
689 if ( j==0 && mIsPeriodicInY && top_proc < nJ )
691 local_boxes.insert( nI*nJ - 1 );
694 else if ( mIsPeriodicInY )
696 local_boxes.insert( nI-1 );
700 if ( j == bottom_proc && j > 0 )
702 local_boxes.insert( j*nI - 1 );
707 mLocalBoxes.push_back(local_boxes);
719 unsigned k_start = CalculateGridIndices(mMinBoxIndex)(2);
720 unsigned k_end = CalculateGridIndices(mMaxBoxIndex)(2);
723 unsigned nI = mNumBoxesEachDirection(0);
724 unsigned nJ = mNumBoxesEachDirection(1);
725 unsigned nK = mNumBoxesEachDirection(2);
728 unsigned bottom_proc = mpDistributedBoxStackFactory->GetLow();
729 unsigned top_proc = mpDistributedBoxStackFactory->GetHigh();
732 for (
unsigned k = k_start; k <= k_end; k++ )
735 for (
unsigned j = 0; j < nJ; j++ )
738 for (
unsigned i = 0; i < nI; i++ )
740 std::set<unsigned> local_boxes;
743 unsigned z_offset = k*nI*nJ;
746 for (
int dj = 0; dj < std::min((
int)nJ-j_mod,2); dj++ )
748 for (
int boxi = std::max((
int)i-1*dj,0);
749 boxi < std::min((
int)i+2,(
int)nI); boxi++ )
751 local_boxes.insert( z_offset + (j_mod+dj)*nI + boxi );
753 if ( i==(nI-1) && mIsPeriodicInX )
755 local_boxes.insert( z_offset + (j_mod+dj)*nI );
757 if ( mIsPeriodicInY && j == (nJ-1) )
763 if ( mIsPeriodicInX && i == 0 )
767 local_boxes.insert( z_offset + (j+2)*nI - 1 );
769 else if ( mIsPeriodicInY )
771 local_boxes.insert( z_offset + nI-1 );
776 std::vector<unsigned> k_offset;
779 k_offset.push_back(k+1);
781 if ( k == bottom_proc && k > 0 )
784 k_offset.push_back(k-1);
786 if ( mIsPeriodicInZ && k == (nK-1) )
788 k_offset.push_back(0);
790 else if ( mIsPeriodicInZ && k==0 && (top_proc < nK) )
792 k_offset.push_back(nK-1);
795 for ( std::vector<unsigned>::iterator k_offset_it = k_offset.begin(); k_offset_it != k_offset.end(); ++k_offset_it )
797 z_offset = (*k_offset_it)*nI*nJ;
799 int pX = (int) mIsPeriodicInX;
800 int pY = (int) mIsPeriodicInY;
801 for (
int boxi = std::max((
int)i-1,-1*pX); boxi < std::min((
int)i+2,(
int)nI+pX); boxi++ )
803 for (
int boxj = std::max((
int)j-1,-1*pY); boxj < std::min((
int)j+2,(
int)nJ+pY); boxj++ )
805 int box_to_add = z_offset + (boxj*(int)nI) + boxi;
814 local_boxes.insert( box_to_add );
820 mLocalBoxes.push_back(local_boxes);
829 mAreLocalBoxesSet=
true;
836 mAreLocalBoxesSet =
true;
841 for (
unsigned i=mMinBoxIndex; i<mMaxBoxIndex+1; i++)
843 std::set<unsigned> local_boxes;
845 local_boxes.insert(i);
850 local_boxes.insert(i-1);
852 if (i+1 != mNumBoxesEachDirection(0))
854 local_boxes.insert(i+1);
857 mLocalBoxes.push_back(local_boxes);
865 unsigned M = mNumBoxesEachDirection(0);
866 unsigned N = mNumBoxesEachDirection(1);
868 std::vector<bool> is_xmin(N*M);
869 std::vector<bool> is_xmax(N*M);
870 std::vector<bool> is_ymin(N*M);
871 std::vector<bool> is_ymax(N*M);
873 for (
unsigned i=0; i<M*N; i++)
875 is_xmin[i] = (i%M==0);
876 is_xmax[i] = ((i+1)%M==0);
877 is_ymin[i] = (i%(M*N)<M);
878 is_ymax[i] = (i%(M*N)>=(N-1)*M);
881 for (
unsigned i=mMinBoxIndex; i<mMaxBoxIndex+1; i++)
883 std::set<unsigned> local_boxes;
885 local_boxes.insert(i);
890 local_boxes.insert(i-1);
896 local_boxes.insert(i+M-1);
903 local_boxes.insert(i+1);
909 local_boxes.insert(i-M+1);
916 local_boxes.insert(i-M);
922 local_boxes.insert(i+(N-1)*M);
929 local_boxes.insert(i+M);
935 local_boxes.insert(i-(N-1)*M);
941 if ( (!is_xmin[i]) && (!is_ymin[i]) )
943 local_boxes.insert(i-1-M);
945 if ( (!is_xmin[i]) && (!is_ymax[i]) )
947 local_boxes.insert(i-1+M);
949 if ( (!is_xmax[i]) && (!is_ymin[i]) )
951 local_boxes.insert(i+1-M);
953 if ( (!is_xmax[i]) && (!is_ymax[i]) )
955 local_boxes.insert(i+1+M);
961 if ( (is_xmin[i]) && (!is_ymin[i]) )
963 local_boxes.insert(i-1);
965 if ( (is_xmin[i]) && (!is_ymax[i]) )
967 local_boxes.insert(i-1+2*M);
969 if ( (is_xmax[i]) && (!is_ymin[i]) )
971 local_boxes.insert(i+1-2*M);
973 if ( (is_xmax[i]) && (!is_ymax[i]) )
975 local_boxes.insert(i+1);
980 if( (is_ymin[i]) && !(is_xmin[i]) )
982 local_boxes.insert(i+(N-1)*M-1);
984 if( (is_ymin[i]) && !(is_xmax[i]) )
986 local_boxes.insert(i+(N-1)*M+1);
988 if( (is_ymax[i]) && !(is_xmin[i]) )
990 local_boxes.insert(i-(N-1)*M-1);
992 if( (is_ymax[i]) && !(is_xmax[i]) )
994 local_boxes.insert(i-(N-1)*M+1);
997 if(mIsPeriodicInX && mIsPeriodicInY)
1001 local_boxes.insert(M*N-1);
1005 local_boxes.insert(M*(N-1));
1007 else if( i==(M*(N-1)) )
1009 local_boxes.insert(M-1);
1011 else if( i==(M*N-1) )
1013 local_boxes.insert(0);
1017 mLocalBoxes.push_back(local_boxes);
1023 mLocalBoxes.clear();
1025 unsigned M = mNumBoxesEachDirection(0);
1026 unsigned N = mNumBoxesEachDirection(1);
1027 unsigned P = mNumBoxesEachDirection(2);
1029 std::vector<bool> is_xmin(N*M*P);
1030 std::vector<bool> is_xmax(N*M*P);
1031 std::vector<bool> is_ymin(N*M*P);
1032 std::vector<bool> is_ymax(N*M*P);
1033 std::vector<bool> is_zmin(N*M*P);
1034 std::vector<bool> is_zmax(N*M*P);
1036 for (
unsigned i=0; i<M*N*P; i++)
1038 is_xmin[i] = (i%M==0);
1039 is_xmax[i] = ((i+1)%M==0);
1040 is_ymin[i] = (i%(M*N)<M);
1041 is_ymax[i] = (i%(M*N)>=(N-1)*M);
1042 is_zmin[i] = (i<M*N);
1043 is_zmax[i] = (i>=M*N*(P-1));
1046 for (
unsigned i=mMinBoxIndex; i<mMaxBoxIndex+1; i++)
1048 std::set<unsigned> local_boxes;
1051 local_boxes.insert(i);
1058 local_boxes.insert(i-1);
1063 local_boxes.insert(i-1-M);
1068 local_boxes.insert(i-1+M);
1073 local_boxes.insert(i-1-M*N);
1078 local_boxes.insert(i-1+M*N);
1085 local_boxes.insert(i+1);
1090 local_boxes.insert(i+1-M);
1095 local_boxes.insert(i+1+M);
1100 local_boxes.insert(i+1-M*N);
1105 local_boxes.insert(i+1+M*N);
1112 local_boxes.insert(i-M);
1117 local_boxes.insert(i-M-M*N);
1122 local_boxes.insert(i-M+M*N);
1129 local_boxes.insert(i+M);
1134 local_boxes.insert(i+M-M*N);
1139 local_boxes.insert(i+M+M*N);
1146 local_boxes.insert(i-N*M);
1152 local_boxes.insert(i+N*M);
1157 if ( (!is_xmin[i]) && (!is_ymin[i]) && (!is_zmin[i]) )
1159 local_boxes.insert(i-1-M-M*N);
1162 if ( (!is_xmin[i]) && (!is_ymin[i]) && (!is_zmax[i]) )
1164 local_boxes.insert(i-1-M+M*N);
1167 if ( (!is_xmin[i]) && (!is_ymax[i]) && (!is_zmin[i]) )
1169 local_boxes.insert(i-1+M-M*N);
1172 if ( (!is_xmin[i]) && (!is_ymax[i]) && (!is_zmax[i]) )
1174 local_boxes.insert(i-1+M+M*N);
1177 if ( (!is_xmax[i]) && (!is_ymin[i]) && (!is_zmin[i]) )
1179 local_boxes.insert(i+1-M-M*N);
1182 if ( (!is_xmax[i]) && (!is_ymin[i]) && (!is_zmax[i]) )
1184 local_boxes.insert(i+1-M+M*N);
1187 if ( (!is_xmax[i]) && (!is_ymax[i]) && (!is_zmin[i]) )
1189 local_boxes.insert(i+1+M-M*N);
1192 if ( (!is_xmax[i]) && (!is_ymax[i]) && (!is_zmax[i]) )
1194 local_boxes.insert(i+1+M+M*N);
1198 if (mIsPeriodicInX && ( is_xmin[i] || is_xmax[i]) )
1202 std::vector< int > z_i_offsets(1,0);
1205 z_i_offsets.push_back(-M*N);
1209 z_i_offsets.push_back(M*N);
1216 for ( std::vector<int>::iterator it = z_i_offsets.begin(); it != z_i_offsets.end(); it++ )
1218 local_boxes.insert( i + (*it) + (M-1) );
1222 local_boxes.insert( i + (*it) - 1 );
1226 local_boxes.insert( i + (*it) + (2*M-1) );
1232 else if ( is_xmax[i] )
1235 for ( std::vector<int>::iterator it = z_i_offsets.begin(); it != z_i_offsets.end(); it++ )
1237 local_boxes.insert( i + (*it) - (M-1) );
1241 local_boxes.insert( i + (*it) - (2*M-1) );
1245 local_boxes.insert( i + (*it) + 1 );
1251 if ( mIsPeriodicInY && (is_ymax[i] || is_ymin[i]) )
1255 std::vector<unsigned> opp_box_i(0);
1258 opp_box_i.push_back(i + (N-1)*M);
1261 opp_box_i.push_back( i - M );
1265 opp_box_i.push_back(i + 2*M*N - M);
1268 else if ( is_ymax[i] )
1270 opp_box_i.push_back( i - (N-1)*M );
1273 opp_box_i.push_back( i - 2*M*N + M );
1277 opp_box_i.push_back( i + M );
1282 for ( std::vector<unsigned>::iterator it_opp_box = opp_box_i.begin(); it_opp_box != opp_box_i.end(); it_opp_box++ )
1284 local_boxes.insert( *it_opp_box );
1287 local_boxes.insert( *it_opp_box - 1 );
1291 local_boxes.insert( *it_opp_box + 1 );
1296 if ( mIsPeriodicInX && mIsPeriodicInY )
1299 if ( is_xmin[i] && is_ymin[i] )
1302 local_boxes.insert(i+M*N-1);
1306 local_boxes.insert(i+2*M*N-1);
1311 local_boxes.insert(i-1);
1314 if ( is_xmax[i] && is_ymin[i] )
1316 local_boxes.insert(i + (N-2)*M + 1);
1319 local_boxes.insert(i + M*N + (N-2)*M + 1);
1324 local_boxes.insert(i-2*M+1);
1327 if ( is_xmin[i] && is_ymax[i] )
1329 local_boxes.insert(i + (N-2)*M - 1);
1333 local_boxes.insert(i - 2*M - 1);
1338 local_boxes.insert(i-2*(N-1)*M-1);
1342 if ( is_xmax[i] && is_ymax[i] )
1347 local_boxes.insert(i + 1);
1352 local_boxes.insert(i - 2*M*N + 1);
1358 if (mIsPeriodicInZ && (is_zmin[i] || is_zmax[i]))
1363 unsigned above_box = i+(P-1)*M*N;
1364 local_boxes.insert(above_box);
1368 local_boxes.insert(above_box-1);
1371 local_boxes.insert(above_box+M-1);
1376 local_boxes.insert(above_box-M-1);
1379 else if ( mIsPeriodicInX )
1382 local_boxes.insert(above_box+M-1);
1385 local_boxes.insert(above_box+2*M-1);
1387 else if ( mIsPeriodicInY )
1390 local_boxes.insert(above_box+M*N-1);
1394 local_boxes.insert(above_box+2*M-1);
1396 else if ( mIsPeriodicInY )
1399 local_boxes.insert(above_box-M*(N-2)-1);
1406 local_boxes.insert(above_box+1);
1409 local_boxes.insert(above_box+M+1);
1413 local_boxes.insert(above_box-M+1);
1416 else if ( mIsPeriodicInX )
1419 local_boxes.insert(above_box - M+1);
1420 if ( mIsPeriodicInY )
1425 local_boxes.insert(above_box+M*(N-2)+1);
1427 else if ( is_ymax[i] )
1430 local_boxes.insert(above_box-M*N+1);
1438 local_boxes.insert(above_box+M);
1440 else if ( mIsPeriodicInY )
1442 local_boxes.insert(above_box-M*(N-1));
1445 local_boxes.insert(above_box-M*(N-1)-1);
1449 local_boxes.insert(above_box-M*(N-1)+1);
1454 local_boxes.insert(above_box-M);
1456 else if ( mIsPeriodicInY )
1458 local_boxes.insert(above_box+M*(N-1));
1461 local_boxes.insert(above_box+M*(N-1)-1);
1465 local_boxes.insert(above_box+M*(N-1)+1);
1469 else if ( is_zmax[i] )
1472 unsigned below_box = i-(P-1)*M*N;
1473 local_boxes.insert(below_box);
1477 local_boxes.insert(below_box-1);
1480 local_boxes.insert(below_box+M-1);
1485 local_boxes.insert(below_box-M-1);
1488 else if ( mIsPeriodicInX )
1491 local_boxes.insert(below_box+M-1);
1492 if ( mIsPeriodicInY )
1497 local_boxes.insert(below_box+M*N-1);
1499 else if ( is_ymax[i] )
1502 local_boxes.insert(below_box-M*(N-2)-1);
1510 local_boxes.insert(below_box+1);
1513 local_boxes.insert(below_box+M+1);
1517 local_boxes.insert(below_box-M+1);
1520 else if ( mIsPeriodicInX )
1523 local_boxes.insert(below_box - M+1);
1524 if ( mIsPeriodicInY )
1529 local_boxes.insert(below_box+M*(N-2)+1);
1531 else if ( is_ymax[i] )
1534 local_boxes.insert(below_box-M*N+1);
1542 local_boxes.insert(below_box+M);
1544 else if ( mIsPeriodicInY )
1546 local_boxes.insert(below_box-M*(N-1));
1549 local_boxes.insert(below_box-M*(N-1)+1);
1553 local_boxes.insert(below_box-M*(N-1)-1);
1558 local_boxes.insert(below_box-M);
1560 else if ( mIsPeriodicInY )
1562 local_boxes.insert(below_box+M*(N-1));
1565 local_boxes.insert(below_box+M*(N-1)+1);
1569 local_boxes.insert(below_box+M*(N-1)-1);
1575 mLocalBoxes.push_back(local_boxes);