509 int new_rows = localDistribution.size();
514 unsigned rows_on_left_process = 0;
515 std::vector<int> node_distr_on_left_process;
517 unsigned num_local_rows = localDistribution.size();
519 MPI_Send(&num_local_rows, 1, MPI_UNSIGNED, proc_right, 123, PETSC_COMM_WORLD);
520 MPI_Recv(&rows_on_left_process, 1, MPI_UNSIGNED, proc_left, 123, PETSC_COMM_WORLD, &status);
522 node_distr_on_left_process.resize(rows_on_left_process > 0 ? rows_on_left_process : 1);
524 MPI_Send(&localDistribution[0], num_local_rows, MPI_INT, proc_right, 123, PETSC_COMM_WORLD);
525 MPI_Recv(&node_distr_on_left_process[0], rows_on_left_process, MPI_INT, proc_left, 123, PETSC_COMM_WORLD, &status);
531 for (
unsigned i=0; i<localDistribution.size(); i++)
533 local_load += localDistribution[i];
535 int load_on_left_proc = 0;
536 for (
unsigned i=0; i<node_distr_on_left_process.size(); i++)
538 load_on_left_proc += node_distr_on_left_process[i];
545 int local_to_left_sq = (local_load - load_on_left_proc) * (local_load - load_on_left_proc);
546 int delta_left = ( (local_load + node_distr_on_left_process[node_distr_on_left_process.size() - 1]) - (load_on_left_proc - node_distr_on_left_process[node_distr_on_left_process.size() - 1]) );
547 delta_left = delta_left*delta_left - local_to_left_sq;
549 int delta_right = ( (local_load - localDistribution[0]) - (load_on_left_proc + localDistribution[0]));
550 delta_right = delta_right*delta_right - local_to_left_sq;
553 int local_change = 0;
554 bool move_left = (!(delta_left > 0) && (node_distr_on_left_process.size() > 1));
560 bool move_right = !(delta_right > 0) && (localDistribution.size() > 2);
566 if (move_left && move_right)
568 local_change = (fabs((
double)delta_right) > fabs((
double)delta_left)) ? -1 : 1;
572 new_rows += local_change;
575 MPI_Send(&local_change, 1, MPI_INT, proc_left, 123, PETSC_COMM_WORLD);
579 int remote_change = 0;
580 MPI_Recv(&remote_change, 1, MPI_INT, proc_right, 123, PETSC_COMM_WORLD, &status);
583 new_rows -= remote_change;
591 if (mAreLocalBoxesSet)
593 EXCEPTION(
"Local Boxes Are Already Set");
605 for (
unsigned global_index = mMinBoxIndex; global_index<mMaxBoxIndex+1; global_index++)
607 std::set<unsigned> local_boxes;
610 local_boxes.insert(global_index);
613 bool right = (global_index==mNumBoxesEachDirection(0)-1);
614 bool left = (global_index == 0);
615 bool proc_left = (global_index == mpDistributedBoxStackFactory->GetLow());
620 local_boxes.insert(global_index+1);
623 if (proc_left && !left)
625 local_boxes.insert(global_index-1);
628 mLocalBoxes.push_back(local_boxes);
638 unsigned j_start = CalculateGridIndices(mMinBoxIndex)(1);
639 unsigned j_end = CalculateGridIndices(mMaxBoxIndex)(1);
641 unsigned nI = mNumBoxesEachDirection(0);
642 unsigned nJ = mNumBoxesEachDirection(1);
644 unsigned bottom_proc = mpDistributedBoxStackFactory->GetLow();
645 unsigned top_proc = mpDistributedBoxStackFactory->GetHigh();
648 for (
unsigned j = j_start; j < (j_end+1); j++ )
651 for (
unsigned i = 0; i < nI; i++ )
653 std::set<unsigned> local_boxes;
660 int dj = -1 * (int)(j == bottom_proc && (j > 0 || (mIsPeriodicInY && top_proc < nJ)) );
663 for (; dj < std::min((
int)nJ-j_mod,(
int)2); dj++ )
667 if ( mIsPeriodicInY && j == 0 && dj < 1 && top_proc < nJ )
669 j_mod_2 = ( dj < 0 ) ? nJ : 0;
674 int boxi = std::max((
int) i-1*std::abs(dj),(
int) 0);
675 for ( ; boxi < std::min((
int)i+2,(
int)nI); boxi++ )
677 local_boxes.insert( (j_mod+dj+j_mod_2)*nI + boxi );
680 if ( i==(nI-1) && mIsPeriodicInX )
682 local_boxes.insert( (j_mod+dj+j_mod_2)*nI );
686 if ( mIsPeriodicInY && j == (nJ-1) && dj == 0 )
692 if ( mIsPeriodicInX && i == 0 )
696 local_boxes.insert( (j+2)*nI - 1 );
697 if ( j==0 && mIsPeriodicInY && top_proc < nJ )
699 local_boxes.insert( nI*nJ - 1 );
702 else if ( mIsPeriodicInY )
704 local_boxes.insert( nI-1 );
708 if ( j == bottom_proc && j > 0 )
710 local_boxes.insert( j*nI - 1 );
715 mLocalBoxes.push_back(local_boxes);
727 unsigned k_start = CalculateGridIndices(mMinBoxIndex)(2);
728 unsigned k_end = CalculateGridIndices(mMaxBoxIndex)(2);
731 unsigned nI = mNumBoxesEachDirection(0);
732 unsigned nJ = mNumBoxesEachDirection(1);
733 unsigned nK = mNumBoxesEachDirection(2);
736 unsigned bottom_proc = mpDistributedBoxStackFactory->GetLow();
737 unsigned top_proc = mpDistributedBoxStackFactory->GetHigh();
740 for (
unsigned k = k_start; k <= k_end; k++ )
743 for (
unsigned j = 0; j < nJ; j++ )
746 for (
unsigned i = 0; i < nI; i++ )
748 std::set<unsigned> local_boxes;
751 unsigned z_offset = k*nI*nJ;
754 for (
int dj = 0; dj < std::min((
int)nJ-j_mod,2); dj++ )
756 for (
int boxi = std::max((
int)i-1*dj,0);
757 boxi < std::min((
int)i+2,(
int)nI); boxi++ )
759 local_boxes.insert( z_offset + (j_mod+dj)*nI + boxi );
761 if ( i==(nI-1) && mIsPeriodicInX )
763 local_boxes.insert( z_offset + (j_mod+dj)*nI );
765 if ( mIsPeriodicInY && j == (nJ-1) )
771 if ( mIsPeriodicInX && i == 0 )
775 local_boxes.insert( z_offset + (j+2)*nI - 1 );
777 else if ( mIsPeriodicInY )
779 local_boxes.insert( z_offset + nI-1 );
784 std::vector<unsigned> k_offset;
787 k_offset.push_back(k+1);
789 if ( k == bottom_proc && k > 0 )
792 k_offset.push_back(k-1);
794 if ( mIsPeriodicInZ && k == (nK-1) )
796 k_offset.push_back(0);
798 else if ( mIsPeriodicInZ && k==0 && (top_proc < nK) )
800 k_offset.push_back(nK-1);
803 for ( std::vector<unsigned>::iterator k_offset_it = k_offset.begin(); k_offset_it != k_offset.end(); ++k_offset_it )
805 z_offset = (*k_offset_it)*nI*nJ;
807 int pX = (int) mIsPeriodicInX;
808 int pY = (int) mIsPeriodicInY;
809 for (
int boxi = std::max((
int)i-1,-1*pX); boxi < std::min((
int)i+2,(
int)nI+pX); boxi++ )
811 for (
int boxj = std::max((
int)j-1,-1*pY); boxj < std::min((
int)j+2,(
int)nJ+pY); boxj++ )
813 int box_to_add = z_offset + (boxj*(int)nI) + boxi;
822 local_boxes.insert( box_to_add );
828 mLocalBoxes.push_back(local_boxes);
837 mAreLocalBoxesSet=
true;
844 mAreLocalBoxesSet =
true;
849 for (
unsigned i=mMinBoxIndex; i<mMaxBoxIndex+1; i++)
851 std::set<unsigned> local_boxes;
853 local_boxes.insert(i);
858 local_boxes.insert(i-1);
860 if (i+1 != mNumBoxesEachDirection(0))
862 local_boxes.insert(i+1);
865 mLocalBoxes.push_back(local_boxes);
873 unsigned M = mNumBoxesEachDirection(0);
874 unsigned N = mNumBoxesEachDirection(1);
876 std::vector<bool> is_xmin(N*M);
877 std::vector<bool> is_xmax(N*M);
878 std::vector<bool> is_ymin(N*M);
879 std::vector<bool> is_ymax(N*M);
881 for (
unsigned i=0; i<M*N; i++)
883 is_xmin[i] = (i%M==0);
884 is_xmax[i] = ((i+1)%M==0);
885 is_ymin[i] = (i%(M*N)<M);
886 is_ymax[i] = (i%(M*N)>=(N-1)*M);
889 for (
unsigned i=mMinBoxIndex; i<mMaxBoxIndex+1; i++)
891 std::set<unsigned> local_boxes;
893 local_boxes.insert(i);
898 local_boxes.insert(i-1);
904 local_boxes.insert(i+M-1);
911 local_boxes.insert(i+1);
917 local_boxes.insert(i-M+1);
924 local_boxes.insert(i-M);
930 local_boxes.insert(i+(N-1)*M);
937 local_boxes.insert(i+M);
943 local_boxes.insert(i-(N-1)*M);
949 if ( (!is_xmin[i]) && (!is_ymin[i]) )
951 local_boxes.insert(i-1-M);
953 if ( (!is_xmin[i]) && (!is_ymax[i]) )
955 local_boxes.insert(i-1+M);
957 if ( (!is_xmax[i]) && (!is_ymin[i]) )
959 local_boxes.insert(i+1-M);
961 if ( (!is_xmax[i]) && (!is_ymax[i]) )
963 local_boxes.insert(i+1+M);
969 if ( (is_xmin[i]) && (!is_ymin[i]) )
971 local_boxes.insert(i-1);
973 if ( (is_xmin[i]) && (!is_ymax[i]) )
975 local_boxes.insert(i-1+2*M);
977 if ( (is_xmax[i]) && (!is_ymin[i]) )
979 local_boxes.insert(i+1-2*M);
981 if ( (is_xmax[i]) && (!is_ymax[i]) )
983 local_boxes.insert(i+1);
988 if( (is_ymin[i]) && !(is_xmin[i]) )
990 local_boxes.insert(i+(N-1)*M-1);
992 if( (is_ymin[i]) && !(is_xmax[i]) )
994 local_boxes.insert(i+(N-1)*M+1);
996 if( (is_ymax[i]) && !(is_xmin[i]) )
998 local_boxes.insert(i-(N-1)*M-1);
1000 if( (is_ymax[i]) && !(is_xmax[i]) )
1002 local_boxes.insert(i-(N-1)*M+1);
1005 if(mIsPeriodicInX && mIsPeriodicInY)
1009 local_boxes.insert(M*N-1);
1013 local_boxes.insert(M*(N-1));
1015 else if( i==(M*(N-1)) )
1017 local_boxes.insert(M-1);
1019 else if( i==(M*N-1) )
1021 local_boxes.insert(0);
1025 mLocalBoxes.push_back(local_boxes);
1031 mLocalBoxes.clear();
1033 unsigned M = mNumBoxesEachDirection(0);
1034 unsigned N = mNumBoxesEachDirection(1);
1035 unsigned P = mNumBoxesEachDirection(2);
1037 std::vector<bool> is_xmin(N*M*P);
1038 std::vector<bool> is_xmax(N*M*P);
1039 std::vector<bool> is_ymin(N*M*P);
1040 std::vector<bool> is_ymax(N*M*P);
1041 std::vector<bool> is_zmin(N*M*P);
1042 std::vector<bool> is_zmax(N*M*P);
1044 for (
unsigned i=0; i<M*N*P; i++)
1046 is_xmin[i] = (i%M==0);
1047 is_xmax[i] = ((i+1)%M==0);
1048 is_ymin[i] = (i%(M*N)<M);
1049 is_ymax[i] = (i%(M*N)>=(N-1)*M);
1050 is_zmin[i] = (i<M*N);
1051 is_zmax[i] = (i>=M*N*(P-1));
1054 for (
unsigned i=mMinBoxIndex; i<mMaxBoxIndex+1; i++)
1056 std::set<unsigned> local_boxes;
1059 local_boxes.insert(i);
1066 local_boxes.insert(i-1);
1071 local_boxes.insert(i-1-M);
1076 local_boxes.insert(i-1+M);
1081 local_boxes.insert(i-1-M*N);
1086 local_boxes.insert(i-1+M*N);
1093 local_boxes.insert(i+1);
1098 local_boxes.insert(i+1-M);
1103 local_boxes.insert(i+1+M);
1108 local_boxes.insert(i+1-M*N);
1113 local_boxes.insert(i+1+M*N);
1120 local_boxes.insert(i-M);
1125 local_boxes.insert(i-M-M*N);
1130 local_boxes.insert(i-M+M*N);
1137 local_boxes.insert(i+M);
1142 local_boxes.insert(i+M-M*N);
1147 local_boxes.insert(i+M+M*N);
1154 local_boxes.insert(i-N*M);
1160 local_boxes.insert(i+N*M);
1165 if ( (!is_xmin[i]) && (!is_ymin[i]) && (!is_zmin[i]) )
1167 local_boxes.insert(i-1-M-M*N);
1170 if ( (!is_xmin[i]) && (!is_ymin[i]) && (!is_zmax[i]) )
1172 local_boxes.insert(i-1-M+M*N);
1175 if ( (!is_xmin[i]) && (!is_ymax[i]) && (!is_zmin[i]) )
1177 local_boxes.insert(i-1+M-M*N);
1180 if ( (!is_xmin[i]) && (!is_ymax[i]) && (!is_zmax[i]) )
1182 local_boxes.insert(i-1+M+M*N);
1185 if ( (!is_xmax[i]) && (!is_ymin[i]) && (!is_zmin[i]) )
1187 local_boxes.insert(i+1-M-M*N);
1190 if ( (!is_xmax[i]) && (!is_ymin[i]) && (!is_zmax[i]) )
1192 local_boxes.insert(i+1-M+M*N);
1195 if ( (!is_xmax[i]) && (!is_ymax[i]) && (!is_zmin[i]) )
1197 local_boxes.insert(i+1+M-M*N);
1200 if ( (!is_xmax[i]) && (!is_ymax[i]) && (!is_zmax[i]) )
1202 local_boxes.insert(i+1+M+M*N);
1206 if (mIsPeriodicInX && ( is_xmin[i] || is_xmax[i]) )
1210 std::vector< int > z_i_offsets(1,0);
1213 z_i_offsets.push_back(-M*N);
1217 z_i_offsets.push_back(M*N);
1224 for ( std::vector<int>::iterator it = z_i_offsets.begin(); it != z_i_offsets.end(); it++ )
1226 local_boxes.insert( i + (*it) + (M-1) );
1230 local_boxes.insert( i + (*it) - 1 );
1234 local_boxes.insert( i + (*it) + (2*M-1) );
1240 else if ( is_xmax[i] )
1243 for ( std::vector<int>::iterator it = z_i_offsets.begin(); it != z_i_offsets.end(); it++ )
1245 local_boxes.insert( i + (*it) - (M-1) );
1249 local_boxes.insert( i + (*it) - (2*M-1) );
1253 local_boxes.insert( i + (*it) + 1 );
1259 if ( mIsPeriodicInY && (is_ymax[i] || is_ymin[i]) )
1263 std::vector<unsigned> opp_box_i(0);
1266 opp_box_i.push_back(i + (N-1)*M);
1269 opp_box_i.push_back( i - M );
1273 opp_box_i.push_back(i + 2*M*N - M);
1276 else if ( is_ymax[i] )
1278 opp_box_i.push_back( i - (N-1)*M );
1281 opp_box_i.push_back( i - 2*M*N + M );
1285 opp_box_i.push_back( i + M );
1290 for ( std::vector<unsigned>::iterator it_opp_box = opp_box_i.begin(); it_opp_box != opp_box_i.end(); it_opp_box++ )
1292 local_boxes.insert( *it_opp_box );
1295 local_boxes.insert( *it_opp_box - 1 );
1299 local_boxes.insert( *it_opp_box + 1 );
1304 if ( mIsPeriodicInX && mIsPeriodicInY )
1307 if ( is_xmin[i] && is_ymin[i] )
1310 local_boxes.insert(i+M*N-1);
1314 local_boxes.insert(i+2*M*N-1);
1319 local_boxes.insert(i-1);
1322 if ( is_xmax[i] && is_ymin[i] )
1324 local_boxes.insert(i + (N-2)*M + 1);
1327 local_boxes.insert(i + M*N + (N-2)*M + 1);
1332 local_boxes.insert(i-2*M+1);
1335 if ( is_xmin[i] && is_ymax[i] )
1337 local_boxes.insert(i + (N-2)*M - 1);
1341 local_boxes.insert(i - 2*M - 1);
1346 local_boxes.insert(i-2*(N-1)*M-1);
1350 if ( is_xmax[i] && is_ymax[i] )
1355 local_boxes.insert(i + 1);
1360 local_boxes.insert(i - 2*M*N + 1);
1366 if (mIsPeriodicInZ && (is_zmin[i] || is_zmax[i]))
1371 unsigned above_box = i+(P-1)*M*N;
1372 local_boxes.insert(above_box);
1376 local_boxes.insert(above_box-1);
1379 local_boxes.insert(above_box+M-1);
1384 local_boxes.insert(above_box-M-1);
1387 else if ( mIsPeriodicInX )
1390 local_boxes.insert(above_box+M-1);
1393 local_boxes.insert(above_box+2*M-1);
1395 else if ( mIsPeriodicInY )
1398 local_boxes.insert(above_box+M*N-1);
1402 local_boxes.insert(above_box+2*M-1);
1404 else if ( mIsPeriodicInY )
1407 local_boxes.insert(above_box-M*(N-2)-1);
1414 local_boxes.insert(above_box+1);
1417 local_boxes.insert(above_box+M+1);
1421 local_boxes.insert(above_box-M+1);
1424 else if ( mIsPeriodicInX )
1427 local_boxes.insert(above_box - M+1);
1428 if ( mIsPeriodicInY )
1433 local_boxes.insert(above_box+M*(N-2)+1);
1435 else if ( is_ymax[i] )
1438 local_boxes.insert(above_box-M*N+1);
1446 local_boxes.insert(above_box+M);
1448 else if ( mIsPeriodicInY )
1450 local_boxes.insert(above_box-M*(N-1));
1453 local_boxes.insert(above_box-M*(N-1)-1);
1457 local_boxes.insert(above_box-M*(N-1)+1);
1462 local_boxes.insert(above_box-M);
1464 else if ( mIsPeriodicInY )
1466 local_boxes.insert(above_box+M*(N-1));
1469 local_boxes.insert(above_box+M*(N-1)-1);
1473 local_boxes.insert(above_box+M*(N-1)+1);
1477 else if ( is_zmax[i] )
1480 unsigned below_box = i-(P-1)*M*N;
1481 local_boxes.insert(below_box);
1485 local_boxes.insert(below_box-1);
1488 local_boxes.insert(below_box+M-1);
1493 local_boxes.insert(below_box-M-1);
1496 else if ( mIsPeriodicInX )
1499 local_boxes.insert(below_box+M-1);
1500 if ( mIsPeriodicInY )
1505 local_boxes.insert(below_box+M*N-1);
1507 else if ( is_ymax[i] )
1510 local_boxes.insert(below_box-M*(N-2)-1);
1518 local_boxes.insert(below_box+1);
1521 local_boxes.insert(below_box+M+1);
1525 local_boxes.insert(below_box-M+1);
1528 else if ( mIsPeriodicInX )
1531 local_boxes.insert(below_box - M+1);
1532 if ( mIsPeriodicInY )
1537 local_boxes.insert(below_box+M*(N-2)+1);
1539 else if ( is_ymax[i] )
1542 local_boxes.insert(below_box-M*N+1);
1550 local_boxes.insert(below_box+M);
1552 else if ( mIsPeriodicInY )
1554 local_boxes.insert(below_box-M*(N-1));
1557 local_boxes.insert(below_box-M*(N-1)+1);
1561 local_boxes.insert(below_box-M*(N-1)-1);
1566 local_boxes.insert(below_box-M);
1568 else if ( mIsPeriodicInY )
1570 local_boxes.insert(below_box+M*(N-1));
1573 local_boxes.insert(below_box+M*(N-1)+1);
1577 local_boxes.insert(below_box+M*(N-1)-1);
1583 mLocalBoxes.push_back(local_boxes);