Actual source code: jostle.c

 2:  #include src/mat/impls/adj/mpi/mpiadj.h

  4: #ifdef PETSC_HAVE_UNISTD_H
  5: #include <unistd.h>
  6: #endif

  8: #ifdef PETSC_HAVE_STDLIB_H
  9: #include <stdlib.h>
 10: #endif

 12: #include "petscfix.h"

 14: EXTERN_C_BEGIN

 16: #include "jostle.h"
 17: /* this function is not declared in 'jostle.h' */
 18: extern void pjostle_comm(MPI_Comm * comm);

 20: EXTERN_C_END

 22: typedef struct {
 23:     int output;
 24:     int coarse_seq;
 25:     int nbvtxcoarsed;           /* number of vertices for the coarse graph */
 26:     char *mesg_log;
 27: } MatPartitioning_Jostle;

 29: #define SIZE_LOG 10000          /* size of buffer for msg_log */

 33: static int MatPartitioningApply_Jostle(MatPartitioning part, IS * partitioning)
 34: {
 35:     int ierr, size, rank, i;
 36:     Mat mat = part->adj, matMPI;
 37:     Mat_MPIAdj *adj = (Mat_MPIAdj *) mat->data;
 38:     MatPartitioning_Jostle *jostle_struct =
 39:         (MatPartitioning_Jostle *) part->data;
 40:     PetscTruth flg;
 41: #ifdef PETSC_HAVE_UNISTD_H
 42:     int fd_stdout, fd_pipe[2], count;
 43: #endif


 47:     /* check that the number of partitions is equal to the number of processors */
 48:     MPI_Comm_rank(mat->comm, &rank);
 49:     MPI_Comm_size(mat->comm, &size);
 50:     if (part->n != size) {
 51:         SETERRQ(PETSC_ERR_SUP, "Supports exactly one domain per processor");
 52:     }

 54:     /* convert adjacency matrix to MPIAdj if needed*/
 55:     PetscTypeCompare((PetscObject) mat, MATMPIADJ, &flg);
 56:     if (!flg) {
 57:         MatConvert(mat, MATMPIADJ, &matMPI);
 58:     } else
 59:         matMPI = mat;

 61:     adj = (Mat_MPIAdj *) matMPI->data;  /* adj contains adjacency graph */
 62:     {
 63:         /* definition of Jostle library arguments */
 64:         int nnodes = matMPI->M; /* number of vertices in full graph */
 65:         int offset = 0;         /* 0 for C array indexing */
 66:         int core = matMPI->m;
 67:         int halo = 0;           /* obsolete with contiguous format */
 68:         int *index_jostle;      /* contribution of each processor */
 69:         int nparts = part->n;
 70:         int *part_wt = NULL;

 72:         int *partition;         /* set number of each vtx (length n) */
 73:         int *degree;            /* degree for each core nodes */
 74:         int *edges = adj->j;
 75:         int *node_wt = NULL;    /* nodes weights */
 76:         int *edge_wt = NULL;    /* edges weights */
 77:         double *coords = NULL;  /* not used (cf jostle documentation) */

 79:         int local_nedges = adj->nz;
 80:         int dimension = 0;      /* not used */
 81:         int output_level = jostle_struct->output;
 82:         char env_str[256];

 84:         /* allocate index_jostle */
 85:         PetscMalloc(nparts * sizeof(int), &index_jostle);

 87:         /* compute number of core nodes for each one */
 88:         for (i = 0; i < nparts - 1; i++)
 89:             index_jostle[i] = adj->rowners[i + 1] - adj->rowners[i];
 90:         index_jostle[nparts - 1] = nnodes - adj->rowners[nparts - 1];

 92:         /* allocate the partition vector */
 93:         PetscMalloc(core * sizeof(int), &partition);

 95:         /* build the degree vector and the local_nedges value */
 96:         PetscMalloc(core * sizeof(int), &degree);
 97:         for (i = 0; i < core; i++)
 98:             degree[i] = adj->i[i + 1] - adj->i[i];

100:         /* library call */
101:         pjostle_init(&size, &rank);
102:         pjostle_comm(&matMPI->comm);
103:         jostle_env("format = contiguous");
104:         jostle_env("timer = off");

106:         sprintf(env_str, "threshold = %d", jostle_struct->nbvtxcoarsed);
107:         jostle_env(env_str);

109:         if (jostle_struct->coarse_seq)
110:             jostle_env("matching = local");

112:         /* redirect output */
113: #ifdef PETSC_HAVE_UNISTD_H
114:         fd_stdout = dup(1);
115:         pipe(fd_pipe);
116:         close(1);
117:         dup2(fd_pipe[1], 1);
118: #endif

120:         pjostle(&nnodes, &offset, &core, &halo, index_jostle, degree, node_wt,
121:             partition, &local_nedges, edges, edge_wt, &nparts,
122:             part_wt, &output_level, &dimension, coords);

124:         printf("Jostle Partitioner statistics\ncut : %d, balance : %f, runtime : %f, mem used : %d\n",
125:             jostle_cut(), jostle_bal(), jostle_tim(), jostle_mem());

127: #ifdef PETSC_HAVE_UNISTD_H
128:         PetscMalloc(SIZE_LOG * sizeof(char), &(jostle_struct->mesg_log));
129:         fflush(stdout);
130:         count = read(fd_pipe[0], jostle_struct->mesg_log, (SIZE_LOG - 1) * sizeof(char));
131:         if (count < 0)
132:             count = 0;
133:         jostle_struct->mesg_log[count] = 0;
134:         close(1);
135:         dup2(fd_stdout, 1);
136:         close(fd_stdout);
137:         close(fd_pipe[0]);
138:         close(fd_pipe[1]);
139: #endif

141:         /* We free the memory used by jostle */
142:         PetscFree(index_jostle);
143:         PetscFree(degree);

145:         /* Creation of the index set */
146:         ISCreateGeneral(part->comm, mat->m, partition, partitioning);

148:         if (matMPI != mat) {
149:             MatDestroy(matMPI);
150:         }

152:         PetscFree(partition);
153:     }

155:     return(0);
156: }


161: int MatPartitioningView_Jostle(MatPartitioning part, PetscViewer viewer)
162: {

164:     MatPartitioning_Jostle *jostle_struct =
165:         (MatPartitioning_Jostle *) part->data;
167:     PetscTruth isascii;


171:     PetscTypeCompare((PetscObject) viewer, PETSC_VIEWER_ASCII, &isascii);
172:     if (isascii) {
173:         if (jostle_struct->mesg_log) {
174:             PetscViewerASCIIPrintf(viewer, "%s\n", jostle_struct->mesg_log);
175:         }
176:     } else {
177:         SETERRQ1(1, "Viewer type %s not supported for this Jostle partitioner",
178:             ((PetscObject) viewer)->type_name);
179:     }

181:     return(0);
182: }

186: /*@
187:     MatPartitioningJostleSetCoarseLevel - Set the coarse level 
188:     
189:   Input Parameter:
190: .  part - the partitioning context
191: .  level - the coarse level in range [0.0,1.0]

193:    Level: advanced

195: @*/
196: int MatPartitioningJostleSetCoarseLevel(MatPartitioning part, PetscReal level)
197: {
198:     MatPartitioning_Jostle *jostle_struct =
199:         (MatPartitioning_Jostle *) part->data;


203:     if (level < 0.0 || level > 1.0) {
204:         SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,
205:             "Jostle: level of coarsening out of range [0.0-1.0]");
206:     } else
207:         jostle_struct->nbvtxcoarsed = part->adj->N * level;

209:     if (jostle_struct->nbvtxcoarsed < 20)
210:         jostle_struct->nbvtxcoarsed = 20;

212:     return(0);
213: }

217: /*@
218:      MatPartitioningJostleSetCoarseSequential - Use the sequential code to 
219:          do the partitioning of the coarse grid.

221:   Input Parameter:
222: .  part - the partitioning context

224:    Level: advanced

226: @*/
227: int MatPartitioningJostleSetCoarseSequential(MatPartitioning part)
228: {
229:     MatPartitioning_Jostle *jostle_struct =
230:         (MatPartitioning_Jostle *) part->data;
232:     jostle_struct->coarse_seq = 1;
233:     return(0);
234: }

238: int MatPartitioningSetFromOptions_Jostle(MatPartitioning part)
239: {
241:     PetscTruth flag;
242:     PetscReal level;

245:     PetscOptionsHead("Set Jostle partitioning options");

247:     PetscOptionsReal("-mat_partitioning_jostle_coarse_level",
248:         "Coarse level", "MatPartitioningJostleSetCoarseLevel", 0,
249:         &level, &flag);
250:     if (flag)
251:         MatPartitioningJostleSetCoarseLevel(part, level);

253:     PetscOptionsName("-mat_partitioning_jostle_coarse_sequential",
254:         "Use sequential coarse partitioner",
255:         "MatPartitioningJostleSetCoarseSequential", &flag);
256:     if (flag) {
257:         MatPartitioningJostleSetCoarseSequential(part);
258:     }

260:     PetscOptionsTail();
261:     return(0);
262: }


267: int MatPartitioningDestroy_Jostle(MatPartitioning part)
268: {
269:     MatPartitioning_Jostle *jostle_struct =
270:         (MatPartitioning_Jostle *) part->data;


275:     if (jostle_struct->mesg_log) {
276:         PetscFree(jostle_struct->mesg_log);
277:     }
278:     PetscFree(jostle_struct);

280:     return(0);
281: }

283: EXTERN_C_BEGIN
286: int MatPartitioningCreate_Jostle(MatPartitioning part)
287: {
289:     MatPartitioning_Jostle *jostle_struct;

292:     PetscNew(MatPartitioning_Jostle, &jostle_struct);

294:     jostle_struct->nbvtxcoarsed = 20;
295:     jostle_struct->output = 0;
296:     jostle_struct->coarse_seq = 0;
297:     jostle_struct->mesg_log = NULL;

299:     part->ops->apply = MatPartitioningApply_Jostle;
300:     part->ops->view = MatPartitioningView_Jostle;
301:     part->ops->destroy = MatPartitioningDestroy_Jostle;
302:     part->ops->setfromoptions = MatPartitioningSetFromOptions_Jostle;
303:     part->data = (void *) jostle_struct;

305:     return(0);
306: }

308: EXTERN_C_END