Actual source code: scotch.c

  1: #include "src/mat/impls/adj/mpi/mpiadj.h"       /*I "petscmat.h" I*/

  3: #ifdef PETSC_HAVE_UNISTD_H
  4: #include <unistd.h>
  5: #endif

  7: #ifdef PETSC_HAVE_STDLIB_H
  8: #include <stdlib.h>
  9: #endif

 11: #include "petscfix.h"

 13: /* 
 14:    Currently using Scotch-3.4
 15: */
 16: EXTERN_C_BEGIN
 17: #include "scotch.h"
 18: EXTERN_C_END

 20: typedef struct {
 21:     char arch[PETSC_MAX_PATH_LEN];
 22:     int multilevel;
 23:     char strategy[30];
 24:     int global_method;          /* global method */
 25:     int local_method;           /* local method */
 26:     int nbvtxcoarsed;           /* number of vertices for the coarse graph */
 27:     int map;                    /* to know if we map on archptr or just partionate the graph */
 28:     char *mesg_log;
 29:     char host_list[PETSC_MAX_PATH_LEN];
 30: } MatPartitioning_Scotch;

 32: #define SIZE_LOG 10000          /* size of buffer for msg_log */

 36: static int MatPartitioningApply_Scotch(MatPartitioning part, IS * partitioning)
 37: {
 38:     int ierr, *parttab, *locals = NULL, rank, i, size, j;
 39:     Mat mat = part->adj, matMPI, matSeq;
 40:     int nb_locals = mat->m;
 41:     Mat_MPIAdj *adj = (Mat_MPIAdj *) mat->data;
 42:     MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;
 43:     PetscTruth flg;
 44: #ifdef PETSC_HAVE_UNISTD_H
 45:     int fd_stdout, fd_pipe[2], count;
 46: #endif


 50:     /* check if the matrix is sequential, use MatGetSubMatrices if necessary */
 51:     MPI_Comm_size(mat->comm, &size);
 52:     PetscTypeCompare((PetscObject) mat, MATMPIADJ, &flg);
 53:     if (size > 1) {
 54:         int M, N;
 55:         IS isrow, iscol;
 56:         Mat *A;

 58:         if (flg) {
 59:             SETERRQ(0, "Distributed matrix format MPIAdj is not supported for sequential partitioners");
 60:         }
 61:         PetscPrintf(part->comm, "Converting distributed matrix to sequential: this could be a performance loss\n");

 63:         MatGetSize(mat, &M, &N);
 64:         ISCreateStride(PETSC_COMM_SELF, M, 0, 1, &isrow);
 65:         ISCreateStride(PETSC_COMM_SELF, N, 0, 1, &iscol);
 66:         MatGetSubMatrices(mat, 1, &isrow, &iscol, MAT_INITIAL_MATRIX, &A);
 67:         ISDestroy(isrow);
 68:         ISDestroy(iscol);
 69:         matSeq = *A;
 70:     } else
 71:         matSeq = mat;

 73:     /* convert the the matrix to MPIADJ type if necessary */
 74:     if (!flg) {
 75:         MatConvert(matSeq, MATMPIADJ, &matMPI);
 76:     } else
 77:         matMPI = matSeq;

 79:     adj = (Mat_MPIAdj *) matMPI->data;  /* finaly adj contains adjacency graph */

 81:     MPI_Comm_rank(part->comm, &rank);

 83:     {
 84:         /* definition of Scotch library arguments */
 85:         SCOTCH_Strat stratptr;  /* scotch strategy */
 86:         SCOTCH_Graph grafptr;   /* scotch graph */
 87:         SCOTCH_Mapping mappptr; /* scotch mapping format */
 88:         int vertnbr = mat->M;   /* number of vertices in full graph */
 89:         int *verttab = adj->i;  /* start of edge list for each vertex */
 90:         int *edgetab = adj->j;  /* edge list data */
 91:         int edgenbr = adj->nz;  /* number of edges */
 92:         int *velotab = NULL;    /* not used by petsc interface */
 93:         int *vlbltab = NULL;
 94:         int *edlotab = NULL;
 95:         int baseval = 0;        /* 0 for C array indexing */
 96:         int flagval = 3;        /* (cf doc scotch no weight edge & vertices) */
 97:         char strategy[256];

 99:         PetscMalloc((mat->M) * sizeof(int), &parttab);

101:         /* redirect output to buffer scotch -> mesg_log */
102: #ifdef PETSC_HAVE_UNISTD_H
103:         fd_stdout = dup(1);
104:         pipe(fd_pipe);
105:         close(1);
106:         dup2(fd_pipe[1], 1);
107:         PetscMalloc(SIZE_LOG * sizeof(char), &(scotch->mesg_log));
108: #endif

110:         /* library call */

112:         /* Construction of the scotch graph object */
113:         SCOTCH_graphInit(&grafptr);
114:         SCOTCH_graphBuild(&grafptr, vertnbr, verttab, velotab,
115:             vlbltab, edgenbr, edgetab, edlotab, baseval, flagval);
116:         SCOTCH_graphCheck(&grafptr);

118:         /* Construction of the strategy */
119:         if (scotch->strategy[0] != 0)   /* strcmp(scotch->strategy,"") */
120:             PetscStrcpy(strategy, scotch->strategy);
121:         else {
122:             PetscStrcpy(strategy, "b{strat=");

124:             if (scotch->multilevel) {
125:                 /* PetscStrcat(strategy,"m{vert=");
126:                    sprintf(strategy+strlen(strategy),"%d",scotch->nbvtxcoarsed);
127:                    PetscStrcat(strategy,",asc="); */
128:                 sprintf(strategy, "b{strat=m{vert=%d,asc=",
129:                     scotch->nbvtxcoarsed);
130:             } else
131:                 PetscStrcpy(strategy, "b{strat=");

133:             switch (scotch->global_method) {
134:             case MP_SCOTCH_GREEDY:
135:                 PetscStrcat(strategy, "h");
136:                 break;
137:             case MP_SCOTCH_GPS:
138:                 PetscStrcat(strategy, "g");
139:                 break;
140:             case MP_SCOTCH_GR_GPS:
141:                 PetscStrcat(strategy, "g|h");
142:             }

144:             switch (scotch->local_method) {
145:             case MP_SCOTCH_KERNIGHAN_LIN:
146:                 if (scotch->multilevel)
147:                     PetscStrcat(strategy, ",low=f}");
148:                 else
149:                     PetscStrcat(strategy, " f");
150:                 break;
151:             case MP_SCOTCH_NONE:
152:                 if (scotch->multilevel)
153:                     PetscStrcat(strategy, ",asc=x}");
154:             default:
155:                 break;
156:             }

158:             PetscStrcat(strategy, " x}");
159:         }

161:         PetscPrintf(part->comm, "strategy=[%s]\n", strategy);

163:         SCOTCH_stratInit(&stratptr);
164:         SCOTCH_stratMap(&stratptr, strategy);

166:         /* check for option mapping */
167:         if (!scotch->map) {
168:             SCOTCH_graphPart(&grafptr, &stratptr, part->n, parttab);
169:             PetscPrintf(PETSC_COMM_SELF, "Partition simple without mapping\n");
170:         } else {
171:             SCOTCH_Graph grafarch;
172:             SCOTCH_Num *listtab;
173:             SCOTCH_Num listnbr = 0;
174:             SCOTCH_Arch archptr;        /* file in scotch architecture format */
175:             SCOTCH_Strat archstrat;
176:             int arch_total_size, *parttab_tmp;
177:             int cpt;
178:             char buf[256];
179:             FILE *file1, *file2;
180:             char host_buf[256];

182:             /* generate the graph that represents the arch */
183:             file1 = fopen(scotch->arch, "r");
184:             if (!file1)
185:                 SETERRQ1(1, "Scotch: unable to open architecture file %s", scotch->arch);

187:             SCOTCH_graphInit(&grafarch);
188:             SCOTCH_graphLoad(&grafarch, file1, baseval, 3);

190:             SCOTCH_graphCheck(&grafarch);
191:             SCOTCH_graphSize(&grafarch, &arch_total_size, &cpt);

193:             fclose(file1);
194:             printf("total size = %d\n", arch_total_size);

196:             /* generate the list of nodes currently working */
197:             PetscGetHostName(host_buf, 256);
198:             PetscStrlen(host_buf, &j);

200:             file2 = fopen(scotch->host_list, "r");
201:             if (!file2)
202:                 SETERRQ1(1, "Scotch: unable to open host list file %s", scotch->host_list);

204:             i = -1;
205:             flg = 0;
206:             while (!feof(file2) && !flg) {
207:                 i++;
208:                 fgets(buf, 256, file2);
209:                 PetscStrncmp(buf, host_buf, j, &flg);
210:             }
211:             fclose(file2);
212:             if (!flg) {
213:                 SETERRQ1(1, "Scotch: unable to find '%s' in host list file", host_buf);
214:             }

216:             listnbr = size;
217:             PetscMalloc(sizeof(SCOTCH_Num) * listnbr, &listtab);

219:             MPI_Allgather(&i, 1, MPI_INT, listtab, 1, MPI_INT, part->comm);

221:             printf("listnbr = %d, listtab = ", listnbr);
222:             for (i = 0; i < listnbr; i++)
223:                 printf("%d ", listtab[i]);

225:             printf("\n");
226:             fflush(stdout);

228:             SCOTCH_stratInit(&archstrat);
229:             SCOTCH_stratBipart(&archstrat, "fx");

231:             SCOTCH_archInit(&archptr);
232:             SCOTCH_archBuild(&archptr, &grafarch, listnbr, listtab,
233:                 &archstrat);

235:             PetscMalloc((mat->M) * sizeof(int), &parttab_tmp);
236:             SCOTCH_mapInit(&mappptr, &grafptr, &archptr, parttab_tmp);

238:             SCOTCH_mapCompute(&mappptr, &stratptr);

240:             SCOTCH_mapView(&mappptr, stdout);

242:             /* now we have to set in the real parttab at the good place */
243:             /* because the ranks order are different than position in */
244:             /* the arch graph */
245:             for (i = 0; i < mat->M; i++) {
246:                 parttab[i] = parttab_tmp[i];
247:             }

249:             PetscFree(listtab);
250:             SCOTCH_archExit(&archptr);
251:             SCOTCH_mapExit(&mappptr);
252:             SCOTCH_stratExit(&archstrat);
253:         }

255:         /* dump to mesg_log... */
256: #ifdef PETSC_HAVE_UNISTD_H
257:         fflush(stdout);
258:         count = read(fd_pipe[0], scotch->mesg_log, (SIZE_LOG - 1) * sizeof(char));
259:         if (count < 0)
260:             count = 0;
261:         scotch->mesg_log[count] = 0;
262:         close(1);
263:         dup2(fd_stdout, 1);
264:         close(fd_stdout);
265:         close(fd_pipe[0]);
266:         close(fd_pipe[1]);
267: #endif

269:         SCOTCH_graphExit(&grafptr);
270:         SCOTCH_stratExit(&stratptr);
271:     }

273:     if (ierr)
274:         SETERRQ(1, scotch->mesg_log);

276:     /* Creation of the index set */

278:     MPI_Comm_rank(part->comm, &rank);
279:     MPI_Comm_size(part->comm, &size);
280:     nb_locals = mat->M / size;
281:     locals = parttab + rank * nb_locals;
282:     if (rank < mat->M % size) {
283:         nb_locals++;
284:         locals += rank;
285:     } else
286:         locals += mat->M % size;
287:     ISCreateGeneral(part->comm, nb_locals, locals, partitioning);

289:     /* destroying old objects */
290:     PetscFree(parttab);
291:     if (matSeq != mat) {
292:         MatDestroy(matSeq);
293:     }
294:     if (matMPI != mat) {
295:         MatDestroy(matMPI);
296:     }

298:     return(0);
299: }


304: int MatPartitioningView_Scotch(MatPartitioning part, PetscViewer viewer)
305: {
306:     MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;
307:     int ierr, rank;
308:     PetscTruth isascii;


312:     MPI_Comm_rank(part->comm, &rank);
313:     PetscTypeCompare((PetscObject) viewer, PETSC_VIEWER_ASCII, &isascii);
314:     if (isascii) {
315:         if (!rank && scotch->mesg_log) {
316:             PetscViewerASCIIPrintf(viewer, "%s\n", scotch->mesg_log);
317:         }
318:     } else {
319:         SETERRQ1(1, "Viewer type %s not supported for this Scotch partitioner",
320:             ((PetscObject) viewer)->type_name);
321:     }

323:     return(0);
324: }

328: /*@
329:      MatPartitioningScotchSetGlobal - Set method for global partitioning.

331:   Input Parameter:
332: .  part - the partitioning context
333: .  method - MP_SCOTCH_GREED, MP_SCOTCH_GIBBS or MP_SCOTCH_GR_GI (the combination of two)
334:    Level: advanced

336: @*/
337: int MatPartitioningScotchSetGlobal(MatPartitioning part,
338:     MPScotchGlobalType global)
339: {
340:     MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;


344:     switch (global) {
345:     case MP_SCOTCH_GREEDY:
346:     case MP_SCOTCH_GPS:
347:     case MP_SCOTCH_GR_GPS:
348:         scotch->global_method = global;
349:         break;
350:     default:
351:         SETERRQ(PETSC_ERR_SUP, "Scotch: Unknown or unsupported option");
352:     }

354:     return(0);
355: }

359: /*@
360:     MatPartitioningScotchSetCoarseLevel - Set the coarse level 
361:     
362:   Input Parameter:
363: .  part - the partitioning context
364: .  level - the coarse level in range [0.0,1.0]

366:    Level: advanced

368: @*/
369: int MatPartitioningScotchSetCoarseLevel(MatPartitioning part, PetscReal level)
370: {
371:     MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;


375:     if (level < 0 || level > 1.0) {
376:         SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,
377:             "Scocth: level of coarsening out of range [0.0-1.0]");
378:     } else
379:         scotch->nbvtxcoarsed = part->adj->N * level;

381:     if (scotch->nbvtxcoarsed < 20)
382:         scotch->nbvtxcoarsed = 20;

384:     return(0);
385: }

389: /*@C
390:     MatPartitioningScotchSetStrategy - Set the strategy to be used by Scotch.
391:     This is an alternative way of specifying the global method, the local
392:     method, the coarse level and the multilevel option.
393:     
394:   Input Parameter:
395: .  part - the partitioning context
396: .  level - the strategy in Scotch format. Check Scotch documentation.

398:    Level: advanced

400: .seealso: MatPartitioningScotchSetGlobal(), MatPartitioningScotchSetLocal(), MatPartitioningScotchSetCoarseLevel(), MatPartitioningScotchSetMultilevel(), 
401: @*/
402: int MatPartitioningScotchSetStrategy(MatPartitioning part, char *strat)
403: {
404:     MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;


408:     PetscStrcpy(scotch->strategy, strat);
409:     return(0);
410: }


415: /*@
416:      MatPartitioningScotchSetLocal - Set method for local partitioning.

418:   Input Parameter:
419: .  part - the partitioning context
420: .  method - MP_SCOTCH_KERNIGHAN_LIN or MP_SCOTCH_NONE

422:    Level: advanced

424: @*/
425: int MatPartitioningScotchSetLocal(MatPartitioning part, MPScotchLocalType local)
426: {
427:     MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;


431:     switch (local) {
432:     case MP_SCOTCH_KERNIGHAN_LIN:
433:     case MP_SCOTCH_NONE:
434:         scotch->local_method = local;
435:         break;
436:     default:
437:         SETERRQ(PETSC_ERR_ARG_CORRUPT, "Scotch: Unknown or unsupported option");
438:     }

440:     return(0);
441: }

445: /*@C
446:      MatPartitioningScotchSetArch - Specify the file that describes the
447:      architecture used for mapping. The format of this file is documented in
448:      the Scotch manual.

450:   Input Parameter:
451: .  part - the partitioning context
452: .  file - the name of file
453:    Level: advanced

455:   Note:
456:   If the name is not set, then the default "archgraph.src" is used.

458: .seealso: MatPartitioningScotchSetHostList(),MatPartitioningScotchSetMapping()
459: @*/
460: int MatPartitioningScotchSetArch(MatPartitioning part, const char *filename)
461: {
462:     MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;


466:     PetscStrcpy(scotch->arch, filename);

468:     return(0);
469: }

473: /*@C
474:      MatPartitioningScotchSetHostList - Specify host list file for mapping.

476:   Input Parameter:
477: .  part - the partitioning context
478: .  file - the name of file

480:    Level: advanced

482:   Notes:
483:   The file must consist in a list of hostnames (one per line). These hosts
484:   are the ones referred to in the architecture file (see 
485:   MatPartitioningScotchSetArch()): the first host corresponds to index 0,
486:   the second one to index 1, and so on.
487:   
488:   If the name is not set, then the default "host_list" is used.
489:   
490: .seealso: MatPartitioningScotchSetArch(), MatPartitioningScotchSetMapping()
491: @*/
492: int MatPartitioningScotchSetHostList(MatPartitioning part, const char *filename)
493: {
494:     MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;


498:     PetscStrcpy(scotch->host_list, filename);

500:     return(0);
501: }

505: /*@
506:      MatPartitioningScotchSetMultilevel - Activates multilevel partitioning.

508:   Input Parameter:
509: .  part - the partitioning context

511:    Level: advanced

513: @*/
514: int MatPartitioningScotchSetMultilevel(MatPartitioning part)
515: {
516:     MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;


520:     scotch->multilevel = 1;

522:     return(0);
523: }


528: /*@
529:      MatPartitioningScotchSetMapping - Activates architecture mapping for the 
530:      partitioning algorithm. Architecture mapping tries to enhance the quality
531:      of partitioning by using network topology information. 

533:   Input Parameter:
534: .  part - the partitioning context

536:    Level: advanced

538: .seealso: MatPartitioningScotchSetArch(),MatPartitioningScotchSetHostList()
539: @*/
540: int MatPartitioningScotchSetMapping(MatPartitioning part)
541: {
542:     MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;


546:     scotch->map = 1;

548:     return(0);
549: }

553: int MatPartitioningSetFromOptions_Scotch(MatPartitioning part)
554: {
556:     PetscTruth flag;
557:     char name[PETSC_MAX_PATH_LEN];
558:     int i;
559:     PetscReal r;

561:     const char *global[] = { "greedy", "gps", "gr_gps" };
562:     const char *local[] = { "kernighan-lin", "none" };

565:     PetscOptionsHead("Set Scotch partitioning options");

567:     PetscOptionsEList("-mat_partitioning_scotch_global",
568:         "Global method to use", "MatPartitioningScotchSetGlobal", global, 3,
569:         global[0], &i, &flag);
570:     if (flag)
571:         MatPartitioningScotchSetGlobal(part, i);

573:     PetscOptionsEList("-mat_partitioning_scotch_local",
574:         "Local method to use", "MatPartitioningScotchSetLocal", local, 2,
575:         local[0], &i, &flag);
576:     if (flag)
577:         MatPartitioningScotchSetLocal(part, i);

579:     PetscOptionsName("-mat_partitioning_scotch_mapping", "Use mapping",
580:         "MatPartitioningScotchSetMapping", &flag);
581:     if (flag)
582:         MatPartitioningScotchSetMapping(part);

584:     PetscOptionsString("-mat_partitioning_scotch_arch",
585:         "architecture file in scotch format", "MatPartitioningScotchSetArch",
586:         "archgraph.src", name, PETSC_MAX_PATH_LEN, &flag);
587:     if (flag)
588:         MatPartitioningScotchSetArch(part, name);

590:     PetscOptionsString("-mat_partitioning_scotch_hosts",
591:         "host list filename", "MatPartitioningScotchSetHostList",
592:         "host_list", name, PETSC_MAX_PATH_LEN, &flag);
593:     if (flag)
594:         MatPartitioningScotchSetHostList(part, name);

596:     PetscOptionsReal("-mat_partitioning_scotch_coarse_level",
597:         "coarse level", "MatPartitioningScotchSetCoarseLevel", 0, &r,
598:         &flag);
599:     if (flag)
600:         MatPartitioningScotchSetCoarseLevel(part, r);

602:     PetscOptionsName("-mat_partitioning_scotch_mul", "Use coarse level",
603:         "MatPartitioningScotchSetMultilevel", &flag);
604:     if (flag)
605:         MatPartitioningScotchSetMultilevel(part);

607:     PetscOptionsString("-mat_partitioning_scotch_strategy",
608:         "Scotch strategy string",
609:         "MatPartitioningScotchSetStrategy", "", name, PETSC_MAX_PATH_LEN,
610:         &flag);
611:     if (flag)
612:         MatPartitioningScotchSetStrategy(part, name);

614:     PetscOptionsTail();
615:     return(0);
616: }


621: int MatPartitioningDestroy_Scotch(MatPartitioning part)
622: {
623:     MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;


628:     if (scotch->mesg_log) {
629:         PetscFree(scotch->mesg_log);
630:     }
631:     PetscFree(scotch);

633:     return(0);
634: }

636: EXTERN_C_BEGIN
639: int MatPartitioningCreate_Scotch(MatPartitioning part)
640: {
642:     MatPartitioning_Scotch *scotch;

645:     PetscNew(MatPartitioning_Scotch, &scotch);

647:     scotch->map = 0;
648:     scotch->global_method = MP_SCOTCH_GR_GPS;
649:     scotch->local_method = MP_SCOTCH_KERNIGHAN_LIN;
650:     PetscStrcpy(scotch->arch, "archgraph.src");
651:     scotch->nbvtxcoarsed = 200;
652:     PetscStrcpy(scotch->strategy, "");
653:     scotch->multilevel = 0;
654:     scotch->mesg_log = NULL;

656:     PetscStrcpy(scotch->host_list, "host_list");

658:     part->ops->apply = MatPartitioningApply_Scotch;
659:     part->ops->view = MatPartitioningView_Scotch;
660:     part->ops->destroy = MatPartitioningDestroy_Scotch;
661:     part->ops->setfromoptions = MatPartitioningSetFromOptions_Scotch;
662:     part->data = (void *) scotch;

664:     return(0);
665: }

667: EXTERN_C_END