Actual source code: scotch.c
1: #include "src/mat/impls/adj/mpi/mpiadj.h" /*I "petscmat.h" I*/
3: #ifdef PETSC_HAVE_UNISTD_H
4: #include <unistd.h>
5: #endif
7: #ifdef PETSC_HAVE_STDLIB_H
8: #include <stdlib.h>
9: #endif
11: #include "petscfix.h"
13: /*
14: Currently using Scotch-3.4
15: */
16: EXTERN_C_BEGIN
17: #include "scotch.h"
18: EXTERN_C_END
20: typedef struct {
21: char arch[PETSC_MAX_PATH_LEN];
22: int multilevel;
23: char strategy[30];
24: int global_method; /* global method */
25: int local_method; /* local method */
26: int nbvtxcoarsed; /* number of vertices for the coarse graph */
27: int map; /* to know if we map on archptr or just partionate the graph */
28: char *mesg_log;
29: char host_list[PETSC_MAX_PATH_LEN];
30: } MatPartitioning_Scotch;
32: #define SIZE_LOG 10000 /* size of buffer for msg_log */
36: static int MatPartitioningApply_Scotch(MatPartitioning part, IS * partitioning)
37: {
38: int ierr, *parttab, *locals = NULL, rank, i, size, j;
39: Mat mat = part->adj, matMPI, matSeq;
40: int nb_locals = mat->m;
41: Mat_MPIAdj *adj = (Mat_MPIAdj *) mat->data;
42: MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;
43: PetscTruth flg;
44: #ifdef PETSC_HAVE_UNISTD_H
45: int fd_stdout, fd_pipe[2], count;
46: #endif
50: /* check if the matrix is sequential, use MatGetSubMatrices if necessary */
51: MPI_Comm_size(mat->comm, &size);
52: PetscTypeCompare((PetscObject) mat, MATMPIADJ, &flg);
53: if (size > 1) {
54: int M, N;
55: IS isrow, iscol;
56: Mat *A;
58: if (flg) {
59: SETERRQ(0, "Distributed matrix format MPIAdj is not supported for sequential partitioners");
60: }
61: PetscPrintf(part->comm, "Converting distributed matrix to sequential: this could be a performance loss\n");
63: MatGetSize(mat, &M, &N);
64: ISCreateStride(PETSC_COMM_SELF, M, 0, 1, &isrow);
65: ISCreateStride(PETSC_COMM_SELF, N, 0, 1, &iscol);
66: MatGetSubMatrices(mat, 1, &isrow, &iscol, MAT_INITIAL_MATRIX, &A);
67: ISDestroy(isrow);
68: ISDestroy(iscol);
69: matSeq = *A;
70: } else
71: matSeq = mat;
73: /* convert the the matrix to MPIADJ type if necessary */
74: if (!flg) {
75: MatConvert(matSeq, MATMPIADJ, &matMPI);
76: } else
77: matMPI = matSeq;
79: adj = (Mat_MPIAdj *) matMPI->data; /* finaly adj contains adjacency graph */
81: MPI_Comm_rank(part->comm, &rank);
83: {
84: /* definition of Scotch library arguments */
85: SCOTCH_Strat stratptr; /* scotch strategy */
86: SCOTCH_Graph grafptr; /* scotch graph */
87: SCOTCH_Mapping mappptr; /* scotch mapping format */
88: int vertnbr = mat->M; /* number of vertices in full graph */
89: int *verttab = adj->i; /* start of edge list for each vertex */
90: int *edgetab = adj->j; /* edge list data */
91: int edgenbr = adj->nz; /* number of edges */
92: int *velotab = NULL; /* not used by petsc interface */
93: int *vlbltab = NULL;
94: int *edlotab = NULL;
95: int baseval = 0; /* 0 for C array indexing */
96: int flagval = 3; /* (cf doc scotch no weight edge & vertices) */
97: char strategy[256];
99: PetscMalloc((mat->M) * sizeof(int), &parttab);
101: /* redirect output to buffer scotch -> mesg_log */
102: #ifdef PETSC_HAVE_UNISTD_H
103: fd_stdout = dup(1);
104: pipe(fd_pipe);
105: close(1);
106: dup2(fd_pipe[1], 1);
107: PetscMalloc(SIZE_LOG * sizeof(char), &(scotch->mesg_log));
108: #endif
110: /* library call */
112: /* Construction of the scotch graph object */
113: SCOTCH_graphInit(&grafptr);
114: SCOTCH_graphBuild(&grafptr, vertnbr, verttab, velotab,
115: vlbltab, edgenbr, edgetab, edlotab, baseval, flagval);
116: SCOTCH_graphCheck(&grafptr);
118: /* Construction of the strategy */
119: if (scotch->strategy[0] != 0) /* strcmp(scotch->strategy,"") */
120: PetscStrcpy(strategy, scotch->strategy);
121: else {
122: PetscStrcpy(strategy, "b{strat=");
124: if (scotch->multilevel) {
125: /* PetscStrcat(strategy,"m{vert=");
126: sprintf(strategy+strlen(strategy),"%d",scotch->nbvtxcoarsed);
127: PetscStrcat(strategy,",asc="); */
128: sprintf(strategy, "b{strat=m{vert=%d,asc=",
129: scotch->nbvtxcoarsed);
130: } else
131: PetscStrcpy(strategy, "b{strat=");
133: switch (scotch->global_method) {
134: case MP_SCOTCH_GREEDY:
135: PetscStrcat(strategy, "h");
136: break;
137: case MP_SCOTCH_GPS:
138: PetscStrcat(strategy, "g");
139: break;
140: case MP_SCOTCH_GR_GPS:
141: PetscStrcat(strategy, "g|h");
142: }
144: switch (scotch->local_method) {
145: case MP_SCOTCH_KERNIGHAN_LIN:
146: if (scotch->multilevel)
147: PetscStrcat(strategy, ",low=f}");
148: else
149: PetscStrcat(strategy, " f");
150: break;
151: case MP_SCOTCH_NONE:
152: if (scotch->multilevel)
153: PetscStrcat(strategy, ",asc=x}");
154: default:
155: break;
156: }
158: PetscStrcat(strategy, " x}");
159: }
161: PetscPrintf(part->comm, "strategy=[%s]\n", strategy);
163: SCOTCH_stratInit(&stratptr);
164: SCOTCH_stratMap(&stratptr, strategy);
166: /* check for option mapping */
167: if (!scotch->map) {
168: SCOTCH_graphPart(&grafptr, &stratptr, part->n, parttab);
169: PetscPrintf(PETSC_COMM_SELF, "Partition simple without mapping\n");
170: } else {
171: SCOTCH_Graph grafarch;
172: SCOTCH_Num *listtab;
173: SCOTCH_Num listnbr = 0;
174: SCOTCH_Arch archptr; /* file in scotch architecture format */
175: SCOTCH_Strat archstrat;
176: int arch_total_size, *parttab_tmp;
177: int cpt;
178: char buf[256];
179: FILE *file1, *file2;
180: char host_buf[256];
182: /* generate the graph that represents the arch */
183: file1 = fopen(scotch->arch, "r");
184: if (!file1)
185: SETERRQ1(1, "Scotch: unable to open architecture file %s", scotch->arch);
187: SCOTCH_graphInit(&grafarch);
188: SCOTCH_graphLoad(&grafarch, file1, baseval, 3);
190: SCOTCH_graphCheck(&grafarch);
191: SCOTCH_graphSize(&grafarch, &arch_total_size, &cpt);
193: fclose(file1);
194: printf("total size = %d\n", arch_total_size);
196: /* generate the list of nodes currently working */
197: PetscGetHostName(host_buf, 256);
198: PetscStrlen(host_buf, &j);
200: file2 = fopen(scotch->host_list, "r");
201: if (!file2)
202: SETERRQ1(1, "Scotch: unable to open host list file %s", scotch->host_list);
204: i = -1;
205: flg = 0;
206: while (!feof(file2) && !flg) {
207: i++;
208: fgets(buf, 256, file2);
209: PetscStrncmp(buf, host_buf, j, &flg);
210: }
211: fclose(file2);
212: if (!flg) {
213: SETERRQ1(1, "Scotch: unable to find '%s' in host list file", host_buf);
214: }
216: listnbr = size;
217: PetscMalloc(sizeof(SCOTCH_Num) * listnbr, &listtab);
219: MPI_Allgather(&i, 1, MPI_INT, listtab, 1, MPI_INT, part->comm);
221: printf("listnbr = %d, listtab = ", listnbr);
222: for (i = 0; i < listnbr; i++)
223: printf("%d ", listtab[i]);
225: printf("\n");
226: fflush(stdout);
228: SCOTCH_stratInit(&archstrat);
229: SCOTCH_stratBipart(&archstrat, "fx");
231: SCOTCH_archInit(&archptr);
232: SCOTCH_archBuild(&archptr, &grafarch, listnbr, listtab,
233: &archstrat);
235: PetscMalloc((mat->M) * sizeof(int), &parttab_tmp);
236: SCOTCH_mapInit(&mappptr, &grafptr, &archptr, parttab_tmp);
238: SCOTCH_mapCompute(&mappptr, &stratptr);
240: SCOTCH_mapView(&mappptr, stdout);
242: /* now we have to set in the real parttab at the good place */
243: /* because the ranks order are different than position in */
244: /* the arch graph */
245: for (i = 0; i < mat->M; i++) {
246: parttab[i] = parttab_tmp[i];
247: }
249: PetscFree(listtab);
250: SCOTCH_archExit(&archptr);
251: SCOTCH_mapExit(&mappptr);
252: SCOTCH_stratExit(&archstrat);
253: }
255: /* dump to mesg_log... */
256: #ifdef PETSC_HAVE_UNISTD_H
257: fflush(stdout);
258: count = read(fd_pipe[0], scotch->mesg_log, (SIZE_LOG - 1) * sizeof(char));
259: if (count < 0)
260: count = 0;
261: scotch->mesg_log[count] = 0;
262: close(1);
263: dup2(fd_stdout, 1);
264: close(fd_stdout);
265: close(fd_pipe[0]);
266: close(fd_pipe[1]);
267: #endif
269: SCOTCH_graphExit(&grafptr);
270: SCOTCH_stratExit(&stratptr);
271: }
273: if (ierr)
274: SETERRQ(1, scotch->mesg_log);
276: /* Creation of the index set */
278: MPI_Comm_rank(part->comm, &rank);
279: MPI_Comm_size(part->comm, &size);
280: nb_locals = mat->M / size;
281: locals = parttab + rank * nb_locals;
282: if (rank < mat->M % size) {
283: nb_locals++;
284: locals += rank;
285: } else
286: locals += mat->M % size;
287: ISCreateGeneral(part->comm, nb_locals, locals, partitioning);
289: /* destroying old objects */
290: PetscFree(parttab);
291: if (matSeq != mat) {
292: MatDestroy(matSeq);
293: }
294: if (matMPI != mat) {
295: MatDestroy(matMPI);
296: }
298: return(0);
299: }
304: int MatPartitioningView_Scotch(MatPartitioning part, PetscViewer viewer)
305: {
306: MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;
307: int ierr, rank;
308: PetscTruth isascii;
312: MPI_Comm_rank(part->comm, &rank);
313: PetscTypeCompare((PetscObject) viewer, PETSC_VIEWER_ASCII, &isascii);
314: if (isascii) {
315: if (!rank && scotch->mesg_log) {
316: PetscViewerASCIIPrintf(viewer, "%s\n", scotch->mesg_log);
317: }
318: } else {
319: SETERRQ1(1, "Viewer type %s not supported for this Scotch partitioner",
320: ((PetscObject) viewer)->type_name);
321: }
323: return(0);
324: }
328: /*@
329: MatPartitioningScotchSetGlobal - Set method for global partitioning.
331: Input Parameter:
332: . part - the partitioning context
333: . method - MP_SCOTCH_GREED, MP_SCOTCH_GIBBS or MP_SCOTCH_GR_GI (the combination of two)
334: Level: advanced
336: @*/
337: int MatPartitioningScotchSetGlobal(MatPartitioning part,
338: MPScotchGlobalType global)
339: {
340: MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;
344: switch (global) {
345: case MP_SCOTCH_GREEDY:
346: case MP_SCOTCH_GPS:
347: case MP_SCOTCH_GR_GPS:
348: scotch->global_method = global;
349: break;
350: default:
351: SETERRQ(PETSC_ERR_SUP, "Scotch: Unknown or unsupported option");
352: }
354: return(0);
355: }
359: /*@
360: MatPartitioningScotchSetCoarseLevel - Set the coarse level
361:
362: Input Parameter:
363: . part - the partitioning context
364: . level - the coarse level in range [0.0,1.0]
366: Level: advanced
368: @*/
369: int MatPartitioningScotchSetCoarseLevel(MatPartitioning part, PetscReal level)
370: {
371: MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;
375: if (level < 0 || level > 1.0) {
376: SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,
377: "Scocth: level of coarsening out of range [0.0-1.0]");
378: } else
379: scotch->nbvtxcoarsed = part->adj->N * level;
381: if (scotch->nbvtxcoarsed < 20)
382: scotch->nbvtxcoarsed = 20;
384: return(0);
385: }
389: /*@C
390: MatPartitioningScotchSetStrategy - Set the strategy to be used by Scotch.
391: This is an alternative way of specifying the global method, the local
392: method, the coarse level and the multilevel option.
393:
394: Input Parameter:
395: . part - the partitioning context
396: . level - the strategy in Scotch format. Check Scotch documentation.
398: Level: advanced
400: .seealso: MatPartitioningScotchSetGlobal(), MatPartitioningScotchSetLocal(), MatPartitioningScotchSetCoarseLevel(), MatPartitioningScotchSetMultilevel(),
401: @*/
402: int MatPartitioningScotchSetStrategy(MatPartitioning part, char *strat)
403: {
404: MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;
408: PetscStrcpy(scotch->strategy, strat);
409: return(0);
410: }
415: /*@
416: MatPartitioningScotchSetLocal - Set method for local partitioning.
418: Input Parameter:
419: . part - the partitioning context
420: . method - MP_SCOTCH_KERNIGHAN_LIN or MP_SCOTCH_NONE
422: Level: advanced
424: @*/
425: int MatPartitioningScotchSetLocal(MatPartitioning part, MPScotchLocalType local)
426: {
427: MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;
431: switch (local) {
432: case MP_SCOTCH_KERNIGHAN_LIN:
433: case MP_SCOTCH_NONE:
434: scotch->local_method = local;
435: break;
436: default:
437: SETERRQ(PETSC_ERR_ARG_CORRUPT, "Scotch: Unknown or unsupported option");
438: }
440: return(0);
441: }
445: /*@C
446: MatPartitioningScotchSetArch - Specify the file that describes the
447: architecture used for mapping. The format of this file is documented in
448: the Scotch manual.
450: Input Parameter:
451: . part - the partitioning context
452: . file - the name of file
453: Level: advanced
455: Note:
456: If the name is not set, then the default "archgraph.src" is used.
458: .seealso: MatPartitioningScotchSetHostList(),MatPartitioningScotchSetMapping()
459: @*/
460: int MatPartitioningScotchSetArch(MatPartitioning part, const char *filename)
461: {
462: MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;
466: PetscStrcpy(scotch->arch, filename);
468: return(0);
469: }
473: /*@C
474: MatPartitioningScotchSetHostList - Specify host list file for mapping.
476: Input Parameter:
477: . part - the partitioning context
478: . file - the name of file
480: Level: advanced
482: Notes:
483: The file must consist in a list of hostnames (one per line). These hosts
484: are the ones referred to in the architecture file (see
485: MatPartitioningScotchSetArch()): the first host corresponds to index 0,
486: the second one to index 1, and so on.
487:
488: If the name is not set, then the default "host_list" is used.
489:
490: .seealso: MatPartitioningScotchSetArch(), MatPartitioningScotchSetMapping()
491: @*/
492: int MatPartitioningScotchSetHostList(MatPartitioning part, const char *filename)
493: {
494: MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;
498: PetscStrcpy(scotch->host_list, filename);
500: return(0);
501: }
505: /*@
506: MatPartitioningScotchSetMultilevel - Activates multilevel partitioning.
508: Input Parameter:
509: . part - the partitioning context
511: Level: advanced
513: @*/
514: int MatPartitioningScotchSetMultilevel(MatPartitioning part)
515: {
516: MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;
520: scotch->multilevel = 1;
522: return(0);
523: }
528: /*@
529: MatPartitioningScotchSetMapping - Activates architecture mapping for the
530: partitioning algorithm. Architecture mapping tries to enhance the quality
531: of partitioning by using network topology information.
533: Input Parameter:
534: . part - the partitioning context
536: Level: advanced
538: .seealso: MatPartitioningScotchSetArch(),MatPartitioningScotchSetHostList()
539: @*/
540: int MatPartitioningScotchSetMapping(MatPartitioning part)
541: {
542: MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;
546: scotch->map = 1;
548: return(0);
549: }
553: int MatPartitioningSetFromOptions_Scotch(MatPartitioning part)
554: {
556: PetscTruth flag;
557: char name[PETSC_MAX_PATH_LEN];
558: int i;
559: PetscReal r;
561: const char *global[] = { "greedy", "gps", "gr_gps" };
562: const char *local[] = { "kernighan-lin", "none" };
565: PetscOptionsHead("Set Scotch partitioning options");
567: PetscOptionsEList("-mat_partitioning_scotch_global",
568: "Global method to use", "MatPartitioningScotchSetGlobal", global, 3,
569: global[0], &i, &flag);
570: if (flag)
571: MatPartitioningScotchSetGlobal(part, i);
573: PetscOptionsEList("-mat_partitioning_scotch_local",
574: "Local method to use", "MatPartitioningScotchSetLocal", local, 2,
575: local[0], &i, &flag);
576: if (flag)
577: MatPartitioningScotchSetLocal(part, i);
579: PetscOptionsName("-mat_partitioning_scotch_mapping", "Use mapping",
580: "MatPartitioningScotchSetMapping", &flag);
581: if (flag)
582: MatPartitioningScotchSetMapping(part);
584: PetscOptionsString("-mat_partitioning_scotch_arch",
585: "architecture file in scotch format", "MatPartitioningScotchSetArch",
586: "archgraph.src", name, PETSC_MAX_PATH_LEN, &flag);
587: if (flag)
588: MatPartitioningScotchSetArch(part, name);
590: PetscOptionsString("-mat_partitioning_scotch_hosts",
591: "host list filename", "MatPartitioningScotchSetHostList",
592: "host_list", name, PETSC_MAX_PATH_LEN, &flag);
593: if (flag)
594: MatPartitioningScotchSetHostList(part, name);
596: PetscOptionsReal("-mat_partitioning_scotch_coarse_level",
597: "coarse level", "MatPartitioningScotchSetCoarseLevel", 0, &r,
598: &flag);
599: if (flag)
600: MatPartitioningScotchSetCoarseLevel(part, r);
602: PetscOptionsName("-mat_partitioning_scotch_mul", "Use coarse level",
603: "MatPartitioningScotchSetMultilevel", &flag);
604: if (flag)
605: MatPartitioningScotchSetMultilevel(part);
607: PetscOptionsString("-mat_partitioning_scotch_strategy",
608: "Scotch strategy string",
609: "MatPartitioningScotchSetStrategy", "", name, PETSC_MAX_PATH_LEN,
610: &flag);
611: if (flag)
612: MatPartitioningScotchSetStrategy(part, name);
614: PetscOptionsTail();
615: return(0);
616: }
621: int MatPartitioningDestroy_Scotch(MatPartitioning part)
622: {
623: MatPartitioning_Scotch *scotch = (MatPartitioning_Scotch *) part->data;
628: if (scotch->mesg_log) {
629: PetscFree(scotch->mesg_log);
630: }
631: PetscFree(scotch);
633: return(0);
634: }
636: EXTERN_C_BEGIN
639: int MatPartitioningCreate_Scotch(MatPartitioning part)
640: {
642: MatPartitioning_Scotch *scotch;
645: PetscNew(MatPartitioning_Scotch, &scotch);
647: scotch->map = 0;
648: scotch->global_method = MP_SCOTCH_GR_GPS;
649: scotch->local_method = MP_SCOTCH_KERNIGHAN_LIN;
650: PetscStrcpy(scotch->arch, "archgraph.src");
651: scotch->nbvtxcoarsed = 200;
652: PetscStrcpy(scotch->strategy, "");
653: scotch->multilevel = 0;
654: scotch->mesg_log = NULL;
656: PetscStrcpy(scotch->host_list, "host_list");
658: part->ops->apply = MatPartitioningApply_Scotch;
659: part->ops->view = MatPartitioningView_Scotch;
660: part->ops->destroy = MatPartitioningDestroy_Scotch;
661: part->ops->setfromoptions = MatPartitioningSetFromOptions_Scotch;
662: part->data = (void *) scotch;
664: return(0);
665: }
667: EXTERN_C_END