Actual source code: bjacobi.c
1: /*$Id: bjacobi.c,v 1.160 2001/08/07 03:03:35 balay Exp $*/
2: /*
3: Defines a block Jacobi preconditioner.
4: */
5: #include src/mat/matimpl.h
6: #include src/ksp/pc/pcimpl.h
7: #include src/ksp/pc/impls/bjacobi/bjacobi.h
9: static int PCSetUp_BJacobi_Singleblock(PC,Mat,Mat);
10: static int PCSetUp_BJacobi_Multiblock(PC,Mat,Mat);
14: static int PCSetUp_BJacobi(PC pc)
15: {
16: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
17: Mat mat = pc->mat,pmat = pc->pmat;
18: int ierr,N,M,start,i,rank,size,sum,end;
19: int bs,i_start=-1,i_end=-1;
20: char *pprefix,*mprefix;
21: int (*f)(Mat,PetscTruth*,MatReuse,Mat*);
24: MPI_Comm_rank(pc->comm,&rank);
25: MPI_Comm_size(pc->comm,&size);
26: MatGetLocalSize(pc->pmat,&M,&N);
27: MatGetBlockSize(pc->pmat,&bs);
29: /* ----------
30: Determines the number of blocks assigned to each processor
31: */
33: /* local block count given */
34: if (jac->n_local > 0 && jac->n < 0) {
35: MPI_Allreduce(&jac->n_local,&jac->n,1,MPI_INT,MPI_SUM,pc->comm);
36: if (jac->l_lens) { /* check that user set these correctly */
37: sum = 0;
38: for (i=0; i<jac->n_local; i++) {
39: if (jac->l_lens[i]/bs*bs !=jac->l_lens[i]) {
40: SETERRQ(PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
41: }
42: sum += jac->l_lens[i];
43: }
44: if (sum != M) SETERRQ(PETSC_ERR_ARG_SIZ,"Local lens sent incorrectly");
45: } else {
46: PetscMalloc(jac->n_local*sizeof(int),&jac->l_lens);
47: for (i=0; i<jac->n_local; i++) {
48: jac->l_lens[i] = bs*((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i));
49: }
50: }
51: } else if (jac->n > 0 && jac->n_local < 0) { /* global block count given */
52: /* global blocks given: determine which ones are local */
53: if (jac->g_lens) {
54: /* check if the g_lens is has valid entries */
55: for (i=0; i<jac->n; i++) {
56: if (!jac->g_lens[i]) SETERRQ(PETSC_ERR_ARG_SIZ,"Zero block not allowed");
57: if (jac->g_lens[i]/bs*bs != jac->g_lens[i]) {
58: SETERRQ(PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
59: }
60: }
61: if (size == 1) {
62: jac->n_local = jac->n;
63: PetscMalloc(jac->n_local*sizeof(int),&jac->l_lens);
64: PetscMemcpy(jac->l_lens,jac->g_lens,jac->n_local*sizeof(int));
65: /* check that user set these correctly */
66: sum = 0;
67: for (i=0; i<jac->n_local; i++) sum += jac->l_lens[i];
68: if (sum != M) SETERRQ(PETSC_ERR_ARG_SIZ,"Global lens sent incorrectly");
69: } else {
70: MatGetOwnershipRange(pc->pmat,&start,&end);
71: /* loop over blocks determing first one owned by me */
72: sum = 0;
73: for (i=0; i<jac->n+1; i++) {
74: if (sum == start) { i_start = i; goto start_1;}
75: if (i < jac->n) sum += jac->g_lens[i];
76: }
77: SETERRQ(PETSC_ERR_ARG_SIZ,"Block sizes\n\
78: used in PCBJacobiSetTotalBlocks()\n\
79: are not compatible with parallel matrix layout");
80: start_1:
81: for (i=i_start; i<jac->n+1; i++) {
82: if (sum == end) { i_end = i; goto end_1; }
83: if (i < jac->n) sum += jac->g_lens[i];
84: }
85: SETERRQ(PETSC_ERR_ARG_SIZ,"Block sizes\n\
86: used in PCBJacobiSetTotalBlocks()\n\
87: are not compatible with parallel matrix layout");
88: end_1:
89: jac->n_local = i_end - i_start;
90: PetscMalloc(jac->n_local*sizeof(int),&jac->l_lens);
91: PetscMemcpy(jac->l_lens,jac->g_lens+i_start,jac->n_local*sizeof(int));
92: }
93: } else { /* no global blocks given, determine then using default layout */
94: jac->n_local = jac->n/size + ((jac->n % size) > rank);
95: PetscMalloc(jac->n_local*sizeof(int),&jac->l_lens);
96: for (i=0; i<jac->n_local; i++) {
97: jac->l_lens[i] = ((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i))*bs;
98: if (!jac->l_lens[i]) SETERRQ(PETSC_ERR_ARG_SIZ,"Too many blocks given");
99: }
100: }
101: } else if (jac->n < 0 && jac->n_local < 0) { /* no blocks given */
102: jac->n = size;
103: jac->n_local = 1;
104: PetscMalloc(sizeof(int),&jac->l_lens);
105: jac->l_lens[0] = M;
106: }
108: MPI_Comm_size(pc->comm,&size);
109: PetscObjectQueryFunction((PetscObject)pc->mat,"MatGetDiagonalBlock_C",(void (**)(void))&f);
110: if (size == 1 && !f) {
111: mat = pc->mat;
112: pmat = pc->pmat;
113: } else {
114: PetscTruth iscopy;
115: MatReuse scall;
117: if (jac->use_true_local) {
118: scall = MAT_INITIAL_MATRIX;
119: if (pc->setupcalled) {
120: if (pc->flag == SAME_NONZERO_PATTERN) {
121: if (jac->tp_mat) {
122: scall = MAT_REUSE_MATRIX;
123: mat = jac->tp_mat;
124: }
125: } else {
126: if (jac->tp_mat) {
127: MatDestroy(jac->tp_mat);
128: }
129: }
130: }
131: if (!f) {
132: SETERRQ(PETSC_ERR_SUP,"This matrix does not support getting diagonal block");
133: }
134: (*f)(pc->mat,&iscopy,scall,&mat);
135: /* make submatrix have same prefix as entire matrix */
136: PetscObjectGetOptionsPrefix((PetscObject)pc->mat,&mprefix);
137: PetscObjectSetOptionsPrefix((PetscObject)mat,mprefix);
138: if (iscopy) {
139: jac->tp_mat = mat;
140: }
141: }
142: if (pc->pmat != pc->mat || !jac->use_true_local) {
143: scall = MAT_INITIAL_MATRIX;
144: if (pc->setupcalled) {
145: if (pc->flag == SAME_NONZERO_PATTERN) {
146: if (jac->tp_pmat) {
147: scall = MAT_REUSE_MATRIX;
148: pmat = jac->tp_pmat;
149: }
150: } else {
151: if (jac->tp_pmat) {
152: MatDestroy(jac->tp_pmat);
153: }
154: }
155: }
156: PetscObjectQueryFunction((PetscObject)pc->pmat,"MatGetDiagonalBlock_C",(void (**)(void))&f);
157: if (!f) {
158: SETERRQ(PETSC_ERR_SUP,"This matrix does not support getting diagonal block");
159: }
160: (*f)(pc->pmat,&iscopy,scall,&pmat);
161: /* make submatrix have same prefix as entire matrix */
162: PetscObjectGetOptionsPrefix((PetscObject)pc->pmat,&pprefix);
163: PetscObjectSetOptionsPrefix((PetscObject)pmat,pprefix);
164: if (iscopy) {
165: jac->tp_pmat = pmat;
166: }
167: } else {
168: pmat = mat;
169: }
170: }
172: /* ------
173: Setup code depends on the number of blocks
174: */
175: if (jac->n_local == 1) {
176: PCSetUp_BJacobi_Singleblock(pc,mat,pmat);
177: } else {
178: PCSetUp_BJacobi_Multiblock(pc,mat,pmat);
179: }
180: return(0);
181: }
183: /* Default destroy, if it has never been setup */
186: static int PCDestroy_BJacobi(PC pc)
187: {
188: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
192: if (jac->g_lens) {PetscFree(jac->g_lens);}
193: if (jac->l_lens) {PetscFree(jac->l_lens);}
194: PetscFree(jac);
195: return(0);
196: }
200: static int PCSetFromOptions_BJacobi(PC pc)
201: {
202: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
203: int blocks,ierr;
204: PetscTruth flg;
207: PetscOptionsHead("Block Jacobi options");
208: PetscOptionsInt("-pc_bjacobi_blocks","Total number of blocks","PCBJacobiSetTotalBlocks",jac->n,&blocks,&flg);
209: if (flg) {
210: PCBJacobiSetTotalBlocks(pc,blocks,PETSC_NULL);
211: }
212: PetscOptionsName("-pc_bjacobi_truelocal","Use the true matrix, not preconditioner matrix to define matrix vector product in sub-problems","PCBJacobiSetUseTrueLocal",&flg);
213: if (flg) {
214: PCBJacobiSetUseTrueLocal(pc);
215: }
216: PetscOptionsTail();
217: return(0);
218: }
222: static int PCView_BJacobi(PC pc,PetscViewer viewer)
223: {
224: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
225: int rank,ierr,i;
226: PetscTruth isascii,isstring;
227: PetscViewer sviewer;
230: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&isascii);
231: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_STRING,&isstring);
232: if (isascii) {
233: if (jac->use_true_local) {
234: PetscViewerASCIIPrintf(viewer," block Jacobi: using true local matrix, number of blocks = %d\n",jac->n);
235: }
236: PetscViewerASCIIPrintf(viewer," block Jacobi: number of blocks = %d\n",jac->n);
237: MPI_Comm_rank(pc->comm,&rank);
238: if (jac->same_local_solves) {
239: PetscViewerASCIIPrintf(viewer," Local solve is same for all blocks, in the following KSP and PC objects:\n");
240: PetscViewerGetSingleton(viewer,&sviewer);
241: if (!rank && jac->ksp) {
242: PetscViewerASCIIPushTab(viewer);
243: KSPView(jac->ksp[0],sviewer);
244: PetscViewerASCIIPopTab(viewer);
245: }
246: PetscViewerRestoreSingleton(viewer,&sviewer);
247: } else {
249: PetscViewerASCIIPrintf(viewer," Local solve info for each block is in the following KSP and PC objects:\n");
250: PetscViewerASCIISynchronizedPrintf(viewer,"Proc %d: number of local blocks = %d, first local block number = %d\n",
251: rank,jac->n_local,jac->first_local);
252: PetscViewerASCIIPushTab(viewer);
253: for (i=0; i<jac->n_local; i++) {
254: PetscViewerASCIISynchronizedPrintf(viewer,"Proc %d: local block number %d\n",rank,i);
255: PetscViewerGetSingleton(viewer,&sviewer);
256: KSPView(jac->ksp[i],sviewer);
257: if (i != jac->n_local-1) {
258: PetscViewerASCIISynchronizedPrintf(viewer,"- - - - - - - - - - - - - - - - - -\n");
259: }
260: PetscViewerRestoreSingleton(viewer,&sviewer);
261: }
262: PetscViewerASCIIPopTab(viewer);
263: PetscViewerFlush(viewer);
264: }
265: } else if (isstring) {
266: PetscViewerStringSPrintf(viewer," blks=%d",jac->n);
267: PetscViewerGetSingleton(viewer,&sviewer);
268: if (jac->ksp) {KSPView(jac->ksp[0],sviewer);}
269: PetscViewerRestoreSingleton(viewer,&sviewer);
270: } else {
271: SETERRQ1(1,"Viewer type %s not supported for block Jacobi",((PetscObject)viewer)->type_name);
272: }
273: return(0);
274: }
276: /* -------------------------------------------------------------------------------------*/
278: EXTERN_C_BEGIN
281: int PCBJacobiSetUseTrueLocal_BJacobi(PC pc)
282: {
283: PC_BJacobi *jac;
286: jac = (PC_BJacobi*)pc->data;
287: jac->use_true_local = PETSC_TRUE;
288: return(0);
289: }
290: EXTERN_C_END
292: EXTERN_C_BEGIN
295: int PCBJacobiGetSubKSP_BJacobi(PC pc,int *n_local,int *first_local,KSP **ksp)
296: {
297: PC_BJacobi *jac = (PC_BJacobi*)pc->data;;
300: if (!pc->setupcalled) SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"Must call KSPSetUp() or PCSetUp() first");
302: if (n_local) *n_local = jac->n_local;
303: if (first_local) *first_local = jac->first_local;
304: *ksp = jac->ksp;
305: jac->same_local_solves = PETSC_FALSE; /* Assume that local solves are now different;
306: not necessarily true though! This flag is
307: used only for PCView_BJacobi() */
308: return(0);
309: }
310: EXTERN_C_END
312: EXTERN_C_BEGIN
315: int PCBJacobiSetTotalBlocks_BJacobi(PC pc,int blocks,int *lens)
316: {
317: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
318: int ierr;
322: if (pc->setupcalled > 0) SETERRQ(1,"Cannot set number of blocks after PCSetUp()/KSPSetUp() has been called");
323: jac->n = blocks;
324: if (!lens) {
325: jac->g_lens = 0;
326: } else {
327: PetscMalloc(blocks*sizeof(int),&jac->g_lens);
328: PetscLogObjectMemory(pc,blocks*sizeof(int));
329: PetscMemcpy(jac->g_lens,lens,blocks*sizeof(int));
330: }
331: return(0);
332: }
333: EXTERN_C_END
335: EXTERN_C_BEGIN
338: int PCBJacobiGetTotalBlocks_BJacobi(PC pc, int *blocks, const int *lens[])
339: {
340: PC_BJacobi *jac = (PC_BJacobi*) pc->data;
343: *blocks = jac->n;
344: if (lens) *lens = jac->g_lens;
345: return(0);
346: }
347: EXTERN_C_END
349: EXTERN_C_BEGIN
352: int PCBJacobiSetLocalBlocks_BJacobi(PC pc,int blocks,const int lens[])
353: {
354: PC_BJacobi *jac;
355: int ierr;
358: jac = (PC_BJacobi*)pc->data;
360: jac->n_local = blocks;
361: if (!lens) {
362: jac->l_lens = 0;
363: } else {
364: PetscMalloc(blocks*sizeof(int),&jac->l_lens);
365: PetscLogObjectMemory(pc,blocks*sizeof(int));
366: PetscMemcpy(jac->l_lens,lens,blocks*sizeof(int));
367: }
368: return(0);
369: }
370: EXTERN_C_END
372: EXTERN_C_BEGIN
375: int PCBJacobiGetLocalBlocks_BJacobi(PC pc, int *blocks, const int *lens[])
376: {
377: PC_BJacobi *jac = (PC_BJacobi*) pc->data;
380: *blocks = jac->n_local;
381: if (lens) *lens = jac->l_lens;
382: return(0);
383: }
384: EXTERN_C_END
386: /* -------------------------------------------------------------------------------------*/
390: /*@
391: PCBJacobiSetUseTrueLocal - Sets a flag to indicate that the block
392: problem is associated with the linear system matrix instead of the
393: default (where it is associated with the preconditioning matrix).
394: That is, if the local system is solved iteratively then it iterates
395: on the block from the matrix using the block from the preconditioner
396: as the preconditioner for the local block.
398: Collective on PC
400: Input Parameters:
401: . pc - the preconditioner context
403: Options Database Key:
404: . -pc_bjacobi_truelocal - Activates PCBJacobiSetUseTrueLocal()
406: Notes:
407: For the common case in which the preconditioning and linear
408: system matrices are identical, this routine is unnecessary.
410: Level: intermediate
412: .keywords: block, Jacobi, set, true, local, flag
414: .seealso: PCSetOperators(), PCBJacobiSetLocalBlocks()
415: @*/
416: int PCBJacobiSetUseTrueLocal(PC pc)
417: {
418: int ierr,(*f)(PC);
422: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetUseTrueLocal_C",(void (**)(void))&f);
423: if (f) {
424: (*f)(pc);
425: }
427: return(0);
428: }
432: /*@C
433: PCBJacobiGetSubKSP - Gets the local KSP contexts for all blocks on
434: this processor.
435:
436: Note Collective
438: Input Parameter:
439: . pc - the preconditioner context
441: Output Parameters:
442: + n_local - the number of blocks on this processor, or PETSC_NULL
443: . first_local - the global number of the first block on this processor, or PETSC_NULL
444: - ksp - the array of KSP contexts
446: Notes:
447: After PCBJacobiGetSubKSP() the array of KSP contexts is not to be freed.
448:
449: Currently for some matrix implementations only 1 block per processor
450: is supported.
451:
452: You must call KSPSetUp() or PCSetUp() before calling PCBJacobiGetSubKSP().
454: Level: advanced
456: .keywords: block, Jacobi, get, sub, KSP, context
458: .seealso: PCBJacobiGetSubKSP()
459: @*/
460: int PCBJacobiGetSubKSP(PC pc,int *n_local,int *first_local,KSP *ksp[])
461: {
462: int ierr,(*f)(PC,int *,int *,KSP **);
466: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetSubKSP_C",(void (**)(void))&f);
467: if (f) {
468: (*f)(pc,n_local,first_local,ksp);
469: } else {
470: SETERRQ(1,"Cannot get subsolvers for this preconditioner");
471: }
472: return(0);
473: }
477: /*@
478: PCBJacobiSetTotalBlocks - Sets the global number of blocks for the block
479: Jacobi preconditioner.
481: Collective on PC
483: Input Parameters:
484: + pc - the preconditioner context
485: . blocks - the number of blocks
486: - lens - [optional] integer array containing the size of each block
488: Options Database Key:
489: . -pc_bjacobi_blocks <blocks> - Sets the number of global blocks
491: Notes:
492: Currently only a limited number of blocking configurations are supported.
493: All processors sharing the PC must call this routine with the same data.
495: Level: intermediate
497: .keywords: set, number, Jacobi, global, total, blocks
499: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiSetLocalBlocks()
500: @*/
501: int PCBJacobiSetTotalBlocks(PC pc,int blocks,const int lens[])
502: {
503: int ierr,(*f)(PC,int,const int[]);
507: if (blocks <= 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have positive blocks");
508: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetTotalBlocks_C",(void (**)(void))&f);
509: if (f) {
510: (*f)(pc,blocks,lens);
511: }
512: return(0);
513: }
517: /*@C
518: PCBJacobiGetTotalBlocks - Gets the global number of blocks for the block
519: Jacobi preconditioner.
521: Collective on PC
523: Input Parameter:
524: . pc - the preconditioner context
526: Output parameters:
527: + blocks - the number of blocks
528: - lens - integer array containing the size of each block
530: Level: intermediate
532: .keywords: get, number, Jacobi, global, total, blocks
534: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiGetLocalBlocks()
535: @*/
536: int PCBJacobiGetTotalBlocks(PC pc, int *blocks, const int *lens[])
537: {
538: int ierr,(*f)(PC,int*, const int *[]);
543: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetTotalBlocks_C",(void (**)(void))&f);
544: if (f) {
545: (*f)(pc,blocks,lens);
546: }
547: return(0);
548: }
549:
552: /*@
553: PCBJacobiSetLocalBlocks - Sets the local number of blocks for the block
554: Jacobi preconditioner.
556: Not Collective
558: Input Parameters:
559: + pc - the preconditioner context
560: . blocks - the number of blocks
561: - lens - [optional] integer array containing size of each block
563: Note:
564: Currently only a limited number of blocking configurations are supported.
566: Level: intermediate
568: .keywords: PC, set, number, Jacobi, local, blocks
570: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiSetTotalBlocks()
571: @*/
572: int PCBJacobiSetLocalBlocks(PC pc,int blocks,const int lens[])
573: {
574: int ierr,(*f)(PC,int,const int []);
578: if (blocks < 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have nonegative blocks");
579: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetLocalBlocks_C",(void (**)(void))&f);
580: if (f) {
581: (*f)(pc,blocks,lens);
582: }
583: return(0);
584: }
585:
588: /*@C
589: PCBJacobiGetLocalBlocks - Gets the local number of blocks for the block
590: Jacobi preconditioner.
592: Not Collective
594: Input Parameters:
595: + pc - the preconditioner context
596: . blocks - the number of blocks
597: - lens - [optional] integer array containing size of each block
599: Note:
600: Currently only a limited number of blocking configurations are supported.
602: Level: intermediate
604: .keywords: PC, get, number, Jacobi, local, blocks
606: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiGetTotalBlocks()
607: @*/
608: int PCBJacobiGetLocalBlocks(PC pc, int *blocks, const int *lens[])
609: {
610: int ierr,(*f)(PC,int*, const int *[]);
615: PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetLocalBlocks_C",(void (**)(void))&f);
616: if (f) {
617: (*f)(pc,blocks,lens);
618: }
619: return(0);
620: }
622: /* -----------------------------------------------------------------------------------*/
624: /*MC
625: PCBJACOBI - Use block Jacobi preconditioning, each block is (approximately) solved with
626: its own KSP object.
628: Options Database Keys:
629: . -pc_bjacobi_truelocal - Activates PCBJacobiSetUseTrueLocal()
631: Notes: Each processor can have one or more blocks, but a block cannot be shared by more
632: than one processor. Defaults to one block per processor.
634: To set options on the solvers for each block append -sub_ to all the KSP, KSP, and PC
635: options database keys. For example, -sub_pc_type ilu -sub_pc_ilu_levels 1 -sub_ksp_type preonly
636:
637: To set the options on the solvers seperate for each block call PCBJacobiGetSubKSP()
638: and set the options directly on the resulting KSP object (you can access its PC
639: KSPGetPC())
641: Level: beginner
643: Concepts: block Jacobi
645: .seealso: PCCreate(), PCSetType(), PCType (for list of available types), PC,
646: PCASM, PCBJacobiSetUseTrueLocal(), PCBJacobiGetSubKSP(), PCBJacobiSetTotalBlocks(),
647: PCBJacobiSetLocalBlocks(), PCSetModifySubmatrices()
648: M*/
650: EXTERN_C_BEGIN
653: int PCCreate_BJacobi(PC pc)
654: {
655: int rank,ierr;
656: PC_BJacobi *jac;
659: PetscNew(PC_BJacobi,&jac);
660: PetscLogObjectMemory(pc,sizeof(PC_BJacobi));
661: MPI_Comm_rank(pc->comm,&rank);
662: pc->ops->apply = 0;
663: pc->ops->applytranspose = 0;
664: pc->ops->setup = PCSetUp_BJacobi;
665: pc->ops->destroy = PCDestroy_BJacobi;
666: pc->ops->setfromoptions = PCSetFromOptions_BJacobi;
667: pc->ops->view = PCView_BJacobi;
668: pc->ops->applyrichardson = 0;
670: pc->data = (void*)jac;
671: jac->n = -1;
672: jac->n_local = -1;
673: jac->first_local = rank;
674: jac->ksp = 0;
675: jac->use_true_local = PETSC_FALSE;
676: jac->same_local_solves = PETSC_TRUE;
677: jac->g_lens = 0;
678: jac->l_lens = 0;
679: jac->tp_mat = 0;
680: jac->tp_pmat = 0;
682: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetUseTrueLocal_C",
683: "PCBJacobiSetUseTrueLocal_BJacobi",
684: PCBJacobiSetUseTrueLocal_BJacobi);
685: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetSubKSP_C","PCBJacobiGetSubKSP_BJacobi",
686: PCBJacobiGetSubKSP_BJacobi);
687: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetTotalBlocks_C","PCBJacobiSetTotalBlocks_BJacobi",
688: PCBJacobiSetTotalBlocks_BJacobi);
689: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetTotalBlocks_C","PCBJacobiGetTotalBlocks_BJacobi",
690: PCBJacobiGetTotalBlocks_BJacobi);
691: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetLocalBlocks_C","PCBJacobiSetLocalBlocks_BJacobi",
692: PCBJacobiSetLocalBlocks_BJacobi);
693: PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetLocalBlocks_C","PCBJacobiGetLocalBlocks_BJacobi",
694: PCBJacobiGetLocalBlocks_BJacobi);
696: return(0);
697: }
698: EXTERN_C_END
700: /* --------------------------------------------------------------------------------------------*/
701: /*
702: These are for a single block per processor; works for AIJ, BAIJ; Seq and MPI
703: */
706: int PCDestroy_BJacobi_Singleblock(PC pc)
707: {
708: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
709: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
710: int ierr;
713: /*
714: If the on processor block had to be generated via a MatGetDiagonalBlock()
715: that creates a copy (for example MPIBDiag matrices do), this frees the space
716: */
717: if (jac->tp_mat) {
718: MatDestroy(jac->tp_mat);
719: }
720: if (jac->tp_pmat) {
721: MatDestroy(jac->tp_pmat);
722: }
724: KSPDestroy(jac->ksp[0]);
725: PetscFree(jac->ksp);
726: VecDestroy(bjac->x);
727: VecDestroy(bjac->y);
728: if (jac->l_lens) {PetscFree(jac->l_lens);}
729: if (jac->g_lens) {PetscFree(jac->g_lens);}
730: PetscFree(bjac);
731: PetscFree(jac);
732: return(0);
733: }
737: int PCSetUpOnBlocks_BJacobi_Singleblock(PC pc)
738: {
739: int ierr;
740: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
741: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
744: KSPSetRhs(jac->ksp[0],bjac->x);
745: KSPSetSolution(jac->ksp[0],bjac->y);
746: KSPSetUp(jac->ksp[0]);
747: return(0);
748: }
752: int PCApply_BJacobi_Singleblock(PC pc,Vec x,Vec y)
753: {
754: int ierr;
755: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
756: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
757: PetscScalar *x_array,*y_array;
760: /*
761: The VecPlaceArray() is to avoid having to copy the
762: y vector into the bjac->x vector. The reason for
763: the bjac->x vector is that we need a sequential vector
764: for the sequential solve.
765: */
766: VecGetArray(x,&x_array);
767: VecGetArray(y,&y_array);
768: VecPlaceArray(bjac->x,x_array);
769: VecPlaceArray(bjac->y,y_array);
770: KSPSolve(jac->ksp[0]);
771: VecRestoreArray(x,&x_array);
772: VecRestoreArray(y,&y_array);
773: return(0);
774: }
778: int PCApplySymmetricLeft_BJacobi_Singleblock(PC pc,Vec x,Vec y)
779: {
780: int ierr;
781: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
782: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
783: PetscScalar *x_array,*y_array;
784: PC subpc;
787: /*
788: The VecPlaceArray() is to avoid having to copy the
789: y vector into the bjac->x vector. The reason for
790: the bjac->x vector is that we need a sequential vector
791: for the sequential solve.
792: */
793: VecGetArray(x,&x_array);
794: VecGetArray(y,&y_array);
795: VecPlaceArray(bjac->x,x_array);
796: VecPlaceArray(bjac->y,y_array);
798: /* apply the symmetric left portion of the inner PC operator */
799: /* note this by-passes the inner KSP and its options completely */
801: KSPGetPC(jac->ksp[0],&subpc);
802: PCApplySymmetricLeft(subpc,bjac->x,bjac->y);
804: VecRestoreArray(x,&x_array);
805: VecRestoreArray(y,&y_array);
806: return(0);
807: }
811: int PCApplySymmetricRight_BJacobi_Singleblock(PC pc,Vec x,Vec y)
812: {
813: int ierr;
814: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
815: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
816: PetscScalar *x_array,*y_array;
817: PC subpc;
820: /*
821: The VecPlaceArray() is to avoid having to copy the
822: y vector into the bjac->x vector. The reason for
823: the bjac->x vector is that we need a sequential vector
824: for the sequential solve.
825: */
826: VecGetArray(x,&x_array);
827: VecGetArray(y,&y_array);
828: VecPlaceArray(bjac->x,x_array);
829: VecPlaceArray(bjac->y,y_array);
831: /* apply the symmetric right portion of the inner PC operator */
832: /* note this by-passes the inner KSP and its options completely */
834: KSPGetPC(jac->ksp[0],&subpc);
835: PCApplySymmetricRight(subpc,bjac->x,bjac->y);
837: VecRestoreArray(x,&x_array);
838: VecRestoreArray(y,&y_array);
839: return(0);
840: }
844: int PCApplyTranspose_BJacobi_Singleblock(PC pc,Vec x,Vec y)
845: {
846: int ierr;
847: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
848: PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
849: PetscScalar *x_array,*y_array;
852: /*
853: The VecPlaceArray() is to avoid having to copy the
854: y vector into the bjac->x vector. The reason for
855: the bjac->x vector is that we need a sequential vector
856: for the sequential solve.
857: */
858: VecGetArray(x,&x_array);
859: VecGetArray(y,&y_array);
860: VecPlaceArray(bjac->x,x_array);
861: VecPlaceArray(bjac->y,y_array);
862: KSPSolveTranspose(jac->ksp[0]);
863: VecRestoreArray(x,&x_array);
864: VecRestoreArray(y,&y_array);
865: return(0);
866: }
870: static int PCSetUp_BJacobi_Singleblock(PC pc,Mat mat,Mat pmat)
871: {
872: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
873: int ierr,m;
874: KSP ksp;
875: Vec x,y;
876: PC_BJacobi_Singleblock *bjac;
877: PC subpc;
881: /* set default direct solver with no Krylov method */
882: if (!pc->setupcalled) {
883: char *prefix;
884: KSPCreate(PETSC_COMM_SELF,&ksp);
885: PetscLogObjectParent(pc,ksp);
886: KSPSetType(ksp,KSPPREONLY);
887: KSPGetPC(ksp,&subpc);
888: PCSetType(subpc,PCILU);
889: PCGetOptionsPrefix(pc,&prefix);
890: KSPSetOptionsPrefix(ksp,prefix);
891: KSPAppendOptionsPrefix(ksp,"sub_");
892: KSPSetFromOptions(ksp);
893: /*
894: The reason we need to generate these vectors is to serve
895: as the right-hand side and solution vector for the solve on the
896: block. We do not need to allocate space for the vectors since
897: that is provided via VecPlaceArray() just before the call to
898: KSPSolve() on the block.
899: */
900: MatGetSize(pmat,&m,&m);
901: VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&x);
902: VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&y);
903: PetscLogObjectParent(pc,x);
904: PetscLogObjectParent(pc,y);
906: pc->ops->destroy = PCDestroy_BJacobi_Singleblock;
907: pc->ops->apply = PCApply_BJacobi_Singleblock;
908: pc->ops->applysymmetricleft = PCApplySymmetricLeft_BJacobi_Singleblock;
909: pc->ops->applysymmetricright = PCApplySymmetricRight_BJacobi_Singleblock;
910: pc->ops->applytranspose = PCApplyTranspose_BJacobi_Singleblock;
911: pc->ops->setuponblocks = PCSetUpOnBlocks_BJacobi_Singleblock;
913: PetscMalloc(sizeof(PC_BJacobi_Singleblock),&bjac);
914: PetscLogObjectMemory(pc,sizeof(PC_BJacobi_Singleblock));
915: bjac->x = x;
916: bjac->y = y;
918: PetscMalloc(sizeof(KSP),&jac->ksp);
919: jac->ksp[0] = ksp;
920: jac->data = (void*)bjac;
921: } else {
922: ksp = jac->ksp[0];
923: bjac = (PC_BJacobi_Singleblock *)jac->data;
924: }
925: if (jac->use_true_local) {
926: KSPSetOperators(ksp,mat,pmat,pc->flag);
927: } else {
928: KSPSetOperators(ksp,pmat,pmat,pc->flag);
929: }
930: return(0);
931: }
933: /* ---------------------------------------------------------------------------------------------*/
937: int PCDestroy_BJacobi_Multiblock(PC pc)
938: {
939: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
940: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
941: int i,ierr;
944: MatDestroyMatrices(jac->n_local,&bjac->pmat);
945: if (jac->use_true_local) {
946: MatDestroyMatrices(jac->n_local,&bjac->mat);
947: }
949: /*
950: If the on processor block had to be generated via a MatGetDiagonalBlock()
951: that creates a copy (for example MPIBDiag matrices do), this frees the space
952: */
953: if (jac->tp_mat) {
954: MatDestroy(jac->tp_mat);
955: }
956: if (jac->tp_pmat) {
957: MatDestroy(jac->tp_pmat);
958: }
960: for (i=0; i<jac->n_local; i++) {
961: KSPDestroy(jac->ksp[i]);
962: VecDestroy(bjac->x[i]);
963: VecDestroy(bjac->y[i]);
964: ISDestroy(bjac->is[i]);
965: }
966: PetscFree(jac->ksp);
967: PetscFree(bjac->x);
968: PetscFree(bjac->starts);
969: PetscFree(bjac->is);
970: PetscFree(bjac);
971: if (jac->l_lens) {PetscFree(jac->l_lens);}
972: if (jac->g_lens) {PetscFree(jac->g_lens);}
973: PetscFree(jac);
974: return(0);
975: }
979: int PCSetUpOnBlocks_BJacobi_Multiblock(PC pc)
980: {
981: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
982: int ierr,i,n_local = jac->n_local;
983: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
986: for (i=0; i<n_local; i++) {
987: KSPSetRhs(jac->ksp[i],bjac->x[i]);
988: KSPSetSolution(jac->ksp[i],bjac->y[i]);
989: KSPSetUp(jac->ksp[i]);
990: }
991: return(0);
992: }
994: /*
995: Preconditioner for block Jacobi
996: */
999: int PCApply_BJacobi_Multiblock(PC pc,Vec x,Vec y)
1000: {
1001: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
1002: int ierr,i,n_local = jac->n_local;
1003: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
1004: PetscScalar *xin,*yin;
1005: static PetscTruth flag = PETSC_TRUE;
1006: static int SUBKspSolve;
1009: if (flag) {
1010: PetscLogEventRegister(&SUBKspSolve,"SubKspSolve",KSP_COOKIE);
1011: flag = PETSC_FALSE;
1012: }
1013: VecGetArray(x,&xin);
1014: VecGetArray(y,&yin);
1015: for (i=0; i<n_local; i++) {
1016: /*
1017: To avoid copying the subvector from x into a workspace we instead
1018: make the workspace vector array point to the subpart of the array of
1019: the global vector.
1020: */
1021: VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
1022: VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);
1024: PetscLogEventBegin(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1025: KSPSolve(jac->ksp[i]);
1026: PetscLogEventEnd(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1027: }
1028: VecRestoreArray(x,&xin);
1029: VecRestoreArray(y,&yin);
1030: return(0);
1031: }
1033: /*
1034: Preconditioner for block Jacobi
1035: */
1038: int PCApplyTranspose_BJacobi_Multiblock(PC pc,Vec x,Vec y)
1039: {
1040: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
1041: int ierr,i,n_local = jac->n_local;
1042: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
1043: PetscScalar *xin,*yin;
1044: static PetscTruth flag = PETSC_TRUE;
1045: static int SUBKspSolve;
1048: if (flag) {
1049: PetscLogEventRegister(&SUBKspSolve,"SubKspSolveTranspose",KSP_COOKIE);
1050: flag = PETSC_FALSE;
1051: }
1052: VecGetArray(x,&xin);
1053: VecGetArray(y,&yin);
1054: for (i=0; i<n_local; i++) {
1055: /*
1056: To avoid copying the subvector from x into a workspace we instead
1057: make the workspace vector array point to the subpart of the array of
1058: the global vector.
1059: */
1060: VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
1061: VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);
1063: PetscLogEventBegin(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1064: KSPSolveTranspose(jac->ksp[i]);
1065: PetscLogEventEnd(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1066: }
1067: VecRestoreArray(x,&xin);
1068: VecRestoreArray(y,&yin);
1069: return(0);
1070: }
1074: static int PCSetUp_BJacobi_Multiblock(PC pc,Mat mat,Mat pmat)
1075: {
1076: PC_BJacobi *jac = (PC_BJacobi*)pc->data;
1077: int ierr,m,n_local,N,M,start,i;
1078: char *prefix,*pprefix,*mprefix;
1079: KSP ksp;
1080: Vec x,y;
1081: PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
1082: PC subpc;
1083: IS is;
1084: MatReuse scall = MAT_REUSE_MATRIX;
1087: MatGetLocalSize(pc->pmat,&M,&N);
1089: n_local = jac->n_local;
1091: if (jac->use_true_local) {
1092: if (mat->type != pmat->type) SETERRQ(PETSC_ERR_ARG_INCOMP,"Matrices not of same type");
1093: }
1095: /* set default direct solver with no Krylov method */
1096: if (!pc->setupcalled) {
1097: scall = MAT_INITIAL_MATRIX;
1098: pc->ops->destroy = PCDestroy_BJacobi_Multiblock;
1099: pc->ops->apply = PCApply_BJacobi_Multiblock;
1100: pc->ops->applytranspose= PCApplyTranspose_BJacobi_Multiblock;
1101: pc->ops->setuponblocks = PCSetUpOnBlocks_BJacobi_Multiblock;
1103: PetscMalloc(sizeof(PC_BJacobi_Multiblock),&bjac);
1104: PetscLogObjectMemory(pc,sizeof(PC_BJacobi_Multiblock));
1105: PetscMalloc(n_local*sizeof(KSP),&jac->ksp);
1106: PetscLogObjectMemory(pc,sizeof(n_local*sizeof(KSP)));
1107: PetscMalloc(2*n_local*sizeof(Vec),&bjac->x);
1108: PetscLogObjectMemory(pc,sizeof(2*n_local*sizeof(Vec)));
1109: bjac->y = bjac->x + n_local;
1110: PetscMalloc(n_local*sizeof(PetscScalar),&bjac->starts);
1111: PetscLogObjectMemory(pc,sizeof(n_local*sizeof(PetscScalar)));
1112:
1113: jac->data = (void*)bjac;
1114: PetscMalloc(n_local*sizeof(IS),&bjac->is);
1115: PetscLogObjectMemory(pc,sizeof(n_local*sizeof(IS)));
1117: start = 0;
1118: for (i=0; i<n_local; i++) {
1119: KSPCreate(PETSC_COMM_SELF,&ksp);
1120: PetscLogObjectParent(pc,ksp);
1121: KSPSetType(ksp,KSPPREONLY);
1122: KSPGetPC(ksp,&subpc);
1123: PCSetType(subpc,PCILU);
1124: PCGetOptionsPrefix(pc,&prefix);
1125: KSPSetOptionsPrefix(ksp,prefix);
1126: KSPAppendOptionsPrefix(ksp,"sub_");
1127: KSPSetFromOptions(ksp);
1129: m = jac->l_lens[i];
1131: /*
1132: The reason we need to generate these vectors is to serve
1133: as the right-hand side and solution vector for the solve on the
1134: block. We do not need to allocate space for the vectors since
1135: that is provided via VecPlaceArray() just before the call to
1136: KSPSolve() on the block.
1138: */
1139: VecCreateSeq(PETSC_COMM_SELF,m,&x);
1140: VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&y);
1141: PetscLogObjectParent(pc,x);
1142: PetscLogObjectParent(pc,y);
1143: bjac->x[i] = x;
1144: bjac->y[i] = y;
1145: bjac->starts[i] = start;
1146: jac->ksp[i] = ksp;
1148: ISCreateStride(PETSC_COMM_SELF,m,start,1,&is);
1149: bjac->is[i] = is;
1150: PetscLogObjectParent(pc,is);
1152: start += m;
1153: }
1154: } else {
1155: bjac = (PC_BJacobi_Multiblock*)jac->data;
1156: /*
1157: Destroy the blocks from the previous iteration
1158: */
1159: if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
1160: MatDestroyMatrices(n_local,&bjac->pmat);
1161: if (jac->use_true_local) {
1162: MatDestroyMatrices(n_local,&bjac->mat);
1163: }
1164: scall = MAT_INITIAL_MATRIX;
1165: }
1166: }
1168: MatGetSubMatrices(pmat,n_local,bjac->is,bjac->is,scall,&bjac->pmat);
1169: if (jac->use_true_local) {
1170: PetscObjectGetOptionsPrefix((PetscObject)mat,&mprefix);
1171: MatGetSubMatrices(mat,n_local,bjac->is,bjac->is,scall,&bjac->mat);
1172: }
1173: /* Return control to the user so that the submatrices can be modified (e.g., to apply
1174: different boundary conditions for the submatrices than for the global problem) */
1175: PCModifySubMatrices(pc,n_local,bjac->is,bjac->is,bjac->pmat,pc->modifysubmatricesP);
1177: PetscObjectGetOptionsPrefix((PetscObject)pmat,&pprefix);
1178: for (i=0; i<n_local; i++) {
1179: PetscLogObjectParent(pc,bjac->pmat[i]);
1180: PetscObjectSetOptionsPrefix((PetscObject)bjac->pmat[i],pprefix);
1181: if (jac->use_true_local) {
1182: PetscLogObjectParent(pc,bjac->mat[i]);
1183: PetscObjectSetOptionsPrefix((PetscObject)bjac->mat[i],mprefix);
1184: KSPSetOperators(jac->ksp[i],bjac->mat[i],bjac->pmat[i],pc->flag);
1185: } else {
1186: KSPSetOperators(jac->ksp[i],bjac->pmat[i],bjac->pmat[i],pc->flag);
1187: }
1188: }
1190: return(0);
1191: }