Actual source code: bjacobi.c

  1: /*$Id: bjacobi.c,v 1.160 2001/08/07 03:03:35 balay Exp $*/
  2: /*
  3:    Defines a block Jacobi preconditioner.
  4: */
 5:  #include src/mat/matimpl.h
 6:  #include src/ksp/pc/pcimpl.h
 7:  #include src/ksp/pc/impls/bjacobi/bjacobi.h

  9: static int PCSetUp_BJacobi_Singleblock(PC,Mat,Mat);
 10: static int PCSetUp_BJacobi_Multiblock(PC,Mat,Mat);

 14: static int PCSetUp_BJacobi(PC pc)
 15: {
 16:   PC_BJacobi      *jac = (PC_BJacobi*)pc->data;
 17:   Mat             mat = pc->mat,pmat = pc->pmat;
 18:   int             ierr,N,M,start,i,rank,size,sum,end;
 19:   int             bs,i_start=-1,i_end=-1;
 20:   char            *pprefix,*mprefix;
 21:   int             (*f)(Mat,PetscTruth*,MatReuse,Mat*);

 24:   MPI_Comm_rank(pc->comm,&rank);
 25:   MPI_Comm_size(pc->comm,&size);
 26:   MatGetLocalSize(pc->pmat,&M,&N);
 27:   MatGetBlockSize(pc->pmat,&bs);

 29:   /* ----------
 30:       Determines the number of blocks assigned to each processor 
 31:   */

 33:   /*   local block count  given */
 34:   if (jac->n_local > 0 && jac->n < 0) {
 35:     MPI_Allreduce(&jac->n_local,&jac->n,1,MPI_INT,MPI_SUM,pc->comm);
 36:     if (jac->l_lens) { /* check that user set these correctly */
 37:       sum = 0;
 38:       for (i=0; i<jac->n_local; i++) {
 39:         if (jac->l_lens[i]/bs*bs !=jac->l_lens[i]) {
 40:           SETERRQ(PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
 41:         }
 42:         sum += jac->l_lens[i];
 43:       }
 44:       if (sum != M) SETERRQ(PETSC_ERR_ARG_SIZ,"Local lens sent incorrectly");
 45:     } else {
 46:       PetscMalloc(jac->n_local*sizeof(int),&jac->l_lens);
 47:       for (i=0; i<jac->n_local; i++) {
 48:         jac->l_lens[i] = bs*((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i));
 49:       }
 50:     }
 51:   } else if (jac->n > 0 && jac->n_local < 0) { /* global block count given */
 52:     /* global blocks given: determine which ones are local */
 53:     if (jac->g_lens) {
 54:       /* check if the g_lens is has valid entries */
 55:       for (i=0; i<jac->n; i++) {
 56:         if (!jac->g_lens[i]) SETERRQ(PETSC_ERR_ARG_SIZ,"Zero block not allowed");
 57:         if (jac->g_lens[i]/bs*bs != jac->g_lens[i]) {
 58:           SETERRQ(PETSC_ERR_ARG_SIZ,"Mat blocksize doesn't match block Jacobi layout");
 59:         }
 60:       }
 61:       if (size == 1) {
 62:         jac->n_local = jac->n;
 63:         PetscMalloc(jac->n_local*sizeof(int),&jac->l_lens);
 64:         PetscMemcpy(jac->l_lens,jac->g_lens,jac->n_local*sizeof(int));
 65:         /* check that user set these correctly */
 66:         sum = 0;
 67:         for (i=0; i<jac->n_local; i++) sum += jac->l_lens[i];
 68:         if (sum != M) SETERRQ(PETSC_ERR_ARG_SIZ,"Global lens sent incorrectly");
 69:       } else {
 70:         MatGetOwnershipRange(pc->pmat,&start,&end);
 71:         /* loop over blocks determing first one owned by me */
 72:         sum = 0;
 73:         for (i=0; i<jac->n+1; i++) {
 74:           if (sum == start) { i_start = i; goto start_1;}
 75:           if (i < jac->n) sum += jac->g_lens[i];
 76:         }
 77:         SETERRQ(PETSC_ERR_ARG_SIZ,"Block sizes\n\
 78:                    used in PCBJacobiSetTotalBlocks()\n\
 79:                    are not compatible with parallel matrix layout");
 80:  start_1:
 81:         for (i=i_start; i<jac->n+1; i++) {
 82:           if (sum == end) { i_end = i; goto end_1; }
 83:           if (i < jac->n) sum += jac->g_lens[i];
 84:         }
 85:         SETERRQ(PETSC_ERR_ARG_SIZ,"Block sizes\n\
 86:                       used in PCBJacobiSetTotalBlocks()\n\
 87:                       are not compatible with parallel matrix layout");
 88:  end_1:
 89:         jac->n_local = i_end - i_start;
 90:         PetscMalloc(jac->n_local*sizeof(int),&jac->l_lens);
 91:         PetscMemcpy(jac->l_lens,jac->g_lens+i_start,jac->n_local*sizeof(int));
 92:       }
 93:     } else { /* no global blocks given, determine then using default layout */
 94:       jac->n_local = jac->n/size + ((jac->n % size) > rank);
 95:       PetscMalloc(jac->n_local*sizeof(int),&jac->l_lens);
 96:       for (i=0; i<jac->n_local; i++) {
 97:         jac->l_lens[i] = ((M/bs)/jac->n_local + (((M/bs) % jac->n_local) > i))*bs;
 98:         if (!jac->l_lens[i]) SETERRQ(PETSC_ERR_ARG_SIZ,"Too many blocks given");
 99:       }
100:     }
101:   } else if (jac->n < 0 && jac->n_local < 0) { /* no blocks given */
102:     jac->n         = size;
103:     jac->n_local   = 1;
104:     PetscMalloc(sizeof(int),&jac->l_lens);
105:     jac->l_lens[0] = M;
106:   }

108:   MPI_Comm_size(pc->comm,&size);
109:   PetscObjectQueryFunction((PetscObject)pc->mat,"MatGetDiagonalBlock_C",(void (**)(void))&f);
110:   if (size == 1 && !f) {
111:     mat  = pc->mat;
112:     pmat = pc->pmat;
113:   } else {
114:     PetscTruth iscopy;
115:     MatReuse   scall;

117:     if (jac->use_true_local) {
118:       scall = MAT_INITIAL_MATRIX;
119:       if (pc->setupcalled) {
120:         if (pc->flag == SAME_NONZERO_PATTERN) {
121:           if (jac->tp_mat) {
122:             scall = MAT_REUSE_MATRIX;
123:             mat   = jac->tp_mat;
124:           }
125:         } else {
126:           if (jac->tp_mat)  {
127:             MatDestroy(jac->tp_mat);
128:           }
129:         }
130:       }
131:       if (!f) {
132:         SETERRQ(PETSC_ERR_SUP,"This matrix does not support getting diagonal block");
133:       }
134:       (*f)(pc->mat,&iscopy,scall,&mat);
135:       /* make submatrix have same prefix as entire matrix */
136:       PetscObjectGetOptionsPrefix((PetscObject)pc->mat,&mprefix);
137:       PetscObjectSetOptionsPrefix((PetscObject)mat,mprefix);
138:       if (iscopy) {
139:         jac->tp_mat = mat;
140:       }
141:     }
142:     if (pc->pmat != pc->mat || !jac->use_true_local) {
143:       scall = MAT_INITIAL_MATRIX;
144:       if (pc->setupcalled) {
145:         if (pc->flag == SAME_NONZERO_PATTERN) {
146:           if (jac->tp_pmat) {
147:             scall = MAT_REUSE_MATRIX;
148:             pmat   = jac->tp_pmat;
149:           }
150:         } else {
151:           if (jac->tp_pmat)  {
152:             MatDestroy(jac->tp_pmat);
153:           }
154:         }
155:       }
156:       PetscObjectQueryFunction((PetscObject)pc->pmat,"MatGetDiagonalBlock_C",(void (**)(void))&f);
157:       if (!f) {
158:         SETERRQ(PETSC_ERR_SUP,"This matrix does not support getting diagonal block");
159:       }
160:       (*f)(pc->pmat,&iscopy,scall,&pmat);
161:       /* make submatrix have same prefix as entire matrix */
162:       PetscObjectGetOptionsPrefix((PetscObject)pc->pmat,&pprefix);
163:       PetscObjectSetOptionsPrefix((PetscObject)pmat,pprefix);
164:       if (iscopy) {
165:         jac->tp_pmat = pmat;
166:       }
167:     } else {
168:       pmat = mat;
169:     }
170:   }

172:   /* ------
173:      Setup code depends on the number of blocks 
174:   */
175:   if (jac->n_local == 1) {
176:     PCSetUp_BJacobi_Singleblock(pc,mat,pmat);
177:   } else {
178:     PCSetUp_BJacobi_Multiblock(pc,mat,pmat);
179:   }
180:   return(0);
181: }

183: /* Default destroy, if it has never been setup */
186: static int PCDestroy_BJacobi(PC pc)
187: {
188:   PC_BJacobi *jac = (PC_BJacobi*)pc->data;

192:   if (jac->g_lens) {PetscFree(jac->g_lens);}
193:   if (jac->l_lens) {PetscFree(jac->l_lens);}
194:   PetscFree(jac);
195:   return(0);
196: }

200: static int PCSetFromOptions_BJacobi(PC pc)
201: {
202:   PC_BJacobi *jac = (PC_BJacobi*)pc->data;
203:   int        blocks,ierr;
204:   PetscTruth flg;

207:   PetscOptionsHead("Block Jacobi options");
208:     PetscOptionsInt("-pc_bjacobi_blocks","Total number of blocks","PCBJacobiSetTotalBlocks",jac->n,&blocks,&flg);
209:     if (flg) {
210:       PCBJacobiSetTotalBlocks(pc,blocks,PETSC_NULL);
211:     }
212:     PetscOptionsName("-pc_bjacobi_truelocal","Use the true matrix, not preconditioner matrix to define matrix vector product in sub-problems","PCBJacobiSetUseTrueLocal",&flg);
213:     if (flg) {
214:       PCBJacobiSetUseTrueLocal(pc);
215:     }
216:   PetscOptionsTail();
217:   return(0);
218: }

222: static int PCView_BJacobi(PC pc,PetscViewer viewer)
223: {
224:   PC_BJacobi  *jac = (PC_BJacobi*)pc->data;
225:   int         rank,ierr,i;
226:   PetscTruth  isascii,isstring;
227:   PetscViewer sviewer;

230:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&isascii);
231:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_STRING,&isstring);
232:   if (isascii) {
233:     if (jac->use_true_local) {
234:       PetscViewerASCIIPrintf(viewer,"  block Jacobi: using true local matrix, number of blocks = %d\n",jac->n);
235:     }
236:     PetscViewerASCIIPrintf(viewer,"  block Jacobi: number of blocks = %d\n",jac->n);
237:     MPI_Comm_rank(pc->comm,&rank);
238:     if (jac->same_local_solves) {
239:       PetscViewerASCIIPrintf(viewer,"  Local solve is same for all blocks, in the following KSP and PC objects:\n");
240:       PetscViewerGetSingleton(viewer,&sviewer);
241:       if (!rank && jac->ksp) {
242:         PetscViewerASCIIPushTab(viewer);
243:         KSPView(jac->ksp[0],sviewer);
244:         PetscViewerASCIIPopTab(viewer);
245:       }
246:       PetscViewerRestoreSingleton(viewer,&sviewer);
247:     } else {

249:       PetscViewerASCIIPrintf(viewer,"  Local solve info for each block is in the following KSP and PC objects:\n");
250:       PetscViewerASCIISynchronizedPrintf(viewer,"Proc %d: number of local blocks = %d, first local block number = %d\n",
251:                    rank,jac->n_local,jac->first_local);
252:       PetscViewerASCIIPushTab(viewer);
253:       for (i=0; i<jac->n_local; i++) {
254:         PetscViewerASCIISynchronizedPrintf(viewer,"Proc %d: local block number %d\n",rank,i);
255:         PetscViewerGetSingleton(viewer,&sviewer);
256:         KSPView(jac->ksp[i],sviewer);
257:         if (i != jac->n_local-1) {
258:           PetscViewerASCIISynchronizedPrintf(viewer,"- - - - - - - - - - - - - - - - - -\n");
259:         }
260:         PetscViewerRestoreSingleton(viewer,&sviewer);
261:       }
262:       PetscViewerASCIIPopTab(viewer);
263:       PetscViewerFlush(viewer);
264:     }
265:   } else if (isstring) {
266:     PetscViewerStringSPrintf(viewer," blks=%d",jac->n);
267:     PetscViewerGetSingleton(viewer,&sviewer);
268:     if (jac->ksp) {KSPView(jac->ksp[0],sviewer);}
269:     PetscViewerRestoreSingleton(viewer,&sviewer);
270:   } else {
271:     SETERRQ1(1,"Viewer type %s not supported for block Jacobi",((PetscObject)viewer)->type_name);
272:   }
273:   return(0);
274: }

276: /* -------------------------------------------------------------------------------------*/

278: EXTERN_C_BEGIN
281: int PCBJacobiSetUseTrueLocal_BJacobi(PC pc)
282: {
283:   PC_BJacobi   *jac;

286:   jac                 = (PC_BJacobi*)pc->data;
287:   jac->use_true_local = PETSC_TRUE;
288:   return(0);
289: }
290: EXTERN_C_END

292: EXTERN_C_BEGIN
295: int PCBJacobiGetSubKSP_BJacobi(PC pc,int *n_local,int *first_local,KSP **ksp)
296: {
297:   PC_BJacobi   *jac = (PC_BJacobi*)pc->data;;

300:   if (!pc->setupcalled) SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"Must call KSPSetUp() or PCSetUp() first");

302:   if (n_local)     *n_local     = jac->n_local;
303:   if (first_local) *first_local = jac->first_local;
304:   *ksp                         = jac->ksp;
305:   jac->same_local_solves        = PETSC_FALSE; /* Assume that local solves are now different;
306:                                                   not necessarily true though!  This flag is 
307:                                                   used only for PCView_BJacobi() */
308:   return(0);
309: }
310: EXTERN_C_END

312: EXTERN_C_BEGIN
315: int PCBJacobiSetTotalBlocks_BJacobi(PC pc,int blocks,int *lens)
316: {
317:   PC_BJacobi *jac = (PC_BJacobi*)pc->data;
318:   int        ierr;


322:   if (pc->setupcalled > 0) SETERRQ(1,"Cannot set number of blocks after PCSetUp()/KSPSetUp() has been called");
323:   jac->n = blocks;
324:   if (!lens) {
325:     jac->g_lens = 0;
326:   } else {
327:     PetscMalloc(blocks*sizeof(int),&jac->g_lens);
328:     PetscLogObjectMemory(pc,blocks*sizeof(int));
329:     PetscMemcpy(jac->g_lens,lens,blocks*sizeof(int));
330:   }
331:   return(0);
332: }
333: EXTERN_C_END

335: EXTERN_C_BEGIN
338: int PCBJacobiGetTotalBlocks_BJacobi(PC pc, int *blocks, const int *lens[])
339: {
340:   PC_BJacobi *jac = (PC_BJacobi*) pc->data;

343:   *blocks = jac->n;
344:   if (lens) *lens = jac->g_lens;
345:   return(0);
346: }
347: EXTERN_C_END

349: EXTERN_C_BEGIN
352: int PCBJacobiSetLocalBlocks_BJacobi(PC pc,int blocks,const int lens[])
353: {
354:   PC_BJacobi *jac;
355:   int        ierr;

358:   jac = (PC_BJacobi*)pc->data;

360:   jac->n_local = blocks;
361:   if (!lens) {
362:     jac->l_lens = 0;
363:   } else {
364:     PetscMalloc(blocks*sizeof(int),&jac->l_lens);
365:     PetscLogObjectMemory(pc,blocks*sizeof(int));
366:     PetscMemcpy(jac->l_lens,lens,blocks*sizeof(int));
367:   }
368:   return(0);
369: }
370: EXTERN_C_END

372: EXTERN_C_BEGIN
375: int PCBJacobiGetLocalBlocks_BJacobi(PC pc, int *blocks, const int *lens[])
376: {
377:   PC_BJacobi *jac = (PC_BJacobi*) pc->data;

380:   *blocks = jac->n_local;
381:   if (lens) *lens = jac->l_lens;
382:   return(0);
383: }
384: EXTERN_C_END

386: /* -------------------------------------------------------------------------------------*/

390: /*@
391:    PCBJacobiSetUseTrueLocal - Sets a flag to indicate that the block 
392:    problem is associated with the linear system matrix instead of the
393:    default (where it is associated with the preconditioning matrix).
394:    That is, if the local system is solved iteratively then it iterates
395:    on the block from the matrix using the block from the preconditioner
396:    as the preconditioner for the local block.

398:    Collective on PC

400:    Input Parameters:
401: .  pc - the preconditioner context

403:    Options Database Key:
404: .  -pc_bjacobi_truelocal - Activates PCBJacobiSetUseTrueLocal()

406:    Notes:
407:    For the common case in which the preconditioning and linear 
408:    system matrices are identical, this routine is unnecessary.

410:    Level: intermediate

412: .keywords:  block, Jacobi, set, true, local, flag

414: .seealso: PCSetOperators(), PCBJacobiSetLocalBlocks()
415: @*/
416: int PCBJacobiSetUseTrueLocal(PC pc)
417: {
418:   int ierr,(*f)(PC);

422:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetUseTrueLocal_C",(void (**)(void))&f);
423:   if (f) {
424:     (*f)(pc);
425:   }

427:   return(0);
428: }

432: /*@C
433:    PCBJacobiGetSubKSP - Gets the local KSP contexts for all blocks on
434:    this processor.
435:    
436:    Note Collective

438:    Input Parameter:
439: .  pc - the preconditioner context

441:    Output Parameters:
442: +  n_local - the number of blocks on this processor, or PETSC_NULL
443: .  first_local - the global number of the first block on this processor, or PETSC_NULL
444: -  ksp - the array of KSP contexts

446:    Notes:  
447:    After PCBJacobiGetSubKSP() the array of KSP contexts is not to be freed.
448:    
449:    Currently for some matrix implementations only 1 block per processor 
450:    is supported.
451:    
452:    You must call KSPSetUp() or PCSetUp() before calling PCBJacobiGetSubKSP().

454:    Level: advanced

456: .keywords:  block, Jacobi, get, sub, KSP, context

458: .seealso: PCBJacobiGetSubKSP()
459: @*/
460: int PCBJacobiGetSubKSP(PC pc,int *n_local,int *first_local,KSP *ksp[])
461: {
462:   int ierr,(*f)(PC,int *,int *,KSP **);

466:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetSubKSP_C",(void (**)(void))&f);
467:   if (f) {
468:     (*f)(pc,n_local,first_local,ksp);
469:   } else {
470:     SETERRQ(1,"Cannot get subsolvers for this preconditioner");
471:   }
472:   return(0);
473: }

477: /*@
478:    PCBJacobiSetTotalBlocks - Sets the global number of blocks for the block
479:    Jacobi preconditioner.

481:    Collective on PC

483:    Input Parameters:
484: +  pc - the preconditioner context
485: .  blocks - the number of blocks
486: -  lens - [optional] integer array containing the size of each block

488:    Options Database Key:
489: .  -pc_bjacobi_blocks <blocks> - Sets the number of global blocks

491:    Notes:  
492:    Currently only a limited number of blocking configurations are supported.
493:    All processors sharing the PC must call this routine with the same data.

495:    Level: intermediate

497: .keywords:  set, number, Jacobi, global, total, blocks

499: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiSetLocalBlocks()
500: @*/
501: int PCBJacobiSetTotalBlocks(PC pc,int blocks,const int lens[])
502: {
503:   int ierr,(*f)(PC,int,const int[]);

507:   if (blocks <= 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have positive blocks");
508:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetTotalBlocks_C",(void (**)(void))&f);
509:   if (f) {
510:     (*f)(pc,blocks,lens);
511:   }
512:   return(0);
513: }

517: /*@C
518:    PCBJacobiGetTotalBlocks - Gets the global number of blocks for the block
519:    Jacobi preconditioner.

521:    Collective on PC

523:    Input Parameter:
524: .  pc - the preconditioner context

526:    Output parameters:
527: +  blocks - the number of blocks
528: -  lens - integer array containing the size of each block

530:    Level: intermediate

532: .keywords:  get, number, Jacobi, global, total, blocks

534: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiGetLocalBlocks()
535: @*/
536: int PCBJacobiGetTotalBlocks(PC pc, int *blocks, const int *lens[])
537: {
538:   int ierr,(*f)(PC,int*, const int *[]);

543:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetTotalBlocks_C",(void (**)(void))&f);
544:   if (f) {
545:     (*f)(pc,blocks,lens);
546:   }
547:   return(0);
548: }
549: 
552: /*@
553:    PCBJacobiSetLocalBlocks - Sets the local number of blocks for the block
554:    Jacobi preconditioner.

556:    Not Collective

558:    Input Parameters:
559: +  pc - the preconditioner context
560: .  blocks - the number of blocks
561: -  lens - [optional] integer array containing size of each block

563:    Note:  
564:    Currently only a limited number of blocking configurations are supported.

566:    Level: intermediate

568: .keywords: PC, set, number, Jacobi, local, blocks

570: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiSetTotalBlocks()
571: @*/
572: int PCBJacobiSetLocalBlocks(PC pc,int blocks,const int lens[])
573: {
574:   int ierr,(*f)(PC,int,const int []);

578:   if (blocks < 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Must have nonegative blocks");
579:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiSetLocalBlocks_C",(void (**)(void))&f);
580:   if (f) {
581:     (*f)(pc,blocks,lens);
582:   }
583:   return(0);
584: }
585: 
588: /*@C
589:    PCBJacobiGetLocalBlocks - Gets the local number of blocks for the block
590:    Jacobi preconditioner.

592:    Not Collective

594:    Input Parameters:
595: +  pc - the preconditioner context
596: .  blocks - the number of blocks
597: -  lens - [optional] integer array containing size of each block

599:    Note:  
600:    Currently only a limited number of blocking configurations are supported.

602:    Level: intermediate

604: .keywords: PC, get, number, Jacobi, local, blocks

606: .seealso: PCBJacobiSetUseTrueLocal(), PCBJacobiGetTotalBlocks()
607: @*/
608: int PCBJacobiGetLocalBlocks(PC pc, int *blocks, const int *lens[])
609: {
610:   int ierr,(*f)(PC,int*, const int *[]);

615:   PetscObjectQueryFunction((PetscObject)pc,"PCBJacobiGetLocalBlocks_C",(void (**)(void))&f);
616:   if (f) {
617:     (*f)(pc,blocks,lens);
618:   }
619:   return(0);
620: }

622: /* -----------------------------------------------------------------------------------*/

624: /*MC
625:    PCBJACOBI - Use block Jacobi preconditioning, each block is (approximately) solved with 
626:            its own KSP object.

628:    Options Database Keys:
629: .  -pc_bjacobi_truelocal - Activates PCBJacobiSetUseTrueLocal()

631:    Notes: Each processor can have one or more blocks, but a block cannot be shared by more
632:      than one processor. Defaults to one block per processor.

634:      To set options on the solvers for each block append -sub_ to all the KSP, KSP, and PC
635:         options database keys. For example, -sub_pc_type ilu -sub_pc_ilu_levels 1 -sub_ksp_type preonly
636:         
637:      To set the options on the solvers seperate for each block call PCBJacobiGetSubKSP()
638:          and set the options directly on the resulting KSP object (you can access its PC
639:          KSPGetPC())

641:    Level: beginner

643:    Concepts: block Jacobi

645: .seealso:  PCCreate(), PCSetType(), PCType (for list of available types), PC,
646:            PCASM, PCBJacobiSetUseTrueLocal(), PCBJacobiGetSubKSP(), PCBJacobiSetTotalBlocks(),
647:            PCBJacobiSetLocalBlocks(), PCSetModifySubmatrices()
648: M*/

650: EXTERN_C_BEGIN
653: int PCCreate_BJacobi(PC pc)
654: {
655:   int          rank,ierr;
656:   PC_BJacobi   *jac;

659:   PetscNew(PC_BJacobi,&jac);
660:   PetscLogObjectMemory(pc,sizeof(PC_BJacobi));
661:   MPI_Comm_rank(pc->comm,&rank);
662:   pc->ops->apply              = 0;
663:   pc->ops->applytranspose     = 0;
664:   pc->ops->setup              = PCSetUp_BJacobi;
665:   pc->ops->destroy            = PCDestroy_BJacobi;
666:   pc->ops->setfromoptions     = PCSetFromOptions_BJacobi;
667:   pc->ops->view               = PCView_BJacobi;
668:   pc->ops->applyrichardson    = 0;

670:   pc->data               = (void*)jac;
671:   jac->n                 = -1;
672:   jac->n_local           = -1;
673:   jac->first_local       = rank;
674:   jac->ksp              = 0;
675:   jac->use_true_local    = PETSC_FALSE;
676:   jac->same_local_solves = PETSC_TRUE;
677:   jac->g_lens            = 0;
678:   jac->l_lens            = 0;
679:   jac->tp_mat            = 0;
680:   jac->tp_pmat           = 0;

682:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetUseTrueLocal_C",
683:                     "PCBJacobiSetUseTrueLocal_BJacobi",
684:                     PCBJacobiSetUseTrueLocal_BJacobi);
685:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetSubKSP_C","PCBJacobiGetSubKSP_BJacobi",
686:                     PCBJacobiGetSubKSP_BJacobi);
687:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetTotalBlocks_C","PCBJacobiSetTotalBlocks_BJacobi",
688:                     PCBJacobiSetTotalBlocks_BJacobi);
689:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetTotalBlocks_C","PCBJacobiGetTotalBlocks_BJacobi",
690:                     PCBJacobiGetTotalBlocks_BJacobi);
691:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiSetLocalBlocks_C","PCBJacobiSetLocalBlocks_BJacobi",
692:                     PCBJacobiSetLocalBlocks_BJacobi);
693:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCBJacobiGetLocalBlocks_C","PCBJacobiGetLocalBlocks_BJacobi",
694:                     PCBJacobiGetLocalBlocks_BJacobi);

696:   return(0);
697: }
698: EXTERN_C_END

700: /* --------------------------------------------------------------------------------------------*/
701: /*
702:         These are for a single block per processor; works for AIJ, BAIJ; Seq and MPI
703: */
706: int PCDestroy_BJacobi_Singleblock(PC pc)
707: {
708:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
709:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
710:   int                    ierr;

713:   /*
714:         If the on processor block had to be generated via a MatGetDiagonalBlock()
715:      that creates a copy (for example MPIBDiag matrices do), this frees the space
716:   */
717:   if (jac->tp_mat) {
718:     MatDestroy(jac->tp_mat);
719:   }
720:   if (jac->tp_pmat) {
721:     MatDestroy(jac->tp_pmat);
722:   }

724:   KSPDestroy(jac->ksp[0]);
725:   PetscFree(jac->ksp);
726:   VecDestroy(bjac->x);
727:   VecDestroy(bjac->y);
728:   if (jac->l_lens) {PetscFree(jac->l_lens);}
729:   if (jac->g_lens) {PetscFree(jac->g_lens);}
730:   PetscFree(bjac);
731:   PetscFree(jac);
732:   return(0);
733: }

737: int PCSetUpOnBlocks_BJacobi_Singleblock(PC pc)
738: {
739:   int                    ierr;
740:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
741:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;

744:   KSPSetRhs(jac->ksp[0],bjac->x);
745:   KSPSetSolution(jac->ksp[0],bjac->y);
746:   KSPSetUp(jac->ksp[0]);
747:   return(0);
748: }

752: int PCApply_BJacobi_Singleblock(PC pc,Vec x,Vec y)
753: {
754:   int                    ierr;
755:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
756:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
757:   PetscScalar            *x_array,*y_array;

760:   /* 
761:       The VecPlaceArray() is to avoid having to copy the 
762:     y vector into the bjac->x vector. The reason for 
763:     the bjac->x vector is that we need a sequential vector
764:     for the sequential solve.
765:   */
766:   VecGetArray(x,&x_array);
767:   VecGetArray(y,&y_array);
768:   VecPlaceArray(bjac->x,x_array);
769:   VecPlaceArray(bjac->y,y_array);
770:   KSPSolve(jac->ksp[0]);
771:   VecRestoreArray(x,&x_array);
772:   VecRestoreArray(y,&y_array);
773:   return(0);
774: }

778: int PCApplySymmetricLeft_BJacobi_Singleblock(PC pc,Vec x,Vec y)
779: {
780:   int                    ierr;
781:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
782:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
783:   PetscScalar            *x_array,*y_array;
784:   PC                     subpc;

787:   /* 
788:       The VecPlaceArray() is to avoid having to copy the 
789:     y vector into the bjac->x vector. The reason for 
790:     the bjac->x vector is that we need a sequential vector
791:     for the sequential solve.
792:   */
793:   VecGetArray(x,&x_array);
794:   VecGetArray(y,&y_array);
795:   VecPlaceArray(bjac->x,x_array);
796:   VecPlaceArray(bjac->y,y_array);

798:   /* apply the symmetric left portion of the inner PC operator */
799:   /* note this by-passes the inner KSP and its options completely */

801:   KSPGetPC(jac->ksp[0],&subpc);
802:   PCApplySymmetricLeft(subpc,bjac->x,bjac->y);

804:   VecRestoreArray(x,&x_array);
805:   VecRestoreArray(y,&y_array);
806:   return(0);
807: }

811: int PCApplySymmetricRight_BJacobi_Singleblock(PC pc,Vec x,Vec y)
812: {
813:   int                    ierr;
814:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
815:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
816:   PetscScalar            *x_array,*y_array;
817:   PC                     subpc;

820:   /* 
821:       The VecPlaceArray() is to avoid having to copy the 
822:     y vector into the bjac->x vector. The reason for 
823:     the bjac->x vector is that we need a sequential vector
824:     for the sequential solve.
825:   */
826:   VecGetArray(x,&x_array);
827:   VecGetArray(y,&y_array);
828:   VecPlaceArray(bjac->x,x_array);
829:   VecPlaceArray(bjac->y,y_array);

831:   /* apply the symmetric right portion of the inner PC operator */
832:   /* note this by-passes the inner KSP and its options completely */

834:   KSPGetPC(jac->ksp[0],&subpc);
835:   PCApplySymmetricRight(subpc,bjac->x,bjac->y);

837:   VecRestoreArray(x,&x_array);
838:   VecRestoreArray(y,&y_array);
839:   return(0);
840: }

844: int PCApplyTranspose_BJacobi_Singleblock(PC pc,Vec x,Vec y)
845: {
846:   int                    ierr;
847:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
848:   PC_BJacobi_Singleblock *bjac = (PC_BJacobi_Singleblock*)jac->data;
849:   PetscScalar            *x_array,*y_array;

852:   /* 
853:       The VecPlaceArray() is to avoid having to copy the 
854:     y vector into the bjac->x vector. The reason for 
855:     the bjac->x vector is that we need a sequential vector
856:     for the sequential solve.
857:   */
858:   VecGetArray(x,&x_array);
859:   VecGetArray(y,&y_array);
860:   VecPlaceArray(bjac->x,x_array);
861:   VecPlaceArray(bjac->y,y_array);
862:   KSPSolveTranspose(jac->ksp[0]);
863:   VecRestoreArray(x,&x_array);
864:   VecRestoreArray(y,&y_array);
865:   return(0);
866: }

870: static int PCSetUp_BJacobi_Singleblock(PC pc,Mat mat,Mat pmat)
871: {
872:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
873:   int                    ierr,m;
874:   KSP                   ksp;
875:   Vec                    x,y;
876:   PC_BJacobi_Singleblock *bjac;
877:   PC                     subpc;


881:   /* set default direct solver with no Krylov method */
882:   if (!pc->setupcalled) {
883:     char *prefix;
884:     KSPCreate(PETSC_COMM_SELF,&ksp);
885:     PetscLogObjectParent(pc,ksp);
886:     KSPSetType(ksp,KSPPREONLY);
887:     KSPGetPC(ksp,&subpc);
888:     PCSetType(subpc,PCILU);
889:     PCGetOptionsPrefix(pc,&prefix);
890:     KSPSetOptionsPrefix(ksp,prefix);
891:     KSPAppendOptionsPrefix(ksp,"sub_");
892:     KSPSetFromOptions(ksp);
893:     /*
894:       The reason we need to generate these vectors is to serve 
895:       as the right-hand side and solution vector for the solve on the 
896:       block. We do not need to allocate space for the vectors since
897:       that is provided via VecPlaceArray() just before the call to 
898:       KSPSolve() on the block.
899:     */
900:     MatGetSize(pmat,&m,&m);
901:     VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&x);
902:     VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&y);
903:     PetscLogObjectParent(pc,x);
904:     PetscLogObjectParent(pc,y);

906:     pc->ops->destroy             = PCDestroy_BJacobi_Singleblock;
907:     pc->ops->apply               = PCApply_BJacobi_Singleblock;
908:     pc->ops->applysymmetricleft  = PCApplySymmetricLeft_BJacobi_Singleblock;
909:     pc->ops->applysymmetricright = PCApplySymmetricRight_BJacobi_Singleblock;
910:     pc->ops->applytranspose      = PCApplyTranspose_BJacobi_Singleblock;
911:     pc->ops->setuponblocks       = PCSetUpOnBlocks_BJacobi_Singleblock;

913:     PetscMalloc(sizeof(PC_BJacobi_Singleblock),&bjac);
914:     PetscLogObjectMemory(pc,sizeof(PC_BJacobi_Singleblock));
915:     bjac->x      = x;
916:     bjac->y      = y;

918:     PetscMalloc(sizeof(KSP),&jac->ksp);
919:     jac->ksp[0] = ksp;
920:     jac->data    = (void*)bjac;
921:   } else {
922:     ksp = jac->ksp[0];
923:     bjac = (PC_BJacobi_Singleblock *)jac->data;
924:   }
925:   if (jac->use_true_local) {
926:     KSPSetOperators(ksp,mat,pmat,pc->flag);
927:   }  else {
928:     KSPSetOperators(ksp,pmat,pmat,pc->flag);
929:   }
930:   return(0);
931: }

933: /* ---------------------------------------------------------------------------------------------*/

937: int PCDestroy_BJacobi_Multiblock(PC pc)
938: {
939:   PC_BJacobi            *jac = (PC_BJacobi*)pc->data;
940:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
941:   int                   i,ierr;

944:   MatDestroyMatrices(jac->n_local,&bjac->pmat);
945:   if (jac->use_true_local) {
946:     MatDestroyMatrices(jac->n_local,&bjac->mat);
947:   }

949:   /*
950:         If the on processor block had to be generated via a MatGetDiagonalBlock()
951:      that creates a copy (for example MPIBDiag matrices do), this frees the space
952:   */
953:   if (jac->tp_mat) {
954:     MatDestroy(jac->tp_mat);
955:   }
956:   if (jac->tp_pmat) {
957:     MatDestroy(jac->tp_pmat);
958:   }

960:   for (i=0; i<jac->n_local; i++) {
961:     KSPDestroy(jac->ksp[i]);
962:     VecDestroy(bjac->x[i]);
963:     VecDestroy(bjac->y[i]);
964:     ISDestroy(bjac->is[i]);
965:   }
966:   PetscFree(jac->ksp);
967:   PetscFree(bjac->x);
968:   PetscFree(bjac->starts);
969:   PetscFree(bjac->is);
970:   PetscFree(bjac);
971:   if (jac->l_lens) {PetscFree(jac->l_lens);}
972:   if (jac->g_lens) {PetscFree(jac->g_lens);}
973:   PetscFree(jac);
974:   return(0);
975: }

979: int PCSetUpOnBlocks_BJacobi_Multiblock(PC pc)
980: {
981:   PC_BJacobi            *jac = (PC_BJacobi*)pc->data;
982:   int                   ierr,i,n_local = jac->n_local;
983:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;

986:   for (i=0; i<n_local; i++) {
987:     KSPSetRhs(jac->ksp[i],bjac->x[i]);
988:     KSPSetSolution(jac->ksp[i],bjac->y[i]);
989:     KSPSetUp(jac->ksp[i]);
990:   }
991:   return(0);
992: }

994: /*
995:       Preconditioner for block Jacobi 
996: */
999: int PCApply_BJacobi_Multiblock(PC pc,Vec x,Vec y)
1000: {
1001:   PC_BJacobi            *jac = (PC_BJacobi*)pc->data;
1002:   int                   ierr,i,n_local = jac->n_local;
1003:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
1004:   PetscScalar           *xin,*yin;
1005:   static PetscTruth     flag = PETSC_TRUE;
1006:   static int            SUBKspSolve;

1009:   if (flag) {
1010:     PetscLogEventRegister(&SUBKspSolve,"SubKspSolve",KSP_COOKIE);
1011:     flag = PETSC_FALSE;
1012:   }
1013:   VecGetArray(x,&xin);
1014:   VecGetArray(y,&yin);
1015:   for (i=0; i<n_local; i++) {
1016:     /* 
1017:        To avoid copying the subvector from x into a workspace we instead 
1018:        make the workspace vector array point to the subpart of the array of
1019:        the global vector.
1020:     */
1021:     VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
1022:     VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);

1024:     PetscLogEventBegin(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1025:     KSPSolve(jac->ksp[i]);
1026:     PetscLogEventEnd(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1027:   }
1028:   VecRestoreArray(x,&xin);
1029:   VecRestoreArray(y,&yin);
1030:   return(0);
1031: }

1033: /*
1034:       Preconditioner for block Jacobi 
1035: */
1038: int PCApplyTranspose_BJacobi_Multiblock(PC pc,Vec x,Vec y)
1039: {
1040:   PC_BJacobi            *jac = (PC_BJacobi*)pc->data;
1041:   int                   ierr,i,n_local = jac->n_local;
1042:   PC_BJacobi_Multiblock *bjac = (PC_BJacobi_Multiblock*)jac->data;
1043:   PetscScalar           *xin,*yin;
1044:   static PetscTruth     flag = PETSC_TRUE;
1045:   static int            SUBKspSolve;

1048:   if (flag) {
1049:     PetscLogEventRegister(&SUBKspSolve,"SubKspSolveTranspose",KSP_COOKIE);
1050:     flag = PETSC_FALSE;
1051:   }
1052:   VecGetArray(x,&xin);
1053:   VecGetArray(y,&yin);
1054:   for (i=0; i<n_local; i++) {
1055:     /* 
1056:        To avoid copying the subvector from x into a workspace we instead 
1057:        make the workspace vector array point to the subpart of the array of
1058:        the global vector.
1059:     */
1060:     VecPlaceArray(bjac->x[i],xin+bjac->starts[i]);
1061:     VecPlaceArray(bjac->y[i],yin+bjac->starts[i]);

1063:     PetscLogEventBegin(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1064:     KSPSolveTranspose(jac->ksp[i]);
1065:     PetscLogEventEnd(SUBKspSolve,jac->ksp[i],bjac->x[i],bjac->y[i],0);
1066:   }
1067:   VecRestoreArray(x,&xin);
1068:   VecRestoreArray(y,&yin);
1069:   return(0);
1070: }

1074: static int PCSetUp_BJacobi_Multiblock(PC pc,Mat mat,Mat pmat)
1075: {
1076:   PC_BJacobi             *jac = (PC_BJacobi*)pc->data;
1077:   int                    ierr,m,n_local,N,M,start,i;
1078:   char                   *prefix,*pprefix,*mprefix;
1079:   KSP                    ksp;
1080:   Vec                    x,y;
1081:   PC_BJacobi_Multiblock  *bjac = (PC_BJacobi_Multiblock*)jac->data;
1082:   PC                     subpc;
1083:   IS                     is;
1084:   MatReuse               scall = MAT_REUSE_MATRIX;

1087:   MatGetLocalSize(pc->pmat,&M,&N);

1089:   n_local = jac->n_local;

1091:   if (jac->use_true_local) {
1092:     if (mat->type != pmat->type) SETERRQ(PETSC_ERR_ARG_INCOMP,"Matrices not of same type");
1093:   }

1095:   /* set default direct solver with no Krylov method */
1096:   if (!pc->setupcalled) {
1097:     scall                  = MAT_INITIAL_MATRIX;
1098:     pc->ops->destroy       = PCDestroy_BJacobi_Multiblock;
1099:     pc->ops->apply         = PCApply_BJacobi_Multiblock;
1100:     pc->ops->applytranspose= PCApplyTranspose_BJacobi_Multiblock;
1101:     pc->ops->setuponblocks = PCSetUpOnBlocks_BJacobi_Multiblock;

1103:     PetscMalloc(sizeof(PC_BJacobi_Multiblock),&bjac);
1104:     PetscLogObjectMemory(pc,sizeof(PC_BJacobi_Multiblock));
1105:     PetscMalloc(n_local*sizeof(KSP),&jac->ksp);
1106:     PetscLogObjectMemory(pc,sizeof(n_local*sizeof(KSP)));
1107:     PetscMalloc(2*n_local*sizeof(Vec),&bjac->x);
1108:     PetscLogObjectMemory(pc,sizeof(2*n_local*sizeof(Vec)));
1109:     bjac->y      = bjac->x + n_local;
1110:     PetscMalloc(n_local*sizeof(PetscScalar),&bjac->starts);
1111:     PetscLogObjectMemory(pc,sizeof(n_local*sizeof(PetscScalar)));
1112: 
1113:     jac->data    = (void*)bjac;
1114:     PetscMalloc(n_local*sizeof(IS),&bjac->is);
1115:     PetscLogObjectMemory(pc,sizeof(n_local*sizeof(IS)));

1117:     start = 0;
1118:     for (i=0; i<n_local; i++) {
1119:       KSPCreate(PETSC_COMM_SELF,&ksp);
1120:       PetscLogObjectParent(pc,ksp);
1121:       KSPSetType(ksp,KSPPREONLY);
1122:       KSPGetPC(ksp,&subpc);
1123:       PCSetType(subpc,PCILU);
1124:       PCGetOptionsPrefix(pc,&prefix);
1125:       KSPSetOptionsPrefix(ksp,prefix);
1126:       KSPAppendOptionsPrefix(ksp,"sub_");
1127:       KSPSetFromOptions(ksp);

1129:       m = jac->l_lens[i];

1131:       /*
1132:       The reason we need to generate these vectors is to serve 
1133:       as the right-hand side and solution vector for the solve on the 
1134:       block. We do not need to allocate space for the vectors since
1135:       that is provided via VecPlaceArray() just before the call to 
1136:       KSPSolve() on the block.

1138:       */
1139:       VecCreateSeq(PETSC_COMM_SELF,m,&x);
1140:       VecCreateSeqWithArray(PETSC_COMM_SELF,m,PETSC_NULL,&y);
1141:       PetscLogObjectParent(pc,x);
1142:       PetscLogObjectParent(pc,y);
1143:       bjac->x[i]      = x;
1144:       bjac->y[i]      = y;
1145:       bjac->starts[i] = start;
1146:       jac->ksp[i]    = ksp;

1148:       ISCreateStride(PETSC_COMM_SELF,m,start,1,&is);
1149:       bjac->is[i] = is;
1150:       PetscLogObjectParent(pc,is);

1152:       start += m;
1153:     }
1154:   } else {
1155:     bjac = (PC_BJacobi_Multiblock*)jac->data;
1156:     /* 
1157:        Destroy the blocks from the previous iteration
1158:     */
1159:     if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
1160:       MatDestroyMatrices(n_local,&bjac->pmat);
1161:       if (jac->use_true_local) {
1162:         MatDestroyMatrices(n_local,&bjac->mat);
1163:       }
1164:       scall = MAT_INITIAL_MATRIX;
1165:     }
1166:   }

1168:   MatGetSubMatrices(pmat,n_local,bjac->is,bjac->is,scall,&bjac->pmat);
1169:   if (jac->use_true_local) {
1170:     PetscObjectGetOptionsPrefix((PetscObject)mat,&mprefix);
1171:     MatGetSubMatrices(mat,n_local,bjac->is,bjac->is,scall,&bjac->mat);
1172:   }
1173:   /* Return control to the user so that the submatrices can be modified (e.g., to apply
1174:      different boundary conditions for the submatrices than for the global problem) */
1175:   PCModifySubMatrices(pc,n_local,bjac->is,bjac->is,bjac->pmat,pc->modifysubmatricesP);

1177:   PetscObjectGetOptionsPrefix((PetscObject)pmat,&pprefix);
1178:   for (i=0; i<n_local; i++) {
1179:     PetscLogObjectParent(pc,bjac->pmat[i]);
1180:     PetscObjectSetOptionsPrefix((PetscObject)bjac->pmat[i],pprefix);
1181:     if (jac->use_true_local) {
1182:       PetscLogObjectParent(pc,bjac->mat[i]);
1183:       PetscObjectSetOptionsPrefix((PetscObject)bjac->mat[i],mprefix);
1184:       KSPSetOperators(jac->ksp[i],bjac->mat[i],bjac->pmat[i],pc->flag);
1185:     } else {
1186:       KSPSetOperators(jac->ksp[i],bjac->pmat[i],bjac->pmat[i],pc->flag);
1187:     }
1188:   }

1190:   return(0);
1191: }