Actual source code: fdda.c

  1: /*$Id: fdda.c,v 1.75 2001/08/07 21:31:51 bsmith Exp $*/
  2: 
 3:  #include src/dm/da/daimpl.h
 4:  #include petscmat.h


  7: EXTERN int DAGetColoring1d_MPIAIJ(DA,ISColoringType,ISColoring *);
  8: EXTERN int DAGetColoring2d_MPIAIJ(DA,ISColoringType,ISColoring *);
  9: EXTERN int DAGetColoring2d_5pt_MPIAIJ(DA,ISColoringType,ISColoring *);
 10: EXTERN int DAGetColoring3d_MPIAIJ(DA,ISColoringType,ISColoring *);

 12: /*
 13:    For ghost i that may be negative or greater than the upper bound this
 14:   maps it into the 0:m-1 range using periodicity
 15: */
 16: #define SetInRange(i,m) ((i < 0) ? m+i:((i >= m) ? i-m:i))

 20: static int DASetBlockFills_Private(int *dfill,int w,int **rfill)
 21: {
 22:   int ierr,i,j,nz,*fill;

 25:   /* count number nonzeros */
 26:   nz = 0;
 27:   for (i=0; i<w; i++) {
 28:     for (j=0; j<w; j++) {
 29:       if (dfill[w*i+j]) nz++;
 30:     }
 31:   }
 32:   PetscMalloc((nz + w + 1)*sizeof(int),&fill);
 33:   /* construct modified CSR storage of nonzero structure */
 34:   nz = w + 1;
 35:   for (i=0; i<w; i++) {
 36:     fill[i] = nz;
 37:     for (j=0; j<w; j++) {
 38:       if (dfill[w*i+j]) {
 39:         fill[nz] = j;
 40:         nz++;
 41:       }
 42:     }
 43:   }
 44:   fill[w] = nz;
 45: 
 46:   *rfill = fill;
 47:   return(0);
 48: }

 52: /*@C
 53:     DASetBlockFills - Sets the fill pattern in each block for a multi-component problem
 54:     of the matrix returned by DAGetMatrix().

 56:     Collective on DA

 58:     Input Parameter:
 59: +   da - the distributed array
 60: .   dfill - the fill pattern in the diagonal block (may be PETSC_NULL, means use dense block)
 61: -   ofill - the fill pattern in the off-diagonal blocks


 64:     Level: developer

 66:     Notes: This only makes sense when you are doing multicomponent problems but using the
 67:        MPIAIJ matrix format

 69:            The format for dfill and ofill is a 2 dimensional dof by dof matrix with 1 entries
 70:        representing coupling and 0 entries for missing coupling. For example 
 71: $             dfill[3][3] = {1, 0, 0,
 72: $                            1, 1, 0,
 73: $                            0, 1, 1} 
 74:        means that row 0 is coupled with only itself in the diagonal block, row 1 is coupled with 
 75:        itself and row 0 (in the diagonal block) and row 2 is coupled with itself and row 1 (in the 
 76:        diagonal block.

 78:      DASetGetMatrix() allows you to provide general code for those more complicated nonzero patterns then
 79:      can be represented in the dfill, ofill format

 81:    Contributed by Glenn Hammond

 83: .seealso DAGetMatrix(), DASetGetMatrix()

 85: @*/
 86: int DASetBlockFills(DA da,int *dfill,int *ofill)
 87: {

 91:   if (dfill) {
 92:     DASetBlockFills_Private(dfill,da->w,&da->dfill);
 93:   }
 94:   DASetBlockFills_Private(ofill,da->w,&da->ofill);
 95:   return(0);
 96: }


101: /*@C
102:     DAGetColoring - Gets the coloring required for computing the Jacobian via
103:     finite differences on a function defined using a stencil on the DA.

105:     Collective on DA

107:     Input Parameter:
108: +   da - the distributed array
109: -   ctype - IS_COLORING_LOCAL or IS_COLORING_GHOSTED

111:     Output Parameters:
112: .   coloring - matrix coloring for use in computing Jacobians (or PETSC_NULL if not needed)

114:     Level: advanced

116:     Notes: These compute the graph coloring of the graph of A^{T}A. The coloring used 
117:    for efficient (parallel or thread based) triangular solves etc is NOT yet 
118:    available. 


121: .seealso ISColoringView(), ISColoringGetIS(), MatFDColoringCreate(), ISColoringType, ISColoring

123: @*/
124: int DAGetColoring(DA da,ISColoringType ctype,ISColoring *coloring)
125: {
126:   int        ierr,dim;

129:   /*
130:                                   m
131:           ------------------------------------------------------
132:          |                                                     |
133:          |                                                     |
134:          |               ----------------------                |
135:          |               |                    |                |
136:       n  |           yn  |                    |                |
137:          |               |                    |                |
138:          |               .---------------------                |
139:          |             (xs,ys)     xn                          |
140:          |            .                                        |
141:          |         (gxs,gys)                                   |
142:          |                                                     |
143:           -----------------------------------------------------
144:   */

146:   /*     
147:          nc - number of components per grid point 
148:          col - number of colors needed in one direction for single component problem
149:   
150:   */
151:   DAGetInfo(da,&dim,0,0,0,0,0,0,0,0,0,0);

153:   /*
154:      We do not provide a getcoloring function in the DA operations because 
155:    the basic DA does not know about matrices. We think of DA as being more 
156:    more low-level then matrices.
157:   */
158:   if (dim == 1) {
159:     DAGetColoring1d_MPIAIJ(da,ctype,coloring);
160:   } else if (dim == 2) {
161:      DAGetColoring2d_MPIAIJ(da,ctype,coloring);
162:   } else if (dim == 3) {
163:      DAGetColoring3d_MPIAIJ(da,ctype,coloring);
164:   } else {
165:       SETERRQ1(1,"Not done for %d dimension, send us mail petsc-maint@mcs.anl.gov for code",dim);
166:   }
167:   return(0);
168: }

170: /* ---------------------------------------------------------------------------------*/

174: int DAGetColoring2d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
175: {
176:   int                    ierr,xs,ys,nx,ny,i,j,ii,gxs,gys,gnx,gny;
177:   int                    m,n,M,N,dim,s,k,nc,col,size;
178:   MPI_Comm               comm;
179:   DAPeriodicType         wrap;
180:   DAStencilType          st;
181:   ISColoringValue        *colors;

184:   /*     
185:          nc - number of components per grid point 
186:          col - number of colors needed in one direction for single component problem
187:   
188:   */
189:   DAGetInfo(da,&dim,&m,&n,0,&M,&N,0,&nc,&s,&wrap,&st);
190:   col    = 2*s + 1;
191:   DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
192:   DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
193:   PetscObjectGetComm((PetscObject)da,&comm);
194:   MPI_Comm_size(comm,&size);

196:   /* special case as taught to us by Paul Hovland */
197:   if (st == DA_STENCIL_STAR && s == 1) {
198:     DAGetColoring2d_5pt_MPIAIJ(da,ctype,coloring);
199:   } else {

201:     if (DAXPeriodic(wrap) && (m % col)){
202:       SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
203:                  by 2*stencil_width + 1\n");
204:     }
205:     if (DAYPeriodic(wrap) && (n % col)){
206:       SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
207:                  by 2*stencil_width + 1\n");
208:     }
209:     if (ctype == IS_COLORING_LOCAL) {
210:       if (!da->localcoloring) {
211:         PetscMalloc(nc*nx*ny*sizeof(ISColoringValue),&colors);
212:         ii = 0;
213:         for (j=ys; j<ys+ny; j++) {
214:           for (i=xs; i<xs+nx; i++) {
215:             for (k=0; k<nc; k++) {
216:               colors[ii++] = k + nc*((i % col) + col*(j % col));
217:             }
218:           }
219:         }
220:         ISColoringCreate(comm,nc*nx*ny,colors,&da->localcoloring);
221:       }
222:       *coloring = da->localcoloring;
223:     } else if (ctype == IS_COLORING_GHOSTED) {
224:       if (!da->ghostedcoloring) {
225:         PetscMalloc(nc*gnx*gny*sizeof(ISColoringValue),&colors);
226:         ii = 0;
227:         for (j=gys; j<gys+gny; j++) {
228:           for (i=gxs; i<gxs+gnx; i++) {
229:             for (k=0; k<nc; k++) {
230:               /* the complicated stuff is to handle periodic boundaries */
231:               colors[ii++] = k + nc*((SetInRange(i,m) % col) + col*(SetInRange(j,n) % col));
232:             }
233:           }
234:         }
235:         ISColoringCreate(comm,nc*gnx*gny,colors,&da->ghostedcoloring);
236:         ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
237:       }
238:       *coloring = da->ghostedcoloring;
239:     } else SETERRQ1(1,"Unknown ISColoringType %d",ctype);
240:   }
241:   ISColoringReference(*coloring);
242:   return(0);
243: }

245: /* ---------------------------------------------------------------------------------*/

249: int DAGetColoring3d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
250: {
251:   int                    ierr,xs,ys,nx,ny,i,j,gxs,gys,gnx,gny;
252:   int                    m,n,p,dim,s,k,nc,col,size,zs,gzs,ii,l,nz,gnz,M,N,P;
253:   MPI_Comm               comm;
254:   DAPeriodicType         wrap;
255:   DAStencilType          st;
256:   ISColoringValue        *colors;

259:   /*     
260:          nc - number of components per grid point 
261:          col - number of colors needed in one direction for single component problem
262:   
263:   */
264:   DAGetInfo(da,&dim,&m,&n,&p,&M,&N,&P,&nc,&s,&wrap,&st);
265:   col    = 2*s + 1;
266:   if (DAXPeriodic(wrap) && (m % col)){
267:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
268:                  by 2*stencil_width + 1\n");
269:   }
270:   if (DAYPeriodic(wrap) && (n % col)){
271:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
272:                  by 2*stencil_width + 1\n");
273:   }
274:   if (DAZPeriodic(wrap) && (p % col)){
275:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Z is divisible\n\
276:                  by 2*stencil_width + 1\n");
277:   }

279:   DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
280:   DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
281:   PetscObjectGetComm((PetscObject)da,&comm);
282:   MPI_Comm_size(comm,&size);

284:   /* create the coloring */
285:   if (ctype == IS_COLORING_LOCAL) {
286:     if (!da->localcoloring) {
287:       PetscMalloc(nc*nx*ny*nz*sizeof(ISColoringValue),&colors);
288:       ii = 0;
289:       for (k=zs; k<zs+nz; k++) {
290:         for (j=ys; j<ys+ny; j++) {
291:           for (i=xs; i<xs+nx; i++) {
292:             for (l=0; l<nc; l++) {
293:               colors[ii++] = l + nc*((i % col) + col*(j % col) + col*col*(k % col));
294:             }
295:           }
296:         }
297:       }
298:       ISColoringCreate(comm,nc*nx*ny*nz,colors,&da->localcoloring);
299:     }
300:     *coloring = da->localcoloring;
301:   } else if (ctype == IS_COLORING_GHOSTED) {
302:     if (!da->ghostedcoloring) {
303:       PetscMalloc(nc*gnx*gny*gnz*sizeof(ISColoringValue),&colors);
304:       ii = 0;
305:       for (k=gzs; k<gzs+gnz; k++) {
306:         for (j=gys; j<gys+gny; j++) {
307:           for (i=gxs; i<gxs+gnx; i++) {
308:             for (l=0; l<nc; l++) {
309:               /* the complicated stuff is to handle periodic boundaries */
310:               colors[ii++] = l + nc*((SetInRange(i,m) % col) + col*(SetInRange(j,n) % col) + col*col*(SetInRange(k,p) % col));
311:             }
312:           }
313:         }
314:       }
315:       ISColoringCreate(comm,nc*gnx*gny*gnz,colors,&da->ghostedcoloring);
316:       ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
317:     }
318:     *coloring = da->ghostedcoloring;
319:   } else SETERRQ1(1,"Unknown ISColoringType %d",ctype);
320:   ISColoringReference(*coloring);
321:   return(0);
322: }

324: /* ---------------------------------------------------------------------------------*/

328: int DAGetColoring1d_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
329: {
330:   int                    ierr,xs,nx,i,i1,gxs,gnx,l;
331:   int                    m,M,dim,s,nc,col,size;
332:   MPI_Comm               comm;
333:   DAPeriodicType         wrap;
334:   ISColoringValue        *colors;

337:   /*     
338:          nc - number of components per grid point 
339:          col - number of colors needed in one direction for single component problem
340:   
341:   */
342:   DAGetInfo(da,&dim,&m,0,0,&M,0,0,&nc,&s,&wrap,0);
343:   col    = 2*s + 1;

345:   if (DAXPeriodic(wrap) && (m % col)) {
346:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points is divisible\n\
347:                  by 2*stencil_width + 1\n");
348:   }

350:   DAGetCorners(da,&xs,0,0,&nx,0,0);
351:   DAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);
352:   PetscObjectGetComm((PetscObject)da,&comm);
353:   MPI_Comm_size(comm,&size);

355:   /* create the coloring */
356:   if (ctype == IS_COLORING_LOCAL) {
357:     if (!da->localcoloring) {
358:       PetscMalloc(nc*nx*sizeof(ISColoringValue),&colors);
359:       i1 = 0;
360:       for (i=xs; i<xs+nx; i++) {
361:         for (l=0; l<nc; l++) {
362:           colors[i1++] = l + nc*(i % col);
363:         }
364:       }
365:       ISColoringCreate(comm,nc*nx,colors,&da->localcoloring);
366:     }
367:     *coloring = da->localcoloring;
368:   } else if (ctype == IS_COLORING_GHOSTED) {
369:     if (!da->ghostedcoloring) {
370:       PetscMalloc(nc*gnx*sizeof(ISColoringValue),&colors);
371:       i1 = 0;
372:       for (i=gxs; i<gxs+gnx; i++) {
373:         for (l=0; l<nc; l++) {
374:           /* the complicated stuff is to handle periodic boundaries */
375:           colors[i1++] = l + nc*(SetInRange(i,m) % col);
376:         }
377:       }
378:       ISColoringCreate(comm,nc*gnx,colors,&da->ghostedcoloring);
379:       ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
380:     }
381:     *coloring = da->ghostedcoloring;
382:   } else SETERRQ1(1,"Unknown ISColoringType %d",ctype);
383:   ISColoringReference(*coloring);
384:   return(0);
385: }

389: int DAGetColoring2d_5pt_MPIAIJ(DA da,ISColoringType ctype,ISColoring *coloring)
390: {
391:   int            ierr,xs,ys,nx,ny,i,j,ii,gxs,gys,gnx,gny;
392:   int            m,n,dim,s,k,nc;
393:   MPI_Comm       comm;
394:   DAPeriodicType wrap;
395:   ISColoringValue        *colors;

398:   /*     
399:          nc - number of components per grid point 
400:          col - number of colors needed in one direction for single component problem
401:   
402:   */
403:   DAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&wrap,0);
404:   DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
405:   DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
406:   PetscObjectGetComm((PetscObject)da,&comm);

408:   if (DAXPeriodic(wrap) && (m % 5)){
409:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
410:                  by 5\n");
411:   }
412:   if (DAYPeriodic(wrap) && (n % 5)){
413:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
414:                  by 5\n");
415:   }

417:   /* create the coloring */
418:   if (ctype == IS_COLORING_LOCAL) {
419:     if (!da->localcoloring) {
420:       PetscMalloc(nc*nx*ny*sizeof(ISColoringValue),&colors);
421:       ii = 0;
422:       for (j=ys; j<ys+ny; j++) {
423:         for (i=xs; i<xs+nx; i++) {
424:           for (k=0; k<nc; k++) {
425:             colors[ii++] = k + nc*((3*j+i) % 5);
426:           }
427:         }
428:       }
429:       ISColoringCreate(comm,nc*nx*ny,colors,&da->localcoloring);
430:     }
431:     *coloring = da->localcoloring;
432:   } else if (ctype == IS_COLORING_GHOSTED) {
433:     if (!da->ghostedcoloring) {
434:       PetscMalloc(nc*gnx*gny*sizeof(ISColoringValue),&colors);
435:       ii = 0;
436:       for (j=gys; j<gys+gny; j++) {
437:         for (i=gxs; i<gxs+gnx; i++) {
438:           for (k=0; k<nc; k++) {
439:             colors[ii++] = k + nc*((3*SetInRange(j,n) + SetInRange(i,m)) % 5);
440:           }
441:         }
442:       }
443:       ISColoringCreate(comm,nc*gnx*gny,colors,&da->ghostedcoloring);
444:       ISColoringSetType(da->ghostedcoloring,IS_COLORING_GHOSTED);
445:     }
446:     *coloring = da->ghostedcoloring;
447:   } else SETERRQ1(1,"Unknown ISColoringType %d",ctype);
448:   return(0);
449: }

451: /* =========================================================================== */
452: EXTERN int DAGetMatrix1d_MPIAIJ(DA,Mat);
453: EXTERN int DAGetMatrix2d_MPIAIJ(DA,Mat);
454: EXTERN int DAGetMatrix2d_MPIAIJ_Fill(DA,Mat);
455: EXTERN int DAGetMatrix3d_MPIAIJ(DA,Mat);
456: EXTERN int DAGetMatrix3d_MPIAIJ_Fill(DA,Mat);
457: EXTERN int DAGetMatrix3d_MPIBAIJ(DA,Mat);
458: EXTERN int DAGetMatrix3d_MPISBAIJ(DA,Mat);

462: /*@C
463:     DAGetMatrix - Creates a matrix with the correct parallel layout and nonzero structure required for 
464:       computing the Jacobian on a function defined using the stencil set in the DA.

466:     Collective on DA

468:     Input Parameter:
469: +   da - the distributed array
470: -   mtype - Supported types are MATSEQAIJ, MATMPIAIJ, MATSEQBAIJ, MATMPIBAIJ, or
471:             any type which inherits from one of these (such as MATAIJ, MATLUSOL, etc.).

473:     Output Parameters:
474: .   J  - matrix with the correct nonzero structure
475:         (obviously without the correct Jacobian values)

477:     Level: advanced

479:     Notes: This properly preallocates the number of nonzeros in the sparse matrix so you 
480:        do not need to do it yourself.

482: .seealso ISColoringView(), ISColoringGetIS(), MatFDColoringCreate()

484: @*/
485: int DAGetMatrix(DA da,const MatType mtype,Mat *J)
486: {
487:   int      ierr,dim,dof,nx,ny,nz,dims[3],starts[3];
488:   Mat      A;
489:   MPI_Comm comm;
490:   MatType  Atype;
491:   void     (*aij)(void)=PETSC_NULL,(*baij)(void)=PETSC_NULL,(*sbaij)(void)=PETSC_NULL;

494:   /*
495:                                   m
496:           ------------------------------------------------------
497:          |                                                     |
498:          |                                                     |
499:          |               ----------------------                |
500:          |               |                    |                |
501:       n  |           yn  |                    |                |
502:          |               |                    |                |
503:          |               .---------------------                |
504:          |             (xs,ys)     xn                          |
505:          |            .                                        |
506:          |         (gxs,gys)                                   |
507:          |                                                     |
508:           -----------------------------------------------------
509:   */

511:   /*     
512:          nc - number of components per grid point 
513:          col - number of colors needed in one direction for single component problem
514:   
515:   */
516:   DAGetInfo(da,&dim,0,0,0,0,0,0,&dof,0,0,0);
517:   DAGetCorners(da,0,0,0,&nx,&ny,&nz);
518:   PetscObjectGetComm((PetscObject)da,&comm);
519:   MatCreate(comm,dof*nx*ny*nz,dof*nx*ny*nz,PETSC_DECIDE,PETSC_DECIDE,&A);
520:   MatSetType(A,mtype);
521:   MatSetFromOptions(A);
522:   MatGetType(A,&Atype);
523:   /*
524:      We do not provide a getmatrix function in the DA operations because 
525:    the basic DA does not know about matrices. We think of DA as being more 
526:    more low-level then matrices. This is kind of cheating but, cause sometimes 
527:    we think of DA has higher level then matrices.

529:      We could switch based on Atype (or mtype), but we do not since the
530:    specialized setting routines depend only the particular preallocation
531:    details of the matrix, not the type itself.
532:   */
533:   PetscObjectQueryFunction((PetscObject)A,"MatMPIAIJSetPreallocation_C",&aij);
534:   if (!aij) {
535:     PetscObjectQueryFunction((PetscObject)A,"MatSeqAIJSetPreallocation_C",&aij);
536:   }
537:   if (aij) {
538:     if (dim == 1) {
539:       DAGetMatrix1d_MPIAIJ(da,A);
540:     } else if (dim == 2) {
541:       if (da->ofill) {
542:         DAGetMatrix2d_MPIAIJ_Fill(da,A);
543:       } else {
544:         DAGetMatrix2d_MPIAIJ(da,A);
545:       }
546:     } else if (dim == 3) {
547:       if (da->ofill) {
548:         DAGetMatrix3d_MPIAIJ_Fill(da,A);
549:       } else {
550:         DAGetMatrix3d_MPIAIJ(da,A);
551:       }
552:     }
553:   } else if (dim == 3) {
554:     PetscObjectQueryFunction((PetscObject)A,"MatMPIBAIJSetPreallocation_C",&baij);
555:     if (!baij) {
556:       PetscObjectQueryFunction((PetscObject)A,"MatSeqBAIJSetPreallocation_C",&baij);
557:     }
558:     if (baij) {
559:       DAGetMatrix3d_MPIBAIJ(da,A);
560:     } else {
561:       PetscObjectQueryFunction((PetscObject)A,"MatMPISBAIJSetPreallocation_C",&sbaij);
562:       if (!sbaij) {
563:         PetscObjectQueryFunction((PetscObject)A,"MatSeqSBAIJSetPreallocation_C",&sbaij);
564:       }
565:       if (sbaij) {
566:         DAGetMatrix3d_MPISBAIJ(da,A);
567:       } else {
568:         SETERRQ2(PETSC_ERR_SUP,"Not implemented for %d dimension and Matrix Type: %s!\n" \
569:                                "Send mail to petsc-maint@mcs.anl.gov for code",
570:                                dim,Atype);
571:       }
572:     }
573:   } else {
574:     SETERRQ2(PETSC_ERR_SUP,"Not implemented for %d dimension and matrix type: %s!\n" \
575:                            "Send mail to petsc-maint@mcs.anl.gov for code",dim,Atype);
576:   }
577:   DAGetGhostCorners(da,&starts[0],&starts[1],&starts[2],&dims[0],&dims[1],&dims[2]);
578:   MatSetStencil(A,dim,dims,starts,dof);
579:   *J = A;
580:   return(0);
581: }

583: /* ---------------------------------------------------------------------------------*/
586: int DAGetMatrix2d_MPIAIJ(DA da,Mat J)
587: {
588:   int                    ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
589:   int                    m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p;
590:   int                    lstart,lend,pstart,pend,*dnz,*onz;
591:   MPI_Comm               comm;
592:   PetscScalar            *values;
593:   DAPeriodicType         wrap;
594:   ISLocalToGlobalMapping ltog;
595:   DAStencilType          st;

598:   /*     
599:          nc - number of components per grid point 
600:          col - number of colors needed in one direction for single component problem
601:   
602:   */
603:   DAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&wrap,&st);
604:   col = 2*s + 1;
605:   DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
606:   DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
607:   PetscObjectGetComm((PetscObject)da,&comm);

609:   PetscMalloc(col*col*nc*nc*sizeof(PetscScalar),&values);
610:   PetscMemzero(values,col*col*nc*nc*sizeof(PetscScalar));
611:   PetscMalloc(nc*sizeof(int),&rows);
612:   PetscMalloc(col*col*nc*nc*sizeof(int),&cols);
613:   DAGetISLocalToGlobalMapping(da,&ltog);
614: 
615:   /* determine the matrix preallocation information */
616:   MatPreallocateInitialize(comm,nc*nx*ny,nc*nx*ny,dnz,onz);
617:   for (i=xs; i<xs+nx; i++) {

619:     pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
620:     pend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));

622:     for (j=ys; j<ys+ny; j++) {
623:       slot = i - gxs + gnx*(j - gys);

625:       lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
626:       lend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));

628:       cnt  = 0;
629:       for (k=0; k<nc; k++) {
630:         for (l=lstart; l<lend+1; l++) {
631:           for (p=pstart; p<pend+1; p++) {
632:             if ((st == DA_STENCIL_BOX) || (!l || !p)) {  /* entries on star have either l = 0 or p = 0 */
633:               cols[cnt++]  = k + nc*(slot + gnx*l + p);
634:             }
635:           }
636:         }
637:         rows[k] = k + nc*(slot);
638:       }
639:       MatPreallocateSetLocal(ltog,nc,rows,cnt,cols,dnz,onz);
640:     }
641:   }
642:   MatSeqAIJSetPreallocation(J,0,dnz);
643:   MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
644:   MatPreallocateFinalize(dnz,onz);

646:   MatSetLocalToGlobalMapping(J,ltog);

648:   /*
649:     For each node in the grid: we get the neighbors in the local (on processor ordering
650:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
651:     PETSc ordering.
652:   */
653:   for (i=xs; i<xs+nx; i++) {
654: 
655:     pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
656:     pend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
657: 
658:     for (j=ys; j<ys+ny; j++) {
659:       slot = i - gxs + gnx*(j - gys);
660: 
661:       lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
662:       lend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));

664:       cnt  = 0;
665:       for (k=0; k<nc; k++) {
666:         for (l=lstart; l<lend+1; l++) {
667:           for (p=pstart; p<pend+1; p++) {
668:             if ((st == DA_STENCIL_BOX) || (!l || !p)) {  /* entries on star have either l = 0 or p = 0 */
669:               cols[cnt++]  = k + nc*(slot + gnx*l + p);
670:             }
671:           }
672:         }
673:         rows[k]      = k + nc*(slot);
674:       }
675:       MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
676:     }
677:   }
678:   PetscFree(values);
679:   PetscFree(rows);
680:   PetscFree(cols);
681:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
682:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
683:   return(0);
684: }

688: int DAGetMatrix2d_MPIAIJ_Fill(DA da,Mat J)
689: {
690:   int                    ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
691:   int                    m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p;
692:   int                    lstart,lend,pstart,pend,*dnz,*onz;
693:   int                    ifill_col,*ofill = da->ofill, *dfill = da->dfill;
694:   MPI_Comm               comm;
695:   PetscScalar            *values;
696:   DAPeriodicType         wrap;
697:   ISLocalToGlobalMapping ltog;
698:   DAStencilType          st;

701:   /*     
702:          nc - number of components per grid point 
703:          col - number of colors needed in one direction for single component problem
704:   
705:   */
706:   DAGetInfo(da,&dim,&m,&n,0,0,0,0,&nc,&s,&wrap,&st);
707:   col = 2*s + 1;
708:   DAGetCorners(da,&xs,&ys,0,&nx,&ny,0);
709:   DAGetGhostCorners(da,&gxs,&gys,0,&gnx,&gny,0);
710:   PetscObjectGetComm((PetscObject)da,&comm);

712:   PetscMalloc(col*col*nc*nc*sizeof(PetscScalar),&values);
713:   PetscMemzero(values,col*col*nc*nc*sizeof(PetscScalar));
714:   PetscMalloc(nc*sizeof(int),&rows);
715:   PetscMalloc(col*col*nc*nc*sizeof(int),&cols);
716:   DAGetISLocalToGlobalMapping(da,&ltog);
717: 
718:   /* determine the matrix preallocation information */
719:   MatPreallocateInitialize(comm,nc*nx*ny,nc*nx*ny,dnz,onz);
720:   for (i=xs; i<xs+nx; i++) {

722:     pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
723:     pend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));

725:     for (j=ys; j<ys+ny; j++) {
726:       slot = i - gxs + gnx*(j - gys);

728:       lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
729:       lend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));

731:       for (k=0; k<nc; k++) {
732:         cnt  = 0;
733:         for (l=lstart; l<lend+1; l++) {
734:           for (p=pstart; p<pend+1; p++) {
735:             if (l || p) {
736:               if ((st == DA_STENCIL_BOX) || (!l || !p)) {  /* entries on star */
737:                 for (ifill_col=ofill[k]; ifill_col<ofill[k+1]; ifill_col++)
738:                   cols[cnt++]  = ofill[ifill_col] + nc*(slot + gnx*l + p);
739:               }
740:             } else {
741:               if (dfill) {
742:                 for (ifill_col=dfill[k]; ifill_col<dfill[k+1]; ifill_col++)
743:                   cols[cnt++]  = dfill[ifill_col] + nc*(slot + gnx*l + p);
744:               } else {
745:                 for (ifill_col=0; ifill_col<nc; ifill_col++)
746:                   cols[cnt++]  = ifill_col + nc*(slot + gnx*l + p);
747:               }
748:             }
749:           }
750:         }
751:         rows[0] = k + nc*(slot);
752:         MatPreallocateSetLocal(ltog,1,rows,cnt,cols,dnz,onz);
753:       }
754:     }
755:   }
756:   MatSeqAIJSetPreallocation(J,0,dnz);
757:   MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
758:   MatPreallocateFinalize(dnz,onz);

760:   MatSetLocalToGlobalMapping(J,ltog);

762:   /*
763:     For each node in the grid: we get the neighbors in the local (on processor ordering
764:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
765:     PETSc ordering.
766:   */
767:   for (i=xs; i<xs+nx; i++) {
768: 
769:     pstart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
770:     pend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
771: 
772:     for (j=ys; j<ys+ny; j++) {
773:       slot = i - gxs + gnx*(j - gys);
774: 
775:       lstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
776:       lend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));

778:       for (k=0; k<nc; k++) {
779:         cnt  = 0;
780:         for (l=lstart; l<lend+1; l++) {
781:           for (p=pstart; p<pend+1; p++) {
782:             if (l || p) {
783:               if ((st == DA_STENCIL_BOX) || (!l || !p)) {  /* entries on star */
784:                 for (ifill_col=ofill[k]; ifill_col<ofill[k+1]; ifill_col++)
785:                   cols[cnt++]  = ofill[ifill_col] + nc*(slot + gnx*l + p);
786:               }
787:             } else {
788:               if (dfill) {
789:                 for (ifill_col=dfill[k]; ifill_col<dfill[k+1]; ifill_col++)
790:                   cols[cnt++]  = dfill[ifill_col] + nc*(slot + gnx*l + p);
791:               } else {
792:                 for (ifill_col=0; ifill_col<nc; ifill_col++)
793:                   cols[cnt++]  = ifill_col + nc*(slot + gnx*l + p);
794:               }
795:             }
796:           }
797:         }
798:         rows[0]      = k + nc*(slot);
799:         MatSetValuesLocal(J,1,rows,cnt,cols,values,INSERT_VALUES);
800:       }
801:     }
802:   }
803:   PetscFree(values);
804:   PetscFree(rows);
805:   PetscFree(cols);
806:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
807:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
808:   return(0);
809: }

811: /* ---------------------------------------------------------------------------------*/

815: int DAGetMatrix3d_MPIAIJ(DA da,Mat J)
816: {
817:   int                    ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
818:   int                    m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p,*dnz,*onz;
819:   int                    istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
820:   MPI_Comm               comm;
821:   PetscScalar            *values;
822:   DAPeriodicType         wrap;
823:   ISLocalToGlobalMapping ltog;
824:   DAStencilType          st;

827:   /*     
828:          nc - number of components per grid point 
829:          col - number of colors needed in one direction for single component problem
830:   
831:   */
832:   DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
833:   col    = 2*s + 1;

835:   DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
836:   DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
837:   PetscObjectGetComm((PetscObject)da,&comm);

839:   PetscMalloc(col*col*col*nc*nc*nc*sizeof(PetscScalar),&values);
840:   PetscMemzero(values,col*col*col*nc*nc*nc*sizeof(PetscScalar));
841:   PetscMalloc(nc*sizeof(int),&rows);
842:   PetscMalloc(col*col*col*nc*sizeof(int),&cols);
843:   DAGetISLocalToGlobalMapping(da,&ltog);

845:   /* determine the matrix preallocation information */
846:   MatPreallocateInitialize(comm,nc*nx*ny*nz,nc*nx*ny*nz,dnz,onz);
847:   for (i=xs; i<xs+nx; i++) {
848:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
849:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
850:     for (j=ys; j<ys+ny; j++) {
851:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
852:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
853:       for (k=zs; k<zs+nz; k++) {
854:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
855:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
856: 
857:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
858: 
859:         cnt  = 0;
860:         for (l=0; l<nc; l++) {
861:           for (ii=istart; ii<iend+1; ii++) {
862:             for (jj=jstart; jj<jend+1; jj++) {
863:               for (kk=kstart; kk<kend+1; kk++) {
864:                 if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
865:                   cols[cnt++]  = l + nc*(slot + ii + gnx*jj + gnx*gny*kk);
866:                 }
867:               }
868:             }
869:           }
870:           rows[l] = l + nc*(slot);
871:         }
872:         MatPreallocateSetLocal(ltog,nc,rows,cnt,cols,dnz,onz);
873:       }
874:     }
875:   }
876:   MatSeqAIJSetPreallocation(J,0,dnz);
877:   MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
878:   MatPreallocateFinalize(dnz,onz);

880:   MatSetLocalToGlobalMapping(J,ltog);

882:   /*
883:     For each node in the grid: we get the neighbors in the local (on processor ordering
884:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
885:     PETSc ordering.
886:   */
887:   for (i=xs; i<xs+nx; i++) {
888:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
889:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
890:     for (j=ys; j<ys+ny; j++) {
891:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
892:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
893:       for (k=zs; k<zs+nz; k++) {
894:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
895:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
896: 
897:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
898: 
899:         cnt  = 0;
900:         for (l=0; l<nc; l++) {
901:           for (ii=istart; ii<iend+1; ii++) {
902:             for (jj=jstart; jj<jend+1; jj++) {
903:               for (kk=kstart; kk<kend+1; kk++) {
904:                 if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
905:                   cols[cnt++]  = l + nc*(slot + ii + gnx*jj + gnx*gny*kk);
906:                 }
907:               }
908:             }
909:           }
910:           rows[l]      = l + nc*(slot);
911:         }
912:         MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
913:       }
914:     }
915:   }
916:   PetscFree(values);
917:   PetscFree(rows);
918:   PetscFree(cols);
919:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
920:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
921:   return(0);
922: }

924: /* ---------------------------------------------------------------------------------*/

928: int DAGetMatrix1d_MPIAIJ(DA da,Mat J)
929: {
930:   int                    ierr,xs,nx,i,i1,slot,gxs,gnx;
931:   int                    m,dim,s,*cols,nc,*rows,col,cnt,l;
932:   int                    istart,iend;
933:   PetscScalar            *values;
934:   DAPeriodicType         wrap;
935:   ISLocalToGlobalMapping ltog;

938:   /*     
939:          nc - number of components per grid point 
940:          col - number of colors needed in one direction for single component problem
941:   
942:   */
943:   DAGetInfo(da,&dim,&m,0,0,0,0,0,&nc,&s,&wrap,0);
944:   col    = 2*s + 1;

946:   DAGetCorners(da,&xs,0,0,&nx,0,0);
947:   DAGetGhostCorners(da,&gxs,0,0,&gnx,0,0);

949:   MatSeqAIJSetPreallocation(J,col*nc,0);
950:   MatMPIAIJSetPreallocation(J,col*nc,0,0,0);

952:   PetscMalloc(col*nc*nc*sizeof(PetscScalar),&values);
953:   PetscMemzero(values,col*nc*nc*sizeof(PetscScalar));
954:   PetscMalloc(nc*sizeof(int),&rows);
955:   PetscMalloc(col*nc*sizeof(int),&cols);
956: 
957:   DAGetISLocalToGlobalMapping(da,&ltog);
958:   MatSetLocalToGlobalMapping(J,ltog);
959: 
960:   /*
961:     For each node in the grid: we get the neighbors in the local (on processor ordering
962:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
963:     PETSc ordering.
964:   */
965:   for (i=xs; i<xs+nx; i++) {
966:     istart = PetscMax(-s,gxs - i);
967:     iend   = PetscMin(s,gxs + gnx - i - 1);
968:     slot   = i - gxs;
969: 
970:     cnt  = 0;
971:     for (l=0; l<nc; l++) {
972:       for (i1=istart; i1<iend+1; i1++) {
973:         cols[cnt++] = l + nc*(slot + i1);
974:       }
975:       rows[l]      = l + nc*(slot);
976:     }
977:     MatSetValuesLocal(J,nc,rows,cnt,cols,values,INSERT_VALUES);
978:   }
979:   PetscFree(values);
980:   PetscFree(rows);
981:   PetscFree(cols);
982:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
983:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
984:   return(0);
985: }

989: int DAGetMatrix3d_MPIBAIJ(DA da,Mat J)
990: {
991:   int                    ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
992:   int                    m,n,dim,s,*cols,k,nc,col,cnt,p,*dnz,*onz;
993:   int                    istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
994:   MPI_Comm               comm;
995:   PetscScalar            *values;
996:   DAPeriodicType         wrap;
997:   DAStencilType          st;
998:   ISLocalToGlobalMapping ltog;

1001:   /*     
1002:          nc - number of components per grid point 
1003:          col - number of colors needed in one direction for single component problem
1004:   
1005:   */
1006:   DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
1007:   if (wrap != DA_NONPERIODIC) SETERRQ(PETSC_ERR_SUP,"Currently no support for periodic");
1008:   col    = 2*s + 1;

1010:   DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1011:   DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1012:   PetscObjectGetComm((PetscObject)da,&comm);

1014:   PetscMalloc(col*col*col*nc*nc*sizeof(PetscScalar),&values);
1015:   PetscMemzero(values,col*col*col*nc*nc*sizeof(PetscScalar));
1016:   PetscMalloc(col*col*col*sizeof(int),&cols);

1018:   DAGetISLocalToGlobalMappingBlck(da,&ltog);

1020:   /* determine the matrix preallocation information */
1021:   MatPreallocateInitialize(comm,nx*ny*nz,nx*ny*nz,dnz,onz);
1022:   for (i=xs; i<xs+nx; i++) {
1023:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1024:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
1025:     for (j=ys; j<ys+ny; j++) {
1026:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1027:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
1028:       for (k=zs; k<zs+nz; k++) {
1029:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1030:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));

1032:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);

1034:         /* Find block columns in block row */
1035:         cnt  = 0;
1036:         if (st == DA_STENCIL_BOX) {   /* if using BOX stencil */
1037:           for (ii=istart; ii<iend+1; ii++) {
1038:             for (jj=jstart; jj<jend+1; jj++) {
1039:               for (kk=kstart; kk<kend+1; kk++) {
1040:                 cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk;
1041:               }
1042:             }
1043:           }
1044:         } else {  /* Star stencil */
1045:           cnt  = 0;
1046:           for (ii=istart; ii<iend+1; ii++) {
1047:             if (ii) {
1048:               /* jj and kk must be zero */
1049:               /* cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk; */
1050:               cols[cnt++]  = slot + ii;
1051:             } else {
1052:               for (jj=jstart; jj<jend+1; jj++) {
1053:                 if (jj) {
1054:                   /* ii and kk must be zero */
1055:                   cols[cnt++]  = slot + gnx*jj;
1056:                 } else {
1057:                   /* ii and jj must be zero */
1058:                   for (kk=kstart; kk<kend+1; kk++) {
1059:                     cols[cnt++]  = slot + gnx*gny*kk;
1060:                   }
1061:                 }
1062:               }
1063:             }
1064:           }
1065:         }
1066:         MatPreallocateSetLocal(ltog,1,&slot,cnt,cols,dnz,onz);
1067:       }
1068:     }
1069:   }
1070:   MatSeqBAIJSetPreallocation(J,nc,0,dnz);
1071:   MatMPIBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1072:   MatPreallocateFinalize(dnz,onz);

1074:   MatSetLocalToGlobalMappingBlock(J,ltog);

1076:   /*
1077:     For each node in the grid: we get the neighbors in the local (on processor ordering
1078:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1079:     PETSc ordering.
1080:   */

1082:   for (i=xs; i<xs+nx; i++) {
1083:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1084:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
1085:     for (j=ys; j<ys+ny; j++) {
1086:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1087:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
1088:       for (k=zs; k<zs+nz; k++) {
1089:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1090:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
1091: 
1092:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1093: 
1094:         cnt  = 0;
1095:         if (st == DA_STENCIL_BOX) {   /* if using BOX stencil */
1096:           for (ii=istart; ii<iend+1; ii++) {
1097:             for (jj=jstart; jj<jend+1; jj++) {
1098:               for (kk=kstart; kk<kend+1; kk++) {
1099:                 cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk;
1100:               }
1101:             }
1102:           }
1103:         } else {  /* Star stencil */
1104:           cnt  = 0;
1105:           for (ii=istart; ii<iend+1; ii++) {
1106:             if (ii) {
1107:               /* jj and kk must be zero */
1108:               cols[cnt++]  = slot + ii;
1109:             } else {
1110:               for (jj=jstart; jj<jend+1; jj++) {
1111:                 if (jj) {
1112:                   /* ii and kk must be zero */
1113:                   cols[cnt++]  = slot + gnx*jj;
1114:                 } else {
1115:                   /* ii and jj must be zero */
1116:                   for (kk=kstart; kk<kend+1; kk++) {
1117:                     cols[cnt++]  = slot + gnx*gny*kk;
1118:                   }
1119:                 }
1120:               }
1121:             }
1122:           }
1123:         }
1124:         MatSetValuesBlockedLocal(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1125:       }
1126:     }
1127:   }
1128:   PetscFree(values);
1129:   PetscFree(cols);
1130:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1131:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1132:   return(0);
1133: }

1135: /* BAD! Almost identical to the BAIJ one */
1138: int DAGetMatrix3d_MPISBAIJ(DA da,Mat J)
1139: {
1140:   int                    ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1141:   int                    m,n,dim,s,*cols,k,nc,col,cnt,p,*dnz,*onz;
1142:   int                    istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1143:   MPI_Comm               comm;
1144:   PetscScalar            *values;
1145:   DAPeriodicType         wrap;
1146:   DAStencilType          st;
1147:   ISLocalToGlobalMapping ltog;

1150:   /*     
1151:          nc - number of components per grid point 
1152:          col - number of colors needed in one direction for single component problem
1153:   
1154:   */
1155:   DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
1156:   if (wrap != DA_NONPERIODIC) SETERRQ(PETSC_ERR_SUP,"Currently no support for periodic");
1157:   col    = 2*s + 1;

1159:   DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1160:   DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1161:   PetscObjectGetComm((PetscObject)da,&comm);

1163:   /* create the matrix */
1164:   PetscMalloc(col*col*col*nc*nc*sizeof(PetscScalar),&values);
1165:   PetscMemzero(values,col*col*col*nc*nc*sizeof(PetscScalar));
1166:   PetscMalloc(col*col*col*sizeof(int),&cols);

1168:   DAGetISLocalToGlobalMappingBlck(da,&ltog);

1170:   /* determine the matrix preallocation information */
1171:   MatPreallocateSymmetricInitialize(comm,nx*ny*nz,nx*ny*nz,dnz,onz);
1172:   for (i=xs; i<xs+nx; i++) {
1173:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1174:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
1175:     for (j=ys; j<ys+ny; j++) {
1176:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1177:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
1178:       for (k=zs; k<zs+nz; k++) {
1179:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1180:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));

1182:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);

1184:         /* Find block columns in block row */
1185:         cnt  = 0;
1186:         if (st == DA_STENCIL_BOX) {   /* if using BOX stencil */
1187:           for (ii=istart; ii<iend+1; ii++) {
1188:             for (jj=jstart; jj<jend+1; jj++) {
1189:               for (kk=kstart; kk<kend+1; kk++) {
1190:                 cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk;
1191:               }
1192:             }
1193:           }
1194:         } else {  /* Star stencil */
1195:           cnt  = 0;
1196:           for (ii=istart; ii<iend+1; ii++) {
1197:             if (ii) {
1198:               /* jj and kk must be zero */
1199:               /* cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk; */
1200:               cols[cnt++]  = slot + ii;
1201:             } else {
1202:               for (jj=jstart; jj<jend+1; jj++) {
1203:                 if (jj) {
1204:                   /* ii and kk must be zero */
1205:                   cols[cnt++]  = slot + gnx*jj;
1206:                 } else {
1207:                   /* ii and jj must be zero */
1208:                   for (kk=kstart; kk<kend+1; kk++) {
1209:                     cols[cnt++]  = slot + gnx*gny*kk;
1210:                   }
1211:                 }
1212:               }
1213:             }
1214:           }
1215:         }
1216:         MatPreallocateSymmetricSetLocal(ltog,1,&slot,cnt,cols,dnz,onz);
1217:       }
1218:     }
1219:   }
1220:   MatSeqSBAIJSetPreallocation(J,nc,0,dnz);
1221:   MatMPISBAIJSetPreallocation(J,nc,0,dnz,0,onz);
1222:   MatPreallocateFinalize(dnz,onz);

1224:   MatSetLocalToGlobalMappingBlock(J,ltog);

1226:   /*
1227:     For each node in the grid: we get the neighbors in the local (on processor ordering
1228:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1229:     PETSc ordering.
1230:   */

1232:   for (i=xs; i<xs+nx; i++) {
1233:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1234:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
1235:     for (j=ys; j<ys+ny; j++) {
1236:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1237:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
1238:       for (k=zs; k<zs+nz; k++) {
1239:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1240:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
1241: 
1242:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1243: 
1244:         cnt  = 0;
1245:         if (st == DA_STENCIL_BOX) {   /* if using BOX stencil */
1246:           for (ii=istart; ii<iend+1; ii++) {
1247:             for (jj=jstart; jj<jend+1; jj++) {
1248:               for (kk=kstart; kk<kend+1; kk++) {
1249:                 cols[cnt++]  = slot + ii + gnx*jj + gnx*gny*kk;
1250:               }
1251:             }
1252:           }
1253:         } else {  /* Star stencil */
1254:           cnt  = 0;
1255:           for (ii=istart; ii<iend+1; ii++) {
1256:             if (ii) {
1257:               /* jj and kk must be zero */
1258:               cols[cnt++]  = slot + ii;
1259:             } else {
1260:               for (jj=jstart; jj<jend+1; jj++) {
1261:                 if (jj) {
1262:                   /* ii and kk must be zero */
1263:                   cols[cnt++]  = slot + gnx*jj;
1264:                 } else {
1265:                   /* ii and jj must be zero */
1266:                   for (kk=kstart; kk<kend+1; kk++) {
1267:                     cols[cnt++]  = slot + gnx*gny*kk;
1268:                   }
1269:                 }
1270:               }
1271:             }
1272:           }
1273:         }
1274:         MatSetValuesBlockedLocal(J,1,&slot,cnt,cols,values,INSERT_VALUES);
1275:       }
1276:     }
1277:   }
1278:   PetscFree(values);
1279:   PetscFree(cols);
1280:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1281:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1282:   return(0);
1283: }


1286: /* ---------------------------------------------------------------------------------*/

1290: int DAGetMatrix3d_MPIAIJ_Fill(DA da,Mat J)
1291: {
1292:   int                    ierr,xs,ys,nx,ny,i,j,slot,gxs,gys,gnx,gny;
1293:   int                    m,n,dim,s,*cols,k,nc,*rows,col,cnt,l,p,*dnz,*onz;
1294:   int                    istart,iend,jstart,jend,kstart,kend,zs,nz,gzs,gnz,ii,jj,kk;
1295:   int                    ifill_col,*dfill = da->dfill,*ofill = da->ofill;
1296:   MPI_Comm               comm;
1297:   PetscScalar            *values;
1298:   DAPeriodicType         wrap;
1299:   ISLocalToGlobalMapping ltog;
1300:   DAStencilType          st;

1303:   /*     
1304:          nc - number of components per grid point 
1305:          col - number of colors needed in one direction for single component problem
1306:   
1307:   */
1308:   DAGetInfo(da,&dim,&m,&n,&p,0,0,0,&nc,&s,&wrap,&st);
1309:   col    = 2*s + 1;
1310:   if (DAXPeriodic(wrap) && (m % col)){
1311:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in X is divisible\n\
1312:                  by 2*stencil_width + 1\n");
1313:   }
1314:   if (DAYPeriodic(wrap) && (n % col)){
1315:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Y is divisible\n\
1316:                  by 2*stencil_width + 1\n");
1317:   }
1318:   if (DAZPeriodic(wrap) && (p % col)){
1319:     SETERRQ(PETSC_ERR_SUP,"For coloring efficiency ensure number of grid points in Z is divisible\n\
1320:                  by 2*stencil_width + 1\n");
1321:   }

1323:   DAGetCorners(da,&xs,&ys,&zs,&nx,&ny,&nz);
1324:   DAGetGhostCorners(da,&gxs,&gys,&gzs,&gnx,&gny,&gnz);
1325:   PetscObjectGetComm((PetscObject)da,&comm);

1327:   PetscMalloc(col*col*col*nc*nc*nc*sizeof(PetscScalar),&values);
1328:   PetscMemzero(values,col*col*col*nc*nc*nc*sizeof(PetscScalar));
1329:   PetscMalloc(nc*sizeof(int),&rows);
1330:   PetscMalloc(col*col*col*nc*sizeof(int),&cols);
1331:   DAGetISLocalToGlobalMapping(da,&ltog);

1333:   /* determine the matrix preallocation information */
1334:   MatPreallocateInitialize(comm,nc*nx*ny*nz,nc*nx*ny*nz,dnz,onz);


1337:   for (i=xs; i<xs+nx; i++) {
1338:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1339:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
1340:     for (j=ys; j<ys+ny; j++) {
1341:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1342:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
1343:       for (k=zs; k<zs+nz; k++) {
1344:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1345:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
1346: 
1347:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1348: 
1349:         for (l=0; l<nc; l++) {
1350:           cnt  = 0;
1351:           for (ii=istart; ii<iend+1; ii++) {
1352:             for (jj=jstart; jj<jend+1; jj++) {
1353:               for (kk=kstart; kk<kend+1; kk++) {
1354:                 if (ii || jj || kk) {
1355:                   if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1356:                     for (ifill_col=ofill[l]; ifill_col<ofill[l+1]; ifill_col++)
1357:                       cols[cnt++]  = ofill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1358:                   }
1359:                 } else {
1360:                   if (dfill) {
1361:                     for (ifill_col=dfill[l]; ifill_col<dfill[l+1]; ifill_col++)
1362:                       cols[cnt++]  = dfill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1363:                   } else {
1364:                     for (ifill_col=0; ifill_col<nc; ifill_col++)
1365:                       cols[cnt++]  = ifill_col + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1366:                   }
1367:                 }
1368:               }
1369:             }
1370:           }
1371:           rows[0] = l + nc*(slot);
1372:           MatPreallocateSetLocal(ltog,1,rows,cnt,cols,dnz,onz);
1373:         }
1374:       }
1375:     }
1376:   }
1377:   MatSeqAIJSetPreallocation(J,0,dnz);
1378:   MatMPIAIJSetPreallocation(J,0,dnz,0,onz);
1379:   MatPreallocateFinalize(dnz,onz);

1381:   MatSetLocalToGlobalMapping(J,ltog);

1383:   /*
1384:     For each node in the grid: we get the neighbors in the local (on processor ordering
1385:     that includes the ghost points) then MatSetValuesLocal() maps those indices to the global
1386:     PETSc ordering.
1387:   */
1388:   for (i=xs; i<xs+nx; i++) {
1389:     istart = DAXPeriodic(wrap) ? -s : (PetscMax(-s,-i));
1390:     iend   = DAXPeriodic(wrap) ?  s : (PetscMin(s,m-i-1));
1391:     for (j=ys; j<ys+ny; j++) {
1392:       jstart = DAYPeriodic(wrap) ? -s : (PetscMax(-s,-j));
1393:       jend   = DAYPeriodic(wrap) ?  s : (PetscMin(s,n-j-1));
1394:       for (k=zs; k<zs+nz; k++) {
1395:         kstart = DAZPeriodic(wrap) ? -s : (PetscMax(-s,-k));
1396:         kend   = DAZPeriodic(wrap) ?  s : (PetscMin(s,p-k-1));
1397: 
1398:         slot = i - gxs + gnx*(j - gys) + gnx*gny*(k - gzs);
1399: 
1400:         for (l=0; l<nc; l++) {
1401:           cnt  = 0;
1402:           for (ii=istart; ii<iend+1; ii++) {
1403:             for (jj=jstart; jj<jend+1; jj++) {
1404:               for (kk=kstart; kk<kend+1; kk++) {
1405:                 if (ii || jj || kk) {
1406:                   if ((st == DA_STENCIL_BOX) || ((!ii && !jj) || (!jj && !kk) || (!ii && !kk))) {/* entries on star*/
1407:                     for (ifill_col=ofill[l]; ifill_col<ofill[l+1]; ifill_col++)
1408:                       cols[cnt++]  = ofill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1409:                   }
1410:                 } else {
1411:                   if (dfill) {
1412:                     for (ifill_col=dfill[l]; ifill_col<dfill[l+1]; ifill_col++)
1413:                       cols[cnt++]  = dfill[ifill_col] + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1414:                   } else {
1415:                     for (ifill_col=0; ifill_col<nc; ifill_col++)
1416:                       cols[cnt++]  = ifill_col + nc*(slot + ii + gnx*jj + gnx*gny*kk);
1417:                   }
1418:                 }
1419:               }
1420:             }
1421:           }
1422:           rows[0] = l + nc*(slot);
1423:           MatSetValuesLocal(J,1,rows,cnt,cols,values,INSERT_VALUES);
1424:         }
1425:       }
1426:     }
1427:   }
1428:   PetscFree(values);
1429:   PetscFree(rows);
1430:   PetscFree(cols);
1431:   MatAssemblyBegin(J,MAT_FINAL_ASSEMBLY);
1432:   MatAssemblyEnd(J,MAT_FINAL_ASSEMBLY);
1433:   return(0);
1434: }