Actual source code: samgpetsctools.c

  1: #include "global.h"
 2:  #include petscksp.h
 3:  #include samgfunc.h
 4:  #include petscfunc.h
 5:  #include externc.h

  7: EXTERN_C_BEGIN
  8: void USER_coo(int * i,int * ndim, double * x, double * y, double * z)
  9: {
 10:   printf("in user_coo");
 11: }
 12: EXTERN_C_END

 14: static  double Machine_Precision_Eps = 2.e-16;
 15: /* ------------------------------------------------------------------- */
 18: /*..SamgGetGrid - This routine gets an array of grids
 19:     INPUT:  levels: number of levels created by SAMG 
 20:             numnodes: number of nodes on finest grid 
 21:             numnonzeros: number of nonzeros on coarsest grid 
 22:     OUTPUT: grid  : array of grids                   ..*/
 23: int SamgGetGrid(int levels, int numnodes, int numnonzero, 
 24:                 GridCtx* grid, void* ctx)
 25: {
 26:    int      k;
 27:    int      ia_shift[MAX_LEVELS], ja_shift[MAX_LEVELS], nnu_cg, nna_cg;
 28:    int      iw_shift[MAX_LEVELS], jw_shift[MAX_LEVELS], rows_weights,
 29:             nna_weights, dummy;
 30:    int      ierr;
 31:    MatInfo  info;

 33:    /*..Get coarse grid operators..*/
 34:    /*....Initialize ia_shift, ja_shift, nnu_cg and nna_cg....*/
 35:    ia_shift[1] = 1;
 36:    ja_shift[1] = 1;
 37:    nnu_cg = numnodes;
 38:    nna_cg = numnonzero;

 40:    for (k=2;k<=levels;k++){ /*....We do not get the finest level matrix....*/
 41:        /*....Update ia_shift and ja_shift values with nna_cg and nnu_cg 
 42:              from previous loop....*/
 43:        ia_shift[k] = ia_shift[k-1] + nna_cg ;
 44:        ja_shift[k] = ja_shift[k-1] + nnu_cg ;

 46:        /*....Get coarse grid matrix on level k....*/
 47:        SamgGetCoarseMat(k, ia_shift[k], ja_shift[k], &(grid[k].A),
 48:                                PETSC_NULL);

 50:        /*....Get size and number of nonzeros of coarse grid matrix on
 51:              level k, i.e. get new nna_cg and nnu_cg values....*/
 52:        MatGetSize(grid[k].A, &nnu_cg, &nnu_cg);
 53:        MatGetInfo(grid[k].A, MAT_LOCAL, &info);
 54:        nna_cg = int(info.nz_used);
 55:    }
 56: 
 57:    /*..Get interpolation operators..*/
 58:    /*....Initialize iw_shift, jw_shift and nna_weights....*/
 59:    iw_shift[0] = 1;
 60:    jw_shift[0] = 1;
 61:    nna_weights = 0;
 62:    rows_weights = numnodes;

 64:    for (k=1;k<=levels-1;k++){/*....There's NO interpolation operator 
 65:                                    associated to the coarsest level....*/
 66:        /*....Update iw_shift with nna_weights value from 
 67:              previous loop....*/
 68:        iw_shift[k] = iw_shift[k-1] + nna_weights ;
 69:        /*....Update jw_shift with rows_weights value from 
 70:              current loop....*/
 71:        jw_shift[k] = jw_shift[k-1] + rows_weights ;
 72: 
 73:        /*....Get interpolation from level k+1 to level k....*/
 74:        SamgGetInterpolation(k, iw_shift[k], jw_shift[k],
 75:                                    &(grid[k].Interp), PETSC_NULL) ;

 77:        /*....Get number of collumns and number of nonzeros of 
 78:              interpolation associated to level k. NOTE: The 
 79:              number of collums at this loop equals the number of 
 80:              rows at the next loop...*/
 81:        MatGetSize(grid[k].Interp, &dummy, &rows_weights);
 82:        MatGetInfo(grid[k].Interp, MAT_LOCAL, &info);
 83:        nna_weights = int(info.nz_used);
 84:    }

 86:    return 0;
 87: }
 88: /* ------------------------------------------------------------------- */
 91: /*..SamgGetCoarseMat - This routine gets the coarse level matrix on the 
 92:     level specified at input. 
 93:     WARNING: This routine does not work to get the fine level matrix, 
 94:              i.e. the value of k at input should be at least 2 
 95:     INPUT:  level:    current grid level 
 96:             ia_shift: shift to apply on ia_cg elements 
 97:             ja_shift: shift to apply on ja_cg elements 
 98:     OUTPUT: coarsemat: coarse level matrix  
 99: ..*/

101: int SamgGetCoarseMat(int level, int ia_shift, int ja_shift, 
102:                      Mat* coarsemat, void* ctx)
103: {
104:    int      nnu_k, nna_k; /* size and non-zeros of operator on level k     */
105:    int      *ia_k, *ja_k; /* coarse grid matrix in skyline format          */
106:    double   *a_k;
107:    int      *nnz_per_row; /* integer vector to hold the number of nonzeros */
108:                           /* of each row. This vector will be used to      */
109:                           /* allocate memory for the matrix, and to store  */
110:                           /* elements in the matrix                        */
111:    int      I, ierr;

113:    /*..Get size (nnu_k) and number of non-zeros (nna_k) of operator 
114:      on level k..*/
115:    SAMGPETSC_get_dim_operator(&level, &nnu_k, &nna_k);

117:    /*..Now that nnu_cg and nna_cg are known, we can allocate memory for 
118:      coarse level matrix in compresses skyline format..*/
119:    PetscMalloc(nna_k     * sizeof(double),&a_k);
120:    PetscMalloc((nnu_k+1) * sizeof(int),&ia_k);
121:    PetscMalloc(nna_k     * sizeof(int),&ja_k);

123:    /*..Get coarse grid matrix in skyline format..*/
124:    SAMGPETSC_get_operator(&level, a_k, ia_k, ja_k);

126:    /*..Apply shift on each of the ia_cg and ja_cg elements..*/
127:    SAMGPETSC_apply_shift(ia_k, &nnu_k, &ia_shift,
128:                          ja_k, &nna_k, &ja_shift);

130:    PetscMalloc(nnu_k * sizeof(int),&nnz_per_row);

132:    /*..The numbero f nonzeros entries in row I can be calculated as      
133:        ia[I+1] - 1 - ia[I] + 1 = ia[I+1] - ia[I]                         ..*/
134:    for (I=0;I<nnu_k;I++)
135:        nnz_per_row[I] = ia_k[I+1] - ia_k[I];

137:    /*..Allocate (create) SeqAIJ matrix  for use within PETSc..*/
138:    MatCreate(PETSC_COMM_WORLD,nnu_k,nnu_k,nnu_k,nnu_k,coarsemat);
139:    MatSetType(*coarsemat,MATSEQAIJ);
140:    MatSeqAIJSetPreallocation(*coarsemat,0,nnz_per_row);

142:    /*..Store coarse grid matrix in Petsc Mat object..*/
143:    for (I=0;I<nnu_k;I++){
144:       MatSetValues(*coarsemat,
145:                1,              /* number of rows */
146:                &I,             /* pointer to global row number */
147:                nnz_per_row[I], /* number of collums = number of nonzero ... */
148:                                /* entries in row I                          */
149:                &(ja_k[ ia_k[I] ]),
150:                               /* vector global column indices */
151:                (PetscScalar *) &(a_k[ ia_k[I] ]),
152:                               /* vector of coefficients */
153:         INSERT_VALUES);
154:    }

156:    MatAssemblyBegin(*coarsemat,MAT_FINAL_ASSEMBLY);
157:    MatAssemblyEnd(*coarsemat,MAT_FINAL_ASSEMBLY);

159:    /*..Free memory required by storage in skyline format..*/
160:    PetscFree(a_k);
161:    PetscFree(ia_k);
162:    PetscFree(ja_k);
163:    /*..Free memory for auxilary array..*/
164:    PetscFree(nnz_per_row);
165: 
166:    return 0;
167: }
168: /* ------------------------------------------------------------------- */
171: /*..SamgGetInterpolation - Get interpolation operator that interpolates 
172:     from level k+1 (coarse grid) to k (fine grid), where the input level 
173:     equals k ..*/
174: /*..WARNING: This routine assumes that the input value for level is strictly 
175:     smaller than the number of levels created..*/
176: /*..Implementation notes
177:     o) The interpolation is a rectangular matrix with 
178:        number of rows equal to fine grid dim 
179:                  cols          coarse. 
180: ..*/
181: int SamgGetInterpolation(int level, int iw_shift, int jw_shift,
182:                          Mat* interpolation, void* ctx)
183: {
184:    int     rows_weights, cols_weights, nna_weights;
185:    int     *iweights, *jweights;
186:    double  *weights;
187:    int      *nnz_per_row; /* integer vector to hold the number of nonzeros */
188:                           /* of each row. This vector will be used to      */
189:                           /* allocate memory for the matrix, and to store  */
190:                           /* elements in the matrix                        */
191:    int      I, ierr, coarser_level=level+1, dummy;

193:    /*..Get number of rows and number of nonzeros of interpolation operator..*/
194:    SAMGPETSC_get_dim_interpol(&level, &rows_weights, &nna_weights);

196:    /*..Get number of cols of interpolation operator. NOTE: The number of 
197:        collums of the interpolation on level k equals the size of 
198:        the coarse grid matrix on the next coarsest grid.  
199:        SAMGPETSC_get_dim_interpol does not allow to get the number of 
200:        collumns of next to coarsest grid..*/
201:    SAMGPETSC_get_dim_operator(&coarser_level, &cols_weights, &dummy);

203:    /*..Now that nnu_weights and nna_weights are known, we can allocate 
204:        memory for interpolation operator in compresses skyline format..*/
205:    PetscMalloc(nna_weights  * sizeof(double),&weights);
206: 
207:    PetscMalloc((rows_weights+1) * sizeof(int),&iweights);
208: 
209:    PetscMalloc(nna_weights  * sizeof(int),&jweights);
210: 

212:    /*..Get interpolation operator in compressed skyline format..*/
213:    SAMGPETSC_get_interpol(&level, weights, iweights, jweights);

215:    /*..Apply shift on each of the ia_cg and ja_cg elements..*/
216:    SAMGPETSC_apply_shift(iweights, &rows_weights, &iw_shift,
217:                          jweights, &nna_weights, &jw_shift);

219:    PetscMalloc(rows_weights * sizeof(int),&nnz_per_row);
220: 

222:    /*..The numbero f nonzeros entries in row I can be calculated as      
223:        ia[I+1] - 1 - ia[I] + 1 = ia[I+1] - ia[I]                         ..*/
224:    for (I=0;I<rows_weights;I++)
225:        nnz_per_row[I] = iweights[I+1] - iweights[I];

227:    /*..Allocate (create) SeqAIJ matrix  for use within PETSc..*/
228:    MatCreate(PETSC_COMM_WORLD,rows_weights,cols_weights,rows_weights,cols_weights,interpolation);
229:    MatSetType(*interpolation,MATSEQAIJ);
230:    MatSeqAIJSetPreallocation(*interpolation,0,nnz_per_row);

232:    /*..Store coarse grid matrix in Petsc Mat object..*/
233:    for (I=0;I<rows_weights;I++){
234:       MatSetValues(*interpolation,
235:                1,              /* number of rows */
236:                &I,             /* pointer to global row number */
237:                nnz_per_row[I], /* number of collums = number of nonzero ... */
238:                                /* entries in row I                          */
239:                &(jweights[ iweights[I] ]),
240:                               /* vector global column indices */
241:                (PetscScalar *) &(weights[ iweights[I] ]),
242:                               /* vector of coefficients */
243:         INSERT_VALUES);
244:    }

246:    MatAssemblyBegin(*interpolation,MAT_FINAL_ASSEMBLY);
247:    MatAssemblyEnd(*interpolation,MAT_FINAL_ASSEMBLY);

249:    /*..Free memory required by storage in skyline format..*/
250:    PetscFree(weights);
251:    PetscFree(iweights);
252:    PetscFree(jweights);
253:    /*..Free memory for auxilary array..*/
254:    PetscFree(nnz_per_row);

256:    return 0;
257: }
258: /* ------------------------------------------------------------------- */
261: /*..Write coarser grid operators constructed by SAMG to ASCII file..*/

263: int SamgPetscWriteOperator(const int numnodes, const double* Asky, 
264:                            const int* ia, const int* ja, int extension)
265: {

267:      static char filename[80];
268:      int         I,j,j1,j2;
269:      FILE        *output;
270: 
271:      /*..Switch arrays iacopy and jacopy to C conventions..*/
272:      //     for (j=0;j<=numnodes;j++)
273:      //         iacopy[j]--;
274:      //     for (j=0;j<ia[numnodes];j++)
275:      //         jacopy[j]--;
276: 
277:      /*....Write matrix to file....*/
278:       sprintf(filename,"coarsemat.%02u", extension);
279:       output=fopen(filename,"w");
280:       fprintf(output, "%% \n");
281:       fprintf(output, "%% %d %d \n", numnodes, ia[numnodes] );

283:       for (I=0;I<numnodes;I++){
284:            j1 = ia[I];
285:            j2 = ia[I+1] - 1;
286:            for (j=j1;j<=j2;j++){
287:                fprintf(output, "%d %d %22.18e\n", I+1, ja[j]+1,
288:                                 Asky[j] );
289:                //               printf("%d %d %e \n", I+1, ja[j]+1,
290:                //                              Asky[j] );
291:            }
292:       }
293:       fclose(output);
294: 
295:       return 0;
296: }
297: /* ------------------------------------------------------------------- */
300: /*..Write interpolation operators constructed by SAMG to ASCII file..*/

302: int SamgPetscWriteInterpol(const int numrows, const double* weights, 
303:                   const int* iweights, const int* jweights, int extension)
304: {

306:      static char filename[80];
307:      int         I,j,j1,j2,numcols,numnonzero;
308:      FILE        *output;
309: 
310:      /*..Set number of nonzeros..*/
311:      numnonzero = iweights[numrows];

313:      /*..Determine numcols as the maximum ja value +1..*/
314:      numcols = jweights[0];
315:      for (j=0;j<numnonzero;j++){
316:        if (jweights[j] > numcols) numcols = jweights[j];
317:      }
318:      numcols++;

320:      /*..Write interpolation operator from grid k+1 (coarse grid) grid to k 
321:          (finer grid) to file..*/
322:       sprintf(filename,"interpol.%02u%02u",
323:                         extension+1, extension);
324:       output=fopen(filename,"w");
325:       fprintf(output, "%% \n%% %d %d %d \n", numrows, numcols, iweights[numrows] );
326:       for (I=0;I<numrows;I++){
327:            j1 = iweights[I];
328:            j2 = iweights[I+1] - 1;
329:            for (j=j1;j<=j2;j++){
330:                fprintf(output, "%d %d %22.18e\n", I+1, jweights[j]+1,
331:                                 weights[j] );
332:                //               printf("%d %d %e \n", I+1, jweights[j]+1,
333:                //                                weights[j] );
334:            }
335:       }
336:       fclose(output);

338: return 0;
339: }
340: /* ------------------------------------------------------------------- */
343: /*..SamgCheckGalerkin - This routine offers a check on the correctness 
344:     of how SAMG interpolation and coarse grid operators are parsed to 
345:     PETSc. This routine computes I^H_h A^h I^h_H by PETSc matrix - matrix 
346:     multiplications, and compares this product with A^H..*/
347: 
348: int SamgCheckGalerkin(int levels, Mat A, GridCtx* grid, 
349:                       void* ctx)
350: {
351:    Mat     FineLevelMatrix, Restriction, HalfGalerkin, Galerkin, Diff;
352:    double  normdiff;
353:    int     ierr, k;

355:    for (k=1;k<=levels-1;k++){
356:       if (k==1)
357:           FineLevelMatrix = A;
358:           else
359:           FineLevelMatrix = grid[k].A;
360:       /*....Compute A^h I^h_H....*/
361:       MatMatMult(FineLevelMatrix, grid[k].Interp, &HalfGalerkin);
362:       /*....Get I^h_H....*/
363:       MatTranspose(grid[k].Interp,&Restriction);
364:       /*....Compute I^H_h A^h I^h_H....*/
365:       MatMatMult(Restriction, HalfGalerkin, &Galerkin);
366:       /*....Compute A^H - I^H_h A^h I^h_H....*/
367:       MatSubstract(grid[k+1].A, Galerkin, &Diff);
368:      /*....Compute || A^H - I^H_h A^h I^h_H||_{\infty}....*/
369:       MatNorm(Diff,NORM_INFINITY,&normdiff);

371:       printf("SamgCheckGalerkin :: || A^H - I^H_h A^h I^h_H||_{infty} on level %8d = %e\n",
372:               k+1, normdiff);

374:       MatDestroy(Restriction);
375:       MatDestroy(HalfGalerkin);
376:       MatDestroy(Galerkin);
377:       MatDestroy(Diff);
378:    }
379:    return 0;
380: }
381: /* ------------------------------------------------------------------- */
384: /*..MatSubstract - Computes the difference Term1 - Term2 
385:     INPUT:  Term1, Term2 : The input matrices 
386:     OUTPUT: Prod:          the difference 
387:   NOTE: Memory needed by difference has to freed outside this routine! 
388: ..*/

390: int MatSubstract(Mat Term1, Mat Term2, Mat* Diff)
391: {
392:    Vec          col_vec1, col_vec2, diff_vec;
393:    int          rows1, cols1, rows2, cols2, col, row, ierr;
394:    static PetscScalar dminusone = -1.;
395:    PetscScalar  matrix_element ;
396:    PetscScalar  *vec_getvalues ;
397:    double       inf_norm_diff_vec = 0.0 ;
398:    double       Zero_Element_Treshold = 0.0 ;

400:    /*..Get sizes of terms..*/
401:    MatGetSize(Term1, &rows1, &cols1);
402:    MatGetSize(Term2, &rows2, &cols2);

404:    /*..Check input..*/
405:    if ( (cols1 != rows1) || (cols2 != rows2) ){
406:       SETERRQ(1,"Error in MatMatMult: cols1 <> rows1 or cols1 <> rows1");
407:    }

409:    /*..Create difference of 2 SeqAIJ matrices..*/
410:    MatCreate(PETSC_COMM_WORLD,rows1,cols1,rows1,cols1,Diff);
411:    MatSetType(*Diff,MATSEQAIJ);
412:    MatSeqAIJSetPreallocation(*Diff,0,PETSC_NULL);

414:    /*..Create vectors..*/
415:    VecCreate(MPI_COMM_WORLD,&col_vec1);
416:    VecSetSizes(col_vec1,PETSC_DECIDE,rows1);
417:    VecSetType(col_vec1,VECSEQ);
418:    VecDuplicate(col_vec1, &col_vec2);
419:    VecDuplicate(col_vec1, &diff_vec);

421:    for (col=0;col<cols1;col++){
422:        /*..Get collumns..*/
423:       MatGetColumnVector(Term1,col_vec1,col);
424:       MatGetColumnVector(Term2,col_vec2,col);
425:       /*..Substract collumns..*/
426:       VecWAXPY(&dminusone, col_vec2, col_vec1, diff_vec);
427: 
428: 
429:       /*..Compute norm..*/
430:       VecNorm( diff_vec, NORM_INFINITY, &inf_norm_diff_vec );
431: 
432:       /*..Set threshold..*/
433:       Zero_Element_Treshold = inf_norm_diff_vec * Machine_Precision_Eps ;

435:       /*..Get Term1(:,col) -  Term2(:,col) values..*/
436:       VecGetArray(diff_vec, &vec_getvalues);

438:       for (row=0;row<rows1;row++){
439:           matrix_element = vec_getvalues[row];
440:            if ( PetscAbsScalar( matrix_element ) >= Zero_Element_Treshold ) {
441:                  MatSetValue(*Diff, row, col,matrix_element, INSERT_VALUES );
442:            }
443:       }
444:       VecRestoreArray(diff_vec, &vec_getvalues);
445:    }

447:    MatAssemblyBegin(*Diff,MAT_FINAL_ASSEMBLY);
448:    MatAssemblyEnd(*Diff,MAT_FINAL_ASSEMBLY);

450:    VecDestroy(col_vec1);
451:    VecDestroy(col_vec2);
452:    VecDestroy(diff_vec);

454:    return 0;
455: }
456: /* ------------------------------------------------------------------- */