Skip to content

Commit 53349b4

Browse files
jeremyltjeremylt
committed
wip - bddc
1 parent d8d9c82 commit 53349b4

File tree

10 files changed

+1116
-7
lines changed

10 files changed

+1116
-7
lines changed

‎examples/petsc/Makefile‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,11 @@ area.o = $(area.c:%.c=$(OBJDIR)/%.o)
4646
area: $(area.o) | $(PETSc.pc) $(ceed.pc)
4747
$(call quiet,LINK.o) $(CEED_LDFLAGS) $^ $(LOADLIBES) $(LDLIBS) -o $@
4848

49+
bddc.c := bddc.c $(utils.c)
50+
bddc.o = $(bddc.c:%.c=$(OBJDIR)/%.o)
51+
bddc: $(bddc.o) | $(PETSc.pc) $(ceed.pc)
52+
$(call quiet,LINK.o) $(CEED_LDFLAGS) $^ $(LOADLIBES) $(LDLIBS) -o $@
53+
4954
bps.c := bps.c $(utils.c)
5055
bps.o = $(bps.c:%.c=$(OBJDIR)/%.o)
5156
bps: $(bps.o) | $(PETSc.pc) $(ceed.pc)
@@ -89,7 +94,7 @@ print: $(PETSc.pc) $(ceed.pc)
8994
@true
9095

9196
clean:
92-
$(RM) -r $(OBJDIR) *.vtu area bps bpsraw bpssphere multigrid
97+
$(RM) -r $(OBJDIR) *.vtu area bddc bps bpsraw bpssphere multigrid
9398

9499
$(PETSc.pc):
95100
$(if $(wildcard $@),,$(error \

‎examples/petsc/bddc.c‎

Lines changed: 608 additions & 0 deletions
Large diffs are not rendered by default.

‎examples/petsc/bddc.h‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
// Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at
2+
// the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights
3+
// reserved. See files LICENSE and NOTICE for details.
4+
//
5+
// This file is part of CEED, a collection of benchmarks, miniapps, software
6+
// libraries and APIs for efficient high-order finite element and spectral
7+
// element discretizations for exascale applications. For more information and
8+
// source code availability see http://github.com/ceed.
9+
//
10+
// The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
11+
// a collaborative effort of two U.S. Department of Energy organizations (Office
12+
// of Science and the National Nuclear Security Administration) responsible for
13+
// the planning and preparation of a capable exascale ecosystem, including
14+
// software, applications, hardware, advanced system engineering and early
15+
// testbed platforms, in support of the nation's exascale computing imperative.
16+
17+
#ifndef bddc_h
18+
#define bddc_h
19+
20+
#include "include/bpsproblemdata.h"
21+
#include "include/petscmacros.h"
22+
#include "include/petscutils.h"
23+
#include "include/matops.h"
24+
#include "include/structs.h"
25+
#include "include/libceedsetup.h"
26+
27+
#include <ceed.h>
28+
#include <petsc.h>
29+
#include <petscdmplex.h>
30+
#include <petscfe.h>
31+
#include <petscsys.h>
32+
#include <stdbool.h>
33+
#include <string.h>
34+
35+
#if PETSC_VERSION_LT(3,12,0)
36+
#ifdef PETSC_HAVE_CUDA
37+
#include <petsccuda.h>
38+
// Note: With PETSc prior to version 3.12.0, providing the source path to
39+
// include 'cublas_v2.h' will be needed to use 'petsccuda.h'.
40+
#endif
41+
#endif
42+
43+
// -----------------------------------------------------------------------------
44+
// Command Line Options
45+
// -----------------------------------------------------------------------------
46+
47+
// Coarsening options
48+
typedef enum {
49+
INJECTION_SCALED = 0, INJECTION_HARMONIC = 1
50+
} InjectionType;
51+
static const char *const injection_types [] = {"scaled", "harmonic",
52+
"InjectionType", "INJECTION", 0
53+
};
54+
55+
#endif // bddc_h

‎examples/petsc/include/libceedsetup.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#include "structs.h"
88

99
PetscErrorCode CeedDataDestroy(CeedInt i, CeedData data);
10+
PetscErrorCode CeedDataBDDCDestroy(CeedDataBDDC data);
1011
PetscErrorCode SetupLibceedByDegree(DM dm, Ceed ceed, CeedInt degree,
1112
CeedInt topo_dim, CeedInt q_extra,
1213
PetscInt num_comp_x, PetscInt num_comp_u,
@@ -17,5 +18,9 @@ PetscErrorCode SetupLibceedByDegree(DM dm, Ceed ceed, CeedInt degree,
1718
PetscErrorCode CeedLevelTransferSetup(Ceed ceed, CeedInt num_levels,
1819
CeedInt num_comp_u, CeedData *data, CeedInt *leveldegrees,
1920
CeedQFunction qf_restrict, CeedQFunction qf_prolong);
21+
PetscErrorCode SetupLibceedBDDC(DM dm_vertex, CeedData data_fine,
22+
CeedData data_vertex,
23+
PetscInt g_vertex_size, PetscInt xl_vertex_size,
24+
BPData bp_data);
2025

2126
#endif // libceedsetup_h

‎examples/petsc/include/matops.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@ PetscErrorCode MatMult_Ceed(Mat A, Vec X, Vec Y);
1313
PetscErrorCode FormResidual_Ceed(SNES snes, Vec X, Vec Y, void *ctx);
1414
PetscErrorCode MatMult_Prolong(Mat A, Vec X, Vec Y);
1515
PetscErrorCode MatMult_Restrict(Mat A, Vec X, Vec Y);
16+
PetscErrorCode PCShellApply_BDDC(Mat A, Vec X, Vec Y);
17+
PetscErrorCode PCShellSetup_BDDC(PC pc);
18+
PetscErrorCode MatMult_BDDCSchur(Mat S, Vec X, Vec Y);
1619
PetscErrorCode ComputeErrorMax(UserO user, CeedOperator op_error,
1720
Vec X, CeedVector target, PetscReal *max_error);
1821

‎examples/petsc/include/petscutils.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ typedef PetscErrorCode (*BCFunction)(PetscInt dim, PetscReal time,
1818
PetscErrorCode SetupDMByDegree(DM dm, PetscInt degree, PetscInt num_comp_u,
1919
PetscInt topo_dim,
2020
bool enforce_bc, BCFunction bc_func);
21+
PetscErrorCode SetupVertexDMFromDM(DM dm, DM dm_vertex, PetscInt num_comp_u,
22+
bool enforce_bc, BCFunction bc_func);
2123
PetscErrorCode CreateRestrictionFromPlex(Ceed ceed, DM dm, CeedInt P,
2224
CeedInt topo_dim, CeedInt height, DMLabel domain_label, CeedInt value,
2325
CeedElemRestriction *elem_restr);

‎examples/petsc/include/structs.h‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,16 @@ struct UserProlongRestr_ {
2929
CeedOperator op_prolong, op_restrict;
3030
Ceed ceed;
3131
};
32+
// Data for PETSc PCshell
33+
typedef struct UserBDDC_ *UserBDDC;
34+
struct UserBDDC_ {
35+
MPI_Comm comm;
36+
DM dm, dm_Pi;
37+
SNES snes_Pi;
38+
KSP ksp_S_Pi;
39+
Vec X_loc, Y_loc, X_Pi, Y_Pi, X_Pi_loc, Y_Pi_loc, mult;
40+
CeedDataBDDC ceed_data_bddc;
41+
};
3242

3343
// -----------------------------------------------------------------------------
3444
// libCEED Data Structs
@@ -45,6 +55,17 @@ struct CeedData_ {
4555
CeedVector q_data, x_ceed, y_ceed;
4656
};
4757

58+
// libCEED data struct for BDDC
59+
typedef struct CeedDataBDDC_ *CeedDataBDDC;
60+
struct CeedDataBDDC_ {
61+
CeedBasis basis_Pi;
62+
CeedInt strides[3];
63+
CeedElemRestriction elem_restr_Pi, elem_restr_Pi_r, elem_restr_r;
64+
CeedOperator op_Pi_r, op_r_Pi, op_Pi_Pi, op_r_r, op_r_r_inv,
65+
op_inject_Pi, op_inject_r, op_restrict_Pi, op_restrict_r;
66+
CeedVector x_Pi_ceed, y_Pi_ceed, x_r_ceed, y_r_ceed, mult_ceed;
67+
};
68+
4869
// BP specific data
4970
typedef struct {
5071
CeedInt num_comp_x, num_comp_u, topo_dim, q_data_size, q_extra;

‎examples/petsc/src/libceedsetup.c‎

Lines changed: 180 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,35 @@ PetscErrorCode CeedDataDestroy(CeedInt i, CeedData data) {
2929
PetscFunctionReturn(0);
3030
};
3131

32+
// -----------------------------------------------------------------------------
33+
// Destroy libCEED BDDC objects
34+
// -----------------------------------------------------------------------------
35+
PetscErrorCode CeedDataBDDCDestroy(CeedDataBDDC data) {
36+
int ierr;
37+
38+
CeedBasisDestroy(&data->basis_Pi);
39+
CeedElemRestrictionDestroy(&data->elem_restr_Pi);
40+
CeedElemRestrictionDestroy(&data->elem_restr_Pi_r);
41+
CeedElemRestrictionDestroy(&data->elem_restr_r);
42+
CeedOperatorDestroy(&data->op_Pi_r;);
43+
CeedOperatorDestroy(&data->op_r_Pi);
44+
CeedOperatorDestroy(&data->op_Pi_Pi);
45+
CeedOperatorDestroy(&data->op_r_r);
46+
CeedOperatorDestroy(&data->op_r_r_inv);
47+
CeedOperatorDestroy(&data->op_inject_Pi);
48+
CeedOperatorDestroy(&data->op_inject_r);
49+
CeedOperatorDestroy(&data->op_restrict_Pi);
50+
CeedOperatorDestroy(&data->op_restrict_r);
51+
CeedVectorDestroy(&data->x_Pi_ceed);
52+
CeedVectorDestroy(&data->y_Pi_ceed);
53+
CeedVectorDestroy(&data->x_r_ceed);
54+
CeedVectorDestroy(&data->y_r_ceed);
55+
CeedVectorDestroy(&data->mult_ceed);
56+
ierr = PetscFree(data); CHKERRQ(ierr);
57+
58+
PetscFunctionReturn(0);
59+
};
60+
3261
// -----------------------------------------------------------------------------
3362
// Set up libCEED for a given degree
3463
// -----------------------------------------------------------------------------
@@ -57,11 +86,9 @@ PetscErrorCode SetupLibceedByDegree(DM dm, Ceed ceed, CeedInt degree,
5786
P = degree + 1;
5887
Q = P + q_extra;
5988
CeedBasisCreateTensorH1Lagrange(ceed, topo_dim, num_comp_u, P, Q,
60-
bp_data.q_mode,
61-
&basis_u);
89+
bp_data.q_mode, &basis_u);
6290
CeedBasisCreateTensorH1Lagrange(ceed, topo_dim, num_comp_x, 2, Q,
63-
bp_data.q_mode,
64-
&basis_x);
91+
bp_data.q_mode, &basis_x);
6592
CeedBasisGetNumQuadraturePoints(basis_u, &num_qpts);
6693

6794
// CEED restrictions
@@ -155,8 +182,7 @@ PetscErrorCode SetupLibceedByDegree(DM dm, Ceed ceed, CeedInt degree,
155182
CeedOperatorSetField(op_setup_rhs, "x", elem_restr_x, basis_x,
156183
CEED_VECTOR_ACTIVE);
157184
CeedOperatorSetField(op_setup_rhs, "q_data", elem_restr_qd_i,
158-
CEED_BASIS_COLLOCATED,
159-
q_data);
185+
CEED_BASIS_COLLOCATED, q_data);
160186
CeedOperatorSetField(op_setup_rhs, "true_soln", elem_restr_u_i,
161187
CEED_BASIS_COLLOCATED, *target);
162188
CeedOperatorSetField(op_setup_rhs, "rhs", elem_restr_u, basis_u,
@@ -255,3 +281,151 @@ PetscErrorCode CeedLevelTransferSetup(Ceed ceed, CeedInt num_levels,
255281
};
256282

257283
// -----------------------------------------------------------------------------
284+
// Set up libCEED for BDDC interface vertices
285+
// -----------------------------------------------------------------------------
286+
PetscErrorCode SetupLibceedBDDC(DM dm_vertex, CeedData data_fine,
287+
CeedDataBDDC data_vertex,
288+
PetscInt g_vertex_size, PetscInt xl_vertex_size,
289+
BPData bp_data {
290+
int ierr;
291+
Ceed ceed = data_fine->ceed;
292+
CeedBasis basis_Pi, basis_u = data_fine->basis_u;
293+
CeedElemRestriction elem_restr_Pi, elem_restr_Pi_r, elem_restr_r;
294+
CeedOperator op_Pi_r, op_r_Pi, op_Pi_Pi, op_r_r, op_r_r_inv, op_inject_Pi, op_inject_r, op_restrict_Pi, op_restrict_r;
295+
CeedVector x_Pi_ceed, y_Pi_ceed, x_r_ceed, y_r_ceed, mask_r_ceed, mask_Gamma_ceed, mask_I_ceed;
296+
CeedInt topo_dim, num_comp_u, P, Q, num_qpts, num_elem, elem_size,
297+
q_data_size = bp_data.q_data_size;
298+
299+
// CEED basis
300+
CeedBasisGetDimension(basis_u, &topo_dim);
301+
CeedBasisGetNumComponents(basis_u, &num_comp_u);
302+
CeedBasisGetNumNodes1D(basis_u, &P);
303+
elem_size = CeedIntPow(P, topo_dim);
304+
CeedBasisGetNumQuadraturePoints1D(basis_u, &Q);
305+
CeedBasisGetNumQuadraturePoints(basis_u, &num_qpts);
306+
CeedScalar *interp_1d, *grad_1d, *q_ref_1d, *q_weight_1d;
307+
interp_1d = calloc(2*Q * sizeof(CeedScalar));
308+
CeedScalar *temp;
309+
CeedBasisGetInterp1D(basis_u, &temp);
310+
memcpy(interp_1d, temp, Q * sizeof(CeedScalar));
311+
memcpy(&interp_1d[1*Q], temp[(P-1)*Q], Q * sizeof(CeedScalar));
312+
grad_1d = calloc(2*Q * sizeof(CeedScalar));
313+
CeedBasisGetGrad1D(basis_u, &temp);
314+
memcpy(grad_1d, temp, Q * sizeof(CeedScalar));
315+
memcpy(&grad_1d[1*Q], temp[(P-1)*Q], Q * sizeof(CeedScalar));
316+
q_ref_1d = calloc(Q * sizeof(CeedScalar));
317+
CeedBasisGetQRef(basis_u, &temp);
318+
memcpy(q_ref_1d, temp, Q * sizeof(CeedScalar));
319+
q_weight_1d = calloc(Q * sizeof(CeedScalar));
320+
CeedBasisGetQWeights(basis_u, &temp);
321+
memcpy(q_weight_1d, temp, Q * sizeof(CeedScalar));
322+
CeedBasisCreateTensorH1(ceed, topo_dim, num_comp_u, 1, Q,
323+
interp_1d, grad_1d, q_ref_1d,
324+
q_weight_1d, &basis_Pi);
325+
326+
// CEED restrictions
327+
// -- Interface vertex restriction
328+
ierr = CreateRestrictionFromPlex(ceed, dm_vertex, P, topo_dim, 0, 0, 0, &elem_restr_Pi);
329+
CHKERRQ(ierr);
330+
331+
// -- Subdomain restriction
332+
ierr = DMPlexGetHeightStratum(dm_vertex, 0, &c_start, &c_end); CHKERRQ(ierr);
333+
num_elem = c_end - c_start;
334+
CeedInt strides = [num_comp_u, 1, num_comp_u*elem_size];
335+
CeedElemRestrictionCreateStrided(ceed, num_elem, elem_size, num_comp_u,
336+
num_comp_u*num_elem*elem_size, // **NOPAD**
337+
strides, &elem_restr_r);
338+
339+
// -- Subdomain to interface vertex restriction
340+
CeedInt offsets_Pi_r[num_elem*8];
341+
for (CeedInt e=0; e<num_elem; e++) {
342+
CeedInt elem_stride = e*num_comp_u*elem_size;
343+
offsets_Pi_r[e*8 + 0] = elem_stride + num_comp_u*0;
344+
offsets_Pi_r[e*8 + 1] = elem_stride + num_comp_u*(P-1);
345+
offsets_Pi_r[e*8 + 2] = elem_stride + num_comp_u*(P*(P-1) + 0);
346+
offsets_Pi_r[e*8 + 3] = elem_stride + num_comp_u*(P*(P-1) + P-1);
347+
offsets_Pi_r[e*8 + 4] = elem_stride + num_comp_u*(P*P*(P-1) + 0);
348+
offsets_Pi_r[e*8 + 5] = elem_stride + num_comp_u*(P*P*(P-1) + P-1);
349+
offsets_Pi_r[e*8 + 6] = elem_stride + num_comp_u*(P*P*(P-1) + P*(P-1) + 0);
350+
offsets_Pi_r[e*8 + 7] = elem_stride + num_comp_u*(P*P*(P-1) + P*(P-1) + P-1);
351+
}
352+
CeedElemRestrictionCreate(ceed, num_elem, elem_size, num_comp_u, 1, num_comp_u*num_elem*elem_size, // **NOPAD**
353+
CEED_MEM_HOST, CEED_COPY_VALUES, &offsets_Pi_r, &elem_restr_Pi_r);
354+
355+
// Create the persistent vectors that will be needed
356+
CeedVectorCreate(ceed, xl_vertex_size, &x_Pi_ceed);
357+
CeedVectorCreate(ceed, xl_vertex_size, &y_Pi_ceed);
358+
CeedVectorCreate(ceed, num_comp_u *elem_size*num_elem, &x_r_ceed);
359+
CeedVectorCreate(ceed, num_comp_u *elem_size*num_elem, &y_r_ceed);
360+
361+
// Create the mass or diff operator
362+
CeedQFunction qf_apply = data_fine->qf_apply;
363+
// -- Interface vertices
364+
CeedOperatorCreate(ceed, data_fine->qf_apply, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_Pi_Pi);
365+
CeedOperatorSetField(op_Pi_Pi, "u", elem_restr_Pi, basis_u, CEED_VECTOR_ACTIVE);
366+
CeedOperatorSetField(op_Pi_Pi, "q_data", data_fine->elem_restr_qd_i,
367+
CEED_BASIS_COLLOCATED, data_fine->q_data);
368+
CeedOperatorSetField(op_Pi_Pi, "v", elem_restr_Pi, basis_u, CEED_VECTOR_ACTIVE);
369+
// -- Subdomains to interface vertices
370+
CeedOperatorCreate(ceed, data_fine->qf_apply, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_Pi_r);
371+
CeedOperatorSetField(op_Pi_r, "u", elem_restr_r, basis_u, CEED_VECTOR_ACTIVE);
372+
CeedOperatorSetField(op_Pi_r, "q_data", data_fine->elem_restr_qd_i,
373+
CEED_BASIS_COLLOCATED, data_fine->q_data);
374+
CeedOperatorSetField(op_Pi_r, "v", elem_restr_Pi, basis_u, CEED_VECTOR_ACTIVE);
375+
// -- Interface vertices to subdomains
376+
CeedOperatorCreate(ceed, data_fine->qf_apply, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_r_Pi);
377+
CeedOperatorSetField(op_r_Pi, "u", elem_restr_Pi, basis_u, CEED_VECTOR_ACTIVE);
378+
CeedOperatorSetField(op_r_Pi, "q_data", data_fine->elem_restr_qd_i,
379+
CEED_BASIS_COLLOCATED, data_fine->q_data);
380+
CeedOperatorSetField(op_r_Pi, "v", elem_restr_r, basis_u, CEED_VECTOR_ACTIVE);
381+
// -- Subdomains
382+
CeedOperatorCreate(ceed, data_fine->qf_apply, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_r_r);
383+
CeedOperatorSetField(op_r_r, "u", elem_restr_r, basis_u, CEED_VECTOR_ACTIVE);
384+
CeedOperatorSetField(op_r_r, "q_data", data_fine->elem_restr_qd_i,
385+
CEED_BASIS_COLLOCATED, data_fine->q_data);
386+
CeedOperatorSetField(op_r_r, "v", elem_restr_r, basis_u, CEED_VECTOR_ACTIVE);
387+
// -- Subdomain FDM inverse
388+
CeedOperatorCreateFDMElementInverse(op_r_r, &op_r_r_inv, CEED_REQUEST_IMMEDIATE);
389+
390+
// Injection and restriction operators
391+
CeedQFunction qf_identity;
392+
CeedQFunctionCreateIdentity(ceed, num_comp_u, CEED_EVAL_NONE, CEED_EVAL_NONE,
393+
&qf_identity);
394+
// -- Injection to interface vertices
395+
CeedOperatorCreate(ceed, qf_identity, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_inject_Pi);
396+
CeedOperatorSetField(op_inject_Pi, "input", elem_restr_r, CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); // Note: r to Pi
397+
CeedOperatorSetField(op_inject_Pi, "output", elem_restr_Pi_r, CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
398+
// -- Injection to subdomains
399+
CeedOperatorCreate(ceed, qf_identity, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_inject_r);
400+
CeedOperatorSetField(op_inject_r, "input", data_fine->elem_restr_u, CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
401+
CeedOperatorSetField(op_inject_r, "output", elem_restr_r, CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
402+
// -- Restriction from interface vertices
403+
CeedOperatorCreate(ceed, qf_identity, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_restrict_Pi);
404+
CeedOperatorSetField(op_restrict_Pi, "input", elem_restr_Pi_r, CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
405+
CeedOperatorSetField(op_restrict_Pi, "output", elem_restr_r, CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
406+
// -- Restriction from subdomains
407+
CeedOperatorCreate(ceed, qf_identity, CEED_QFUNCTION_NONE, CEED_QFUNCTION_NONE, &op_restrict_r);
408+
CeedOperatorSetField(op_restrict_r, "input", elem_restr_r, CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE); // Note: Pi to r
409+
CeedOperatorSetField(op_restrict_r, "output", data_fine->elem_restr_u, CEED_BASIS_COLLOCATED, CEED_VECTOR_ACTIVE);
410+
// -- Cleanup
411+
CeedQFunctionDestroy(&qf_identity);
412+
413+
// Save libCEED data required for level
414+
data_vertex->basis_Pi = basis_Pi;
415+
data_vertex->elem_restr_Pi = elem_restr_Pi;
416+
data_vertex->elem_restr_Pi_r = elem_restr_Pi_r;
417+
data_vertex->elem_restr_r = elem_restr_r;
418+
data_vertex->op_Pi_r = op_Pi_r;
419+
data_vertex->op_r_Pi = op_r_Pi;
420+
data_vertex->op_Pi_Pi = op_Pi_Pi;
421+
data_vertex->op_r_r = op_r_r;
422+
data_vertex->op_r_r_inv = op_r_r_inv;
423+
data_vertex->x_Pi_ceed = x_Pi_ceed;
424+
data_vertex->y_Pi_ceed = y_Pi_ceed;
425+
data_vertex->x_r_ceed = x_r_ceed;
426+
data_vertex->y_r_ceed = y_r_ceed;
427+
428+
PetscFunctionReturn(0);
429+
};
430+
431+
// -----------------------------------------------------------------------------

0 commit comments

Comments
 (0)