Skip to content

Commit ab31067

Browse files
t-samplest-samples
authored
Merge pull request #158 from Kingsrook/feature/process-tracers
Feature/process tracers
2 parents ec71355 + c47c39f commit ab31067

File tree

19 files changed

+1586
-58
lines changed

19 files changed

+1586
-58
lines changed

‎qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/processes/RunProcessAction.java‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import java.util.Optional;
3333
import java.util.UUID;
3434
import com.kingsrook.qqq.backend.core.actions.ActionHelper;
35+
import com.kingsrook.qqq.backend.core.actions.customizers.QCodeLoader;
3536
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.NoCodeWidgetRenderer;
3637
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
3738
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
@@ -53,6 +54,7 @@
5354
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
5455
import com.kingsrook.qqq.backend.core.model.data.QRecord;
5556
import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData;
57+
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
5658
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
5759
import com.kingsrook.qqq.backend.core.model.metadata.processes.NoCodeWidgetFrontendComponentMetaData;
5860
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
@@ -63,6 +65,7 @@
6365
import com.kingsrook.qqq.backend.core.model.metadata.processes.QStepMetaData;
6466
import com.kingsrook.qqq.backend.core.model.session.QSession;
6567
import com.kingsrook.qqq.backend.core.processes.implementations.basepull.BasepullConfiguration;
68+
import com.kingsrook.qqq.backend.core.processes.tracing.ProcessTracerInterface;
6669
import com.kingsrook.qqq.backend.core.state.InMemoryStateProvider;
6770
import com.kingsrook.qqq.backend.core.state.StateProviderInterface;
6871
import com.kingsrook.qqq.backend.core.state.StateType;
@@ -71,6 +74,7 @@
7174
import com.kingsrook.qqq.backend.core.utils.StringUtils;
7275
import com.kingsrook.qqq.backend.core.utils.ValueUtils;
7376
import org.apache.commons.lang.BooleanUtils;
77+
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
7478

7579

7680
/*******************************************************************************
@@ -87,12 +91,16 @@ public class RunProcessAction
8791
public static final String BASEPULL_TIMESTAMP_FIELD = "basepullTimestampField";
8892
public static final String BASEPULL_CONFIGURATION = "basepullConfiguration";
8993

94+
public static final String PROCESS_TRACER_CODE_REFERENCE_FIELD = "processTracerCodeReference";
95+
9096
////////////////////////////////////////////////////////////////////////////////////////////////
9197
// indicator that the timestamp field should be updated - e.g., the execute step is finished. //
9298
////////////////////////////////////////////////////////////////////////////////////////////////
9399
public static final String BASEPULL_READY_TO_UPDATE_TIMESTAMP_FIELD = "basepullReadyToUpdateTimestamp";
94100
public static final String BASEPULL_DID_QUERY_USING_TIMESTAMP_FIELD = "basepullDidQueryUsingTimestamp";
95101

102+
private ProcessTracerInterface processTracer;
103+
96104

97105

98106
/*******************************************************************************
@@ -119,6 +127,8 @@ public RunProcessOutput execute(RunProcessInput runProcessInput) throws QExcepti
119127
}
120128
runProcessOutput.setProcessUUID(runProcessInput.getProcessUUID());
121129

130+
traceStartOrResume(runProcessInput, process);
131+
122132
UUIDAndTypeStateKey stateKey = new UUIDAndTypeStateKey(UUID.fromString(runProcessInput.getProcessUUID()), StateType.PROCESS_STATUS);
123133
ProcessState processState = primeProcessState(runProcessInput, stateKey, process);
124134

@@ -166,13 +176,15 @@ public RunProcessOutput execute(RunProcessInput runProcessInput) throws QExcepti
166176
////////////////////////////////////////////////////////////
167177
// upon exception (e.g., one thrown by a step), throw it. //
168178
////////////////////////////////////////////////////////////
179+
traceBreakOrFinish(runProcessInput, runProcessOutput, qe);
169180
throw (qe);
170181
}
171182
catch(Exception e)
172183
{
173184
////////////////////////////////////////////////////////////
174185
// upon exception (e.g., one thrown by a step), throw it. //
175186
////////////////////////////////////////////////////////////
187+
traceBreakOrFinish(runProcessInput, runProcessOutput, e);
176188
throw (new QException("Error running process", e));
177189
}
178190
finally
@@ -183,6 +195,8 @@ public RunProcessOutput execute(RunProcessInput runProcessInput) throws QExcepti
183195
runProcessOutput.setProcessState(processState);
184196
}
185197

198+
traceBreakOrFinish(runProcessInput, runProcessOutput, null);
199+
186200
return (runProcessOutput);
187201
}
188202

@@ -630,6 +644,7 @@ RunBackendStepOutput runBackendStep(RunProcessInput runProcessInput, QProcessMet
630644
runBackendStepInput.setCallback(runProcessInput.getCallback());
631645
runBackendStepInput.setFrontendStepBehavior(runProcessInput.getFrontendStepBehavior());
632646
runBackendStepInput.setAsyncJobCallback(runProcessInput.getAsyncJobCallback());
647+
runBackendStepInput.setProcessTracer(processTracer);
633648

634649
runBackendStepInput.setTableName(process.getTableName());
635650
if(!StringUtils.hasContent(runBackendStepInput.getTableName()))
@@ -651,9 +666,13 @@ RunBackendStepOutput runBackendStep(RunProcessInput runProcessInput, QProcessMet
651666
runBackendStepInput.setBasepullLastRunTime((Instant) runProcessInput.getValues().get(BASEPULL_LAST_RUNTIME_KEY));
652667
}
653668

669+
traceStepStart(runBackendStepInput);
670+
654671
RunBackendStepOutput runBackendStepOutput = new RunBackendStepAction().execute(runBackendStepInput);
655672
storeState(stateKey, runBackendStepOutput.getProcessState());
656673

674+
traceStepFinish(runBackendStepInput, runBackendStepOutput);
675+
657676
if(runBackendStepOutput.getException() != null)
658677
{
659678
runProcessOutput.setException(runBackendStepOutput.getException());
@@ -931,4 +950,153 @@ else if(basepullHoursBackForInitialTimestamp != null)
931950
runProcessInput.getValues().put(BASEPULL_TIMESTAMP_FIELD, basepullConfiguration.getTimestampField());
932951
runProcessInput.getValues().put(BASEPULL_CONFIGURATION, basepullConfiguration);
933952
}
953+
954+
955+
956+
/***************************************************************************
957+
**
958+
***************************************************************************/
959+
private void setupProcessTracer(RunProcessInput runProcessInput, QProcessMetaData process)
960+
{
961+
try
962+
{
963+
if(process.getProcessTracerCodeReference() != null)
964+
{
965+
processTracer = QCodeLoader.getAdHoc(ProcessTracerInterface.class, process.getProcessTracerCodeReference());
966+
}
967+
968+
Serializable processTracerCodeReference = runProcessInput.getValue(PROCESS_TRACER_CODE_REFERENCE_FIELD);
969+
if(processTracerCodeReference != null)
970+
{
971+
if(processTracerCodeReference instanceof QCodeReference codeReference)
972+
{
973+
processTracer = QCodeLoader.getAdHoc(ProcessTracerInterface.class, codeReference);
974+
}
975+
}
976+
}
977+
catch(Exception e)
978+
{
979+
LOG.warn("Error setting up processTracer", e, logPair("processName", runProcessInput.getProcessName()));
980+
}
981+
}
982+
983+
984+
985+
/***************************************************************************
986+
**
987+
***************************************************************************/
988+
private void traceStartOrResume(RunProcessInput runProcessInput, QProcessMetaData process)
989+
{
990+
setupProcessTracer(runProcessInput, process);
991+
992+
try
993+
{
994+
if(processTracer != null)
995+
{
996+
if(StringUtils.hasContent(runProcessInput.getStartAfterStep()) || StringUtils.hasContent(runProcessInput.getStartAtStep()))
997+
{
998+
processTracer.handleProcessResume(runProcessInput);
999+
}
1000+
else
1001+
{
1002+
processTracer.handleProcessStart(runProcessInput);
1003+
}
1004+
}
1005+
}
1006+
catch(Exception e)
1007+
{
1008+
LOG.info("Error in traceStart", e, logPair("processName", runProcessInput.getProcessName()));
1009+
}
1010+
}
1011+
1012+
1013+
1014+
/***************************************************************************
1015+
**
1016+
***************************************************************************/
1017+
private void traceBreakOrFinish(RunProcessInput runProcessInput, RunProcessOutput runProcessOutput, Exception processException)
1018+
{
1019+
try
1020+
{
1021+
if(processTracer != null)
1022+
{
1023+
ProcessState processState = runProcessOutput.getProcessState();
1024+
boolean isBreak = true;
1025+
1026+
/////////////////////////////////////////////////////////////
1027+
// if there's no next step, that means the process is done //
1028+
/////////////////////////////////////////////////////////////
1029+
if(processState.getNextStepName().isEmpty())
1030+
{
1031+
isBreak = false;
1032+
}
1033+
else
1034+
{
1035+
/////////////////////////////////////////////////////////////////
1036+
// or if the next step is the last index, then we're also done //
1037+
/////////////////////////////////////////////////////////////////
1038+
String nextStepName = processState.getNextStepName().get();
1039+
int nextStepIndex = processState.getStepList().indexOf(nextStepName);
1040+
if(nextStepIndex == processState.getStepList().size() - 1)
1041+
{
1042+
isBreak = false;
1043+
}
1044+
}
1045+
1046+
if(isBreak)
1047+
{
1048+
processTracer.handleProcessBreak(runProcessInput, runProcessOutput, processException);
1049+
}
1050+
else
1051+
{
1052+
processTracer.handleProcessFinish(runProcessInput, runProcessOutput, processException);
1053+
}
1054+
}
1055+
}
1056+
catch(Exception e)
1057+
{
1058+
LOG.info("Error in traceProcessFinish", e, logPair("processName", runProcessInput.getProcessName()));
1059+
}
1060+
}
1061+
1062+
1063+
1064+
/***************************************************************************
1065+
**
1066+
***************************************************************************/
1067+
private void traceStepStart(RunBackendStepInput runBackendStepInput)
1068+
{
1069+
try
1070+
{
1071+
if(processTracer != null)
1072+
{
1073+
processTracer.handleStepStart(runBackendStepInput);
1074+
}
1075+
}
1076+
catch(Exception e)
1077+
{
1078+
LOG.info("Error in traceStepFinish", e, logPair("processName", runBackendStepInput.getProcessName()), logPair("stepName", runBackendStepInput.getStepName()));
1079+
}
1080+
}
1081+
1082+
1083+
1084+
/***************************************************************************
1085+
**
1086+
***************************************************************************/
1087+
private void traceStepFinish(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput)
1088+
{
1089+
try
1090+
{
1091+
if(processTracer != null)
1092+
{
1093+
processTracer.handleStepFinish(runBackendStepInput, runBackendStepOutput);
1094+
}
1095+
}
1096+
catch(Exception e)
1097+
{
1098+
LOG.info("Error in traceStepFinish", e, logPair("processName", runBackendStepInput.getProcessName()), logPair("stepName", runBackendStepInput.getStepName()));
1099+
}
1100+
}
1101+
9341102
}

‎qqq-backend-core/src/main/java/com/kingsrook/qqq/backend/core/actions/tables/helpers/QueryStatManager.java‎

Lines changed: 5 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -31,16 +31,12 @@
3131
import java.util.concurrent.ScheduledExecutorService;
3232
import java.util.concurrent.TimeUnit;
3333
import java.util.function.Supplier;
34-
import com.kingsrook.qqq.backend.core.actions.tables.GetAction;
3534
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
3635
import com.kingsrook.qqq.backend.core.context.QContext;
3736
import com.kingsrook.qqq.backend.core.exceptions.QException;
3837
import com.kingsrook.qqq.backend.core.instances.QMetaDataVariableInterpreter;
3938
import com.kingsrook.qqq.backend.core.logging.QLogger;
40-
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetInput;
41-
import com.kingsrook.qqq.backend.core.model.actions.tables.get.GetOutput;
4239
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
43-
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertOutput;
4440
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
4541
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
4642
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
@@ -54,12 +50,10 @@
5450
import com.kingsrook.qqq.backend.core.model.querystats.QueryStatJoinTable;
5551
import com.kingsrook.qqq.backend.core.model.querystats.QueryStatOrderByField;
5652
import com.kingsrook.qqq.backend.core.model.session.QSession;
57-
import com.kingsrook.qqq.backend.core.model.tables.QQQTable;
58-
import com.kingsrook.qqq.backend.core.model.tables.QQQTablesMetaDataProvider;
53+
import com.kingsrook.qqq.backend.core.model.tables.QQQTableTableManager;
5954
import com.kingsrook.qqq.backend.core.utils.CollectionUtils;
6055
import com.kingsrook.qqq.backend.core.utils.PrefixedDefaultThreadFactory;
6156
import com.kingsrook.qqq.backend.core.utils.StringUtils;
62-
import com.kingsrook.qqq.backend.core.utils.collections.MapBuilder;
6357
import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair;
6458

6559

@@ -371,7 +365,7 @@ public void run()
371365
//////////////////////
372366
// set the table id //
373367
//////////////////////
374-
Integer qqqTableId = getQQQTableId(queryStat.getTableName());
368+
Integer qqqTableId = QQQTableTableManager.getQQQTableId(getInstance().qInstance, queryStat.getTableName());
375369
queryStat.setQqqTableId(qqqTableId);
376370

377371
//////////////////////////////
@@ -382,7 +376,7 @@ public void run()
382376
List<QueryStatJoinTable> queryStatJoinTableList = new ArrayList<>();
383377
for(String joinTableName : queryStat.getJoinTableNames())
384378
{
385-
queryStatJoinTableList.add(new QueryStatJoinTable().withQqqTableId(getQQQTableId(joinTableName)));
379+
queryStatJoinTableList.add(new QueryStatJoinTable().withQqqTableId(QQQTableTableManager.getQQQTableId(getInstance().qInstance, joinTableName)));
386380
}
387381
queryStat.setQueryStatJoinTableList(queryStatJoinTableList);
388382
}
@@ -460,7 +454,7 @@ private static void processCriteriaFromFilter(Integer qqqTableId, List<QueryStat
460454
String[] parts = fieldName.split("\\.");
461455
if(parts.length > 1)
462456
{
463-
queryStatCriteriaField.setQqqTableId(getQQQTableId(parts[0]));
457+
queryStatCriteriaField.setQqqTableId(QQQTableTableManager.getQQQTableId(getInstance().qInstance, parts[0]));
464458
queryStatCriteriaField.setName(parts[1]);
465459
}
466460
}
@@ -498,7 +492,7 @@ private static void processOrderByFromFilter(Integer qqqTableId, List<QueryStatO
498492
String[] parts = fieldName.split("\\.");
499493
if(parts.length > 1)
500494
{
501-
queryStatOrderByField.setQqqTableId(getQQQTableId(parts[0]));
495+
queryStatOrderByField.setQqqTableId(QQQTableTableManager.getQQQTableId(getInstance().qInstance, parts[0]));
502496
queryStatOrderByField.setName(parts[1]);
503497
}
504498
}
@@ -512,50 +506,6 @@ private static void processOrderByFromFilter(Integer qqqTableId, List<QueryStatO
512506
}
513507
}
514508
}
515-
516-
517-
518-
/*******************************************************************************
519-
**
520-
*******************************************************************************/
521-
private static Integer getQQQTableId(String tableName) throws QException
522-
{
523-
/////////////////////////////
524-
// look in the cache table //
525-
/////////////////////////////
526-
GetInput getInput = new GetInput();
527-
getInput.setTableName(QQQTablesMetaDataProvider.QQQ_TABLE_CACHE_TABLE_NAME);
528-
getInput.setUniqueKey(MapBuilder.of("name", tableName));
529-
GetOutput getOutput = new GetAction().execute(getInput);
530-
531-
////////////////////////
532-
// upon cache miss... //
533-
////////////////////////
534-
if(getOutput.getRecord() == null)
535-
{
536-
QTableMetaData tableMetaData = getInstance().qInstance.getTable(tableName);
537-
if(tableMetaData == null)
538-
{
539-
LOG.info("No such table", logPair("tableName", tableName));
540-
return (null);
541-
}
542-
543-
///////////////////////////////////////////////////////
544-
// insert the record (into the table, not the cache) //
545-
///////////////////////////////////////////////////////
546-
InsertInput insertInput = new InsertInput();
547-
insertInput.setTableName(QQQTable.TABLE_NAME);
548-
insertInput.setRecords(List.of(new QRecord().withValue("name", tableName).withValue("label", tableMetaData.getLabel())));
549-
InsertOutput insertOutput = new InsertAction().execute(insertInput);
550-
551-
///////////////////////////////////
552-
// repeat the get from the cache //
553-
///////////////////////////////////
554-
getOutput = new GetAction().execute(getInput);
555-
}
556-
557-
return getOutput.getRecord().getValueInteger("id");
558-
}
559509
}
560510

561511

0 commit comments

Comments
 (0)