Skip to content

Commit 62463c3

Browse files
zachwolfezachwolfe
committed
Improve navigations implementation
1 parent dfdfb24 commit 62463c3

1 file changed

Lines changed: 95 additions & 32 deletions

File tree

‎Airtable.EFCore/Query/Internal/AirtableShapedQueryCompilingExpressionVisitor.cs‎

Lines changed: 95 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -461,6 +461,7 @@ protected override Expression VisitShapedQuery(ShapedQueryExpression shapedQuery
461461
if (innerExpression is SelectExpression selectExpression)
462462
{
463463
var shaperLambdaMapping = new Dictionary<string, Expression<Action<AirtableQueryContext, List<AirtableRecord>, NavigationData>>>();
464+
var fieldsMapping = new Dictionary<string, string[]>();
464465
var recordParameter = Expression.Parameter(typeof(AirtableRecord), "record");
465466
var shaperBlock = BuildShaperBlock(selectExpression, shapedQueryExpression.ShaperExpression, recordParameter);
466467
var shaperLambda = Expression.Lambda(
@@ -472,7 +473,7 @@ protected override Expression VisitShapedQuery(ShapedQueryExpression shapedQuery
472473
var navigationResolver = new NavigationResolver();
473474
foreach (var navigation in entityType.GetNavigations().Concat<INavigationBase>(entityType.GetSkipNavigations()))
474475
{
475-
VisitNavigation(shaperLambdaMapping, navigationResolver, entityType, navigation);
476+
VisitNavigation(shaperLambdaMapping, fieldsMapping, navigationResolver, entityType, navigation);
476477
}
477478

478479
var shaperMapping = shaperLambdaMapping.ToDictionary(item => item.Key, item => item.Value.Compile());
@@ -482,6 +483,7 @@ protected override Expression VisitShapedQuery(ShapedQueryExpression shapedQuery
482483
Expression.Constant(selectExpression),
483484
Expression.Constant(shaperLambda.Compile()),
484485
Expression.Constant(shaperMapping),
486+
Expression.Constant(fieldsMapping),
485487
Expression.Constant(navigationResolver),
486488
Expression.Constant(
487489
QueryCompilationContext.QueryTrackingBehavior == QueryTrackingBehavior.NoTrackingWithIdentityResolution)
@@ -605,7 +607,7 @@ private Expression<Action<AirtableQueryContext, List<AirtableRecord>, Navigation
605607
navigationDataParameter);
606608
}
607609

608-
private void VisitNavigation(Dictionary<string, Expression<Action<AirtableQueryContext, List<AirtableRecord>, NavigationData>>> shaperMapping, NavigationResolver navigationResolver, IEntityType declaringEntityType, INavigationBase navigation)
610+
private void VisitNavigation(Dictionary<string, Expression<Action<AirtableQueryContext, List<AirtableRecord>, NavigationData>>> shaperMapping, Dictionary<string, string[]> fieldsMapping, NavigationResolver navigationResolver, IEntityType declaringEntityType, INavigationBase navigation)
609611
{
610612
BuildNavigationLambdas(navigation, declaringEntityType, navigationResolver);
611613

@@ -627,6 +629,9 @@ private void VisitNavigation(Dictionary<string, Expression<Action<AirtableQueryC
627629
var shaperLambda = BuildShaperLambda(navigationSelectExpression, shaper, navigationTableName, targetEntityType.ClrType);
628630
shaperMapping[navigationTableName] = shaperLambda;
629631

632+
// Store which fields are needed for this navigation table
633+
fieldsMapping[navigationTableName] = navigationSelectExpression.GetFields().ToArray();
634+
630635
return;
631636
}
632637

@@ -1066,36 +1071,64 @@ navigation is ISkipNavigation
10661071
entityMapping,
10671072
nameof(EntityMapping<object>.ReferencedRecordIndex))));
10681073

1069-
loopExpressions.Add(
1074+
// Check bounds and null before accessing to handle missing/deleted records
1075+
var boundsCheck = Expression.AndAlso(
1076+
Expression.LessThan(
1077+
referencedRecordIndex,
1078+
Expression.Property(referencedEntities, nameof(List<object>.Count))),
1079+
Expression.GreaterThanOrEqual(
1080+
referencedRecordIndex,
1081+
Expression.Constant(0)));
1082+
1083+
var fixupStatements = new List<Expression>
1084+
{
10701085
Expression.Assign(
10711086
referencedEntity,
10721087
Expression.MakeIndex(
10731088
referencedEntities,
10741089
referencedEntityListType.GetProperty("Item"),
1075-
new[] { referencedRecordIndex })));
1090+
new[] { referencedRecordIndex }))
1091+
};
1092+
1093+
// Add null check after assignment
1094+
var entityNotNullCheck = Expression.NotEqual(
1095+
referencedEntity,
1096+
Expression.Constant(null, targetEntityType.ClrType));
1097+
1098+
var navigationFixupStatements = new List<Expression>();
10761099

10771100
if (navigation is ISkipNavigation)
10781101
{
1079-
loopExpressions.Add(
1102+
navigationFixupStatements.Add(
10801103
Expression.Call(
10811104
Expression.Field(referencingEntity, backingFieldName),
10821105
referencedEntityCollectionAdd,
10831106
new[] { referencedEntity }));
10841107
}
10851108
else
10861109
{
1087-
loopExpressions.Add(
1110+
navigationFixupStatements.Add(
10881111
Expression.Assign(
10891112
Expression.Field(referencingEntity, backingFieldName),
10901113
referencedEntity));
10911114
}
10921115

1093-
loopExpressions.Add(
1116+
navigationFixupStatements.Add(
10941117
Expression.Call(
10951118
Expression.Field(referencedEntity, inverseBackingFieldName),
10961119
referencingEntityCollectionAdd,
10971120
new[] { referencingEntity }));
10981121

1122+
fixupStatements.Add(
1123+
Expression.IfThen(
1124+
entityNotNullCheck,
1125+
Expression.Block(navigationFixupStatements)));
1126+
1127+
loopExpressions.Add(
1128+
Expression.IfThen(
1129+
boundsCheck,
1130+
Expression.Block(fixupStatements)));
1131+
10991132
loopExpressions.Add(Expression.PostIncrementAssign(entityMappingIndex));
11001133
navigationFixupExpressions.Add(
11011134
Expression.Loop(
@@ -1204,13 +1237,28 @@ private class TableData<Entity> : TableDataBase
12041237
public override void MarkEntitiesAsVisited() => FirstUnvisitedEntity = Entities.Count;
12051238
public override void MarkEntitiesAsLoaded()
12061239
{
1240+
// Check for null entities (missing/deleted records) and provide context
1241+
var nullIndices = new List<int>();
12071242
for (var i = FirstUnloadedEntity; i < Entities.Count; i++)
12081243
{
12091244
if (Entities[i] == null)
12101245
{
1211-
throw new InvalidOperationException("Not all entities loaded");
1246+
nullIndices.Add(i);
12121247
}
12131248
}
1249+
1250+
if (nullIndices.Count > 0 && nullIndices.Count < RecordIds.Count)
1251+
{
1252+
// Some records are missing - don't throw, allow navigation fixup to skip missing entities
1253+
// This handles cases where records were deleted in Airtable or have referential integrity issues
1254+
// Navigation fixup will gracefully skip these missing entities
1255+
}
1256+
else if (nullIndices.Count == RecordIds.Count && RecordIds.Count > 0)
1257+
{
1258+
// All records are missing - this is likely an error
1259+
throw new InvalidOperationException($"None of the {RecordIds.Count} referenced entities were found in Airtable. Record IDs may be invalid.");
1260+
}
1261+
12141262
FirstUnloadedEntity = Entities.Count;
12151263
}
12161264

@@ -1272,6 +1320,7 @@ private sealed class QueryingEnumerable<T> : IAsyncEnumerable<T>
12721320
private readonly FormulaGenerator _formulaGenerator;
12731321
private readonly Func<AirtableQueryContext, AirtableRecord, T> _shaper;
12741322
private readonly Dictionary<string, Action<AirtableQueryContext, List<AirtableRecord>, NavigationData>> _shaperMapping;
1323+
private readonly Dictionary<string, string[]> _fieldsMapping;
12751324
private readonly NavigationResolver _navigationResolver;
12761325
private readonly NavigationData _navigationData;
12771326
private readonly bool _standalone;
@@ -1282,6 +1331,7 @@ public QueryingEnumerable(
12821331
SelectExpression selectExpression,
12831332
Func<AirtableQueryContext, AirtableRecord, T> shaper,
12841333
Dictionary<string, Action<AirtableQueryContext, List<AirtableRecord>, NavigationData>> shaperMapping,
1334+
Dictionary<string, string[]> fieldsMapping,
12851335
NavigationResolver navigationResolver,
12861336
bool standalone)
12871337
{
@@ -1290,6 +1340,7 @@ public QueryingEnumerable(
12901340
_formulaGenerator = new FormulaGenerator(airtableQueryContext.Parameters);
12911341
_shaper = shaper;
12921342
_shaperMapping = shaperMapping;
1343+
_fieldsMapping = fieldsMapping;
12931344
_navigationResolver = navigationResolver;
12941345
_navigationData = new();
12951346
_standalone = standalone;
@@ -1468,38 +1519,50 @@ private async IAsyncEnumerable<T> ProcessElements(IEnumerable<AirtableRecord> re
14681519
continue;
14691520
}
14701521

1471-
var filter = new StringBuilder("OR(");
1472-
for (var index = tableData.FirstUnloadedEntity; index < tableData.RecordIds.Count; index++)
1522+
// Airtable has a formula length limit (~10,000 chars), so we need to batch
1523+
// Record IDs are ~17 chars each, plus formula overhead = ~30 chars per ID
1524+
// Use conservative batch size of 300 to stay well under limit
1525+
const int MaxRecordsPerBatch = 300;
1526+
1527+
// Get the fields needed for this table
1528+
var fields = _fieldsMapping.TryGetValue(tableName, out var tableFields) ? tableFields : null;
1529+
1530+
for (var batchStart = tableData.FirstUnloadedEntity; batchStart < tableData.RecordIds.Count; batchStart += MaxRecordsPerBatch)
14731531
{
1474-
if (index > tableData.FirstUnloadedEntity)
1532+
var batchEnd = Math.Min(batchStart + MaxRecordsPerBatch, tableData.RecordIds.Count);
1533+
var filter = new StringBuilder("OR(");
1534+
1535+
for (var index = batchStart; index < batchEnd; index++)
14751536
{
1476-
filter.Append(", ");
1537+
if (index > batchStart)
1538+
{
1539+
filter.Append(", ");
1540+
}
1541+
filter.Append("RECORD_ID() = \"");
1542+
filter.Append(tableData.RecordIds[index]);
1543+
filter.Append('"');
14771544
}
1478-
filter.Append("RECORD_ID() = \"");
1479-
filter.Append(tableData.RecordIds[index]);
1480-
filter.Append('"');
1481-
}
1482-
filter.Append(')');
1545+
filter.Append(')');
14831546

1484-
newRecords.Clear();
1485-
1486-
response = null;
1487-
do
1488-
{
1489-
response = await _base.ListRecords(
1490-
tableName,
1491-
filterByFormula: filter.ToString(),
1492-
offset: response?.Offset);
1547+
response = null;
1548+
do
1549+
{
1550+
response = await _base.ListRecords(
1551+
tableName,
1552+
fields: fields,
1553+
filterByFormula: filter.ToString(),
1554+
offset: response?.Offset);
14931555

1494-
if (response is null)
1495-
throw new InvalidOperationException("Airtable response is null");
1556+
if (response is null)
1557+
throw new InvalidOperationException("Airtable response is null");
14961558

1497-
if (!response.Success)
1498-
throw new InvalidOperationException("Airtable error", response.AirtableApiError);
1559+
if (!response.Success)
1560+
throw new InvalidOperationException("Airtable error", response.AirtableApiError);
14991561

1500-
newRecords.AddRange(response.Records);
1562+
newRecords.AddRange(response.Records);
1563+
}
1564+
while (response.Offset != null);
15011565
}
1502-
while (response.Offset != null);
15031566

15041567
_shaperMapping[tableName](_airtableQueryContext, newRecords, _navigationData);
15051568
}

0 commit comments

Comments
 (0)