Index backfill optimization to only read columns present in index definition

src/postgres/src/backend/access/heap/heapam_handler.c

@@ -47,6 +47,7 @@
 
 /* YB includes */
 #include "access/yb_scan.h"
+#include "pg_yb_utils.h"
 
 static void reform_and_rewrite_tuple(HeapTuple tuple,
 									 Relation OldHeap, Relation NewHeap,
@@ -1261,27 +1262,52 @@ heapam_index_build_range_scan(Relation heapRelation,
 		else
 			snapshot = SnapshotAny;
 
-		scan = table_beginscan_strat(heapRelation,	/* relation */
-									 snapshot,	/* snapshot */
-									 0, /* number of keys */
-									 NULL,	/* scan key */
-									 true,	/* buffer access strategy OK */
-									 allow_sync);	/* syncscan OK? */
+		if (IsYBRelation(heapRelation) && yb_enable_index_backfill_column_projection)
+		{
+			/*
+			 * For YB relations, use the optimized scan function that only
+			 * fetches columns needed by the index. This avoids reading the
+			 * entire row from DocDB during index build/backfill.
+			 */
+			uint32		flags = SO_TYPE_SEQSCAN | SO_ALLOW_PAGEMODE |
+								SO_ALLOW_STRAT;
+
+			if (allow_sync)
+				flags |= SO_ALLOW_SYNC;
+
+			scan = ybc_heap_beginscan_for_index_build(heapRelation,
+													  snapshot,
+													  0, /* number of keys */
+													  NULL, /* scan key */
+													  flags,
+													  indexInfo);
+		}
+		else
+		{
+			scan = table_beginscan_strat(heapRelation,	/* relation */
+										 snapshot,	/* snapshot */
+										 0, /* number of keys */
+										 NULL,	/* scan key */
+										 true,	/* buffer access strategy OK */
+										 allow_sync);	/* syncscan OK? */
+		}
+
 		if (IsYBRelation(heapRelation))
 		{
 			YbcPgExecParameters *exec_params = &estate->yb_exec_params;
 
 			if (bfinfo)
 			{
 				if (bfinfo->bfinstr)
+				{
 					exec_params->bfinstr = pstrdup(bfinfo->bfinstr);
-				exec_params->backfill_read_time = bfinfo->read_time;
-				exec_params->partition_key =
-					pstrdup(bfinfo->row_bounds->partition_key);
-				exec_params->out_param = bfresult;
-				exec_params->is_index_backfill = true;
+					exec_params->backfill_read_time = bfinfo->read_time;
+					exec_params->partition_key =
+						pstrdup(bfinfo->row_bounds->partition_key);
+					exec_params->out_param = bfresult;
+					exec_params->is_index_backfill = true;
+				}
 			}
-
 			((YbScanDesc) scan)->exec_params = exec_params;
 		}
 	}

src/postgres/src/backend/access/yb_access/yb_scan.c

@@ -3340,6 +3340,65 @@ YbApplyMergeSortKeys(YbScanDesc ybScan, Scan *pg_scan_plan)
 	HandleYBStatus(YBCPgDmlSetMergeSortKeys(stmt, sort_info->numCols, yb_sort_keys));
 }
 
+/*
+ * Helper function to allocate and initialize basic scan descriptor fields.
+ * This is shared between ybcBeginScan and ybc_heap_beginscan_for_index_build.
+ */
+static YbScanDesc
+ybcAllocScanDesc(Relation relation, int nkeys, ScanKey keys)
+{
+	YbScanDesc	ybScan = (YbScanDesc) palloc0(sizeof(YbScanDescData));
+	TableScanDesc tsdesc = (TableScanDesc) ybScan;
+
+	tsdesc->rs_rd = relation;
+	tsdesc->rs_key = keys;
+	tsdesc->rs_nkeys = nkeys;
+
+	return ybScan;
+}
+
+/*
+ * Helper function to extract scan keys and check for unsatisfiable conditions.
+ * Returns true if scan keys are valid and scan should continue,
+ * false if scan should quit early (sets quit_scan = true).
+ * This is shared between ybcBeginScan and ybc_heap_beginscan_for_index_build.
+ */
+static bool
+ybcExtractAndCheckScanKeysValid(YbScanDesc ybScan, int nkeys, ScanKey keys)
+{
+	/* Flatten keys and store the results in ybScan */
+	ybExtractScanKeys(keys, nkeys, ybScan);
+
+	if (YbIsUnsatisfiableCondition(ybScan->nkeys, ybScan->keys))
+	{
+		ybScan->quit_scan = true;
+		return false;
+	}
+	return true;
+}
+
+/*
+ * Helper function to bind scan keys and hash keys.
+ * Returns true if binding succeeded, false if scan should quit.
+ * On failure, frees scan_plan bitmapsets and sets quit_scan = true.
+ * This is shared between ybcBeginScan and ybc_heap_beginscan_for_index_build.
+ */
+static bool
+ybcBindKeysValid(YbScanDesc ybScan, YbScanPlanData *scan_plan,
+							Scan *pg_scan_plan, bool is_for_precheck)
+{
+	if (!YbBindScanKeys(ybScan, scan_plan, pg_scan_plan, is_for_precheck) ||
+		!YbBindHashKeys(ybScan))
+	{
+		ybScan->quit_scan = true;
+		bms_free(scan_plan->hash_key);
+		bms_free(scan_plan->primary_key);
+		bms_free(scan_plan->sk_cols);
+		return false;
+	}
+	return true;
+}
+
 /*
  * Begin a scan for
  *   SELECT <Targets> FROM <relation> USING <index> WHERE <Binds>
@@ -3379,21 +3438,11 @@ ybcBeginScan(Relation relation,
 			 bool fetch_ybctids_only)
 {
 	/* Set up Yugabyte scan description */
-	YbScanDesc	ybScan = (YbScanDesc) palloc0(sizeof(YbScanDescData));
-	TableScanDesc tsdesc = (TableScanDesc) ybScan;
+	YbScanDesc	ybScan = ybcAllocScanDesc(relation, nkeys, keys);
 
-	tsdesc->rs_rd = relation;
-	tsdesc->rs_key = keys;
-	tsdesc->rs_nkeys = nkeys;
-
-	/* Flatten keys and store the results in ybScan. */
-	ybExtractScanKeys(keys, nkeys, ybScan);
-
-	if (YbIsUnsatisfiableCondition(ybScan->nkeys, ybScan->keys))
-	{
-		ybScan->quit_scan = true;
+	if (!ybcExtractAndCheckScanKeysValid(ybScan, nkeys, keys))
 		return ybScan;
-	}
+
 	ybScan->exec_params = exec_params;
 	ybScan->index = index;
 	ybScan->quit_scan = false;
@@ -3408,16 +3457,9 @@ ybcBeginScan(Relation relation,
 	ybScan->handle = YbNewSelect(relation, &ybScan->prepare_params);
 
 	/* Set up binds */
-	if (!YbBindScanKeys(ybScan, &scan_plan, pg_scan_plan,
-						false /* is_for_precheck */ ) ||
-		!YbBindHashKeys(ybScan))
-	{
-		ybScan->quit_scan = true;
-		bms_free(scan_plan.hash_key);
-		bms_free(scan_plan.primary_key);
-		bms_free(scan_plan.sk_cols);
+	if (!ybcBindKeysValid(ybScan, &scan_plan, pg_scan_plan,
+									 false /* is_for_precheck */))
 		return ybScan;
-	}
 
 	/*
 	 * Set up targets.  There are two separate cases:
@@ -3820,6 +3862,132 @@ ybc_heap_endscan(TableScanDesc tsdesc)
 	ybc_free_ybscan(ybdesc);
 }
 
+
+/*
+ * ybc_heap_beginscan_for_index_build
+ *		Begin a heap scan specifically for index build/backfill operations.
+ *
+ * Unlike the regular heap scan which fetches all columns, this function
+ * only requests the columns needed for the index:
+ *   - Columns referenced in ii_IndexAttrNumbers (index key and non-key columns)
+ *   - Columns referenced in ii_Expressions (expression index columns)
+ *   - Columns referenced in ii_Predicate (partial index predicate)
+ *   - ybctid (always needed for index entry construction)
+ *
+ * This optimization significantly reduces the amount of data read from
+ * DocDB during concurrent index creation (backfill), especially for tables
+ * with many columns where only a few are indexed.
+ */
+TableScanDesc
+ybc_heap_beginscan_for_index_build(Relation relation,
+								   Snapshot snapshot,
+								   int nkeys,
+								   ScanKey key,
+								   uint32 flags,
+								   IndexInfo *indexInfo)
+{
+	YbScanDesc	ybScan;
+	YbScanPlanData scan_plan;
+	TupleDesc	tupdesc;
+	Bitmapset  *required_attrs = NULL;
+	int			i;
+	int			idx;
+
+	/* Allocate and initialize scan descriptor */
+	ybScan = ybcAllocScanDesc(relation, nkeys, key);
+	TableScanDesc tsdesc = (TableScanDesc) ybScan;
+
+	tsdesc->rs_snapshot = snapshot;
+	tsdesc->rs_flags = flags;
+
+	if (!ybcExtractAndCheckScanKeysValid(ybScan, nkeys, key))
+		return tsdesc;
+
+	ybScan->index = NULL;
+	ybScan->quit_scan = false;
+	ybScan->prepare_params.fetch_ybctids_only = false;
+
+	/* Set up the scan plan */
+	ybcSetupScanPlan(false /* xs_want_itup */, ybScan, &scan_plan);
+	ybcSetupScanKeys(ybScan, &scan_plan);
+
+	ybScan->handle = YbNewSelect(relation, &ybScan->prepare_params);
+
+	/* Bind scan keys */
+	if (!ybcBindKeysValid(ybScan, &scan_plan, NULL /* pg_scan_plan */,
+									 false /* is_for_precheck */))
+		return tsdesc;
+
+	/*
+	 * Build the set of required attribute numbers based on IndexInfo.
+	 * Use FirstLowInvalidHeapAttributeNumber as the offset for system columns.
+	 */
+	tupdesc = RelationGetDescr(relation);
+
+	/* Always need ybctid for index entry construction */
+	required_attrs = bms_add_member(required_attrs,
+									YBTupleIdAttributeNumber -
+									FirstLowInvalidHeapAttributeNumber);
+
+	/* Add columns directly referenced in the index */
+	for (i = 0; i < indexInfo->ii_NumIndexAttrs; i++)
+	{
+		AttrNumber	attnum = indexInfo->ii_IndexAttrNumbers[i];
+
+		/*
+		 * attnum == 0 means this is an expression index column,
+		 * which will be handled by extracting vars from ii_Expressions.
+		 */
+		if (attnum > 0)
+			required_attrs = bms_add_member(required_attrs,
+											attnum -
+											FirstLowInvalidHeapAttributeNumber);
+	}
+
+	/*
+	 * Add columns referenced in index expressions.
+	 * Use varno=1 since this is always scanning the base relation.
+	 */
+	if (indexInfo->ii_Expressions != NIL)
+		pull_varattnos((Node *) indexInfo->ii_Expressions, 1, &required_attrs);
+
+	/*
+	 * Add columns referenced in partial index predicate.
+	 */
+	if (indexInfo->ii_Predicate != NIL)
+		pull_varattnos((Node *) indexInfo->ii_Predicate, 1, &required_attrs);
+
+	/*
+	 * Now set up targets based on required_attrs.
+	 * Iterate through the bitmapset and add each required column.
+	 */
+	idx = -1;
+	while ((idx = bms_next_member(required_attrs, idx)) >= 0)
+	{
+		AttrNumber	attnum = idx + FirstLowInvalidHeapAttributeNumber;
+
+		if (attnum > 0)
+		{
+			/* Regular column - verify it exists and is not dropped */
+			if (attnum <= tupdesc->natts &&
+				!TupleDescAttr(tupdesc, attnum - 1)->attisdropped)
+				YbDmlAppendTargetRegular(tupdesc, attnum, ybScan->handle);
+		}
+		else
+		{
+			/* System column (like ybctid) */
+			YbDmlAppendTargetSystem(attnum, ybScan->handle);
+		}
+	}
+
+	bms_free(required_attrs);
+	bms_free(scan_plan.hash_key);
+	bms_free(scan_plan.primary_key);
+	bms_free(scan_plan.sk_cols);
+
+	return tsdesc;
+}
+
 /* --------------------------------------------------------------------------------------------- */
 
 /*

src/postgres/src/backend/utils/misc/guc.c

@@ -3314,6 +3314,19 @@ static struct config_bool ConfigureNamesBool[] =
 		NULL, NULL, NULL
 	},
 
+	{
+		{"yb_enable_index_backfill_column_projection", PGC_USERSET, QUERY_TUNING_OTHER,
+			gettext_noop("Enables index backfill column projection optimization. "
+						 "If true, index build/backfill only reads columns needed for the index, "
+						 "rather than all columns from the base table."),
+			NULL,
+			GUC_NOT_IN_SAMPLE
+		},
+		&yb_enable_index_backfill_column_projection,
+		true,
+		NULL, NULL, NULL
+	},
+
 	{
 		{"yb_enable_fkey_catcache", PGC_USERSET, DEVELOPER_OPTIONS,
 			gettext_noop("Enable preloading of foreign key information into the relation cache."),

src/postgres/src/backend/utils/misc/pg_yb_utils.c

@@ -2196,6 +2196,7 @@ bool		yb_enable_sequence_pushdown = true;
 bool		yb_disable_wait_for_backends_catalog_version = false;
 bool		yb_enable_base_scans_cost_model = false;
 bool		yb_enable_update_reltuples_after_create_index = false;
+bool		yb_enable_index_backfill_column_projection = true;
 int			yb_wait_for_backends_catalog_version_timeout = 5 * 60 * 1000;	/* 5 min */
 bool		yb_prefer_bnl = false;
 bool		yb_explain_hide_non_deterministic_fields = false;

src/postgres/src/include/access/yb_scan.h

@@ -264,6 +264,18 @@ extern TableScanDesc ybc_heap_beginscan(Relation relation,
 extern HeapTuple ybc_heap_getnext(TableScanDesc scanDesc);
 extern void ybc_heap_endscan(TableScanDesc scanDesc);
 
+/*
+ * Begin a heap scan for index build/backfill that only fetches columns
+ * needed by the index (as defined in IndexInfo).
+ */
+extern TableScanDesc ybc_heap_beginscan_for_index_build(Relation relation,
+														Snapshot snapshot,
+														int nkeys,
+														ScanKey key,
+														uint32 flags,
+														struct IndexInfo *indexInfo);
+
+
 extern void YbBindDatumToColumn(YbcPgStatement stmt,
 								int attr_num,
 								Oid type_id,

src/postgres/src/include/pg_yb_utils.h

@@ -547,6 +547,13 @@ extern bool yb_enable_base_scans_cost_model;
  */
 extern bool yb_enable_update_reltuples_after_create_index;
 
+/*
+ * Enables index backfill column projection optimization.
+ * If true, index build/backfill only reads columns needed for the index,
+ * rather than all columns from the base table.
+ */
+extern bool yb_enable_index_backfill_column_projection;
+
 /*
  * Total timeout for waiting for backends to have up-to-date catalog version.
  */