CNDB-17161: Add more logging to SAI initialization code

src/java/org/apache/cassandra/index/SecondaryIndexManager.java

@@ -300,6 +300,7 @@ public Future<?> buildIndex(final Index index)
         {
             try
             {
+                logger.debug("IndexBuildDecider#onInitialBuild for index {} returned {}", index.getIndexMetadata().name, IndexBuildDecider.instance.onInitialBuild());
                 initialBuildTask = index.shouldSkipInitialization() ? null : index.getInitializationTask();
             }
             catch (Throwable t)
@@ -312,8 +313,12 @@ public Future<?> buildIndex(final Index index)
         // if there's no initialization, just mark as built (if it should be queryable) and return:
         if (initialBuildTask == null)
         {
+            logger.debug("No initialization task for index [{}], assuming index is built", index.getIndexMetadata().name);
+
             if (IndexBuildDecider.instance.isIndexQueryableAfterInitialBuild(baseCfs))
                 markIndexBuilt(index, true);
+            else
+                logger.debug("Not marking index [{}] as built because IndexBuildDecider#isIndexQueryableAfterInitialBuild returned false", index.getIndexMetadata().name);
 
             return Futures.immediateFuture(null);
         }
@@ -332,8 +337,11 @@ public void onFailure(Throwable t)
             @Override
             public void onSuccess(Object o)
             {
+                logger.debug("Initial build task completed successfully for index [{}]", index.getIndexMetadata().name);
                 if (IndexBuildDecider.instance.isIndexQueryableAfterInitialBuild(baseCfs))
                     markIndexBuilt(index, true);
+                else
+                    logger.debug("Not marking index [{}] as built because IndexBuildDecider returned false", index.getIndexMetadata().name);
 
                 initialization.set(o);
             }
@@ -853,12 +861,12 @@ private synchronized void markIndexFailed(Index index, boolean isInitialBuild)
 
     private void logAndMarkIndexesFailed(Set<Index> indexes, Throwable indexBuildFailure, boolean isInitialBuild)
     {
-        JVMStabilityInspector.inspectThrowable(indexBuildFailure);
         if (indexBuildFailure != null)
             logger.warn("Index build of {} failed. Please run full index rebuild to fix it.", Index.joinNames(indexes), indexBuildFailure);
         else
             logger.warn("Index build of {} failed. Please run full index rebuild to fix it.", Index.joinNames(indexes));
         indexes.forEach(i -> this.markIndexFailed(i, isInitialBuild));
+        JVMStabilityInspector.inspectThrowable(indexBuildFailure);
     }
 
     /**
@@ -1923,7 +1931,10 @@ public void makeIndexNonQueryable(Index index, Index.Status status)
         {
             propagateLocalIndexStatus(keyspace.getName(), name, status);
             if (!index.isQueryable(status))
-                queryableIndexes.remove(name);
+            {
+                if (queryableIndexes.remove(name))
+                    logger.info("Index [{}] became non-queryable (status: {}))", name, status);
+            }
         }
     }
 

src/java/org/apache/cassandra/index/sai/IndexContext.java

@@ -925,7 +925,7 @@ public Pair<Set<SSTableIndex>, Set<SSTableContext>> getBuiltIndexes(Collection<S
             var perIndexComponents = perSSTableComponents.indexDescriptor().perIndexComponents(this);
             if (!perSSTableComponents.isComplete() || !perIndexComponents.isComplete())
             {
-                logger.debug(logMessage("An on-disk index build for SSTable {} has not completed (per-index components={})."), context.descriptor(), perIndexComponents.all());
+                logger.debug(logMessage("An on-disk index build for SSTable {} has not completed (per-sstable components={}, per-index components={})."), context.descriptor(), perSSTableComponents.all(), perIndexComponents.all());
                 return;
             }
 

src/java/org/apache/cassandra/index/sai/disk/format/IndexDescriptor.java

@@ -118,6 +118,8 @@ private void initialize(Set<IndexContext> indices, SSTableIndexComponentsState d
     {
         this.perSSTable = initializeGroup(null, discovered.perSSTableBuild());
         initializeIndexes(indices, discovered);
+        logger.debug("Initialized IndexDescriptor for SSTable {} with per-sstable components {} and per-index components: {}",
+                     descriptor, perSSTable, perIndexes);
     }
 
     private void initializeIndexes(Set<IndexContext> indices, SSTableIndexComponentsState discovered)
@@ -229,6 +231,7 @@ public IndexDescriptor reload(SSTableReader sstable, Set<IndexContext> indices)
     {
         Preconditions.checkArgument(sstable.getDescriptor().equals(this.descriptor));
         SSTableIndexComponentsState discovered = IndexComponentDiscovery.instance().discoverComponents(sstable);
+        logger.debug("Discovered index components for sstable {}: {}", sstable.descriptor, discovered);
 
         // We want to make sure the descriptor only has data for the provided `indices` on reload, so we remove any
         // index data that is not in the ones provided. This essentially make sure we don't hold up memory for

src/java/org/apache/cassandra/index/sai/view/IndexViewManager.java

@@ -82,6 +82,8 @@ public Set<SSTableContext> update(Collection<SSTableReader> oldSSTables,
                                       Collection<SSTableContext> newSSTableContexts,
                                       boolean validate)
     {
+        logger.debug("Updating view of index [{}]. Removed: {}. Added: {}.", context.getIndexName(), oldSSTables, newSSTableContexts);
+
         // Valid indexes on the left and invalid SSTable contexts on the right...
         // The valid indexes are referenced as a part of object initialization.
         Pair<Set<SSTableIndex>, Set<SSTableContext>> indexes = context.getBuiltIndexes(newSSTableContexts, validate);