diff --git a/Observer/InvalidateVarnishObserver.php b/Observer/InvalidateVarnishObserver.php
index 927d5de8..74c60b0f 100644
--- a/Observer/InvalidateVarnishObserver.php
+++ b/Observer/InvalidateVarnishObserver.php
@@ -81,6 +81,11 @@ public function execute(\Magento\Framework\Event\Observer $observer): void
                         continue;
                     }
                     $tag = $this->cacheTags->convertCacheTags($tag);
+
+                    if (!$this->isTagAllowed($tag)) {
+                        continue;
+                    }
+
                     if (!in_array($tag, $this->alreadyPurged)) {
                         $tags[] = $tag;
                         $this->alreadyPurged[] = $tag;
@@ -113,4 +118,24 @@ private function canPurgeObject(\Magento\Framework\DataObject\IdentityInterface
         }
         return true;
     }
+
+    /**
+     * Additional validation since IdentityInterface can pass canPurgeObject check (e.g. ProductRuleIndexer), but still
+     * hold tags of products or categories
+     *
+     * @param string $tag
+     * @return bool
+     */
+    private function isTagAllowed(string $tag)
+    {
+        if ($tag === \Magento\Catalog\Model\Category::CACHE_TAG && !$this->config->canPurgeCatalogCategory()) {
+            return false;
+        }
+
+        if ($tag === \Magento\Catalog\Model\Product::CACHE_TAG && !$this->config->canPurgeCatalogProduct()) {
+            return false;
+        }
+
+        return true;
+    }
 }