Added robots configuration

app/code/core/Mage/Adminhtml/Block/Cms/Page/Edit/Tab/Meta.php

@@ -57,6 +57,14 @@ protected function _prepareForm()
             'disabled'  => $isElementDisabled
         ]);
 
+        $fieldset->addField('meta_robots', 'select', array(
+            'name' => 'meta_robots',
+            'label' => Mage::helper('cms')->__('Robots'),
+            'title' => Mage::helper('cms')->__('Meta Robots'),
+            'options'   => $model->getAvailableRobots(),
+            'disabled'  => $isElementDisabled
+        ));
+
         Mage::dispatchEvent('adminhtml_cms_page_edit_tab_meta_prepare_form', ['form' => $form]);
 
         $form->setValues($model->getData());

app/code/core/Mage/Adminhtml/Model/System/Config/Source/Design/Robots.php

@@ -26,6 +26,9 @@ public function toOptionArray()
             ['value' => 'NOINDEX,FOLLOW', 'label' => 'NOINDEX, FOLLOW'],
             ['value' => 'INDEX,NOFOLLOW', 'label' => 'INDEX, NOFOLLOW'],
             ['value' => 'NOINDEX,NOFOLLOW', 'label' => 'NOINDEX, NOFOLLOW'],
+            ['value' => 'INDEX,FOLLOW,NOARCHIVE', 'label' => 'INDEX, FOLLOW, NOARCHIVE'],
+            ['value' => 'INDEX,NOFOLLOW,NOARCHIVE', 'label' => 'INDEX, NOFOLLOW, NOARCHIVE'],
+            ['value' => 'NOINDEX,NOFOLLOW,NOARCHIVE', 'label' => 'NOINDEX, NOFOLLOW, NOARCHIVE'],
         ];
     }
 }

app/code/core/Mage/Catalog/Block/Category/View.php

@@ -47,12 +47,20 @@ protected function _prepareLayout()
 
             /** @var Mage_Catalog_Helper_Category $helper */
             $helper = $this->helper('catalog/category');
+
+            if ($robots = $category->getMetaRobots()) {
+                if ($helper->canUseNoindexFollow() && Mage::helper('catalog/category')->isNotFirstCategoryPage()) {
+                    $headBlock->setRobots(Mage::getSingleton('catalog/category_attribute_source_robots')->getOptionLabel(3));
+                } else {
+                    $headBlock->setRobots(Mage::getSingleton('catalog/category_attribute_source_robots')->getOptionLabel($robots));
+                }
+            }
+
             if ($helper->canUseCanonicalTag()) {
                 $headBlock->addLinkRel('canonical', $category->getUrl());
             }
-            /*
-            want to show rss feed in the url
-            */
+
+            // want to show rss feed in the url
             if ($this->isRssCatalogEnable() && $this->isTopCategory()) {
                 $title = $this->helper('rss')->__('%s RSS Feed', $this->getCurrentCategory()->getName());
                 $headBlock->addItem('rss', $this->getRssLink(), 'title="' . $title . '"');

app/code/core/Mage/Catalog/Block/Product/View.php

@@ -65,6 +65,12 @@ protected function _prepareLayout()
             } else {
                 $headBlock->setDescription(Mage::helper('core/string')->substr($product->getDescription(), 0, 255));
             }
+            $robots = $product->getMetaRobots();
+            if ($robots) {
+                $headBlock->setRobots(Mage::getSingleton('catalog/product_attribute_source_robots')->getOptionLabel($robots));
+            } else {
+                $headBlock->setRobots(Mage::getStoreConfig('design/head/default_robots'));
+            }
 
             /** @var Mage_Catalog_Helper_Product $helper */
             $helper = $this->helper('catalog/product');

app/code/core/Mage/Catalog/Helper/Category.php

@@ -24,6 +24,7 @@ class Mage_Catalog_Helper_Category extends Mage_Core_Helper_Abstract
     public const XML_PATH_CATEGORY_URL_SUFFIX          = 'catalog/seo/category_url_suffix';
     public const XML_PATH_USE_CATEGORY_CANONICAL_TAG   = 'catalog/seo/category_canonical_tag';
     public const XML_PATH_CATEGORY_ROOT_ID             = 'catalog/category/root_id';
+    public const XML_PATH_CHANGE_CATEGORY_PAGES_ROBOTS = 'catalog/seo/category_pages_robots';
 
     protected $_moduleName = 'Mage_Catalog';
 
@@ -173,4 +174,32 @@ public function canUseCanonicalTag($store = null)
     {
         return Mage::getStoreConfig(self::XML_PATH_USE_CATEGORY_CANONICAL_TAG, $store);
     }
+
+    /**
+     * Check if Robots NOINDEX,FOLLOW can be used for category pages
+     *
+     * @param null|string|bool|int|Mage_Core_Model_Store $store
+     * @return bool
+     */
+    public function canUseNoindexFollow($store = null)
+    {
+        return Mage::getStoreConfig(self::XML_PATH_CHANGE_CATEGORY_PAGES_ROBOTS, $store);
+    }
+
+
+    /**
+     * Check if current category is first page
+     *
+     * @return boolean
+     */
+    public function isNotFirstCategoryPage()
+    {
+        $url = Mage::helper('core/url')->getCurrentUrl();
+        $parsedUrl = parse_url($url);
+        if (isset($parsedUrl['query']) && (preg_match("/p=/i", $parsedUrl['query'])) && (!preg_match("/p=1/i", $parsedUrl['query']))) {
+            return true;
+        }
+
+        return false;
+    }
 }

app/code/core/Mage/Catalog/Model/Api2/Product.php

@@ -69,7 +69,7 @@ protected function _isAttributeVisible(Mage_Catalog_Model_Resource_Eav_Attribute
         } else {
             $systemAttributesForNonAdmin = [
                 'sku', 'name', 'short_description', 'description', 'tier_price', 'meta_title', 'meta_description',
-                'meta_keyword',
+                'meta_keyword', 'meta_robot'
             ];
             if ($attribute->getIsUserDefined()) {
                 $isAttributeVisible = $attribute->getIsVisibleOnFront();

app/code/core/Mage/Catalog/Model/Category.php

@@ -56,6 +56,7 @@
  *
  * @method string getMetaDescription()
  * @method string getMetaKeywords()
+ * @method int getMetaRobots()
  * @method string getMetaTitle()
  * @method int getMovedCategoryId()
  *
@@ -93,6 +94,16 @@ class Mage_Catalog_Model_Category extends Mage_Catalog_Model_Abstract
 
     public const CACHE_TAG             = 'catalog_category';
 
+    /**
+     * Category robots
+     */
+    public const META_ROBOTS_INDEX_FOLLOW               = 1;
+    public const META_ROBOTS_INDEX_NOFOLLOW             = 2;
+    public const META_ROBOTS_NOINDEX_FOLLOW             = 3;
+    public const META_ROBOTS_NOINDEX_NOFOLLOW           = 4;
+    public const META_ROBOTS_INDEX_FOLLOW_NOARCHIVE     = 5;
+    public const META_ROBOTS_INDEX_NOFOLLOW_NOARCHIVE   = 6;
+    public const META_ROBOTS_NOINDEX_NOFOLLOW_NOARCHIVE = 7;
     /**
      * Prefix of model events names
      *

app/code/core/Mage/Catalog/Model/Category/Attribute/Source/Robots.php

@@ -0,0 +1,69 @@
+<?php
+/**
+ * OpenMage
+ *
+ * This source file is subject to the Open Software License (OSL 3.0)
+ * that is bundled with this package in the file LICENSE.txt.
+ * It is also available at https://opensource.org/license/osl-3-0-php
+ *
+ * @category   Mage
+ * @package    Mage_Catalog
+ * @copyright  Copyright (c) 2023 The OpenMage Contributors (https://www.openmage.org)
+ * @license    https://opensource.org/licenses/osl-3.0.php  Open Software License (OSL 3.0)
+ */
+
+/**
+ * Catalog category robots attribute source
+ *
+ * @category   Mage
+ * @package    Mage_Catalog
+ */
+class Mage_Catalog_Model_Category_Attribute_Source_Robots extends Mage_Eav_Model_Entity_Attribute_Source_Abstract
+{
+    /**
+     * @return array
+     */
+    public function getAllOptions()
+    {
+        return [
+            Mage_Catalog_Model_Category::META_ROBOTS_INDEX_FOLLOW => [
+                'value' => Mage_Catalog_Model_Category::META_ROBOTS_INDEX_FOLLOW,
+                'label' => Mage::helper('cms')->__('INDEX,FOLLOW')
+            ],
+            Mage_Catalog_Model_Category::META_ROBOTS_INDEX_NOFOLLOW => [
+                'value' => Mage_Catalog_Model_Category::META_ROBOTS_INDEX_NOFOLLOW,
+                'label' => Mage::helper('cms')->__('INDEX,NOFOLLOW')
+            ],
+            Mage_Catalog_Model_Category::META_ROBOTS_NOINDEX_FOLLOW => [
+                'value' => Mage_Catalog_Model_Category::META_ROBOTS_NOINDEX_FOLLOW,
+                'label' => Mage::helper('cms')->__('NOINDEX,FOLLOW')
+            ],
+            Mage_Catalog_Model_Category::META_ROBOTS_NOINDEX_NOFOLLOW => [
+                'value' => Mage_Catalog_Model_Category::META_ROBOTS_NOINDEX_NOFOLLOW,
+                'label' => Mage::helper('cms')->__('NOINDEX,NOFOLLOW')
+            ],
+            Mage_Catalog_Model_Category::META_ROBOTS_INDEX_FOLLOW_NOARCHIVE => [
+                'value' => Mage_Catalog_Model_Category::META_ROBOTS_INDEX_FOLLOW_NOARCHIVE,
+                'label' => Mage::helper('cms')->__('INDEX,FOLLOW,NOARCHIVE')
+            ],
+            Mage_Catalog_Model_Category::META_ROBOTS_INDEX_NOFOLLOW_NOARCHIVE => [
+                'value' => Mage_Catalog_Model_Category::META_ROBOTS_INDEX_NOFOLLOW_NOARCHIVE,
+                'label' => Mage::helper('cms')->__('INDEX,NOFOLLOW,NOARCHIVE')
+            ],
+            Mage_Catalog_Model_Category::META_ROBOTS_NOINDEX_NOFOLLOW_NOARCHIVE => [
+                'value' => Mage_Catalog_Model_Category::META_ROBOTS_NOINDEX_NOFOLLOW_NOARCHIVE,
+                'label' => Mage::helper('cms')->__('NOINDEX,NOFOLLOW,NOARCHIVE')
+            ]
+        ];
+    }
+
+    /**
+     * @param int $key
+     * @return string
+     */
+    public function getOptionLabel($key)
+    {
+        $options = $this->getAllOptions();
+        return $options[$key]['label'];
+    }
+}

app/code/core/Mage/Catalog/Model/Product.php

@@ -135,6 +135,7 @@
  * @method array getMediaGallery()
  * @method $this setMediaGallery(array $value)
  * @method string getMessage()
+ * @method int getMetaRobot()
  * @method string getMetaDescription()
  * @method string getMetaKeyword()
  * @method string getMetaTitle()
@@ -262,6 +263,18 @@ class Mage_Catalog_Model_Product extends Mage_Catalog_Model_Abstract
      */
     public const ENTITY          = 'catalog_product';
     public const CACHE_TAG       = 'catalog_product';
+
+    /**
+     * Product robots
+     */
+    public const META_ROBOTS_INDEX_FOLLOW               = 1;
+    public const META_ROBOTS_INDEX_NOFOLLOW             = 2;
+    public const META_ROBOTS_NOINDEX_FOLLOW             = 3;
+    public const META_ROBOTS_NOINDEX_NOFOLLOW           = 4;
+    public const META_ROBOTS_INDEX_FOLLOW_NOARCHIVE     = 5;
+    public const META_ROBOTS_INDEX_NOFOLLOW_NOARCHIVE   = 6;
+    public const META_ROBOTS_NOINDEX_NOFOLLOW_NOARCHIVE = 7;
+
     protected $_cacheTag         = 'catalog_product';
     protected $_eventPrefix      = 'catalog_product';
     protected $_eventObject      = 'product';

app/code/core/Mage/Catalog/Model/Product/Api.php

@@ -51,6 +51,7 @@ class Mage_Catalog_Model_Product_Api extends Mage_Catalog_Model_Api_Resource
         'meta_title',
         'meta_keyword',
         'meta_description',
+        'meta_robot',
         'custom_design',
         'custom_layout_update',
         'options_container',

app/code/core/Mage/Catalog/Model/Product/Attribute/Source/Robots.php

@@ -0,0 +1,77 @@
+<?php
+/**
+ * OpenMage
+ *
+ * This source file is subject to the Open Software License (OSL 3.0)
+ * that is bundled with this package in the file LICENSE.txt.
+ * It is also available at https://opensource.org/license/osl-3-0-php
+ *
+ * @category   Mage
+ * @package    Mage_Catalog
+ * @copyright  Copyright (c) 2023 The OpenMage Contributors (https://www.openmage.org)
+ * @license    https://opensource.org/licenses/osl-3.0.php  Open Software License (OSL 3.0)
+ */
+
+/**
+ * Catalog product robots attribute source
+ *
+ * @category   Mage
+ * @package    Mage_Catalog
+ */
+class Mage_Catalog_Model_Product_Attribute_Source_Robots extends Mage_Eav_Model_Entity_Attribute_Source_Abstract
+{
+    /**
+     * @return array
+     */
+    public function getAllOptions()
+    {
+        return array(
+            Mage_Catalog_Model_Product::META_ROBOTS_INDEX_FOLLOW =>
+                array(
+                    'value' => Mage_Catalog_Model_Product::META_ROBOTS_INDEX_FOLLOW,
+                    'label' => Mage::helper('cms')->__('INDEX,FOLLOW')
+                ),
+            Mage_Catalog_Model_Product::META_ROBOTS_INDEX_NOFOLLOW =>
+                array(
+                    'value' => Mage_Catalog_Model_Product::META_ROBOTS_INDEX_NOFOLLOW,
+                    'label' => Mage::helper('cms')->__('INDEX,NOFOLLOW')
+                ),
+            Mage_Catalog_Model_Product::META_ROBOTS_NOINDEX_FOLLOW =>
+                array(
+                    'value' => Mage_Catalog_Model_Product::META_ROBOTS_NOINDEX_FOLLOW,
+                    'label' => Mage::helper('cms')->__('NOINDEX,FOLLOW')
+                ),
+            Mage_Catalog_Model_Product::META_ROBOTS_NOINDEX_NOFOLLOW =>
+                array(
+                    'value' => Mage_Catalog_Model_Product::META_ROBOTS_NOINDEX_NOFOLLOW,
+                    'label' => Mage::helper('cms')->__('NOINDEX,NOFOLLOW')
+                ),
+            Mage_Catalog_Model_Product::META_ROBOTS_INDEX_FOLLOW_NOARCHIVE =>
+                array(
+                    'value' => Mage_Catalog_Model_Product::META_ROBOTS_INDEX_FOLLOW_NOARCHIVE,
+                    'label' => Mage::helper('cms')->__('INDEX,FOLLOW,NOARCHIVE')
+                ),
+            Mage_Catalog_Model_Product::META_ROBOTS_INDEX_NOFOLLOW_NOARCHIVE =>
+                array(
+                    'value' => Mage_Catalog_Model_Product::META_ROBOTS_INDEX_NOFOLLOW_NOARCHIVE,
+                    'label' => Mage::helper('cms')->__('INDEX,NOFOLLOW,NOARCHIVE')
+                ),
+            Mage_Catalog_Model_Product::META_ROBOTS_NOINDEX_NOFOLLOW_NOARCHIVE =>
+                array(
+                    'value' => Mage_Catalog_Model_Product::META_ROBOTS_NOINDEX_NOFOLLOW_NOARCHIVE,
+                    'label' => Mage::helper('cms')->__('NOINDEX,NOFOLLOW,NOARCHIVE')
+                ),
+        );
+    }
+
+    /**
+     * @param int $key
+     * @return string
+     */
+    public function getOptionLabel($key)
+    {
+        $options = $this->getAllOptions();
+
+        return $options[$key]['label'];
+    }
+}

app/code/core/Mage/Catalog/Model/Resource/Setup.php

@@ -147,6 +147,16 @@ public function getDefaultEntities()
                         'global'                     => Mage_Catalog_Model_Resource_Eav_Attribute::SCOPE_STORE,
                         'group'                      => 'General Information',
                     ],
+                    'meta_robots'   => [
+                        'type'                       => 'int',
+                        'label'                      => 'Robots',
+                        'input'                      => 'select',
+                        'source'                     => 'catalog/category_attribute_source_robots',
+                        'required'                   => false,
+                        'sort_order'                 => 9,
+                        'global'                     => Mage_Catalog_Model_Resource_Eav_Attribute::SCOPE_STORE,
+                        'group'                      => 'General Information',
+                    ],
                     'display_mode'       => [
                         'type'                       => 'varchar',
                         'label'                      => 'Display Mode',
@@ -516,6 +526,16 @@ public function getDefaultEntities()
                         'global'                     => Mage_Catalog_Model_Resource_Eav_Attribute::SCOPE_STORE,
                         'group'                      => 'Meta Information',
                     ],
+                    'meta_robots'         => [
+                        'type'                       => 'int',
+                        'label'                      => 'Robots',
+                        'input'                      => 'select',
+                        'source'                     => 'catalog/product_attribute_source_robots',
+                        'default'                    => '1',
+                        'sort_order'                 => 4,
+                        'global'                     => Mage_Catalog_Model_Resource_Eav_Attribute::SCOPE_STORE,
+                        'group'                      => 'Meta Information',
+                    ],
                     'image'              => [
                         'type'                       => 'varchar',
                         'label'                      => 'Base Image',

app/code/core/Mage/Catalog/controllers/Product/CompareController.php

@@ -53,6 +53,7 @@ public function indexAction()
         }
 
         $this->loadLayout();
+        $this->getLayout()->getBlock('head')->setRobots(Mage::getStoreConfig('web/robots_configuration/compare'));
         $this->renderLayout();
     }
 

app/code/core/Mage/Catalog/controllers/Seo/SitemapController.php

@@ -50,6 +50,7 @@ public function categoryAction()
         }
         $this->loadLayoutUpdates();
         $this->generateLayoutXml()->generateLayoutBlocks();
+        $this->getLayout()->getBlock('head')->setRobots(Mage::getStoreConfig('web/robots_configuration/sitemap'));
         $this->renderLayout();
     }
 
@@ -60,6 +61,7 @@ public function categoryAction()
     public function productAction()
     {
         $this->loadLayout();
+        $this->getLayout()->getBlock('head')->setRobots(Mage::getStoreConfig('web/robots_configuration/sitemap'));
         $this->renderLayout();
     }
 }

app/code/core/Mage/Catalog/etc/config.xml

@@ -17,7 +17,7 @@
 <config>
     <modules>
         <Mage_Catalog>
-            <version>1.6.0.0.19.1.6</version>
+            <version>1.6.0.0.19.1.7</version>
         </Mage_Catalog>
     </modules>
     <admin>