Prevent logging tracking failure for obviously malformed idsite or token auth

CHANGELOG.md

@@ -9,7 +9,11 @@ The Product Changelog at **[matomo.org/changelog](https://matomo.org/changelog)*
 ### New Features
 
 * Themes can now customize the focus ring colors using `@theme-color-focus-ring` (used globally) and `@theme-color-focus-ring-alternative` (used in header navigation on solid background).
+* New event `PrivacyManager.deleteDataSubjectsForDeletedSites` to enable plugins to be GDPR compliant, when tracking visit unrelated data.
 
+### HTTP Tracking API
+
+* The new Bot Tracking plugin now supports analyzing requests from AI bots. See https://developer.matomo.org/api-reference/tracking-api#tracking-bots for supported tracking parameters.
 
 ## Matomo 5.4.0
 

config/global.ini.php

@@ -1296,6 +1296,7 @@
 Plugins[] = JsTrackerInstallCheck
 Plugins[] = FeatureFlags
 Plugins[] = AIAgents
+Plugins[] = BotTracking
 
 [PluginsInstalled]
 PluginsInstalled[] = Diagnostics

core/Plugin/RequestProcessors.php

@@ -10,10 +10,15 @@
 namespace Piwik\Plugin;
 
 use Piwik\Container\StaticContainer;
+use Piwik\Tracker\BotRequestProcessor;
+use Piwik\Tracker\RequestProcessor;
 
 class RequestProcessors
 {
-    public function getRequestProcessors()
+    /**
+     * @return RequestProcessor[]
+     */
+    public function getRequestProcessors(): array
     {
         $manager    = Manager::getInstance();
         $processors = $manager->findMultipleComponents('Tracker', 'Piwik\\Tracker\\RequestProcessor');
@@ -25,4 +30,20 @@ public function getRequestProcessors()
 
         return $instances;
     }
+
+    /**
+     * @return BotRequestProcessor[]
+     */
+    public function getBotRequestProcessors(): array
+    {
+        $manager    = Manager::getInstance();
+        $processors = $manager->findMultipleComponents('Tracker', 'Piwik\\Tracker\\BotRequestProcessor');
+
+        $instances = [];
+        foreach ($processors as $processor) {
+            $instances[] = StaticContainer::get($processor);
+        }
+
+        return $instances;
+    }
 }

core/Tracker.php

@@ -11,8 +11,10 @@
 
 use Exception;
 use Piwik\Container\StaticContainer;
+use Piwik\DeviceDetector\DeviceDetectorFactory;
 use Piwik\Plugins\BulkTracking\Tracker\Requests;
 use Piwik\Plugins\PrivacyManager\Config as PrivacyManagerConfig;
+use Piwik\Tracker\BotRequest;
 use Piwik\Tracker\Db as TrackerDb;
 use Piwik\Tracker\Db\DbException;
 use Piwik\Tracker\Handler;
@@ -166,16 +168,53 @@ public function trackRequest(Request $request)
                 'date' => date("Y-m-d H:i:s", $request->getCurrentTimestamp()),
             ]);
 
-            $visit = Visit\Factory::make();
-            $visit->setRequest($request);
-            $visit->handle();
+            $isBot = $this->isBotRequest($request);
+
+            /**
+             * Allows overwriting the Bot detection done using Device Detector
+             * Use this event if you want to have a request handled as bot request instead of a normal visit
+             *
+             * @param bool &$isBot Indicates if the request should be handled as Bot
+             * @param Request $request current tracking request
+             */
+            Piwik::postEvent('Tracker.isBotRequest', [&$isBot, $request]);
+
+            $rawParams = $request->getRawParams();
+
+            /**
+             * The recMode param will for now be used to keep BC.
+             * If it is not set, which is currently the case for all tracking requests, it will be processed as Visit only
+             * When set to 1, only bot tracking will be processed. In case the request is not detected as bot, it will be discarded
+             * Setting it to 2 enables auto mode. Meaning it will be either processed as bot request or visit, depending on the detection
+             *
+             * @deprecated Remove this parameter handling with Matomo 6 and decide the tracking method based on the bot detection only.
+             */
+            $recMode = $rawParams['recMode'] ?? null;
+
+            if (((int)$recMode === 1 || (int)$recMode === 2) && $isBot) {
+                $botRequest = StaticContainer::get(BotRequest::class);
+                $botRequest->setRequest($request);
+                $botRequest->handle();
+            }
+
+            if (empty($recMode) || ((int)$recMode === 2 && !$isBot)) {
+                $visit = Visit\Factory::make();
+                $visit->setRequest($request);
+                $visit->handle();
+            }
         }
 
         // increment successfully logged request count. make sure to do this after try-catch,
         // since an excluded visit is considered 'successfully logged'
         ++$this->countOfLoggedRequests;
     }
 
+    private function isBotRequest(Request $request): bool
+    {
+        $deviceDetector = StaticContainer::get(DeviceDetectorFactory::class)->makeInstance($request->getUserAgent(), $request->getClientHints());
+        return $deviceDetector->isBot();
+    }
+
     /**
      * Used to initialize core Piwik components on a piwik.php request
      * Eg. when cache is missed and we will be calling some APIs to generate cache

core/Tracker/BotRequest.php

@@ -0,0 +1,94 @@
+<?php
+
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link    https://matomo.org
+ * @license https://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+declare(strict_types=1);
+
+namespace Piwik\Tracker;
+
+use Piwik\Log\LoggerInterface;
+use Piwik\Plugin\RequestProcessors;
+
+/**
+ * Class used to handle a Bot request.
+ */
+class BotRequest
+{
+    use RequestHandlerTrait;
+
+    /**
+     * @var  Request
+     */
+    protected $request;
+
+    /**
+     * @var BotRequestProcessor[]
+     */
+    protected $botRequestProcessors;
+
+    /**
+     * @var RequestProcessor[]
+     */
+    protected $requestProcessors;
+
+    /**
+     * @var LoggerInterface
+     */
+    protected $logger;
+
+    public function __construct(RequestProcessors $requestProcessors, LoggerInterface $logger)
+    {
+        $this->requestProcessors    = $requestProcessors->getRequestProcessors();
+        $this->botRequestProcessors = $requestProcessors->getBotRequestProcessors();
+        $this->logger = $logger;
+    }
+
+    /**
+     * @param Request $request
+     */
+    public function setRequest(Request $request)
+    {
+        $this->request = $request;
+    }
+
+    public function handle()
+    {
+        $this->checkSiteExists($this->request);
+
+        /**
+         * For BC reasons we iterate over all visit request processors as well, to ensure a possible request manipulation is applied
+         * For Matomo 6 we should remove that and ensure plugins that also should manipulate bot requests implement a BotRequestProcessor for it
+         * @deprecated
+         */
+        foreach ($this->requestProcessors as $processor) {
+            $this->logger->debug("Executing " . get_class($processor) . "::manipulateRequest()...");
+
+            $processor->manipulateRequest($this->request);
+        }
+
+        foreach ($this->botRequestProcessors as $processor) {
+            $this->logger->debug("Executing " . get_class($processor) . "::manipulateRequest()...");
+
+            $processor->manipulateRequest($this->request);
+        }
+
+        $this->validateRequest($this->request);
+
+        $wasHandled = false;
+
+        foreach ($this->botRequestProcessors as $processor) {
+            $this->logger->debug("Executing " . get_class($processor) . "::handleRequest()...");
+
+            $wasHandled |= $processor->handleRequest($this->request);
+        }
+
+        if ($wasHandled) {
+            $this->markArchivedReportsAsInvalidIfArchiveAlreadyFinished($this->request);
+        }
+    }
+}

core/Tracker/BotRequestProcessor.php

@@ -0,0 +1,41 @@
+<?php
+
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link    https://matomo.org
+ * @license https://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+declare(strict_types=1);
+
+namespace Piwik\Tracker;
+
+abstract class BotRequestProcessor
+{
+    /**
+     * This is the first method called when processing a bot request.
+     *
+     * Derived classes can use this method to manipulate a bot request before the request
+     * is handled. Plugins could change the URL, add custom variables, etc.
+     *
+     * @param Request $request
+     */
+    public function manipulateRequest(Request $request): void
+    {
+        // empty
+    }
+
+    /**
+     * This method is called last.
+     *
+     * Derived classes should use this method to insert log data.
+     *
+     * @param Request $request
+     * @return bool return true if the processor handled the request, this will automatically trigger archive invalidation
+     */
+    public function handleRequest(Request $request): bool
+    {
+        return false;
+    }
+}

core/Tracker/Request.php

@@ -206,7 +206,10 @@ protected function authenticateTrackingApi(
             if ($this->isAuthenticated) {
                 Common::printDebug("token_auth is authenticated!");
             } else {
-                StaticContainer::get('Piwik\Tracker\Failures')->logFailure(Failures::FAILURE_ID_NOT_AUTHENTICATED, $this);
+                if (preg_match('/^\w{28,36}$/', $tokenAuth) || empty($tokenAuth)) {
+                    // only log a failure if the token auth looks partial valid or is completely missing
+                    StaticContainer::get('Piwik\Tracker\Failures')->logFailure(Failures::FAILURE_ID_NOT_AUTHENTICATED, $this);
+                }
             }
         } else {
             $this->isAuthenticated = true;

core/Tracker/RequestHandlerTrait.php

@@ -0,0 +1,92 @@
+<?php
+
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link    https://matomo.org
+ * @license https://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+declare(strict_types=1);
+
+namespace Piwik\Tracker;
+
+use Piwik\Container\StaticContainer;
+use Piwik\Date;
+use Piwik\Exception\UnexpectedWebsiteFoundException;
+use Piwik\Plugins\UserCountry\Columns\Base;
+
+trait RequestHandlerTrait
+{
+    protected $fieldsThatRequireAuth = [
+        'city',
+        'region',
+        'country',
+        'lat',
+        'long',
+    ];
+
+    protected function checkSiteExists(Request $request): void
+    {
+        try {
+            $request->getIdSite();
+        } catch (UnexpectedWebsiteFoundException $e) {
+            $idSite = $this->request->getRawParams()['idsite'] ?? null;
+            if (is_numeric($idSite) && (string)(int)$idSite === (string)$idSite && (int)$idSite >= 0) {
+                // only log a failure in case the provided idsite was valid positive integer
+                StaticContainer::get(Failures::class)->logFailure(Failures::FAILURE_ID_INVALID_SITE, $request);
+            }
+
+            throw $e;
+        }
+    }
+
+    protected function validateRequest(Request $request): void
+    {
+        // Check for params that aren't allowed to be included unless the request is authenticated
+        foreach ($this->fieldsThatRequireAuth as $field) {
+            Base::getValueFromUrlParamsIfAllowed($field, $request);
+        }
+
+        // Special logic for timestamp as some overrides are OK without auth and others aren't
+        $request->getCurrentTimestamp();
+    }
+
+    protected function markArchivedReportsAsInvalidIfArchiveAlreadyFinished(Request $request): void
+    {
+        $idSite = (int)$request->getIdSite();
+        $time   = $request->getCurrentTimestamp();
+
+        $timezone = $this->getTimezoneForSite($idSite);
+
+        if (!isset($timezone)) {
+            return;
+        }
+
+        $date = Date::factory((int)$time, $timezone);
+
+        // $date->isToday() is buggy when server and website timezones don't match - so we'll do our own checking
+        $startOfToday         = Date::factoryInTimezone('yesterday', $timezone)->addDay(1);
+        $isLaterThanYesterday = $date->getTimestamp() >= $startOfToday->getTimestamp();
+        if ($isLaterThanYesterday) {
+            return; // don't try to invalidate archives for today or later
+        }
+
+        StaticContainer::get('Piwik\Archive\ArchiveInvalidator')->rememberToInvalidateArchivedReportsLater($idSite, $date);
+    }
+
+    private function getTimezoneForSite(int $idSite): ?string
+    {
+        try {
+            $site = Cache::getCacheWebsiteAttributes($idSite);
+        } catch (UnexpectedWebsiteFoundException $e) {
+            return null;
+        }
+
+        if (!empty($site['timezone'])) {
+            return $site['timezone'];
+        }
+
+        return null;
+    }
+}