Use SHUB_VIRTUAL_SPIDER as spider name when available (#462)

* Use SHUB_VIRTUAL_SPIDER as spider name when available

* Black fixes

* Fix tests

* Refactor spider name extracto into a new utility function

* Add tests

Author: fcanobrash

spidermon/contrib/actions/discord/templates/discord/spider/notifier/macros.jinja

@@ -16,4 +16,4 @@
 
 {% macro render_job_url() %}{% if data.job %} / [view job in Scrapy Cloud](https://app.scrapinghub.com/p/{{ data.job.key }}){% endif %}{% endmacro %}
 {% macro render_url() %}{{ render_job_url() }}{% endmacro %}
-{% macro render_spider_name() %}{% if data.spider %}{{ data.spider.name }}{% elif data.job %}{{ data.job.metadata.get('spider') }}{% else %}??{% endif %}{% endmacro %}
+{% macro render_spider_name() %}{% if data.spider %}{{ data.sc_spider_name }}{% elif data.job %}{{ data.job.metadata.get('spider') }}{% else %}??{% endif %}{% endmacro %}

spidermon/contrib/actions/reports/templates/reports/email/monitors/result.jinja

@@ -168,7 +168,7 @@
                             {% set log_errors_count = data.stats.get('log_count/ERROR', 0) %}
                             {% set running_time = data.stats.get('finish_time') - data.stats.get('start_time') %}
 
-                            {{ render_header_data('Spider', data.spider.name, 'label label-blue') }}
+                            {{ render_header_data('Spider', data.sc_spider_name, 'label label-blue') }}
                             {{ render_header_data('Items', items_count, "badge badge-green") }}
                             {{ render_header_data('Requests', requests_count, "badge") }}
                             {{ render_header_data('Errors', log_errors_count, "badge badge-red") }}

spidermon/contrib/actions/sentry/__init__.py

@@ -66,7 +66,7 @@ def get_title(self):
             "{project_name} | {environment} | Spider {spider_name} notification".format(
                 project_name=self.project_name,
                 environment=self.environment,
-                spider_name=self.data.spider.name,
+                spider_name=self.data.sc_spider_name,
             )
         )
 
@@ -83,7 +83,7 @@ def get_message(self):
             )
 
         if self.data.spider:
-            message["spider_name"] = self.data.spider.name
+            message["spider_name"] = self.data.sc_spider_name
 
         if self.data.stats:
             message["items_count"] = self.data.stats.get("item_scraped_count", 0)

spidermon/contrib/actions/slack/templates/slack/spider/notifier/macros.jinja

@@ -21,4 +21,4 @@
 {% macro render_job_url() %}{% if data.job %} / <https://app.zyte.com/p/{{ data.job.key }}|view job in Scrapy Cloud>{% endif %}{% endmacro %}
 {% macro render_report_url() %}{% if include_report_link and data.meta.reports_links %} / <{{ data.meta.reports_links[report_index] }}|report>{% endif %}{% endmacro %}
 {% macro render_url() %}{{ render_report_url() }}{{ render_job_url() }}{% endmacro %}
-{% macro render_spider_name() %}{% if data.spider %}{{ data.spider.name }}{% elif data.job %}{{ data.job.metadata.get('spider') }}{% else %}??{% endif %}{% endmacro %}
+{% macro render_spider_name() %}{% if data.spider %}{{ data.sc_spider_name }}{% elif data.job %}{{ data.job.metadata.get('spider') }}{% else %}??{% endif %}{% endmacro %}

spidermon/contrib/actions/sns/notifiers.py

@@ -6,7 +6,10 @@ def run_action(self):
         subject = "Spider Started"
         attributes = {
             "EventType": {"DataType": "String", "StringValue": "SpiderStarted"},
-            "SpiderName": {"DataType": "String", "StringValue": self.data.spider.name},
+            "SpiderName": {
+                "DataType": "String",
+                "StringValue": self.data.sc_spider_name,
+            },
             "StartTime": {
                 "DataType": "String",
                 "StringValue": str(self.data.stats.start_time),
@@ -20,7 +23,10 @@ def run_action(self):
         subject = "Spider Finished"
         attributes = {
             "EventType": {"DataType": "String", "StringValue": "SpiderFinished"},
-            "SpiderName": {"DataType": "String", "StringValue": self.data.spider.name},
+            "SpiderName": {
+                "DataType": "String",
+                "StringValue": self.data.sc_spider_name,
+            },
             "ItemsScraped": {
                 "DataType": "Number",
                 "StringValue": str(self.data.stats.item_scraped_count),

spidermon/contrib/actions/telegram/templates/telegram/spider/notifier/macros.jinja

@@ -16,4 +16,4 @@
 
 {% macro render_job_url() %}{% if data.job %} / [view job in Scrapy Cloud](https://app.zyte.com/p/{{ data.job.key }}){% endif %}{% endmacro %}
 {% macro render_url() %}{{ render_job_url() }}{% endmacro %}
-{% macro render_spider_name() %}{% if data.spider %}{{ data.spider.name }}{% elif data.job %}{{ data.job.metadata.get('spider') }}{% else %}??{% endif %}{% endmacro %}
+{% macro render_spider_name() %}{% if data.spider %}{{ data.sc_spider_name }}{% elif data.job %}{{ data.job.metadata.get('spider') }}{% else %}??{% endif %}{% endmacro %}

spidermon/contrib/scrapy/extensions.py

@@ -1,10 +1,13 @@
+import os
+
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.misc import load_object
 from twisted.internet.task import LoopingCall
 from itemadapter import ItemAdapter
 
 from spidermon import MonitorSuite
+from spidermon.contrib.utils.spider import get_spider_name
 from spidermon.contrib.scrapy.runners import SpiderMonitorRunner
 from spidermon.python import factory
 from spidermon.python.monitors import ExpressionsMonitor
@@ -238,5 +241,6 @@ def _generate_data_for_spider(self, spider):
             else [],
             "crawler": self.crawler,
             "spider": spider,
+            "sc_spider_name": get_spider_name(spider),
             "job": self.client.job if self.client.available else None,
         }

spidermon/contrib/stats/statscollectors/local_storage.py

@@ -5,11 +5,14 @@
 from scrapy.statscollectors import StatsCollector
 from scrapy.utils.project import data_path
 
+from spidermon.contrib.utils.spider import get_spider_name
+
 
 class LocalStorageStatsHistoryCollector(StatsCollector):
     def _stats_location(self, spider):
         statsdir = data_path("stats", createdir=True)
-        return os.path.join(statsdir, f"{spider.name}_stats_history")
+        spider_name = get_spider_name(spider)
+        return os.path.join(statsdir, f"{spider_name}_stats_history")
 
     def open_spider(self, spider):
         stats_location = self._stats_location(spider)

spidermon/contrib/stats/statscollectors/sc_collections.py

@@ -5,6 +5,8 @@
 import scrapinghub
 from sh_scrapy.stats import HubStorageStatsCollector
 
+from spidermon.contrib.utils.spider import get_spider_name
+
 logger = logging.getLogger(__name__)
 
 
@@ -18,7 +20,8 @@ def _open_collection(self, spider):
 
         project = sh_client.get_project(proj_id)
         collections = project.collections
-        stats_location = f"{spider.name}_stats_history"
+        spider_name = get_spider_name(spider)
+        stats_location = f"{spider_name}_stats_history"
         store = collections.get_store(stats_location)
         return store
 

spidermon/contrib/utils/spider.py

@@ -0,0 +1,5 @@
+import os
+
+
+def get_spider_name(spider):
+    return os.getenv("SHUB_VIRTUAL_SPIDER") or spider.name