Refactor spider name extracto into a new utility function

Author: fcanobrash

spidermon/contrib/scrapy/extensions.py

@@ -7,6 +7,7 @@
 from itemadapter import ItemAdapter
 
 from spidermon import MonitorSuite
+from spidermon.contrib.utils.spider import get_spider_name
 from spidermon.contrib.scrapy.runners import SpiderMonitorRunner
 from spidermon.python import factory
 from spidermon.python.monitors import ExpressionsMonitor
@@ -240,6 +241,6 @@ def _generate_data_for_spider(self, spider):
             else [],
             "crawler": self.crawler,
             "spider": spider,
-            "sc_spider_name": os.getenv("SHUB_VIRTUAL_SPIDER", spider.name),
+            "sc_spider_name": get_spider_name(spider),
             "job": self.client.job if self.client.available else None,
         }

spidermon/contrib/stats/statscollectors/local_storage.py

@@ -5,11 +5,13 @@
 from scrapy.statscollectors import StatsCollector
 from scrapy.utils.project import data_path
 
+from spidermon.contrib.utils.spider import get_spider_name
+
 
 class LocalStorageStatsHistoryCollector(StatsCollector):
     def _stats_location(self, spider):
         statsdir = data_path("stats", createdir=True)
-        spider_name = os.getenv("SHUB_VIRTUAL_SPIDER", spider.name)
+        spider_name = get_spider_name(spider)
         return os.path.join(statsdir, f"{spider_name}_stats_history")
 
     def open_spider(self, spider):

spidermon/contrib/stats/statscollectors/sc_collections.py

@@ -5,6 +5,8 @@
 import scrapinghub
 from sh_scrapy.stats import HubStorageStatsCollector
 
+from spidermon.contrib.utils.spider import get_spider_name
+
 logger = logging.getLogger(__name__)
 
 
@@ -18,7 +20,7 @@ def _open_collection(self, spider):
 
         project = sh_client.get_project(proj_id)
         collections = project.collections
-        spider_name = os.getenv("SHUB_VIRTUAL_SPIDER", spider.name)
+        spider_name = get_spider_name(spider)
         stats_location = f"{spider_name}_stats_history"
         store = collections.get_store(stats_location)
         return store

spidermon/contrib/utils/spider.py

@@ -0,0 +1,5 @@
+import os
+
+
+def get_spider_name(spider):
+    return os.getenv("SHUB_VIRTUAL_SPIDER") or spider.name