Added: Commands "jobmeta" and "stats"

shub/jobmeta.py

@@ -0,0 +1,39 @@
+from __future__ import absolute_import
+import json
+import click
+
+from shub.utils import get_job
+
+
+HELP = """
+Given a job ID, fetch jobmeta for that job from Scrapy Cloud and output them as
+JSON.
+
+A job ID consists of the Scrapinghub project ID, the numerical spider ID, and
+the job ID, separated by forward slashes, e.g.:
+
+    shub jobmeta 12345/2/15
+
+You can also provide the Scrapinghub job URL instead:
+
+    shub jobmeta https://app.scrapinghub.com/p/12345/2/15
+
+You can omit the project ID if you have a default target defined in your
+scrapinghub.yml:
+
+    shub jobmeta 2/15
+
+Or use any target defined in your scrapinghub.yml:
+
+    shub jobmeta production/2/15
+
+"""
+
+SHORT_HELP = "Fetch jobmeta from Scrapy Cloud"
+
+
+@click.command(help=HELP, short_help=SHORT_HELP)
+@click.argument('job_id')
+def cli(job_id):
+    job = get_job(job_id)
+    click.echo(json.dumps(dict(job.metadata)))

shub/stats.py

@@ -0,0 +1,39 @@
+from __future__ import absolute_import
+import json
+import click
+
+from shub.utils import get_job
+
+
+HELP = """
+Given a job ID, fetch stats for that job from Scrapy Cloud and output them as
+JSON.
+
+A job ID consists of the Scrapinghub project ID, the numerical spider ID, and
+the job ID, separated by forward slashes, e.g.:
+
+    shub stats 12345/2/15
+
+You can also provide the Scrapinghub job URL instead:
+
+    shub stats https://app.scrapinghub.com/p/12345/2/15
+
+You can omit the project ID if you have a default target defined in your
+scrapinghub.yml:
+
+    shub stats 2/15
+
+Or use any target defined in your scrapinghub.yml:
+
+    shub stats production/2/15
+
+"""
+
+SHORT_HELP = "Fetch stats from Scrapy Cloud"
+
+
+@click.command(help=HELP, short_help=SHORT_HELP)
+@click.argument('job_id')
+def cli(job_id):
+    job = get_job(job_id)
+    click.echo(json.dumps(job.metadata.get('scrapystats', {})))

shub/tool.py

@@ -47,6 +47,8 @@ def cli():
     "schedule",
     "log",
     "requests",
+    "jobmeta",
+    "stats",
     "copy_eggs",
     "migrate_eggs",
     "image",

tests/test_jobresource.py

@@ -7,7 +7,7 @@
 
 from click.testing import CliRunner
 
-from shub import items, log, requests
+from shub import items, log, requests, jobmeta, stats
 
 
 class JobResourceTest(unittest.TestCase):
@@ -36,6 +36,15 @@ def _test_forwards_follow(self, cmd_mod):
             self.runner.invoke(cmd_mod.cli, ('1/2/3', '-f'))
             self.assertTrue(mock_jri.call_args[1]['follow'])
 
+    def _test_mock_jobmeta(self, cmd_mod, mocked_value, expected_result):
+        jobid = '1/2/3'
+        with mock.patch.object(cmd_mod, 'get_job', autospec=True) as mock_gj:
+            mock_gj.return_value._metadata_updated = time.time()
+            mock_gj.return_value = mock.PropertyMock(metadata=mocked_value)
+            result = self.runner.invoke(cmd_mod.cli, (jobid,))
+            mock_gj.assert_called_once_with(jobid)
+            self.assertEqual(expected_result, json.loads(result.output))
+
     def test_items(self):
         self._test_prints_objects(items, 'items')
         self._test_forwards_follow(items)
@@ -44,6 +53,19 @@ def test_requests(self):
         self._test_prints_objects(requests, 'requests')
         self._test_forwards_follow(requests)
 
+    def test_jobmeta(self):
+        mocked_jobmeta = {'foo': 'bar'}
+        self._test_mock_jobmeta(jobmeta, mocked_jobmeta, mocked_jobmeta)
+
+    def test_stats(self):
+        # For jobs without metadata, e.g. those are pending or cancelled
+        # before running
+        mocked_jobmeta = {'foo': 'bar'}
+        self._test_mock_jobmeta(stats, mocked_jobmeta, {})
+        # For jobs with metadata
+        mocked_jobmeta['scrapystats'] = {'bar': 'baz'}
+        self._test_mock_jobmeta(stats, mocked_jobmeta, mocked_jobmeta['scrapystats'])
+
     def test_log(self):
         objects = [
             {'time': 0, 'level': 20, 'message': 'message 1'},