Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 28 additions & 6 deletions ckanext/xloader/command.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
from ckanext.xloader.jobs import xloader_data_into_datastore_
from ckanext.xloader.utils import XLoaderFormats, get_xloader_user_apitoken

log = logging.getLogger(__name__)


class XloaderCmd:
def __init__(self, dry_run=False):
Expand Down Expand Up @@ -133,17 +135,37 @@ def _submit_resource(self, resource, user, indent=0, sync=False, queue=None):

def print_status(self):
import ckan.lib.jobs as rq_jobs

jobs = rq_jobs.get_queue().jobs
if not jobs:
print('No jobs currently queued')
return

for job in jobs:
job_params = eval(job.description.replace(
'ckanext.xloader.jobs.xloader_data_into_datastore', ''))
job_metadata = job_params['metadata']
# FIX DEFINITIVO:
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please use English.

# Non usare job.description (non è un canale dati stabile e può essere troncato).
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How does this affect backward compatibility?

# I parametri veri stanno in job.args / job.kwargs.
metadata = {}

try:
if getattr(job, 'args', None) and len(job.args) >= 1:
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Doesn't the truthiness check already ensure it's not empty?

payload = job.args[0] or {}
if isinstance(payload, dict):
metadata = payload.get('metadata') or {}
elif getattr(job, 'kwargs', None) and isinstance(job.kwargs, dict):
payload = job.kwargs.get('data') or job.kwargs
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why would it be "either the 'data' field or the whole thing"?

if isinstance(payload, dict):
metadata = payload.get('metadata') or {}
except Exception:
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we use a narrower exception type? Surely retrieving keys has a limited range of types it can raise.

metadata = {}

res_id = metadata.get('resource_id', 'N/A')
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm dubious about using "N/A" as the default value.

url = metadata.get('original_url') or metadata.get('url') or 'N/A'

print('{id} Enqueued={enqueued:%Y-%m-%d %H:%M} res_id={res_id} '
'url={url}'.format(
id=job._id,
id=getattr(job, '_id', None),
enqueued=job.enqueued_at,
res_id=job_metadata['resource_id'],
url=job_metadata['original_url'],
res_id=res_id,
url=url,
))