Merge pull request #886 from aaxelb/feat/9046-shtrove-rss

[ENG-9046] shtrove cardsearch feeds

Author: aaxelb

TODO.md

@@ -4,13 +4,11 @@ ways to better this mess
 ## better shtrove api experience
 
 - better web-browsing experience
-    - when `Accept` header accepts html, use html regardless of query-params
-        - when query param `acceptMediatype` requests another mediatype, display on page in copy/pastable way
-        - exception: when given `withFileName`, download without html wrapping
-        - exception: `/trove/browse` should still give hypertext with clickable links
     - include more explanatory docs (and better fill out those explanations)
-    - more helpful (less erratic) visual design
+    - even more helpful (less erratic) visual design
     - in each html rendering of an api response, include a `<form>` for adding/editing/viewing query params
+    - in browsable html, replace json literals with rdf rendered like the rest of the page
+    - (perf) add bare-minimal IndexcardDeriver (iris, types, namelikes); use for search-result display
 - better tsv/csv experience
     - set default columns for `index-value-search` (and/or broadly improve `fields` handling)
 - better turtle experience

api/middleware.py

@@ -27,7 +27,7 @@ def process_view(self, request, view_func, view_args, view_kwargs):
 
         if settings.HIDE_DEPRECATED_VIEWS and deprecation_level == DeprecationLevel.HIDDEN:
             return HttpResponse(
-                f'This path ({request.path}) has been removed. If you have built something that relies on it, please email us at [email protected]',
+                f'This path ({request.path}) has been removed. If you have built something that relies on it, please email us at {settings.SHARE_SUPPORT_EMAIL}',
                 status=410,
             )
 

api/views/feeds.py

@@ -1,3 +1,4 @@
+import datetime
 from xml.sax.saxutils import unescape
 import json
 import logging
@@ -10,7 +11,6 @@
 from share.search import index_strategy
 from share.search.exceptions import IndexStrategyError
 from share.util.xml import strip_illegal_xml_chars
-from share.util.fromisoformat import fromisoformat
 
 
 logger = logging.getLogger(__name__)
@@ -108,10 +108,10 @@ def item_author_name(self, item):
         return prepare_string('{}{}'.format(author_name, ' et al.' if len(authors) > 1 else ''))
 
     def item_pubdate(self, item):
-        return fromisoformat(item.get('date_published') or item.get('date_created'))
+        return datetime.datetime.fromisoformat(item.get('date_published') or item.get('date_created'))
 
     def item_updateddate(self, item):
-        return fromisoformat(item.get(self._order))
+        return datetime.datetime.fromisoformat(item.get(self._order))
 
     def item_categories(self, item):
         categories = item.get('subjects', [])

project/settings.py

@@ -445,6 +445,7 @@ def route_urgent_task(name, args, kwargs, options, task=None, **kw):
 PUBLIC_SENTRY_DSN = os.environ.get('PUBLIC_SENTRY_DSN')
 
 SHARE_WEB_URL = os.environ.get('SHARE_WEB_URL', 'http://localhost:8003').rstrip('/') + '/'
+SHARE_SUPPORT_EMAIL = os.environ.get('SHARE_SUPPORT_EMAIL', '[email protected]')
 SHARE_USER_AGENT = os.environ.get('SHARE_USER_AGENT', 'SHAREbot/{} (+{})'.format(VERSION, SHARE_WEB_URL))
 SHARE_ADMIN_USERNAME = os.environ.get('SHARE_ADMIN_USERNAME', 'admin')
 SHARE_ADMIN_PASSWORD = os.environ.get('SHARE_ADMIN_PASSWORD')

share/models/index_backfill.py

@@ -185,5 +185,4 @@ def task__schedule_index_backfill(self, index_backfill_pk):
     except Exception as error:
         _index_backfill.pls_mark_error(error)
         raise error
-    else:
-        _index_backfill.pls_note_scheduling_has_finished()
+    _index_backfill.pls_note_scheduling_has_finished()

share/oaipmh/indexcard_repository.py

@@ -1,15 +1,16 @@
+import datetime
 import uuid
 
 from django.core.exceptions import ValidationError as DjangoValidationError
+from django.conf import settings
 from django.db.models import OuterRef, Subquery, F
 
 from share.oaipmh import errors as oai_errors
 from share.oaipmh.verbs import OAIVerb
 from share.oaipmh.response_renderer import OAIRenderer
-from share.oaipmh.util import format_datetime
-from share.util.fromisoformat import fromisoformat
 from share import models as share_db
 from trove import models as trove_db
+from trove.util.datetime import datetime_isoformat_z as format_datetime
 from trove.vocab.namespaces import OAI_DC
 
 
@@ -18,7 +19,7 @@ class OaiPmhRepository:
     REPOSITORY_IDENTIFIER = 'share.osf.io'
     IDENTIFER_DELIMITER = ':'
     GRANULARITY = 'YYYY-MM-DD'
-    ADMIN_EMAILS = ['[email protected]']
+    ADMIN_EMAILS = [settings.SHARE_SUPPORT_EMAIL]
 
     # TODO better way of structuring this than a bunch of dictionaries?
     # this dictionary's keys are `metadataPrefix` values
@@ -206,7 +207,7 @@ def _get_indexcard_page_queryset(self, kwargs, catch=True, last_id=None):
         )
         if 'from' in kwargs:
             try:
-                _from = fromisoformat(kwargs['from'])
+                _from = datetime.datetime.fromisoformat(kwargs['from'])
             except ValueError:
                 if not catch:
                     raise
@@ -217,7 +218,7 @@ def _get_indexcard_page_queryset(self, kwargs, catch=True, last_id=None):
                 )
         if 'until' in kwargs:
             try:
-                _until = fromisoformat(kwargs['until'])
+                _until = datetime.datetime.fromisoformat(kwargs['until'])
             except ValueError:
                 if not catch:
                     raise
@@ -291,12 +292,12 @@ def _get_resumption_token(self, kwargs, last_id):
         _until = None
         if 'from' in kwargs:
             try:
-                _from = fromisoformat(kwargs['from'])
+                _from = datetime.datetime.fromisoformat(kwargs['from'])
             except ValueError:
                 self.errors.append(oai_errors.BadArgument('Invalid value for', 'from'))
         if 'until' in kwargs:
             try:
-                _until = fromisoformat(kwargs['until'])
+                _until = datetime.datetime.fromisoformat(kwargs['until'])
             except ValueError:
                 self.errors.append(oai_errors.BadArgument('Invalid value for', 'until'))
         _set_spec = kwargs.get('set', '')

share/oaipmh/response_renderer.py

@@ -4,7 +4,8 @@
 
 from django.urls import reverse
 
-from share.oaipmh.util import format_datetime, SubEl, ns, nsmap
+from share.oaipmh.util import SubEl, ns, nsmap
+from trove.util.datetime import datetime_isoformat_z as format_datetime
 
 
 class OAIRenderer:

share/oaipmh/util.py

@@ -1,23 +1,11 @@
-import datetime
 from typing import Any
 
 from lxml import etree
 from primitive_metadata import primitive_rdf
 
-from share.util.fromisoformat import fromisoformat
 from trove.vocab.namespaces import OAI, OAI_DC
 
 
-def format_datetime(dt: datetime.datetime | primitive_rdf.Literal | str) -> str:
-    """OAI-PMH has specific time format requirements -- comply.
-    """
-    if isinstance(dt, primitive_rdf.Literal):
-        dt = dt.unicode_value
-    if isinstance(dt, str):
-        dt = fromisoformat(dt)
-    return dt.strftime('%Y-%m-%dT%H:%M:%SZ')
-
-
 XML_NAMESPACES = {
     'dc': 'http://purl.org/dc/elements/1.1/',
     'oai': str(OAI),

share/util/fromisoformat.py

@@ -15,5 +15,5 @@
 )
 
 
-def strip_illegal_xml_chars(string):
+def strip_illegal_xml_chars(string: str) -> str:
     return RE_XML_ILLEGAL.sub('', string)