Support for the Research Organization Registry (ROR) (#4483)

* First draft of data model #3168

* Build ROR importer #3168

* Updates to data model #3168

* Create standard buttons #3168

* UI for core affiliation management #3168

* Deprecate several more old account templates #4380

* chore #3168 Clean up old prints in testing

* Work through regressions #3168

* Avoid Django ListView paginator bug

* Display primary affiliation; add primary if first #3168

* feat #3168 Add affiliation inline to admin pages

* feat #3168 Migrations for existing data

* feat #3168 Avoid bug that caused duplicate locations to be created

* feat #3168 Add country and location to data migration

* feat #3168 Override get_or_create to handle old affiliation fields

* feat #3168 Utility for matching legacy organizations

* feat #3168 Improve importing process

* feat #3168 Fix a bug where a bool might be unset

* feat #3168 Fix a bug with a URL name

* feat #3168 Display start and end dates on affiliations

* feat #3168 Use function-based views for CRUD interface

* feat #3168 Fix logic that manages affiliations

* Clean up submission test data

* feat #3168 Adjustments from manual testing

* feat #3168 Expand max_length of OrganizationName.value

* feat #3168 Validate exclusive fields on Affiliation

* feat #3168 Fix view bugs and write tests

* feat #3168 Make OrganizationName.value a required field

* feat #3168 Fixes to backwards compatibility

* feat #3168 Reflow migrations after rebase

* feat #3168 Make account affiliation form more secure

* feat #3168 Pass account to form kwargs

* feat #3168 Cleanup requested in review

* feat #3168 Include variable names in translatable strings

* feat #3168 Use safer method in affil create view

* feat #3168 Guard against duplicate affiliations

* feat #3168 Fix bugs

* feat #3168 Rename methods

* feat #3168 Prepend 'is_' to status-related properties

* feat #3168 Log exceptions not just errors

* feat #3168 Refactor exclusive fields check as model constraint

* feat #3168 Account for case where all exclusive fields are blank

* feat #3168 Store non-URI form of ROR ID

* feat #3168 More small fixes

* feat #3168 Reflow migrations again

* feat #3168 Remove accidentally added migration

* feat #3168 Change print statement

* feat #3168 Check defaults dict in backwards compat method

* feat #3168 Backwards compatibility in queryset lookups

* feat #3168 Backwards compatibility in queryset lookups

* feat #3168 Improve regex handling

* feat #3168 Improve logging

* feat #3168 Avoid unnecessary change to test

* feat #3168 Delay importing latitude and longitude

* feat #3168 Fix bug between ISO-639-1 and ISO-639-2/T

* feat #3168 Document odd Account.objects.create custom method

* feat #3168 Move regex patterns to constant

Author: joemull

requirements.txt

@@ -31,6 +31,7 @@ geoip2==4.8.0
 html2text==2017.10.4
 idna==2.7
 ipaddress==1.0.16
+iso639-lang==2.6
 django-ipware==7.0.1
 more-itertools==10.1.0
 jsmin==3.0.1

src/core/admin.py

@@ -40,20 +40,20 @@ class SettingAdmin(admin.ModelAdmin):
 class AccountAdmin(UserAdmin):
     """Displays Account objects in the Django admin interface."""
     list_display = ('id', 'email', 'orcid', 'first_name', 'middle_name',
-                    'last_name', 'institution', '_roles_in', 'last_login')
+                    'last_name', '_roles_in', 'last_login')
     list_display_links = ('id', 'email')
     list_filter = ('accountrole__journal',
                    'repositoryrole__repository__short_name',
                    'is_active', 'is_staff', 'is_admin', 'is_superuser',
                    'last_login')
     search_fields = ('id', 'username', 'email', 'first_name', 'middle_name',
-                     'last_name', 'orcid', 'institution',
+                     'last_name', 'orcid',
                      'biography', 'signature')
 
     fieldsets = UserAdmin.fieldsets + (
         (None, {'fields': (
             'name_prefix', 'middle_name', 'orcid',
-            'institution', 'department', 'country', 'twitter',
+            'twitter',
             'linkedin', 'facebook', 'github', 'website', 'biography', 'enable_public_profile',
             'signature', 'profile_image', 'interest', "preferred_timezone",
         )}),
@@ -71,6 +71,7 @@ class AccountAdmin(UserAdmin):
     raw_id_fields = ('interest',)
 
     inlines = [
+        admin_utils.ControlledAffiliationInline,
         admin_utils.AccountRoleInline,
         admin_utils.RepositoryRoleInline,
         admin_utils.EditorialGroupMemberInline,
@@ -398,6 +399,84 @@ class AccessRequestAdmin(admin.ModelAdmin):
     date_hierarchy = ('requested')
 
 
+class OrganizationAdmin(admin.ModelAdmin):
+    list_display = ('pk', 'ror_id', '_ror_display', '_custom_label',
+                    'website', '_locations', 'ror_status')
+    list_display_links = ('pk', 'ror_id')
+    list_filter = ('ror_status', 'locations__country')
+    search_fields = ('pk', 'ror_display__value', 'custom_label__value', 'labels__value',
+                     'aliases__value', 'acronyms__value', 'website', 'ror_id')
+    raw_id_fields = ('locations', )
+
+    def _ror_display(self, obj):
+        return obj.ror_display if obj and obj.ror_display else ''
+
+    def _locations(self, obj):
+        return '; '.join([str(l) for l in obj.locations.all()]) if obj else ''
+
+    def _custom_label(self, obj):
+        return obj.custom_label if obj and obj.custom_label else ''
+
+
+class OrganizationNameAdmin(admin.ModelAdmin):
+    list_display = ('pk', 'value', 'language')
+    list_display_links = ('pk', 'value')
+    search_fields = ('pk', 'value')
+    raw_id_fields = ('ror_display_for', 'custom_label_for',
+                     'label_for', 'alias_for', 'acronym_for')
+
+    def _ror_display(self, obj):
+        return obj.ror_display if obj and obj.ror_display else ''
+
+    def _locations(self, obj):
+        return '; '.join([str(l) for l in obj.locations.all()]) if obj else ''
+
+    def _custom_label(self, obj):
+        return obj.custom_label if obj and obj.custom_label else ''
+
+
+class LocationAdmin(admin.ModelAdmin):
+    list_display = ('pk', 'name', 'country', 'geonames_id')
+    list_display_links = ('pk', 'name')
+    list_filter = ('country',)
+    search_fields = ('pk', 'name', 'country__code', 'country__name',
+                     'geonames_id')
+
+
+class ControlledAffiliationAdmin(admin.ModelAdmin):
+    list_display = ('pk', '_person', 'organization',
+                    'title', 'department', 'start', 'end')
+    list_display_links = ('pk', '_person')
+    list_filter = ('start', 'end', 'organization__locations__country')
+    search_fields = (
+        'pk',
+        'title',
+        'department',
+        'organization__ror_display__value',
+        'organization__custom_label__value',
+        'organization__labels__value',
+        'organization__aliases__value',
+        'organization__acronyms__value',
+        'account__first_name',
+        'account__last_name',
+        'account__email',
+        'frozen_author__first_name',
+        'frozen_author__last_name',
+        'frozen_author__frozen_email',
+        'preprint_author__account__first_name',
+        'preprint_author__account__last_name',
+        'preprint_author__account__email',
+    )
+    raw_id_fields = ('account', 'frozen_author',
+                     'preprint_author', 'organization')
+
+    def _person(self, obj):
+        if obj:
+            return obj.account or obj.frozen_author or obj.preprint_author
+        else:
+            return ''
+
+
 admin_list = [
     (models.AccountRole, AccountRoleAdmin),
     (models.Account, AccountAdmin),
@@ -427,6 +506,10 @@ class AccessRequestAdmin(admin.ModelAdmin):
     (models.Contacts, ContactsAdmin),
     (models.Contact, ContactAdmin),
     (models.AccessRequest, AccessRequestAdmin),
+    (models.Organization, OrganizationAdmin),
+    (models.OrganizationName, OrganizationNameAdmin),
+    (models.Location, LocationAdmin),
+    (models.ControlledAffiliation, ControlledAffiliationAdmin),
 ]
 
 [admin.site.register(*t) for t in admin_list]

src/core/forms/__init__.py

@@ -1,11 +1,13 @@
 from core.forms.forms import (
     AccessRequestForm,
+    AccountAffiliationForm,
     AccountRoleForm,
     AdminUserForm,
     ArticleMetaImageForm,
     CBVFacetForm,
     ConfirmableForm,
     ConfirmableIfErrorsForm,
+    ConfirmDeleteForm,
     EditAccountForm,
     EditKey,
     EditorialGroupForm,
@@ -24,6 +26,7 @@
     JournalSubmissionForm,
     LoginForm,
     NotificationForm,
+    OrganizationNameForm,
     PasswordResetForm,
     PressJournalAttrForm,
     QuickUserForm,

src/core/forms/forms.py

@@ -183,7 +183,7 @@ class RegistrationForm(forms.ModelForm, CaptchaForm):
     class Meta:
         model = models.Account
         fields = ('email', 'salutation', 'first_name', 'middle_name',
-                  'last_name', 'department', 'institution', 'country', 'orcid',)
+                  'last_name', 'orcid',)
         widgets = {'orcid': forms.HiddenInput() }
 
     def __init__(self, *args, **kwargs):
@@ -532,7 +532,7 @@ def __init__(self, *args, **kwargs):
 class QuickUserForm(forms.ModelForm):
     class Meta:
         model = models.Account
-        fields = ('email', 'salutation', 'first_name', 'last_name', 'institution',)
+        fields = ('email', 'salutation', 'first_name', 'last_name',)
 
 
 class LoginForm(CaptchaForm):
@@ -945,3 +945,58 @@ class AccountRoleForm(forms.ModelForm):
     class Meta:
         model = models.AccountRole
         fields = '__all__'
+
+
+class OrganizationNameForm(forms.ModelForm):
+
+    class Meta:
+        model = models.OrganizationName
+        fields = ('value',)
+
+
+class AccountAffiliationForm(forms.ModelForm):
+    """
+    A form for account holders to edit their own affiliations.
+    Not intended for editing someone else's affiliations.
+    """
+
+    class Meta:
+        model = models.ControlledAffiliation
+        fields = ('title', 'department', 'is_primary', 'start', 'end')
+        widgets = {
+            'start': HTMLDateInput,
+            'end': HTMLDateInput,
+        }
+
+    def __init__(self, *args, **kwargs):
+        self.account = kwargs.pop('account', None)
+        self.organization = kwargs.pop('organization', None)
+        super().__init__(*args, **kwargs)
+
+    def clean(self):
+        cleaned_data = super().clean()
+        query = Q(account=self.account, organization=self.organization)
+        for key, value in cleaned_data.items():
+            query &= Q((key, value))
+        if self._meta.model.objects.filter(query).exists():
+            self.add_error(
+                None,
+                "An affiliation with matching details already exists."
+            )
+        return cleaned_data
+
+    def save(self, commit=True):
+        affiliation = super().save(commit=False)
+        affiliation.account = self.account
+        affiliation.organization = self.organization
+        if commit:
+            affiliation.save()
+        return affiliation
+
+
+class ConfirmDeleteForm(forms.Form):
+    """
+    A generic form for use on confirm-delete pages
+    where a valid form with POST data means yes, delete.
+    """
+    pass

src/core/include_urls.py

@@ -129,6 +129,38 @@
     re_path(r'^manager/user/(?P<user_id>\d+)/edit/$', core_views.user_edit, name='core_user_edit'),
     re_path(r'^manager/user/(?P<user_id>\d+)/history/$', core_views.user_history, name='core_user_history'),
 
+    ## Affiliations
+    re_path(
+        r'^profile/organization/search/$',
+        core_views.OrganizationListView.as_view(),
+        name='core_organization_search'
+    ),
+    re_path(
+        r'^profile/organization_name/create/$',
+        core_views.organization_name_create,
+        name='core_organization_name_create'
+    ),
+    re_path(
+        r'^profile/organization_name/(?P<organization_name_id>\d+)/update/$',
+        core_views.organization_name_update,
+        name='core_organization_name_update'
+    ),
+    re_path(
+        r'^profile/organization/(?P<organization_id>\d+)/affiliation/create/$',
+        core_views.affiliation_create,
+        name='core_affiliation_create'
+    ),
+    re_path(
+        r'^profile/affiliation/(?P<affiliation_id>\d+)/update/$',
+        core_views.affiliation_update,
+        name='core_affiliation_update'
+    ),
+    re_path(
+        r'^profile/affiliation/(?P<affiliation_id>\d+)/delete/$',
+        core_views.affiliation_delete,
+        name='core_affiliation_delete'
+    ),
+
     # Templates
     re_path(r'^manager/templates/$', core_views.email_templates, name='core_email_templates'),