Extend Gitlab CI/CD pipeline

.gitlab/scripts/signposting.py

@@ -0,0 +1,175 @@
+"""
+This script is based on https://github.com/korvoj/signposting/blob/1.0.1/entrypoint.py
+
+It has been modified to cover GitLab Pages instead of GitHub Pages.
+"""
+import json
+import os
+import urllib.parse
+import argparse
+import yaml
+from wcmatch import glob
+import requests
+
+argument_parser = argparse.ArgumentParser(description='Signposting linkset generator')
+argument_parser.add_argument('--default-branch', type=str, default='main')
+argument_parser.add_argument('--default-profile', type=str, required=True)
+argument_parser.add_argument('--exclusions-file', type=str, default='mkdocs.yml')
+argument_parser.add_argument('--root-dir', type=str, default='resources')
+argument_parser.add_argument('--gl-repository-url', type=str, required=True)
+argument_parser.add_argument('--pages-url', type=str, required=True)
+args = argument_parser.parse_args()
+
+DEFAULT_BRANCH = args.default_branch
+PAGES_URL = args.pages_url
+DEFAULT_PROFILE_DISCOVERED_ITEMS = args.default_profile
+EXCLUSIONS_FILE_PATH = args.exclusions_file
+ROOT_DIR_PATH = args.root_dir
+GITLAB_REPOSITORY_URL = args.gl_repository_url
+BLOB_CONTENT_URL = f'{GITLAB_REPOSITORY_URL}/-/blob'
+RAW_CONTENT_URL = f'{GITLAB_REPOSITORY_URL}/-/raw'
+
+
+class SignPost:
+    def __init__(self, href, type=None, profile=None):
+        self.href = href
+        self.type = type
+        self.profile = profile
+
+    def __repr__(self):
+        str_representation = f'href: {self.href}'
+        if self.type is not None:
+            str_representation += f' type: {self.type}'
+        if self.profile is not None:
+            str_representation += f' profile: {self.profile}'
+        return str_representation
+
+    def to_json(self):
+        json_representaton = dict()
+        json_representaton['href'] = self.href
+        if self.type is not None:
+            json_representaton['type'] = self.type
+        if self.profile is not None and str.strip(self.profile) != '':
+            json_representaton['profile'] = self.profile
+        return json_representaton
+
+
+def read_exclusions_file(exclusions_file_path):
+    with open(exclusions_file_path) as stream:
+        try:
+            yaml_exclusions = yaml.safe_load(stream)
+            return yaml_exclusions.get('signposting_exclusions', [])
+        except yaml.YAMLError as err:
+            print('An error has occurred: ', err)
+
+
+def fetch_files(root_dir, exclusions):
+    url_list = []
+    markdown_files = glob.glob(patterns='**/**.md', root_dir=root_dir,
+                               exclude=exclusions, flags=glob.GLOBSTAR)
+    for markdown_file in markdown_files:
+        # print(markdown_file)
+        url_encoded_path = urllib.parse.quote(markdown_file)
+        if root_dir and root_dir != '':
+            url_list.append(
+                f'{BLOB_CONTENT_URL}/{DEFAULT_BRANCH}/{root_dir}/{url_encoded_path}')
+        else:
+            url_list.append(
+                f'{BLOB_CONTENT_URL}/{DEFAULT_BRANCH}/{url_encoded_path}')
+
+    return url_list
+
+
+def parse_citation_cff_authors(citation_cff):
+    orcids = [SignPost(href=i['orcid'], type=None) for i in citation_cff.get('authors', []) if
+              i.get('orcid') is not None]
+    return orcids
+
+
+def parse_citation_cff_license(citation_cff):
+    response = requests.get('https://raw.githubusercontent.com/spdx/license-list-data/main/json/licenses.json')
+    if response.status_code != 200:
+        print('Error fetching license list, status code: ', response.status_code)
+        return
+    license_list = response.json().get('licenses', [])
+    cff_license = citation_cff.get('license')
+    if cff_license is not None:
+        for i in license_list:
+            if i.get('licenseId', '') == cff_license:
+                return SignPost(href=i.get('reference'), type=None)
+    raise Exception('No license mapping to SPDX possible')
+
+
+def parse_citation_cff_repository(citation_cff):
+    repository_url = citation_cff.get('repository')
+    return SignPost(href=repository_url, type='text/html')
+
+
+def parse_citation_cff_related(citation_cff):
+    doi = citation_cff.get('doi', '')
+    doi = f'https://doi.org/{doi}'
+    return SignPost(href=doi, type='text/html')
+
+
+def construct_types():
+    return [
+        SignPost(href='https://schema.org/LearningResource', type=None),
+        SignPost(href='https://schema.org/AboutPage', type=None)
+    ]
+
+
+def construct_described_by():
+    return [
+        SignPost(type='application/yaml', profile='https://citation-file-format.github.io/1.2.0/schema.json',
+                 href=f'{RAW_CONTENT_URL}/{DEFAULT_BRANCH}/CITATION.cff')
+    ]
+
+
+def construct_items(files):
+    signposts = []
+    for file in files:
+        signpost = SignPost(href=file, type='text/markdown', profile=DEFAULT_PROFILE_DISCOVERED_ITEMS)
+        signposts.append(signpost)
+    return signposts
+
+
+def generate_linkset(root_dir, exclusions):
+    citation_cff = ''
+    with open('CITATION.cff') as stream:
+        try:
+            citation_cff = yaml.safe_load(stream)
+        except yaml.YAMLError as err:
+            print('An error has occurred: ', err)
+            return
+    authors = parse_citation_cff_authors(citation_cff)
+    spdx_license = parse_citation_cff_license(citation_cff)
+    item_repository_url = parse_citation_cff_repository(citation_cff)
+    related = parse_citation_cff_related(citation_cff)
+    types = construct_types()
+    described_by = construct_described_by()
+    discovered_files = fetch_files(root_dir=root_dir, exclusions=exclusions)
+    all_items = construct_items(discovered_files)
+    all_items.append(item_repository_url)
+
+    json_linkset = {
+        'linkset': [
+            {
+                'anchor': PAGES_URL,
+                'type': [i.to_json() for i in types],
+                'author': [i.to_json() for i in authors],
+                'item': [i.to_json() for i in all_items],
+                'describedby': [i.to_json() for i in described_by],
+                'license': [spdx_license.to_json()],
+                'related': [related.to_json()]
+            }
+        ]
+    }
+
+    with open('linkset.json', 'w') as f:
+        json.dump(json_linkset, f)
+    # print(json_linkset)
+
+
+if __name__ == '__main__':
+    exclusions = read_exclusions_file(EXCLUSIONS_FILE_PATH)
+    generate_linkset(root_dir=ROOT_DIR_PATH, exclusions=exclusions)

.zenodo.json

@@ -0,0 +1,30 @@
+{
+  "creators": [
+    {
+      "affiliation": "Organization",
+      "name": "Surname, First_Name",
+      "orcid": "1234-5678-9101-1121"
+    }
+  ],
+  "description": "Training description",
+  "keywords": [
+    "thefirstkeyword",
+    "thesecondkeyword",
+    "a third keyword"
+  ],
+  "license": "CC-BY-4.0",
+  "publication_date": "1970-01-01",
+  "title": "Training Name",
+  "version": "0.0.1",
+  "upload_type": "lesson",
+  "communities": [
+    {
+      "identifier": "skills4eosc"
+    }
+  ],
+  "grants": [
+    {
+      "id": "10.13039/501100000780::101058527"
+    }
+  ]
+}

CITATION.cff

@@ -7,8 +7,8 @@ authors:
     affiliation: Organization
 title: "Training Name"
 abstract: "Training description"
-version: 1.0.0
-doi: 10.5070/zenodo.123
+version: 0.0.1
+doi: 10.5072/zenodo.123
 date-released: "1970-01-01"
 license: CC-BY-4.0
 license-url: "https://creativecommons.org/licenses/by/4.0/legalcode.txt"

README.md

@@ -23,6 +23,50 @@ If you are interested in following the training as a learner:
 
 If you want to start developing FAIR-by-Design learning materials based on these templates simply clone this repository.
 
+### Gitlab Pages
+
+This repo contains a `.gitlab-ci.yml` file for automatically deploying the content of this repo to [Gitlab Pages](https://docs.gitlab.com/ee/user/project/pages/).
+
+#### Available workflows
+
+The included `.gitlab-ci.yml` file provides 2 workflows:
+
+##### Push to main branch during development
+
+On each push to the main branch, the CI/CD pipeline will
+
+- automatically synchronise the metadata between `CITATION.cff`, `mkdocs.yml`, `.zenodo.json`, and `linkset.json` and
+- build and deploy the MkDocs document to GitLab pages under `/latest/`.
+
+##### Create a release
+
+If you create a [tag](https://docs.gitlab.com/user/project/repository/tags/) in the [Semantic Versioning](https://semver.org/) format `[number].[number].[number]` (e.g. 1.0.0; see the [Fair-by-Design Train of Trainers unit "Zenodo Publishing"](https://fair-by-design-methodology.github.io/FAIR-by-Design_ToT/latest/Stage%205%20%E2%80%93%20Publish/17-Zenodo%20Publishing/17-Zenodo%20Publishing/) for more information about Semantic Versioning), the CI/CD pipeline will
+
+- reserve a DOI on Zenodo,
+- synchronise the current date and version number from the tag into the `CITATION.cff`,
+- run the above steps from the pipeline that runs on a push to main branch (synchronize metadata and deploy latest version to GitLab pages),
+- build and deploy the MkDocs document to GitLab pages under `/<semantic-version-number>/`, and
+- populate the Zenodo entry with the metadata from the repo and a snapshot of the current contents of this repository.
+
+#### Setup the GitLab CI/CD pipeline
+
+To setup the CI/CD pipeline, you need to complete the following steps:
+
+- Make sure the [project feaures](https://docs.gitlab.com/ee/user/project/settings/) `CI/CD` and `Pages` are activated in your project (*Settings* > *General* > *Visibility, project features, permissions*).
+- Allow the pipeline to push content back to the repository (*Settings* > *CI/CD Settings* > *Job token permissions* > *Additional permissions* > *Allow Git push requests to the repository*).
+- Create a [Zenodo Access Token](https://zenodo.org/account/settings/applications/tokens/new/) and save it in GitLab under *Settings* > *CI/CD* > *Variables* > *CI/CD Variables* > *Add variable* with the following properties:
+  - Type: Variable (default)
+  - Environments: All (default)
+  - Visibility: Masked and hidden
+  - Flags:
+    - Protect variable: No (if you want to increase the security and activate this protection, you need to create a rule that all release tags are marked as [protected tags]())
+    - Expand variable reference: No
+  - Key: ZENODO_ACCESS_TOKEN
+  - Value: `<your-access-token>`
+- If you want to use the Zenodo Sandbox for testing, save the access token for the Sandbox as described above, but with the Key `ZENODO_SANDBOX_ACCESS_TOKEN` and create another variable with the key `ZENODO_USE_SANDBOX` and the value `true`.
+
+If the pipelines are still not working, make sure that there is at least [one active runner](https://docs.gitlab.com/ee/ci/runners/runners_scope.html) (navigate to *Settings* > *CI/CD Settings* > *Runners*).
+
 ---
 
 May your learning materials always be FAIR!

linkset.json

@@ -0,0 +1 @@
+{"linkset": [{"anchor": "https://URL-will-automatically-be-replaced/latest/", "type": [{"href": "https://schema.org/LearningResource"}, {"href": "https://schema.org/AboutPage"}], "author": [{"href": "https://orcid.org/1234-5678-9101-1121"}], "item": [{"href": "https://github.com/citation-file-format/my-research-software", "type": "text/html"}], "describedby": [{"href": "https://URL-will-automatically-be-replaced/-/raw/main/CITATION.cff", "type": "application/yaml", "profile": "https://citation-file-format.github.io/1.2.0/schema.json"}], "license": [{"href": "https://spdx.org/licenses/CC-BY-4.0.html"}], "related": [{"href": "https://doi.org/10.5072/zenodo.123", "type": "text/html"}]}]}

mkdocs.yml

@@ -39,28 +39,29 @@ extra:
     provider: mike
 extra_css:
   - stylesheets/extra.css
-citation:
+citation: # Automatically updated by GitHub action/GitLab CI
   cff-version: 1.2.0
   message: "If you use these learning materials, please cite it as below."
   authors:
     - family-names: Surname
       given-names: First_Name
       orcid: https://orcid.org/1234-5678-9101-1121
+      affiliation: Organization
   title: "Training Name"
   abstract: "Training description"
-  version: 1.0.0
-  doi: 10.5072/zenodo.1234567
-  date-released: 2023-10-17
+  version: 0.0.1 # Automatically updated by GitHub action/GitLab CI
+  doi: 10.5072/zenodo.123 # Automatically updated by GitHub action/GitLab CI
+  date-released: "1970" # Automatically updated by GitHub action/GitLab CI
   license: CC-BY-4.0
-  license-url: "https://creativecommons.org/publicdomain/zero/1.0/legalcode"
+  license-url: "https://creativecommons.org/licenses/by/4.0/legalcode.txt"
   type: generic
   keywords:
-  - thefirstkeyword
-  - thesecondkeyword
-  - "a third keyword"
+    - "thefirstkeyword"
+    - "thesecondkeyword"
+    - "a third keyword"
   repository: "https://github.com/citation-file-format/my-research-software"
-signposting_linkset: https://raw.githubusercontent.com/FAIR-by-Design-ToT/templates/main/linkset.json # do not update manually
-signposting_default_profile: '' # update with an URL towards the profile used for the Markdown pages (optional)
+signposting_linkset: "" # do not update manually
+signposting_default_profile: "" # update with an URL towards the profile used for the Markdown pages (optional)
 # signposting_gitbook_url: https://gitbook.example.com # leave commented (prefixed with `#`) if using GitHub Pages with the default domain.
 signposting_exclusions:
   - 'venv/**'
@@ -76,3 +77,4 @@ signposting_exclusions:
   - '*syllabus.md'
   - '**/*_plan.md'
   - '**/template_content.md'
+site_url: "" # leave empty ("") if GitLab CI should set it up

resources/syllabus.md

@@ -13,7 +13,7 @@ abstract: "A brief synopsis about or description of the learning resource."
 primaryLanguage: "two letter code, Language in which the resource was originally published or made available."
 license: "A license document that applies to this content, typically indicated by URL"
 versionDate: "YYYY-MM-DD Version date for the most recently published or broadcast resource."
-urlToResource: "https://doi.org/10.5072/zenodo.1234567"
+urlToResource: "https://doi.org/10.5072/zenodo.123"
 resourceURLType: "URL"
 targetGroup: "Principle users(s) for which the resource was designed."
 learningResourceType: "The predominant type or kind that characterizes the learning resource."
@@ -118,7 +118,7 @@ Keywords or tags used to describe the training.
 
 ## DOI
 
-[https://doi.org/10.5072/zenodo.1234567](https://doi.org/10.5072/zenodo.1234567)
+[https://doi.org/10.5072/zenodo.123](https://doi.org/10.5072/zenodo.123)
 
 ## Accessibility Mission