454 migrate resources from old cms to aiod platform

connectors/ai4europe_cms/Dockerfile

@@ -0,0 +1,19 @@
+FROM aiod_metadata_catalogue
+
+COPY organisations.sh /opt/connectors/script/organisations.sh
+COPY events.sh /opt/connectors/script/events.sh
+COPY news.sh /opt/connectors/script/news.sh
+COPY entry.sh /opt/connectors/script/entry.sh
+COPY cron /etc/cron.d/aiod
+
+
+USER root
+RUN apt -y install cron
+
+RUN chmod +x /opt/connectors/script/organisations.sh
+RUN chmod +x /opt/connectors/script/events.sh
+RUN chmod +x /opt/connectors/script/news.sh
+RUN chmod +x /opt/connectors/script/entry.sh
+RUN crontab /etc/cron.d/aiod
+
+WORKDIR /app

connectors/ai4europe_cms/cron

@@ -0,0 +1,3 @@
+40 * * * * bash /opt/connectors/script/organisations.sh >> /opt/connectors/data/ai4europe_cms/organisation/cron.log 2>&1
+40 * * * * bash /opt/connectors/script/events.sh >> /opt/connectors/data/ai4europe_cms/events/cron.log 2>&1
+40 * * * * bash /opt/connectors/script/news.sh >> /opt/connectors/data/ai4europe_cms/news/cron.log 2>&1

connectors/ai4europe_cms/entry.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+mkdir -p /opt/connectors/data/ai4europe_cms/organisation
+mkdir -p /opt/connectors/data/ai4europe_cms/event
+mkdir -p /opt/connectors/data/ai4europe_cms/news
+
+/usr/sbin/cron -f -l 4

connectors/ai4europe_cms/events.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+WORK_DIR=/opt/connectors/data/ai4europe_cms/event
+CONNECTOR=connectors.ai4europe_cms.ai4europe_cms_event_connector.AI4EuropeCmsEventConnector
+
+another_instance()
+{
+    echo $(date -u) "This script is already running in a different thread."
+    exit 1
+}
+exec 9< "$0"
+flock -n -x 9 || another_instance
+
+echo $(date -u) "Starting synchronization..."
+PYTHONPATH=/app /usr/local/bin/python3 /app/connectors/synchronization.py \
+      -c $CONNECTOR \
+      -w $WORK_DIR \
+      --save-every 100  >> ${WORK_DIR}/connector.log 2>&1
+echo $(date -u) "Synchronization Done."

connectors/ai4europe_cms/news.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+WORK_DIR=/opt/connectors/data/ai4europe_cms/news
+CONNECTOR=connectors.ai4europe_cms.ai4europe_cms_news_connector.AI4EuropeCmsNewsConnector
+
+another_instance()
+{
+    echo $(date -u) "This script is already running in a different thread."
+    exit 1
+}
+exec 9< "$0"
+flock -n -x 9 || another_instance
+
+echo $(date -u) "Starting synchronization..."
+PYTHONPATH=/app /usr/local/bin/python3 /app/connectors/synchronization.py \
+      -c $CONNECTOR \
+      -w $WORK_DIR \
+      --save-every 100  >> ${WORK_DIR}/connector.log 2>&1
+echo $(date -u) "Synchronization Done."

connectors/ai4europe_cms/organisations.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+WORK_DIR=/opt/connectors/data/ai4europe_cms/organisation
+CONNECTOR=connectors.ai4europe_cms.ai4europe_cms_organisation_connector.AI4EuropeCmsOrganisationConnector
+
+another_instance()
+{
+    echo $(date -u) "This script is already running in a different thread."
+    exit 1
+}
+exec 9< "$0"
+flock -n -x 9 || another_instance
+
+echo $(date -u) "Starting synchronization..."
+PYTHONPATH=/app /usr/local/bin/python3 /app/connectors/synchronization.py \
+      -c $CONNECTOR \
+      -w $WORK_DIR \
+      --save-every 100  >> ${WORK_DIR}/connector.log 2>&1
+echo $(date -u) "Synchronization Done."

docker-compose.yaml

@@ -109,6 +109,24 @@ services:
       app:
         condition: service_healthy
 
+  ai4europe_cms-connector:
+    profiles: ["ai4europe_cms"]
+    build:
+      context: connectors/ai4europe_cms
+      dockerfile: Dockerfile
+    image: aiod_ai4europe_cms_connector
+    container_name: ai4europe_cms-connector
+    environment:
+      - KEYCLOAK_CLIENT_SECRET=$KEYCLOAK_CLIENT_SECRET
+    volumes:
+      - ${DATA_PATH}/connectors:/opt/connectors/data
+      - ./src/config.override.toml:/app/config.override.toml:ro
+    command: >
+      /bin/bash -c "/opt/connectors/script/entry.sh"
+    depends_on:
+      app:
+        condition: service_healthy
+
   aibuilder-connector:
     profiles: ["aibuilder"]
     build:

src/connectors/ai4europe_cms/ai4europe_cms_event_connector.py

@@ -0,0 +1,95 @@
+import requests
+
+from requests.exceptions import HTTPError
+from typing import Iterator
+
+from connectors.abstract.resource_connector import ResourceConnector
+from connectors.record_error import RecordError
+from database.model.platform.platform_names import PlatformName
+from database.model.resource_read_and_create import resource_create
+from database.model.event.event import Event
+from connectors.resource_with_relations import ResourceWithRelations
+from database.model.ai_resource.text import Text
+
+
+class AI4EuropeCmsEventConnector(ResourceConnector[Event]):
+    @property
+    def resource_class(self) -> type[Event]:
+        return Event
+
+    @property
+    def platform_name(self) -> PlatformName:
+        return PlatformName.ai4europe_cms
+
+    def run(self, state: dict, **kwargs) -> Iterator[ResourceWithRelations[Event] | RecordError]:
+        """Fetch resources and update the state"""
+
+        url_data = "https://community-dev-api.aiod.eu/api/events/"
+
+        headers = {"AuthorizationToken": "1234567890"}
+
+        response = requests.get(url_data, headers=headers, timeout=600)
+
+        if not response.ok:
+            status_code = response.status_code
+            msg = response.json()["error"]["message"]
+            err_msg = f"Error while fetching {url_data} from AI4Europe CMS: ({status_code}) {msg}"
+            err = HTTPError(err_msg)
+            yield RecordError(identifier=None, error=err)
+            return
+
+        try:
+            events = response.json()
+        except Exception as e:
+            yield RecordError(identifier=None, error=e)
+            return
+
+        for event in events:
+            pydantic_class = resource_create(Event)
+            desc = event.get("description") or {}
+
+            yield ResourceWithRelations[Event](
+                resource=pydantic_class(
+                    platform_resource_identifier=(
+                        event["platform_resource_identifier"]
+                        if event.get("platform_resource_identifier") is not None
+                        else None
+                    ),
+                    platform=event["platform"] if event.get("platform") is not None else None,
+                    name=event["name"] if event.get("name") is not None else None,
+                    date_published=event["date_published"]
+                    if event.get("date_published") is not None
+                    else None,
+                    start_date=event["start_date"] if event.get("start_date") is not None else None,
+                    end_date=event["end_date"] if event.get("end_date") is not None else None,
+                    registration_link=event["registration_link"]
+                    if event.get("registration_link") is not None
+                    and len(event.get("registration_link")) <= 256
+                    else None,
+                    mode=event["mode"] if event.get("mode") is not None else None,
+                    scientific_domain=[sd for sd in event.get("scientific_domain")]
+                    if event.get("scientific_domain") is not None
+                    else [],
+                    industrial_sector=[ins for ins in event.get("industrial_sector")]
+                    if event.get("industrial_sector") is not None
+                    else [],
+                    relevant_link=[rl for rl in event.get("relevant_link")]
+                    if event.get("relevant_link") is not None
+                    else [],
+                    alternate_name=[an for an in event.get("alternate_name")]
+                    if event.get("alternate_name") is not None
+                    else [],
+                    application_area=[ar for ar in event.get("application_area")]
+                    if event.get("application_area") is not None
+                    else [],
+                    keyword=[k for k in event.get("keyword")]
+                    if event.get("keyword") is not None
+                    else [],
+                    same_as=event["same_as"] if event.get("same_as") is not None else None,
+                    description=Text(
+                        plain=desc.get("plain") or "",
+                        html=desc.get("html") or "",
+                    ),
+                ),
+                resource_ORM_class=Event,
+            )

src/connectors/ai4europe_cms/ai4europe_cms_news_connector.py

@@ -0,0 +1,88 @@
+import requests
+
+from requests.exceptions import HTTPError
+from typing import Iterator
+
+from connectors.abstract.resource_connector import ResourceConnector
+from connectors.record_error import RecordError
+from database.model.platform.platform_names import PlatformName
+from database.model.resource_read_and_create import resource_create
+from database.model.news.news import News
+from connectors.resource_with_relations import ResourceWithRelations
+
+
+class AI4EuropeCmsNewsConnector(ResourceConnector[News]):
+    @property
+    def resource_class(self) -> type[News]:
+        return News
+
+    @property
+    def platform_name(self) -> PlatformName:
+        return PlatformName.ai4europe_cms
+
+    def run(self, state: dict, **kwargs) -> Iterator[ResourceWithRelations[News] | RecordError]:
+        """Fetch resources and update the state"""
+
+        url_data = "https://community-dev-api.aiod.eu/api/news/"
+
+        headers = {"AuthorizationToken": "1234567890"}
+
+        response = requests.get(url_data, headers=headers, timeout=600)
+
+        if not response.ok:
+            status_code = response.status_code
+            msg = response.json()["error"]["message"]
+            err_msg = f"Error while fetching {url_data} from AI4Europe CMS: ({status_code}) {msg}"
+            err = HTTPError(err_msg)
+            yield RecordError(identifier=None, error=err)
+            return
+
+        try:
+            news = response.json()
+        except Exception as e:
+            yield RecordError(identifier=None, error=e)
+            return
+
+        for n in news:
+            pydantic_class = resource_create(News)
+
+            yield ResourceWithRelations[News](
+                resource=pydantic_class(
+                    platform_resource_identifier=(
+                        n["platform_resource_identifier"]
+                        if n.get("platform_resource_identifier") is not None
+                        else None
+                    ),
+                    platform=n["platform"] if n.get("platform") is not None else None,
+                    name=n["name"] if n.get("name") is not None else None,
+                    date_published=n["date_published"]
+                    if n.get("date_published") is not None
+                    else None,
+                    headline=n["headline"] if n.get("headline") is not None else None,
+                    alternative_headline=n["alternative_headline"]
+                    if n.get("alternative_headline") is not None
+                    else None,
+                    category=[cat for cat in n["category"]]
+                    if n.get("category") is not None
+                    else [],
+                    source=n["source"] if n.get("source") is not None else None,
+                    scientific_domain=[sd for sd in n.get("scientific_domain")]
+                    if n.get("scientific_domain") is not None
+                    else [],
+                    industrial_sector=[ins for ins in n.get("industrial_sector")]
+                    if n.get("industrial_sector") is not None
+                    else [],
+                    relevant_link=[rl for rl in n.get("relevant_link")]
+                    if n.get("relevant_link") is not None
+                    else [],
+                    alternate_name=[an for an in n.get("alternate_name")]
+                    if n.get("alternate_name") is not None
+                    else [],
+                    application_area=[ar for ar in n.get("application_area")]
+                    if n.get("application_area") is not None
+                    else [],
+                    keyword=[k for k in n.get("keyword")] if n.get("keyword") is not None else [],
+                    same_as=n["same_as"] if n.get("same_as") is not None else None,
+                ),
+                resource_ORM_class=News,
+            )