Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions jdhapi/admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,10 @@
from .tasks import (
save_article_fingerprint,
save_article_specific_content,
save_citation,
save_libraries,
save_references,
)
from .utils.articles import save_citation
from import_export.admin import ExportActionMixin
from django.utils.html import format_html

Expand All @@ -37,7 +37,7 @@ def save_notebook_specific_cell(modeladmin, request, queryset):

def save_article_citation(modeladmin, request, queryset):
for article in queryset:
save_citation.delay(article_id=article.pk)
save_citation(article_id=article.pk)


save_article_citation.short_description = "3: Generate citation"
Expand Down
10 changes: 8 additions & 2 deletions jdhapi/models/article.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ class Article(models.Model):
Methods:
get_kernel_language(): Returns the kernel language based on the 'tool' tag.
__str__(): Returns the title of the abstract.
send_email_if_peer_review(): Sends an email with a PDF attachment if the article is in peer review status.
"""

class Status(models.TextChoices):
Expand All @@ -63,7 +62,14 @@ class Status(models.TextChoices):
"DESIGN_REVIEW",
"Design review",
)
PUBLISHED = "PUBLISHED", "Published"
COPY_EDITING = (
"COPY_EDITING",
"Copy editing",
)
PUBLISHED = (
"PUBLISHED",
"Published",
)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should add SOCIAL_MEDIA and ARCHIVED.


class CopyrightType(models.TextChoices):
DRAFT = (
Expand Down
2 changes: 1 addition & 1 deletion jdhapi/signals.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from django.dispatch import receiver
from jdhapi.models import Article
from jdhapi.utils.articles import convert_string_to_base64
from jdhapi.utils.run_github_action import trigger_workflow



@receiver(pre_save, sender=Article)
Expand Down
13 changes: 0 additions & 13 deletions jdhapi/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
from jdhapi.utils.articles import (
get_notebook_stats,
get_notebook_specifics_tags,
get_citation,
generate_tags,
generate_narrative_tags,
get_notebook_references_fulltext,
Expand Down Expand Up @@ -75,18 +74,6 @@ def save_article_specific_content(article_id):
article.save()


@shared_task
def save_citation(article_id):
logger.info(f"save_article_citation:{article_id}")
try:
article = Article.objects.get(pk=article_id)
except Article.DoesNotExist:
logger.error(f"save_article_citation:{article_id} not found")
citation = get_citation(raw_url=article.notebook_ipython_url, article=article)
article.citation = citation
article.save()


@shared_task
def save_libraries(article_id):
logger.info(f"save_article_libraries:{article_id}")
Expand Down
2 changes: 2 additions & 0 deletions jdhapi/urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@
path("api/articles/cover", views.get_social_cover_image, name="articles-social-media-cover"),
path("api/articles/ojs", views.send_article_to_ojs, name="articles-send-to-ojs"),
path("api/articles/tweet", views.get_tweet_md_file, name="articles-tweet"),
path("api/articles/docx", views.get_docx, name="article-docx"),
path("api/articles/docx/email", views.send_docx_email, name="article-docx-email"),
path("api/authors/", views.AuthorList.as_view(), name="author-list"),
path("api/authors/<int:pk>/", views.AuthorDetail.as_view(), name="author-detail"),
path(
Expand Down
14 changes: 13 additions & 1 deletion jdhapi/utils/articles.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from django.utils.html import strip_tags

from jdhapi.utils.doi import get_doi_url_formatted_jdh
from jdhapi.models import Author, Tag
from jdhapi.models import Article, Author, Tag

from jdhseo.utils import getReferencesFromJupyterNotebook
from requests.exceptions import HTTPError
Expand Down Expand Up @@ -217,6 +217,18 @@ def get_citation(raw_url, article):
}


def save_citation(article_id):
logger.info(f"save_article_citation:{article_id}")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you want to write the name of the function, write instead 'save_citation', so that for debugging we know exactly what to look for :)

try:
article = Article.objects.get(pk=article_id)
except Article.DoesNotExist:
logger.error(f"save_article_citation:{article_id} not found")
return
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe here do a raise Exception("Article id not found") instead of a return

citation = get_citation(raw_url=article.notebook_ipython_url, article=article)
article.citation = citation
article.save()


def get_raw_from_github(
repository_url, file_type, host="https://raw.githubusercontent.com"
):
Expand Down
148 changes: 130 additions & 18 deletions jdhapi/utils/run_github_action.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,15 @@
#!/usr/bin/env python3
import logging
import os
import sys
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do not forget to clean imports that we are not using :)

import time
from datetime import datetime, timezone
import requests
from pathlib import Path
from urllib.parse import urlparse

logger = logging.getLogger(__name__)


def trigger_workflow(repo_url, workflow_filename, token=None, ref="main"):
"""
Expand All @@ -13,11 +18,112 @@ def trigger_workflow(repo_url, workflow_filename, token=None, ref="main"):
:param workflow_filename: Filename of the workflow in .github/workflows (e.g. "hello-world.yml")
:param ref: Git ref (branch or tag) to run the workflow on
"""
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the doc here does not match with the function parameters

if not token:
from jdh.settings import GITHUB_ACCESS_TOKEN
token = _get_github_token(token)
owner, repo = _parse_owner_repo(repo_url)

url = f"https://api.github.com/repos/{owner}/{repo}/actions/workflows/{workflow_filename}/dispatches"
headers = {
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json",
}
payload = {"ref": ref}
try:
resp = requests.post(url, json=payload, headers=headers, timeout=10)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think res it is more common for the response variable name or response

if resp.status_code == 204:
logger.info(
"Workflow '%s' dispatched on ref '%s' for %s/%s.",
workflow_filename,
ref,
owner,
repo,
)
return

logger.error(
"Failed to dispatch workflow '%s' (%s): %s",
workflow_filename,
resp.status_code,
resp.text,
)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if ... else ... I think ?

resp.raise_for_status()
except requests.RequestException as e:
logger.error("Workflow dispatch failed: %s", e)
raise


def trigger_workflow_and_wait(
repo_url,
workflow_filename,
token=None,
ref="main",
timeout_seconds=600,
poll_interval_seconds=5,
):
token = _get_github_token(token)
owner, repo = _parse_owner_repo(repo_url)
started_at = datetime.now(timezone.utc)

trigger_workflow(repo_url, workflow_filename, token=token, ref=ref)

runs_url = (
f"https://api.github.com/repos/{owner}/{repo}/actions/workflows/"
f"{workflow_filename}/runs"
)
headers = {
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json",
}

deadline = time.time() + timeout_seconds
run_id = None

token = GITHUB_ACCESS_TOKEN
while time.time() < deadline:
try:
resp = requests.get(runs_url, headers=headers, timeout=10)
resp.raise_for_status()
data = resp.json()
except requests.RequestException as e:
logger.error("Failed to list workflow runs: %s", e)
raise

for run in data.get("workflow_runs", []):
created_at = _parse_github_datetime(run.get("created_at"))
if not created_at:
continue
if (
run.get("event") == "workflow_dispatch"
and run.get("head_branch") == ref
and created_at >= started_at
):
run_id = run.get("id")
status = run.get("status")
conclusion = run.get("conclusion")

if status == "completed":
if conclusion == "success":
logger.info(
"Workflow '%s' completed successfully for %s/%s.",
workflow_filename,
owner,
repo,
)
return
raise RuntimeError(
f"Workflow '{workflow_filename}' завершён: {conclusion}"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what does it mean this "russian" word ? ^^

)
break

if run_id is None:
logger.info("Waiting for workflow run to start...")

time.sleep(poll_interval_seconds)

raise TimeoutError(
f"Workflow '{workflow_filename}' did not complete within {timeout_seconds}s"
)


def _parse_owner_repo(repo_url):
parsed = urlparse(repo_url)
path = parsed.path.lstrip("/")

Expand All @@ -26,19 +132,25 @@ def trigger_workflow(repo_url, workflow_filename, token=None, ref="main"):

parts = path.split("/")
if len(parts) >= 2:
owner = parts[0]
repo = parts[1]
return parts[0], parts[1]
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can improve readability with :
owner = parts[0]
repo = parts[1]


url = f"https://api.github.com/repos/{owner}/{repo}/actions/workflows/{workflow_filename}/dispatches"
headers = {
"Authorization": f"Bearer {token}",
"Accept": "application/vnd.github+json",
}
payload = {"ref": ref}
resp = requests.post(url, json=payload, headers=headers)
if resp.status_code == 204:
print(f"Workflow '{workflow_filename}' dispatched on ref '{ref}'.")
else:
print(f"Failed to dispatch workflow: {resp.status_code}")
print(resp.json())
resp.raise_for_status()
raise ValueError(f"Invalid repository URL: '{repo_url}'")


def _get_github_token(token):
if token:
return token
from jdh.settings import GITHUB_ACCESS_TOKEN

return GITHUB_ACCESS_TOKEN
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here we need to add doc + error message



def _parse_github_datetime(value):
if not value:
return None
try:
return datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ").replace(
tzinfo=timezone.utc
)
except ValueError:
return None
1 change: 1 addition & 0 deletions jdhapi/views/articles/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@
from .send_to_ojs import *
from .social_media import *
from .update_article import *
from .copy_editing import *
5 changes: 4 additions & 1 deletion jdhapi/views/articles/articles.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from rest_framework import generics, filters
from rest_framework.permissions import BasePermission
from django.shortcuts import get_object_or_404
from jdhapi.views.articles.status_handlers import TechnicalReviewHandler
from jdhapi.views.articles.status_handlers import *
from rest_framework.permissions import IsAdminUser
from rest_framework.response import Response
from rest_framework.views import APIView
Expand Down Expand Up @@ -94,6 +94,9 @@ class ArticleStatus(APIView):
permission_classes = [IsAdminUser]
STATUS_HANDLERS = {
'TECHNICAL_REVIEW': TechnicalReviewHandler(),
'COPY_EDITING': CopyEditingHandler(),
'PEER_REVIEW': PeerReviewHandler(),
'PUBLISHED' : PublishedHandler()
}

def patch(self, request, abstract__pid):
Expand Down
Loading