New script

Author: mboudet

biocontainers2rseq/cleaner.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+
+import argparse
+import logging
+
+from tool import Tool
+
+logging.basicConfig()
+logging.root.setLevel(logging.INFO)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('input', type=str, help="Path to a Dockerfile")
+    parser.add_argument('output', type=str, help="Output dir (ie, Research-software-ecosystem repository)")
+    parser.add_argument('--dry-run', action='store_true', help="Dry run")
+    args = parser.parse_args()
+
+    tool = Tool(args.input, args.output, args.dry_run)
+    tool.write_yaml()

biocontainers2rseq/cleaner_batch.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+import argparse
+import os
+from pathlib import Path
+from collections import defaultdict
+from git import Repo
+
+from tool import Tool
+
+
+def find_dockerfiles(root_dir):
+    root = Path(root_dir).resolve()
+    dockerfiles = []
+
+    for dirpath, _, filenames in os.walk(root):
+        current = Path(dirpath)
+
+        for f in filenames:
+            if f == "Dockerfile":
+                dockerfiles.append(current / f)
+
+    return root, dockerfiles
+
+def get_tool_name(path, root):
+    rel = path.relative_to(root)
+    return rel.parts[0] if len(rel.parts) > 1 else "unknown"
+
+
+def get_last_commit_time(repo, file_path):
+    """
+    Returns last commit timestamp affecting the file.
+    """
+    commits = list(repo.iter_commits(paths=file_path, max_count=1))
+    if not commits:
+        return 0
+    return commits[0].committed_date
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('biocontainers_repo', type=str, help="Biocontainers reposity")
+    parser.add_argument('rse_repo', type=str, help="Research-software-ecosystem repository")
+    parser.add_argument('--dry-run', action='store_true', help="Dry run")
+
+    args = parser.parse_args()
+
+    repo_path = Path(args.biocontainers_repo).resolve()
+    repo = Repo(repo_path)
+    root, dockerfiles = find_dockerfiles(str(repo_path))
+
+    grouped = defaultdict(list)
+
+    for df in dockerfiles:
+        tool = get_tool_name(df, root)
+        grouped[tool].append(df)
+
+    selected = {}
+
+    for tool, files in grouped.items():
+        print(f"\nProcessing tool: {tool}")
+
+        versions = []
+
+        best_file = None
+        best_ts = -1
+
+        for f in files:
+            versions.append(str(f.relative_to(root)))
+
+            rel = str(f.relative_to(root))
+            ts = get_last_commit_time(repo, rel)
+
+            if ts > best_ts:
+                best_file = f
+                best_ts = ts
+
+        if best_file:
+            selected[tool] = best_file
+            print(best_file)
+            print(f"  SELECTED: {best_file.relative_to(root)} among {versions}")
+
+            tool = Tool(best_file, args.rse_repo, args.dry_run)
+            tool.write_yaml()
+
+        else:
+            raise Exception("Error on tool " + tool)

biocontainers2rseq/requirements.txt

@@ -0,0 +1 @@
+gitpython

biocontainers2rseq/tool.py

@@ -0,0 +1,127 @@
+import os
+import re
+
+from copy import deepcopy
+
+from yaml import dump
+try:
+    from yaml import CDumper as Dumper
+except ImportError:
+    from yaml import Dumper
+
+class Tool:
+
+    def parse_label_string(self, label_body):
+        """
+        Parse Docker LABEL string into key-value pairs.
+        Handles:
+          key=value
+          key="value"
+          key='value'
+        """
+        kv = {}
+
+        # Split while preserving quoted values
+        tokens = re.findall(r'(\S+=".*?"|\S+=\'.*?\'|\S+)', label_body)
+
+        for token in tokens:
+            if "=" not in token:
+                continue
+
+            key, value = token.split("=", 1)
+            key = key.strip()
+            value = value.strip()
+
+            # Remove surrounding quotes
+            if (value.startswith('"') and value.endswith('"')) or (
+                value.startswith("'") and value.endswith("'")
+            ):
+                value = value[1:-1]
+
+            kv[key] = value
+
+        return kv
+
+    def parse_Dockerfile(self, dockerfile_path):
+        LABEL_RE = re.compile(r'LABEL\s+(.*)', re.IGNORECASE)
+        labels = {}
+
+        with open(dockerfile_path, 'r') as f:
+            lines = f.readlines()
+
+        buffer = ""
+        collecting = False
+
+        for line in lines:
+            line = line.strip()
+
+            # handle continuation lines
+            if line.endswith("\\"):
+                buffer += " " + line[:-1].strip()
+                collecting = True
+                continue
+            else:
+                if collecting:
+                    buffer += " " + line
+                    full_line = buffer.strip()
+                    buffer = ""
+                    collecting = False
+                else:
+                    full_line = line
+
+            match = LABEL_RE.match(full_line)
+            if match:
+                parsed = self.parse_label_string(match.group(1))
+                labels.update(parsed)
+
+        if 'software' not in labels:
+            raise Exception("Missing 'software' label in Dockerfile")
+        if 'software.version' not in labels:
+            raise Exception("Missing 'software.version' label in Dockerfile")
+        self.labels = labels
+
+    def __init__(self, dockerfile, output_folder, dry_run=False):
+        self.dockerfile = dockerfile
+        self.output_folder = os.path.abspath(output_folder)
+        self.dry_run = dry_run
+
+        self.parse_Dockerfile(dockerfile)
+
+    def write_yaml(self):
+        name = self.labels['software']
+        version = self.labels['software.version']
+
+        biotools = None
+        if self.labels.get('extra.identifiers.biotools'):
+            biotools = self.labels.get('extra.identifiers.biotools').strip()
+
+        all_tmpdir = os.path.join(self.output_folder, "import/biocontainers/")
+
+        files_to_write = [all_tmpdir + '{}.biocontainers.yaml'.format(name)]
+
+        if not os.path.exists(all_tmpdir):
+            os.makedirs(all_tmpdir)
+
+        if biotools is not None:
+            biotool_tmpdir = self.output_folder + '/data/{}/'.format(biotools)
+            if not os.path.exists(biotool_tmpdir):
+                os.makedirs(biotool_tmpdir)
+            files_to_write.append(biotool_tmpdir + '{}.biocontainers.yaml'.format(name))
+
+        data = {
+                'software': name,
+                "url": "biocontainers/" + name + ":" + version,
+                "version": version,
+                "type": "Container file",
+                'labels': deepcopy(self.labels)
+        }
+
+        softwares = {'softwares': {}}
+        softwares["softwares"][name] = data
+
+        for file_path in files_to_write:
+                print("Will write content to " + file_path)
+                if self.dry_run:
+                    continue
+                with open(file_path, 'w') as fp:
+                    dump(softwares, fp, Dumper=Dumper)