last commit Miguel

Author: Str3am786

__init__.py

@@ -0,0 +1 @@
+__version__ = "0.0.1"

__main__.py

@@ -0,0 +1,7 @@
+from object_creator.pipeline import *
+
+output_directory = "/Users/pingamax2/Documents" #output path
+dois_txt = "./test/dois.txt" #path to txt of dois
+if __name__ == '__main__':
+    dois_txt_to_bidir_json(dois_txt=dois_txt,output_dir=output_directory)
+    dois_txt_to_unidir_json(dois_txt=dois_txt,output_dir=output_directory)

download_pdf/download_pipeline.py

@@ -22,6 +22,7 @@ def _is_arxiv(doi):
     else:
         return False
 
+
 def pdf_download_pipeline(doi, output_directory):
     """
     Input

metadata_extraction/github_extractor_tika.py

@@ -15,6 +15,26 @@ def read_pdf(pdf_path):
     except Exception as e:
         return []
 
+def find_abstract_index(pdf_data):
+    index = 0
+    for line in pdf_data:
+        if "abstract" in line.lower():
+            if index < len(pdf_data):
+                return index
+        index +=1
+def get_possible_abstract(pdf_data):
+    try:
+        index = find_abstract_index(pdf_data)
+        if index:
+            return ''.join(pdf_data[index:index+50])
+    except Exception as e:
+        print(e)
+
+def find_github_in_abstract(pdf_data):
+    abstract = get_possible_abstract(pdf_data)
+    if abstract:
+        return look_for_github_urls(abstract)
+
 #regular expression to get all the urls, returned as a list
 def get_git_urls(text):
     """

metadata_extraction/paper_obj.py

@@ -3,13 +3,14 @@
 from utils.regex import str_to_doiID, str_to_arxivID
 
 class PaperObj:
-    def __init__(self, title, urls, doi, arxiv, file_name, file_path):
+    def __init__(self, title, urls, doi, arxiv, abstract, file_name, file_path):
         self._title = title
         self._urls = urls
         self._doi = str_to_doiID(doi)
         self._arxiv = str_to_arxivID(arxiv)
         self._file_name = file_name
         self._file_path = file_path
+        self._abstract = abstract
 
     @property
     def title(self):
@@ -27,6 +28,14 @@ def urls(self):
     def urls(self, value):
         self._urls = value
 
+    @property
+    def abstract(self):
+        return self._abstract
+
+    @urls.setter
+    def abstract(self, value):
+        self._abstract = value
+
     @property
     def doi(self):
         return self._doi
@@ -62,9 +71,10 @@ def file_path(self, value):
     def to_dict(self):
         return {
             'title': self._title,
-            'urls': self._urls,
+            'urls': self._abstract,
             'doi': self._doi,
             'arxiv': self.arxiv,
+            'abstract': self.abstract,
             'file_name': self._file_name,
             'file_path': self._file_path
         }

modelling/unidirectionality.py

@@ -31,7 +31,7 @@ def find_substring(substring, larger_string):
     return
 
 def _iterate_results(results, string_2_find):
-    if not results:
+    if (not results) or (not string_2_find):
         return False
     for result in results:
         value = safe_dic(safe_dic(result,"result"),'value')
@@ -51,6 +51,14 @@ def is_repo_unidir(paperObj, repo_json):
         #Repo title is close to the repo full title
         results = safe_dic(repo_data,'full_title')
         unidir = _iterate_results(results, paperObj.title)
+    if not unidir:
+        #Repo title is close to the repo full title
+        results = safe_dic(repo_data,'name')
+        unidir = _iterate_results(results, paperObj.abstract)
+    if not unidir:
+        #Repo title is close to the repo full title
+        results = safe_dic(repo_data,'full_title')
+        unidir = _iterate_results(results, paperObj.abstract)
     # See if paper title is within the description
     if not unidir:
         results = safe_dic(repo_data,'description')

object_creator/create_downloadedObj.py

@@ -2,7 +2,7 @@
 from download_pdf.downloaded_obj import DownloadedObj
 import os
 import json
-from .doi_to_metadata import metaDict_to_metaObj
+from .doi_to_metadata import metaDict_to_metaObj, doi_to_metadataObj
 
 
 def meta_to_dwnldd(metadataObj, output_dir):
@@ -20,8 +20,9 @@ def meta_to_dwnldd(metadataObj, output_dir):
         file_path = pdf_download_pipeline(doi=metadataObj.doi,output_directory=output_dir)
         file_name = os.path.basename(file_path)
         return DownloadedObj(title=metadataObj.title,doi=metadataObj.doi,arxiv=metadataObj.arxiv,file_name=file_name,file_path=file_path)
-    except:
+    except Exception as e:
         print("Error while creating the downloaded object")
+        print(str(e))
         return None
 
 
@@ -98,7 +99,55 @@ def metaJson_to_downloadedJson(meta_json, output_dir):
                   ensure_ascii=False)
     return output_path
 
+def doi_to_downloadedObj(doi,output_dir):
+    meta = doi_to_metadataObj(doi)
+    return meta_to_dwnldd(meta,output_dir)
+
+def doi_to_downloadedDic(doi,output_dir):
+    return downloaded_dictionary(doi_to_downloadedObj(doi, output_dir))
+
+
+def dois_to_downloadedDics(dois_list, output_dir):
+    result = {}
+    for doi in dois_list:
+        result.update(doi_to_downloadedDic(doi,output_dir))
+    return result
+def dois_txt_to_downloadedDics(dois_txt,output_dir):
+    try:
+        with open(dois_txt, 'r') as file:
+            dois = file.read().splitlines()
+    except:
+        print("Error while opening the txt")
+    return dois_to_downloadedDics(dois,output_dir)
+
+def doi_to_downloadedJson(doi,output_dir):
+    dict = doi_to_downloadedDic(doi, output_dir)
+    output_path = output_dir + "/" + "downloaded_metadata.json"
+    with open(output_path, 'w+') as out_file:
+        json.dump(dict, out_file, sort_keys=True, indent=4,
+                  ensure_ascii=False)
+    return output_path
+def dois_to_downloadedJson(dois,output_dir):
+    dict = dois_to_downloadedDics(dois, output_dir)
+    output_path = output_dir + "/" + "downloaded_metadata.json"
+    with open(output_path, 'w+') as out_file:
+        json.dump(dict, out_file, sort_keys=True, indent=4,
+                  ensure_ascii=False)
+    return output_path
+def dois_txt_to_downloadedJson(dois_txt,output_dir):
+    dict = dois_txt_to_downloadedDics(dois_txt, output_dir)
+    output_path = output_dir + "/" + "downloaded_metadata.json"
+    with open(output_path, 'w+') as out_file:
+        json.dump(dict, out_file, sort_keys=True, indent=4,
+                  ensure_ascii=False)
+    return output_path
 
+def download_from_doi(doi,output_dir):
+    return doi_to_downloadedJson(doi,output_dir)
+def download_from_doi_list(dois,output_dir):
+    return dois_to_downloadedJson(dois,output_dir)
+def download_from_doi_txt(dois_txt,output_dir):
+    return dois_to_downloadedJson(dois_txt, output_dir)
 def safe_dic(dic, key):
     try:
         return dic[key]

object_creator/downloaded_to_paperObj.py

@@ -1,4 +1,4 @@
-from metadata_extraction.github_extractor_tika import ranked_git_url, read_pdf
+from metadata_extraction.github_extractor_tika import ranked_git_url, read_pdf, get_possible_abstract
 from metadata_extraction.paper_obj import PaperObj
 from object_creator.create_downloadedObj import downloadedDic_to_downloadedObj
 import json
@@ -17,12 +17,13 @@ def downloaded_to_paperObj(downloadedObj):
     try:
         pdf_data = read_pdf(downloadedObj.file_path)
         urls = ranked_git_url(pdf_data)
+        abstract = get_possible_abstract(pdf_data)
         title = downloadedObj.title
         doi = downloadedObj.doi
         arxiv = downloadedObj.arxiv
         file_name = downloadedObj.file_name
         file_path = downloadedObj.file_path
-        return PaperObj(title, urls, doi, arxiv, file_name, file_path)
+        return PaperObj(title, urls, doi, arxiv, abstract, file_name, file_path)
     except Exception as e:
         print(str(e))
         print("Error while trying to read from the pdf")

object_creator/paper_obj_utils.py

@@ -7,7 +7,8 @@ def paperDict_to_paperObj(paper_dict):
     file_name = safe_dic(paper_dict,"file_name")
     file_path = safe_dic(paper_dict,"file_path")
     urls = safe_dic(paper_dict,"urls")
-    return PaperObj(title=title, urls=urls, doi=doi, arxiv=arxiv, file_name=file_name, file_path=file_path)
+    abstract = safe_dic(paper_dict,"abstract")
+    return PaperObj(title=title, urls=urls, doi=doi, arxiv=arxiv, file_name=file_name, file_path=file_path, abstract=abstract)
 
 
 def safe_dic(dic, key):

object_creator/pipeline.py

@@ -28,6 +28,16 @@ def pipeline_single_bidir(doi,output_dir):
     paper = doi_to_paper(doi,output_dir)
     result = check_bidir(paper,output_dir)
     return result
+def pipeline_single_unidir(doi,output_dir):
+    '''
+    @Param doi: doi
+    @Param output_dir: where the pdf will be downloaded to
+    :returns
+    dictionary with doi and the urls found that are unidirectional for that doi
+    '''
+    paper = doi_to_paper(doi,output_dir)
+    result = check_unidir(paper,output_dir)
+    return result
 
 def pipeline_multiple_bidir(list_dois, output_dir):
     '''
@@ -48,6 +58,25 @@ def pipeline_multiple_bidir(list_dois, output_dir):
     except Exception as e:
         print(str(e))
         return None
+def pipeline_multiple_unidir(list_dois, output_dir):
+    '''
+    @Param list_dois: list of dois
+    @Param output_dir: where the pdf will be downloaded to
+    :returns
+    dictionary with dois and the urls found that are unidirectional for that doi
+    '''
+    result = {}
+    try:
+        for doi in list_dois:
+            paper = doi_to_paper(doi,output_dir)
+            if not paper:
+                continue
+            if (unidir:=(check_unidir(paper,output_dir))):
+                result.update(unidir)
+        return result
+    except Exception as e:
+        print(str(e))
+        return None
 
 def pipeline_txt_dois_bidir(dois_txt, output_dir):
     '''
@@ -63,6 +92,20 @@ def pipeline_txt_dois_bidir(dois_txt, output_dir):
         print("Error while opening the txt")
     return pipeline_multiple_bidir(dois,output_dir)
 
+def pipeline_txt_dois_unidir(dois_txt, output_dir):
+    '''
+    @Param dois_txt: dois seperated by \n within a txt
+    @Param output_dir: where the pdf will be downloaded to
+    :returns
+    dictionary with dois and the urls found that are bidirectional for that doi
+    '''
+    try:
+        with open(dois_txt, 'r') as file:
+            dois = file.read().splitlines()
+    except:
+        print("Error while opening the txt")
+    return pipeline_multiple_unidir(dois,output_dir)
+
 def from_papers_json_to_bidir(papers_json, output_dir):
     '''
     @Param papers_json: json of papers, Key: DOI, V: paperObj (as a dictionary)
@@ -112,6 +155,15 @@ def dois_txt_to_bidir_json(dois_txt, output_dir):
     '''
     output_path = os.path.join(output_dir,"bidir.json")
     return dict_to_json(pipeline_txt_dois_bidir(dois_txt,output_dir),output_path)
+def dois_txt_to_unidir_json(dois_txt, output_dir):
+    '''
+    @Param dois_txt: dois seperated by \n within a txt
+    @Param output_dir: where the pdf will be downloaded to
+    :returns
+    path to output JSON
+    '''
+    output_path = os.path.join(output_dir,"unidir_20_07.json")
+    return dict_to_json(pipeline_txt_dois_unidir(dois_txt,output_dir),output_path)
 
 
 def from_papers_json_to_unidir(papers_json, output_dir):
@@ -133,7 +185,7 @@ def from_papers_json_to_unidir(papers_json, output_dir):
         unidir = check_unidir(paper, output_dir)
         if unidir:
             result.update(unidir)
-    return dict_to_json(result,output_path=os.path.join(output_dir,"unidir.json"))
+    return dict_to_json(result,output_path=os.path.join(output_dir,"unidir_20_07.json"))