prod_source_eval_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import all libraries\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "import bibtexparser\n",
    "from dateutil import parser\n",
    "import json\n",
    "import requests\n",
    "import tldextract\n",
    "from collections import defaultdict\n",
    "import re\n",
    "import mwparserfromhell\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Given the DOI, PMID, PMC number, fetch journal's meta data\n",
    "\n",
    "def get_metainfo_doi(doi):\n",
    "    \"\"\"Input: doi string\n",
    "    Output: the journal name and date published of the article. Return None for each value if the can't parsed\n",
    "    \"\"\"\n",
    "    res = requests.get(\"http://dx.doi.org/\"+ doi, headers={\"Accept\": \"application/x-bibtex\"})\n",
    "    res = res.content.decode('utf-8')\n",
    "    bibtext = bibtexparser.loads(res).entries\n",
    "    if len(bibtext) >0 :\n",
    "        journal = bibtext[0][\"journal\"].strip() if \"journal\" in bibtext[0] else None\n",
    "        time_published = \"\"\n",
    "\n",
    "        if \"year\" in bibtext[0]:\n",
    "            time_published += bibtext[0][\"year\"]\n",
    "        if \"month\" in bibtext[0]:\n",
    "            time_published += \" \" + bibtext[0][\"month\"]\n",
    "        if \"day\" in bibtext[0]:\n",
    "            time_published += \" \" + bibtext[0][\"day\"]\n",
    "        if len(time_published) > 0:\n",
    "            time_published = parser.parse(time_published)\n",
    "        else:\n",
    "            time_published = None\n",
    "\n",
    "        return journal, time_published\n",
    "    else:\n",
    "        return None, None\n",
    "\n",
    "\n",
    "def get_metainfo_pmc(pmc):\n",
    "    res = requests.get(\"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pmc&id=\"+pmc+\"&retmode=json\")\n",
    "    res = res.content.decode(\"utf-8\")\n",
    "    res = json.loads(res)\n",
    "    data = res[\"result\"][pmc]\n",
    "    journal, time_published = None, None\n",
    "    if \"error\" in data:\n",
    "        return None, None\n",
    "    else:\n",
    "        journal = data[\"fulljournalname\"].strip()\n",
    "        time_published = parser.parse(data[\"pubdate\"])\n",
    "        return journal, time_published\n",
    "\n",
    "\n",
    "def get_metainfo_pmid(pmid):\n",
    "    res = requests.get(\"https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id=\"+pmid+\"&retmode=json\")\n",
    "    res = res.content.decode(\"utf-8\")\n",
    "    res = json.loads(res)\n",
    "    data = res[\"result\"][pmid]\n",
    "    journal, time_published = None, None\n",
    "    if \"error\" in data:\n",
    "        return None, None\n",
    "    else:\n",
    "        journal = data[\"fulljournalname\"].strip()\n",
    "        time_published = parser.parse(data[\"pubdate\"])\n",
    "        return journal, time_published"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_html(page_url):\n",
    "  \"\"\" This function parse metadata of citations from HTML tag.\n",
    "    Input: wiki_url\n",
    "    Output: a parsed citation list from HTML. Each citation has format key: value\n",
    "        key: the text version of all citation\n",
    "        value: a dictionary with schema \n",
    "            {\"external_link\": str, \"type\": str, \"html_tag\": HTMLElement, \"publisher\": str | None, \"date\": datetime |None}\"\"\"\n",
    "  \n",
    "  citation_types = {'web', 'journal', 'book', 'conference', 'news'}\n",
    "  all_parsed_citations = defaultdict(dict)\n",
    "  response = requests.get(page_url)\n",
    "  soup = BeautifulSoup(response.content, 'html.parser')\n",
    "\n",
    "  # Get all the references\n",
    "  all_ref = []\n",
    "  ordered_ref_lst = soup.find_all(\"ol\", {\"class\": \"references\"})\n",
    "  for each in ordered_ref_lst:\n",
    "    refs = each.find_all(\"li\")\n",
    "    all_ref += refs\n",
    "\n",
    "  for ele in all_ref:\n",
    "    #Check if it has <span class=\"reference text\">\n",
    "    ref = ele.find(\"span\", {\"class\":\"reference-text\"})\n",
    "    \n",
    "    source_type = \"other\" #first default value for source_type\n",
    "\n",
    "    if ref:\n",
    "      # TASK: get all essential information from citation tag\n",
    "      citation_key = ref.get_text()\n",
    "      hyperlink = ref.find(\"a\", {\"class\": \"external text\"})\n",
    "      external_link = hyperlink[\"href\"] if hyperlink else None\n",
    "\n",
    "      # TASK: find source type, ie whether it's 'web', 'journal', 'book', 'conference', 'news'\n",
    "      cite_tag = ref.find(\"cite\") \n",
    "      if cite_tag:\n",
    "        for class_tag in cite_tag[\"class\"]:\n",
    "          if class_tag in citation_types:\n",
    "            source_type = class_tag      \n",
    "            break\n",
    "          \n",
    "      # TASK: get publisher (journal name for journal or conference, domain website for webs, 'work' for news)\n",
    "      #for journal, conference, others look for DOI or PMID or PMC\n",
    "      if source_type in {'journal', 'conference', 'other'}:\n",
    "        has_doi = ref.find(\"a\", {\"title\": \"Doi (identifier)\"})\n",
    "        has_pmc = ref.find(\"a\", {\"title\": \"PMC (identifier)\"})\n",
    "        has_pmid = ref.find(\"a\", {\"title\": \"PMID (identifier)\"})\n",
    "        journal, date = None, None\n",
    "        if has_doi:\n",
    "          doi = has_doi.find_next(\"a\", {\"class\": \"external text\"})\n",
    "          journal, date = get_metainfo_doi(doi.text)\n",
    "        elif has_pmc:\n",
    "          pmc = has_pmc.find_next(\"a\", {\"class\": \"external text\"})\n",
    "          journal, date = get_metainfo_pmc(pmc.text)\n",
    "        elif has_pmid:\n",
    "          pmid = has_pmid.find_next(\"a\", {\"class\": \"external text\"})\n",
    "          journal, date = get_metainfo_pmid(pmid.text)         \n",
    "        \n",
    "        all_parsed_citations[citation_key] = {\"external_link\": external_link, \"type\": source_type, \"html_tag\": ele, \"publisher\": journal, \"date\": date}\n",
    "\n",
    "      # for news, web, other that hasn't been parsed, publisher is the domain of the website \n",
    "      elif source_type in {'news', 'web', 'other'}:\n",
    "        publisher = tldextract.extract(external_link).domain if external_link else None\n",
    "        all_parsed_citations[citation_key] = {\"external_link\": external_link, \"type\": source_type, \"html_tag\": ele, \"publisher\": publisher, \"date\": None}\n",
    "        \n",
    "\n",
    "  return all_parsed_citations\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After finish parsing with HTML tag, we fetch the wikitext version of the page, match it with the HTML tag to extract more information about the citation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_match_wikitext(wiki_url):\n",
    "    \"\"\"\n",
    "    This function parse wikitext version of the citations, match it with the HTML version, \n",
    "    and extract more information, such as publisher and date that weren't extracted on the HTML.\n",
    "    Input: wiki_url\n",
    "    Output: a fully parsed citation list. Each citation has format key: value\n",
    "        key: the text version of all citation\n",
    "        value: a dictionary with schema \n",
    "            {\"external_link\": str, \"type\": str, \"html_tag\": HTMLElement, \"publisher\": str | None, \"date\": datetime |None}\n",
    "    \"\"\"\n",
    "    parsed_citation = parse_html(wiki_url)\n",
    "    print(\"ALL citation\", len(parsed_citation))\n",
    "    wiki_page = wiki_url.split(\"wiki/\")[1]\n",
    "    url = \"https://en.wikipedia.org/w/index.php?title=\" + wiki_page +\"&action=raw\"\n",
    "    response = requests.get(url)\n",
    "    text = response.text\n",
    "    wikicode = mwparserfromhell.parse(text)\n",
    "    \n",
    "\n",
    "    # Create a copy of not fully parsed citation, ie one that lacks publisher or date param\n",
    "    not_fully_parsed = defaultdict(dict)\n",
    "    for key, val in parsed_citation.items():\n",
    "        if not val[\"publisher\"] or not val[\"date\"]:\n",
    "            not_fully_parsed[key] = val\n",
    "\n",
    "    for tpl in wikicode.filter_templates(matches=\"{{cite\"):\n",
    "        #tpl is template, for a template in wikitext\n",
    "        found_match = None\n",
    "        \n",
    "        # Match on external link:\n",
    "        if tpl.has_param(\"url\"):\n",
    "            external_url = tpl.get(\"url\").split(\"=\")[1]\n",
    "            for key, val in not_fully_parsed.items():\n",
    "                if val[\"external_link\"]:\n",
    "                    if val[\"external_link\"].strip() == external_url.strip():\n",
    "                        found_match = key\n",
    "                        break\n",
    "        # if not found match by URL, find by title\n",
    "        if not found_match:\n",
    "            if tpl.has_param(\"title\"):\n",
    "                #Get the title of citation in without formatting text\n",
    "                title = tpl.get(\"title\").split(\"=\")[1] \n",
    "                title = re.sub('[^A-Za-z0-9 ]+', '', title) #filter out extra formatting\n",
    "                for key in not_fully_parsed.keys():\n",
    "                    if title in key:\n",
    "                        found_match = key\n",
    "                        break\n",
    "        \n",
    "        if found_match:\n",
    "            # Fetch publisher/ journal name from wikitext\n",
    "            if not parsed_citation[found_match][\"publisher\"]:\n",
    "                publisher = None\n",
    "                if tpl.has_param(\"journal\"): #for journal name\n",
    "                    publisher = tpl.get(\"journal\").split(\"=\")[1] \n",
    "                elif tpl.has_param(\"publishder\"): #for website or book publisher\n",
    "                    publisher = tpl.get(\"publisher\").split(\"=\")[1]\n",
    "                elif tpl.has_param(\"work\"): #for news/ magazine name\n",
    "                    publisher = tpl.get(\"work\").split(\"=\")[1]\n",
    "                \n",
    "                if publisher:\n",
    "                    publisher = re.sub('[^A-Za-z0-9 ]+', '', publisher)\n",
    "                parsed_citation[found_match][\"publisher\"] = publisher\n",
    "\n",
    "            # Fetch publication date from wikitext\n",
    "            if not parsed_citation[found_match][\"date\"]:\n",
    "                date = None\n",
    "                if tpl.has_param(\"date\"):\n",
    "                    date = tpl.get(\"date\").split(\"=\")[1]\n",
    "                    if len(date) >= 4: #at least 4 digits for year, or yy-mm format\n",
    "                        date = parser.parse(date)\n",
    "                parsed_citation[found_match][\"date\"] = date\n",
    "            \n",
    "\n",
    "    return parsed_citation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_scholarly_sources(citation):\n",
    "    \"\"\"\n",
    "    This function evaluates the tag for a scholarly souces (journal, conference, or other type)\n",
    "    Input: \n",
    "        the citation dictionary, which has format {\"external_link\": str, \"type\": str, \"html_tag\": HTMLElement, \"publisher\": str | None, \"date\": datetime |None}\n",
    "    Output:\n",
    "        the tag for citation (red, green, yellow, unknown)\n",
    "    \"\"\"\n",
    "    # TODO: find check for green, both in domain check and name check\n",
    "    \n",
    "\n",
    "    # read the dictionaries of flags from the json file\n",
    "    with open(\"scholarly_flags.json\", \"r\") as f:\n",
    "        all_flags = json.load(f)\n",
    "\n",
    "    found_tag = False\n",
    "    # Check on the domain of external link\n",
    "    if citation[\"external_link\"]:\n",
    "        domain = tldextract.extract(citation[\"external_link\"]).domain\n",
    "        if domain in all_flags[\"red_scholarly_reverse\"]:\n",
    "            found_tag = True\n",
    "            # print(\"The citation is\", citation, \"which matches the red list item\", red_flags_reverse[domain])\n",
    "            return \"red\"\n",
    "        elif domain in all_flags[\"yellow_scholarly_reverse\"]:\n",
    "            # print(\"The citation is \", citation, \"which matches the yellow list item \", yellow_flags_reverse[domain])\n",
    "            return \"yellow\"\n",
    "    #check on the name dictionary\n",
    "    if not found_tag:\n",
    "        if citation[\"publisher\"] in all_flags[\"red_scholarly\"]:\n",
    "            found_tag = True\n",
    "            # print(\"The citation is \", citation, \"which matches the red list item\", citation[\"publisher\"])\n",
    "            return \"red\"\n",
    "        elif citation[\"publisher\"] in all_flags[\"yellow_scholarly\"]:\n",
    "            found_tag = True\n",
    "            # print(\"The citation is \", citation, \"which matches the yellow list item\", citation[\"publisher\"])\n",
    "            return \"yellow\"\n",
    "\n",
    "    if not found_tag:\n",
    "        return \"unknown\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_non_scholarly_sources(citation, citation_val):\n",
    "    \"\"\"\n",
    "    This function evaluates the tag for a non-scholarly scholarly souces (journal, conference, or other type)\n",
    "    Input: \n",
    "        the citation dictionary, which has format {\"external_link\": str, \"type\": str, \"html_tag\": HTMLElement, \"publisher\": str | None, \"date\": datetime |None}\n",
    "    Output:\n",
    "        the tag for citation (red, green, yellow, unknown)\n",
    "    \"\"\"\n",
    "    with open(\"non_scholarly_flags.json\", \"r\") as f:\n",
    "        non_scholarly_flags = json.load(f)\n",
    "    \n",
    "    # Check if the tag is found in either name or is part of external_link\n",
    "    for key, val in non_scholarly_flags.items():\n",
    "        for source in val:\n",
    "            if source in citation_val[\"external_link\"]:\n",
    "                return key\n",
    "            elif source in citation:\n",
    "                return key\n",
    "    return \"unknown\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_source_quality(wiki_url):\n",
    "    \"\"\"\n",
    "    Go through each parsed citation, check them through the red-yellow-green tag\n",
    "    Return: red, yellow, green lists that include the citations belong to each category\n",
    "    \"\"\"\n",
    "    parsed = parse_match_wikitext(wiki_url)\n",
    "    red_flag_list = []\n",
    "    yellow_flag_list = []\n",
    "    green_flag_list = []\n",
    "    unknown_list = []\n",
    "\n",
    "    for citation, val in parsed.items():\n",
    "        eval = None\n",
    "        # Check for journals/ conference/ other\n",
    "        if val[\"type\"] in {\"journal\", \"conference\", \"other\"}:\n",
    "           eval = eval_scholarly_sources(val)\n",
    "        \n",
    "        elif val[\"type\"] in {\"web\", \"book\", \"news\", \"other\"}:\n",
    "            eval = eval_non_scholarly_sources(citation, val)\n",
    "        \n",
    "        if eval == \"red\":\n",
    "            red_flag_list.append((citation, val[\"publisher\"]))\n",
    "        elif eval == \"yellow\":\n",
    "            yellow_flag_list.append((citation, val[\"publisher\"]))\n",
    "        elif eval == \"green\":\n",
    "            green_flag_list.append((citation, val[\"publisher\"]))\n",
    "        elif eval == \"unknown\":\n",
    "            unknown_list.append((citation, val[\"publisher\"]))\n",
    "    \n",
    "    return red_flag_list, yellow_flag_list, green_flag_list, unknown_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ALL citation 63\n",
      "Red flag source: [('\"The Communist History Of Albania\". (http://thebalkanista.com). The Balkanista. 25 February 2018.', 'thebalkanista'), ('\"Parliamentary elections held on May 26th, 1996 in Albania and the political crises they caused\" (PDF). American Research Journal of History and Culture. 6 (1, 1–9). ISSN\\xa02379-2914.', 'American Research Journal of History and Culture '), ('\"Election - June 2001\". www.globalsecurity.org/. GlobalSecurity.org.', 'globalsecurity')]\n",
      "Yellow flag source:  []\n",
      "Green source:  [('\"Parties and Partners\". European People\\'s Party.', 'epp'), ('\"Republic of Albania: A Technical Assessment of the Election of the Peoples Assembly, March 9 - April 2, 1992\". IFES. 30 April 1992.', 'ifes'), ('\"Albania timeline\". (http://news.bbc.co.uk). BBC United Kingdom. 24 January 2012.', 'bbc'), ('\"Albanian Party Hails Ex-Communist Defeat\". The New York Times. Reuters. 24 March 1992.', 'nytimes'), ('\"World: Europe Violent protests at Albanian PM\\'s office\". news.bbc.co.uk. BBC. 13 September 1998.', 'bbc'), ('Times, Tirana (26 July 2013). \"Basha elected new Democratic Party leader\". Tirana Times. Retrieved 20 March 2017.', 'tiranatimes'), ('\"Albanian protesters rally, pitch tent to demand free elections\". Reuters. 18 February 2017. Retrieved 25 June 2017.', 'reuters'), ('\"Albanian anti-govt protesters try to storm parliament - The Washington Post\". The Washington Post. 25 March 2019. Archived from the original on 25 March 2019. Retrieved 21 June 2022.', 'archive')]\n",
      "Undetermined sources:  [('\"European Election Watch Albania\". Center for Strategic and International Studies. Retrieved 4 January 2022.', 'csis'), ('\"Berisha Claims He\\'s Back in Charge of Albania\\'s Democratic Party\". 23 May 2022.', 'balkaninsight'), ('\"Democrat Asks MP to Not Attend Alibeaj\\'s Meet on New President\".', 'albaniandailynews'), ('\"Albanian opposition won\\'t elect new leader following chairman\\'s resignation\". 23 March 2022.', 'euractiv'), ('\"Flamur Noka is elected General Secretary of the Democratic Party\". politiko.al. Retrieved 29 June 2022.', 'politiko'), ('Robert Elsie (2010) Historical Dictionary of Albania, Scarecrow Press, pxliii', None), ('\"Berisha nga Durrësi: 40 mijë demokratë u ngritën për t\\'i rikthyer vlerën partisë, Prokuroria Speciale është bërë e rrezikshme për Ramën\". 17 May 2022.', 'newsbomb'), ('Nordsieck, Wolfram (2017). \"Albania\". Parties and Elections in Europe.', 'parties-and-elections'), ('\"The Albanian Electoral Systems since 1990\" (PDF). Albanian Elections Observatory Brief. Hellenic Foundation for European & Foreign Policy (1). 19 April 2013. Retrieved 2 July 2013.', 'Albanian Elections Observatory Brief'), ('Doder, Dusko (20 January 1991). \"At Last, Albanians Feel Winds of Change\". The Chicago Tribune. Tirana. Retrieved 5 July 2013.', 'chicagotribune'), ('\"Communist-Era Disappearances Still Haunt Albania\". (https://balkaninsight.com). Balkan Insight. 15 March 2021.', 'balkaninsight'), ('\"Albania - Collapse of communism | Britannica\". www.britannica.com. Retrieved 24 November 2021.', 'britannica'), ('\"Albania Table of Contents\". (http://countrystudies.us). countrystudies.', 'countrystudies'), ('\"Albania - GOVERNMENT\". Mongabay.com. Retrieved 8 May 2021.', 'mongabay'), ('\"Parliamentary elections held on May 26th, 1996 in Albania and the political crises they caused\". American Research Journals. Retrieved 8 May 2021.', 'arjonline'), ('\"OBSERVATION OF THE PARLIAMENTARY ELECTIONS HELD IN THE REPUBLIC OF ALBANIA – MAY 26 and JUNE 2, 1996\" (PDF). Office for Democratic Institutions and Human Rights. Retrieved 8 May 2021.', 'osce'), ('\"Parliamentary Chamber: Kuvendi Popullor – ELECTIONS HELD IN 1996\". Inter-Parliamentary Union. Retrieved 8 May 2021.', 'ipu'), ('Hockstader, Lee. \"ALBANIAN DREAMS SHATTER IN PYRAMID SCHEMES\\' FALL\". Washington Post.', 'washingtonpost'), ('Jarvis, Christopher. \"The Rise and Fall of Albania\\'s Pyramid Schemes\". imf.org. IMF.', 'imf'), ('Skarda, Erin (7 March 2012). \"Albanian Pyramid Schemes\". Time.', None), ('\"Albania 1997 Legislative Election\". binghamton.edul. The Center on Democratic Performance, Binghamton University.', 'binghamton'), ('Human Rights Watch., International Foundation for Election Systems. \"Albania: Voting procedures during the 1997, 2000 and 2001 elections\". refworld.org. Canada: Immigration and Refugee Board of Canada. Retrieved 8 May 2021.', 'refworld'), ('Topalli, Jozefina. \"On the political assassinations, arrests and persecution of the opposition by the neo-communist regime in Albania\". hartford-hwp.com. Retrieved 8 May 2021.', 'hartford-hwp'), ('\"REPUBLIC OF ALBANIA PARLIAMENTARY ELECTIONS 24 June – 19 August 2001\" (PDF). www.osce.org/. Office for Democratic Institutions and Human Rights.', 'osce'), ('\"The Gerdec explosions trial ends with 19 convictions\". country.eiu.com. Retrieved 7 June 2021.', 'eiu'), ('\"United States Concludes Clean Up Effort at Albanian Explosion Site\". U.S. Department of State. Retrieved 7 June 2021.', 'state'), ('\"IPU PARLINE database: ALBANIA (Kuvendi) ELECTIONS IN 2009\". archive.ipu.org. Retrieved 30 April 2022.', 'ipu'), ('Christo, Van. \"The 2009 Elections in Albania | The Frosina Information Network\". Retrieved 30 April 2022.', 'frosina'), ('\"Press corner\". European Commission - European Commission. Retrieved 30 April 2022.', 'europa'), ('\"Basha Elected Head of Albania\\'s Democrats\\xa0:: Balkan Insight\". www.balkaninsight.com. 23 July 2013. Retrieved 20 March 2017.', 'balkaninsight'), ('albania-opposition-party-elects-new-leadership', None), ('\"Basha Announces Thorough Reform of PD - Exit\". Exit. 11 December 2016. Retrieved 20 March 2017.', 'exit'), ('\"Albania Opposition Parties to Boycott June Elections\\xa0:: Balkan Insight\". www.balkaninsight.com. 11 April 2017. Retrieved 25 June 2017.', 'balkaninsight'), ('\"\"Partia Demokratike-Aleanca për Ndryshim\"/ 13 partitë që garojnë bashkë me demokratët më 25 Prill\". Ora News. Retrieved 21 June 2022.', 'oranews'), ('\"Komisioni Qendror i Zgjedhjeve\". kqz.gov.al. Retrieved 21 June 2022.', 'kqz'), ('jalvarado (11 May 2021). \"Albania: April 2021 Parliamentary Election Report\". www.ndi.org. Retrieved 21 June 2022.', 'ndi'), ('\"Public Designation of Albanian Sali Berisha Due to Involvement in Significant Corruption\". United States Department of State. Retrieved 21 June 2022.', 'state'), ('\"Basha përjashton Berishën nga grupi parlamentar PD derisa të zbardhet çështja Non Grata\". Ora News. Retrieved 21 June 2022.', 'oranews'), ('\"Kuvendi i thirrur nga Berisha, shkarkon Bashën dhe Kryesinë e PD\". Euronews Albania. 11 December 2021. Retrieved 21 June 2022.', 'euronews'), ('\"2022 Albania: \"House of Freedom\" vs. Drug cartel\". www.ifimes.org. Retrieved 21 June 2022.', 'ifimes'), ('\"STATEMENT – Tomorrow\\'s Partial Local Elections extremely important for the prevalence of democratic values and principles in the future of the country - KRIIK\". 6 March 2022. Retrieved 21 June 2022.', 'kriik'), ('\"Largohet nga drejtimi i PD Lulzim Basha, REAGON Ambasada Amerikane dhe jep mesazhin e RËNDËSISHËM\". sot.com.al. Retrieved 21 June 2022.', 'sot'), ('\"Alibeaj gathers himself, Berisha: He has only the right that Rama gives him. Immigrant vote warning\". politiko.al. Retrieved 21 June 2022.', 'politiko'), ('\"Belind Këlliçi shuan zërat se pas Evi Kokalarit qëndron Berisha: Shpifjet i përdori i pari Erion Veliaj!\". Lapsi.al (in Albanian). 17 June 2022. Retrieved 22 June 2022.', 'lapsi'), ('\"Clashes between Democrats / Belind Këlliçi sued for defamation, comes the immediate reaction of Evi Kokalari\". sot.com.al. Retrieved 22 June 2022.', 'sot'), ('\"Thousands of Albanians Participate in Opposition-Led Protest against Government\". Exit - Explaining Albania. 8 July 2022. Retrieved 8 July 2022.', 'exit'), ('\"Anti-government protests held in Albania\". www.aa.com.tr. Retrieved 8 July 2022.', 'aa'), ('\"Tirana, capital city of Albania\". www.visitcapitalcity.com. www.visitcapitalcity.com. 8 March 2020.', 'visitcapitalcity'), ('Bakke, Elisabeth (18 February 2010), \"Central and East European party systems since 1989\", Central and Southeast European Politics since 1989, Cambridge University Press, pp.\\xa064–90, retrieved 30 January 2023', None), ('\"Berisha Reacts after Alibeaj\\'s Withdrawal from Appeal\". ALBANIA DAILY NEWS. Retrieved 23 January 2023.', 'albaniandailynews'), ('\"Përjashtimi i deputetëve/ Alibeaj në realitet tjetër: Unë jam drejtuesi i PD\". Lapsi.al (in Albanian). 12 January 2023. Retrieved 23 January 2023.', 'lapsi'), ('\"\"Lufta\" e numrave në PD/ Kush është me Berishën e kush me Alibeajn-Emrat\". CNA. 11 January 2023. Retrieved 23 January 2023.', 'cna')]\n"
     ]
    }
   ],
   "source": [
    "# TEST\n",
    "a = check_source_quality(\"https://en.wikipedia.org/wiki/Democratic_Party_of_Albania\")\n",
    "print(\"Red flag source:\" , a[0])\n",
    "print(\"Yellow flag source: \", a[1])\n",
    "print(\"Green source: \", a[2])\n",
    "print(\"Undetermined sources: \", a[3])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}