extract_non_scholarly_flags.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['70 News',\n",
       " 'ABCnews.com.co',\n",
       " 'American News',\n",
       " 'banned.video',\n",
       " \"Before It's News\",\n",
       " 'bients.com',\n",
       " 'Bipartisan Report',\n",
       " 'bizstandardnews.com',\n",
       " 'Bloomberg.ma',\n",
       " 'The Boston Tribune',\n",
       " 'Breaking-CNN.com',\n",
       " 'BVA News',\n",
       " 'Cairns News',\n",
       " 'Celebtricity',\n",
       " 'CBSnews.com.co',\n",
       " 'cnn-trending.com',\n",
       " 'Conservative 101',\n",
       " 'Conservative Frontline',\n",
       " 'CountyNewsroom.info',\n",
       " 'Daily USA Update',\n",
       " 'Disclose.tv',\n",
       " 'DrudgeReport.com.co',\n",
       " 'Empire Herald',\n",
       " 'Empire News',\n",
       " 'Empire Sports',\n",
       " 'The Exposé',\n",
       " 'Fox-news24.com',\n",
       " 'The Gateway Pundit',\n",
       " 'Global Associated News',\n",
       " 'Globalresearch.ca',\n",
       " 'Gossip Mill Mzansi',\n",
       " 'The Grayzone',\n",
       " 'Guerilla News',\n",
       " 'Gummy Post',\n",
       " 'Houston Chronicle TV',\n",
       " 'Huzlers',\n",
       " 'InfoWars',\n",
       " 'Judicial Watch',\n",
       " 'ΚΒΟΙ2.com',\n",
       " 'KMT 11 News',\n",
       " 'The Last Line of Defense',\n",
       " 'Law Enforcement Today',\n",
       " 'Liberal Society',\n",
       " 'Liberty Writers News',\n",
       " 'LinkBeef',\n",
       " 'MV-media',\n",
       " 'Naha Daily',\n",
       " 'National Insider Politics',\n",
       " 'NationalReport.net',\n",
       " 'Natural News',\n",
       " 'NBCNews.com.co',\n",
       " 'News Breaks Here',\n",
       " 'NewsBuzzDaily',\n",
       " 'News Examiner',\n",
       " 'News Hound',\n",
       " 'The News Nerd',\n",
       " 'NewsPunch',\n",
       " 'NewsWatch33',\n",
       " 'The New York Evening',\n",
       " 'Next News Network',\n",
       " 'Now 8 News',\n",
       " 'Oneworld.press',\n",
       " 'OpIndia',\n",
       " 'Palmer Report',\n",
       " 'Peace Data',\n",
       " 'Postcard News',\n",
       " 'The Predicted',\n",
       " 'Prntly',\n",
       " 'React 365',\n",
       " 'The Reporterz',\n",
       " 'Snoopack',\n",
       " 'Spin Zone',\n",
       " 'St George Gazette',\n",
       " 'Stuppid',\n",
       " 'Super Station 95',\n",
       " 'TruNews',\n",
       " 'TrueTrumpers.com',\n",
       " 'UConservative',\n",
       " 'UndergroundNewsReport.com',\n",
       " 'The Unhived Mind',\n",
       " 'United Media Publishing',\n",
       " 'USA Daily Info',\n",
       " 'usatoday.com.co',\n",
       " 'US Postman',\n",
       " 'washingtonpost.com.co',\n",
       " 'WorldNetDaily',\n",
       " 'World News Daily Report']"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# Read the fake new lists and clean up the names\n",
    "red_news = []\n",
    "data = pd.read_csv(\"fake_news_list.csv\", header=None)\n",
    "for ele in data.iloc[:, 0]:\n",
    "    a = ele.split(\"(\")\n",
    "    red_news.append(a[0].strip())\n",
    "\n",
    "red_news\n",
    "        \n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Scrape from [Perennial Sources list](https://en.wikipedia.org/wiki/Wikipedia:Reliable_sources/Perennial_sources)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapping ={\n",
    "    \"s-gr\": \"green\", #generally reliable\n",
    "    \"s-nc\": \"yellow\", #no consensus\n",
    "    \"s-gu\": \"red\", #generally unreliable\n",
    "    \"s-d\": \"red\", #deprecated\n",
    "    \"s-b\": \"red\" #blacklisted\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'red': ['112 Ukraine',\n",
       "  'Ad Fontes Media',\n",
       "  'Advameg',\n",
       "  'AlterNet',\n",
       "  'Amazon',\n",
       "  'Anadolu Agency ',\n",
       "  'Ancestry.com',\n",
       "  'ANNA News',\n",
       "  'Answers.com',\n",
       "  'Antiwar.com',\n",
       "  'arXiv',\n",
       "  'Baidu Baike',\n",
       "  'bestgore.com',\n",
       "  'Bild',\n",
       "  'Blaze Media',\n",
       "  'Blogger',\n",
       "  'Breitbart News',\n",
       "  'BroadwayWorld',\n",
       "  'The California Globe',\n",
       "  'The Canary',\n",
       "  'CelebrityNetWorth',\n",
       "  'Centre for Research on Globalization',\n",
       "  'CESNUR',\n",
       "  'China Global Television Network',\n",
       "  'CNET ',\n",
       "  'CoinDesk',\n",
       "  'Consortium News',\n",
       "  'CounterPunch',\n",
       "  'Cracked.com',\n",
       "  'Crunchbase',\n",
       "  'The Daily Caller',\n",
       "  'Daily Express',\n",
       "  'Daily Kos',\n",
       "  'Daily Mail',\n",
       "  'Daily Sabah',\n",
       "  'Daily Star',\n",
       "  'The Daily Wire',\n",
       "  'Discogs',\n",
       "  'The Electronic Intifada',\n",
       "  'Encyclopaedia Metallum',\n",
       "  'The Epoch Times',\n",
       "  'Examiner.com',\n",
       "  'Facebook',\n",
       "  'FamilySearch',\n",
       "  'Famous Birthdays',\n",
       "  'Fandom',\n",
       "  'The Federalist',\n",
       "  'Find a Grave',\n",
       "  'Findmypast',\n",
       "  'Flags of the World',\n",
       "  'Flickr',\n",
       "  'Forbes.com contributors',\n",
       "  'Fox News ',\n",
       "  'FrontPage Magazine',\n",
       "  'The Gateway Pundit',\n",
       "  'Gawker',\n",
       "  'Geni.com',\n",
       "  'gnis-class',\n",
       "  'gns-class',\n",
       "  'Global Times',\n",
       "  'GlobalSecurity.org',\n",
       "  'Goodreads',\n",
       "  'The Grayzone',\n",
       "  'Guido Fawkes',\n",
       "  'Heat Street',\n",
       "  'HispanTV',\n",
       "  'History',\n",
       "  'HuffPost contributors',\n",
       "  'IMDb',\n",
       "  'Independent Media Center',\n",
       "  'InfoWars',\n",
       "  'Inquisitr',\n",
       "  'International Business Times',\n",
       "  'Investopedia',\n",
       "  'Jewish Virtual Library',\n",
       "  'Jihad Watch',\n",
       "  'Joshua Project',\n",
       "  'Know Your Meme',\n",
       "  'Last.fm',\n",
       "  'Lenta.ru',\n",
       "  'LifeSiteNews',\n",
       "  'LinkedIn',\n",
       "  'LiveJournal',\n",
       "  'LiveLeak',\n",
       "  'Lulu.com',\n",
       "  'The Mail on Sunday',\n",
       "  \"Marquis Who's Who\",\n",
       "  'Mashable sponsored content',\n",
       "  'Media Bias/Fact Check',\n",
       "  'Media Research Center',\n",
       "  'Medium',\n",
       "  'metal-experience',\n",
       "  'Metro',\n",
       "  'MintPress News',\n",
       "  'MyLife',\n",
       "  'National Enquirer',\n",
       "  'Natural News',\n",
       "  'The New American',\n",
       "  'New Eastern Outlook',\n",
       "  'New York Post',\n",
       "  'News Break',\n",
       "  'NewsBlaze',\n",
       "  'News of the World',\n",
       "  'Newsmax',\n",
       "  'NNDB',\n",
       "  'Occupy Democrats',\n",
       "  'One America News Network',\n",
       "  'The Onion',\n",
       "  'OpIndia',\n",
       "  'Our Campaigns',\n",
       "  'PanAm Post',\n",
       "  'Patheos',\n",
       "  'Peerage websites',\n",
       "  'An Phoblacht',\n",
       "  'The Points Guy',\n",
       "  'The Points Guy ',\n",
       "  'The Post Millennial',\n",
       "  'PR Newswire',\n",
       "  'Press TV',\n",
       "  'Project Veritas',\n",
       "  'Quadrant',\n",
       "  'Quillette',\n",
       "  'Quora',\n",
       "  'Rate Your Music',\n",
       "  'The Raw Story',\n",
       "  'Reddit',\n",
       "  'RedState',\n",
       "  'ResearchGate',\n",
       "  'Republic TV',\n",
       "  'Rolling Stone ',\n",
       "  'Rolling Stone ',\n",
       "  'Royal Central',\n",
       "  'RT',\n",
       "  'Scribd',\n",
       "  'Scriptural texts',\n",
       "  'Sixth Tone ',\n",
       "  'The Skwawkbox',\n",
       "  'SourceWatch',\n",
       "  'Spirit of Metal',\n",
       "  'Sportskeeda',\n",
       "  'Sputnik',\n",
       "  'Stack Exchange',\n",
       "  'starsunfolded.com',\n",
       "  'The Sun',\n",
       "  'Swarajya',\n",
       "  \"Taki's Magazine\",\n",
       "  'TASS',\n",
       "  'Telesur',\n",
       "  'The Truth About Guns',\n",
       "  'TV.com',\n",
       "  'TV Tropes',\n",
       "  'Twitter',\n",
       "  'The Unz Review',\n",
       "  'Urban Dictionary',\n",
       "  'VDARE',\n",
       "  'Venezuelanalysis',\n",
       "  'Veterans Today',\n",
       "  'VGChartz',\n",
       "  'VoC',\n",
       "  'Voltaire Network',\n",
       "  'Weather2Travel',\n",
       "  'The Western Journal',\n",
       "  'We Got This Covered',\n",
       "  'WhatCulture',\n",
       "  \"Who's Who \",\n",
       "  'WhoSampled',\n",
       "  'Wikidata',\n",
       "  'WikiLeaks',\n",
       "  'Wikinews',\n",
       "  'Wikipedia',\n",
       "  'WordPress.com',\n",
       "  'WorldNetDaily',\n",
       "  'Worldometer',\n",
       "  'YouTube',\n",
       "  'Zero Hedge',\n",
       "  'ZoomInfo'],\n",
       " 'green': ['ABC News',\n",
       "  'The Age',\n",
       "  'Agence France-Presse',\n",
       "  'Al Jazeera',\n",
       "  'Amnesty International',\n",
       "  'Anti-Defamation League',\n",
       "  'Aon',\n",
       "  'Ars Technica',\n",
       "  'Associated Press',\n",
       "  'The Atlantic',\n",
       "  'The Australian',\n",
       "  'The A.V. Club',\n",
       "  'AVN',\n",
       "  'Axios',\n",
       "  'BBC',\n",
       "  'Behind the Voice Actors',\n",
       "  'Bellingcat',\n",
       "  'Bloomberg',\n",
       "  \"Burke's Peerage\",\n",
       "  'BuzzFeed News',\n",
       "  'The Christian Science Monitor',\n",
       "  'Climate Feedback',\n",
       "  'CNET ',\n",
       "  'CNN',\n",
       "  'Coda Media',\n",
       "  'Common Sense Media',\n",
       "  'The Conversation',\n",
       "  'The Daily Telegraph',\n",
       "  'Deadline Hollywood',\n",
       "  \"Debrett's\",\n",
       "  'Deseret News',\n",
       "  'Deutsche Welle',\n",
       "  'Digital Spy',\n",
       "  'The Diplomat',\n",
       "  'The Economist',\n",
       "  'Encyclopædia Iranica',\n",
       "  'Engadget',\n",
       "  'Entertainment Weekly',\n",
       "  'Financial Times',\n",
       "  'Forbes',\n",
       "  'Fox News',\n",
       "  'Game Developer',\n",
       "  'Game Informer',\n",
       "  'Gazeta Wyborcza',\n",
       "  'gnis-coord',\n",
       "  'Gizmodo',\n",
       "  'The Globe and Mail',\n",
       "  'The Guardian',\n",
       "  'Haaretz',\n",
       "  'The Hill',\n",
       "  'The Hindu',\n",
       "  'The Hollywood Reporter',\n",
       "  'HuffPost',\n",
       "  'Idolator',\n",
       "  'IGN',\n",
       "  'The Independent',\n",
       "  'The Indian Express',\n",
       "  'Insider culture',\n",
       "  'Inter Press Service',\n",
       "  'The Intercept',\n",
       "  'International Fact-Checking Network',\n",
       "  'Jacobin',\n",
       "  'JAMA',\n",
       "  'The Jewish Chronicle',\n",
       "  'Kirkus Reviews',\n",
       "  'Kommersant',\n",
       "  'Los Angeles Times',\n",
       "  'Mail & Guardian',\n",
       "  'The Mary Sue',\n",
       "  'Metacritic',\n",
       "  'Le Monde diplomatique',\n",
       "  'Mother Jones',\n",
       "  'MSNBC',\n",
       "  'The Nation',\n",
       "  'National Geographic',\n",
       "  'NBC News',\n",
       "  'The New Republic',\n",
       "  'New York',\n",
       "  'New York Daily News',\n",
       "  'The New York Times',\n",
       "  'The New Yorker',\n",
       "  'The New Zealand Herald',\n",
       "  'Newslaundry',\n",
       "  'Newsweek ',\n",
       "  'NPR',\n",
       "  'People',\n",
       "  'Pew Research Center',\n",
       "  'People Make Games',\n",
       "  'PinkNews',\n",
       "  'Playboy',\n",
       "  'Politico',\n",
       "  'PolitiFact',\n",
       "  'Polygon',\n",
       "  'ProPublica',\n",
       "  'Quartz',\n",
       "  'Radio Free Asia',\n",
       "  'Rappler',\n",
       "  'Reason',\n",
       "  'The Register',\n",
       "  'Religion News Service',\n",
       "  'Reuters',\n",
       "  'Rolling Stone',\n",
       "  'Rotten Tomatoes',\n",
       "  'Science-Based Medicine',\n",
       "  'Scientific American',\n",
       "  'SCOTUSblog',\n",
       "  'Sky News UK',\n",
       "  'Snopes',\n",
       "  'SCMP',\n",
       "  'Southern Poverty Law Center',\n",
       "  'Space.com',\n",
       "  'Der Spiegel',\n",
       "  'The Sydney Morning Herald',\n",
       "  'TheWrap',\n",
       "  'Time',\n",
       "  'The Times',\n",
       "  'TorrentFreak',\n",
       "  'TV Guide',\n",
       "  'U.S. News & World Report',\n",
       "  'USA Today',\n",
       "  'Vanity Fair',\n",
       "  'Variety',\n",
       "  'VentureBeat',\n",
       "  'The Verge',\n",
       "  'Vogue',\n",
       "  'Voice of America',\n",
       "  'Vox',\n",
       "  'The Wall Street Journal',\n",
       "  'The Washington Post',\n",
       "  'The Weekly Standard',\n",
       "  'Wired',\n",
       "  'Yahoo News',\n",
       "  'ZDNet'],\n",
       " 'yellow': ['Alexa Internet',\n",
       "  'AllSides',\n",
       "  'The American Conservative',\n",
       "  'Anadolu Agency',\n",
       "  'Apple Daily',\n",
       "  'Arab News',\n",
       "  'Asian News International',\n",
       "  'AskMen',\n",
       "  'Asian News International',\n",
       "  'Ballotpedia',\n",
       "  'Biography.com',\n",
       "  'Bloomberg profiles',\n",
       "  'Boing Boing',\n",
       "  'Bustle',\n",
       "  'BuzzFeed',\n",
       "  'Cato Institute',\n",
       "  'Center for Economic and Policy Research',\n",
       "  'China Daily',\n",
       "  'CliffsNotes',\n",
       "  'CNET ',\n",
       "  'Cosmopolitan',\n",
       "  'The Daily Beast',\n",
       "  'The Daily Dot',\n",
       "  'Daily Mirror',\n",
       "  'Daily NK',\n",
       "  'Democracy Now!',\n",
       "  'Dotdash',\n",
       "  'Encyclopædia Britannica',\n",
       "  'Entrepreneur',\n",
       "  'Evening Standard',\n",
       "  'Fairness and Accuracy in Reporting',\n",
       "  'Fox News ',\n",
       "  'Genius',\n",
       "  'gns-coord',\n",
       "  'Google Maps',\n",
       "  'The Green Papers',\n",
       "  'The Guardian blogs',\n",
       "  'Guinness World Records',\n",
       "  'Hansard',\n",
       "  'Heavy.com',\n",
       "  'Hope not Hate',\n",
       "  'HuffPost ',\n",
       "  'Human Events',\n",
       "  'Independent Journal Review',\n",
       "  'Insider',\n",
       "  'IslamQA.info',\n",
       "  'Jezebel',\n",
       "  'Mashable',\n",
       "  'MDPI',\n",
       "  'Media Matters for America',\n",
       "  'Mediaite',\n",
       "  'MetalSucks',\n",
       "  'Middle East Media Research Institute',\n",
       "  'Mondoweiss',\n",
       "  'Morning Star',\n",
       "  'National Review',\n",
       "  'The Needle Drop',\n",
       "  'Newsweek ',\n",
       "  'The Next Web',\n",
       "  'Pride.com',\n",
       "  'Quackwatch',\n",
       "  'RealClearPolitics',\n",
       "  'RhythmOne',\n",
       "  'RIA Novosti',\n",
       "  'Salon',\n",
       "  'ScienceBlogs',\n",
       "  'Screen Rant',\n",
       "  'Sherdog',\n",
       "  'Sixth Tone',\n",
       "  \"The Skeptic's Dictionary\",\n",
       "  'Sky News Australia',\n",
       "  'SparkNotes',\n",
       "  'The Spectator',\n",
       "  'The Straits Times',\n",
       "  'TechCrunch',\n",
       "  'ThinkProgress',\n",
       "  'The Times of India',\n",
       "  'TMZ',\n",
       "  'Townhall',\n",
       "  'TRT World',\n",
       "  'Us Weekly',\n",
       "  'Vice Media',\n",
       "  'Washington Examiner',\n",
       "  'The Washington Times',\n",
       "  'Wikidata transcluded statements',\n",
       "  'World Socialist Web Site',\n",
       "  'XBIZ',\n",
       "  'Xinhua News Agency']}"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "non_scholarly_flags = {\"red\": [], \"green\":[], \"yellow\": []}\n",
    "\n",
    "url = \"https://en.wikipedia.org/wiki/Wikipedia:Reliable_sources/Perennial_sources\"\n",
    "res = requests.get(url)\n",
    "soup = BeautifulSoup(res.content, \"html.parser\")\n",
    "table = soup.find(\"table\", {\"class\": \"wikitable\"})\n",
    "body = table.find(\"tbody\")\n",
    "all_rows = body.find_all(\"tr\")\n",
    "for row in all_rows:\n",
    "    if row.has_attr(\"id\"):\n",
    "        name = row[\"id\"].replace(\"_\", \" \")\n",
    "        name = name.split(\"(\")[0]\n",
    "        tag = row[\"class\"][0]\n",
    "        flag = mapping[tag]\n",
    "        non_scholarly_flags[flag].append(name)\n",
    "\n",
    "non_scholarly_flags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'red': ['112 Ukraine',\n",
       "  'Ad Fontes Media',\n",
       "  'Advameg',\n",
       "  'AlterNet',\n",
       "  'Amazon',\n",
       "  'Anadolu Agency ',\n",
       "  'Ancestry.com',\n",
       "  'ANNA News',\n",
       "  'Answers.com',\n",
       "  'Antiwar.com',\n",
       "  'arXiv',\n",
       "  'Baidu Baike',\n",
       "  'bestgore.com',\n",
       "  'Bild',\n",
       "  'Blaze Media',\n",
       "  'Blogger',\n",
       "  'Breitbart News',\n",
       "  'BroadwayWorld',\n",
       "  'The California Globe',\n",
       "  'The Canary',\n",
       "  'CelebrityNetWorth',\n",
       "  'Centre for Research on Globalization',\n",
       "  'CESNUR',\n",
       "  'China Global Television Network',\n",
       "  'CNET ',\n",
       "  'CoinDesk',\n",
       "  'Consortium News',\n",
       "  'CounterPunch',\n",
       "  'Cracked.com',\n",
       "  'Crunchbase',\n",
       "  'The Daily Caller',\n",
       "  'Daily Express',\n",
       "  'Daily Kos',\n",
       "  'Daily Mail',\n",
       "  'Daily Sabah',\n",
       "  'Daily Star',\n",
       "  'The Daily Wire',\n",
       "  'Discogs',\n",
       "  'The Electronic Intifada',\n",
       "  'Encyclopaedia Metallum',\n",
       "  'The Epoch Times',\n",
       "  'Examiner.com',\n",
       "  'Facebook',\n",
       "  'FamilySearch',\n",
       "  'Famous Birthdays',\n",
       "  'Fandom',\n",
       "  'The Federalist',\n",
       "  'Find a Grave',\n",
       "  'Findmypast',\n",
       "  'Flags of the World',\n",
       "  'Flickr',\n",
       "  'Forbes.com contributors',\n",
       "  'Fox News ',\n",
       "  'FrontPage Magazine',\n",
       "  'The Gateway Pundit',\n",
       "  'Gawker',\n",
       "  'Geni.com',\n",
       "  'gnis-class',\n",
       "  'gns-class',\n",
       "  'Global Times',\n",
       "  'GlobalSecurity.org',\n",
       "  'Goodreads',\n",
       "  'The Grayzone',\n",
       "  'Guido Fawkes',\n",
       "  'Heat Street',\n",
       "  'HispanTV',\n",
       "  'History',\n",
       "  'HuffPost contributors',\n",
       "  'IMDb',\n",
       "  'Independent Media Center',\n",
       "  'InfoWars',\n",
       "  'Inquisitr',\n",
       "  'International Business Times',\n",
       "  'Investopedia',\n",
       "  'Jewish Virtual Library',\n",
       "  'Jihad Watch',\n",
       "  'Joshua Project',\n",
       "  'Know Your Meme',\n",
       "  'Last.fm',\n",
       "  'Lenta.ru',\n",
       "  'LifeSiteNews',\n",
       "  'LinkedIn',\n",
       "  'LiveJournal',\n",
       "  'LiveLeak',\n",
       "  'Lulu.com',\n",
       "  'The Mail on Sunday',\n",
       "  \"Marquis Who's Who\",\n",
       "  'Mashable sponsored content',\n",
       "  'Media Bias/Fact Check',\n",
       "  'Media Research Center',\n",
       "  'Medium',\n",
       "  'metal-experience',\n",
       "  'Metro',\n",
       "  'MintPress News',\n",
       "  'MyLife',\n",
       "  'National Enquirer',\n",
       "  'Natural News',\n",
       "  'The New American',\n",
       "  'New Eastern Outlook',\n",
       "  'New York Post',\n",
       "  'News Break',\n",
       "  'NewsBlaze',\n",
       "  'News of the World',\n",
       "  'Newsmax',\n",
       "  'NNDB',\n",
       "  'Occupy Democrats',\n",
       "  'One America News Network',\n",
       "  'The Onion',\n",
       "  'OpIndia',\n",
       "  'Our Campaigns',\n",
       "  'PanAm Post',\n",
       "  'Patheos',\n",
       "  'Peerage websites',\n",
       "  'An Phoblacht',\n",
       "  'The Points Guy',\n",
       "  'The Points Guy ',\n",
       "  'The Post Millennial',\n",
       "  'PR Newswire',\n",
       "  'Press TV',\n",
       "  'Project Veritas',\n",
       "  'Quadrant',\n",
       "  'Quillette',\n",
       "  'Quora',\n",
       "  'Rate Your Music',\n",
       "  'The Raw Story',\n",
       "  'Reddit',\n",
       "  'RedState',\n",
       "  'ResearchGate',\n",
       "  'Republic TV',\n",
       "  'Rolling Stone ',\n",
       "  'Rolling Stone ',\n",
       "  'Royal Central',\n",
       "  'RT',\n",
       "  'Scribd',\n",
       "  'Scriptural texts',\n",
       "  'Sixth Tone ',\n",
       "  'The Skwawkbox',\n",
       "  'SourceWatch',\n",
       "  'Spirit of Metal',\n",
       "  'Sportskeeda',\n",
       "  'Sputnik',\n",
       "  'Stack Exchange',\n",
       "  'starsunfolded.com',\n",
       "  'The Sun',\n",
       "  'Swarajya',\n",
       "  \"Taki's Magazine\",\n",
       "  'TASS',\n",
       "  'Telesur',\n",
       "  'The Truth About Guns',\n",
       "  'TV.com',\n",
       "  'TV Tropes',\n",
       "  'Twitter',\n",
       "  'The Unz Review',\n",
       "  'Urban Dictionary',\n",
       "  'VDARE',\n",
       "  'Venezuelanalysis',\n",
       "  'Veterans Today',\n",
       "  'VGChartz',\n",
       "  'VoC',\n",
       "  'Voltaire Network',\n",
       "  'Weather2Travel',\n",
       "  'The Western Journal',\n",
       "  'We Got This Covered',\n",
       "  'WhatCulture',\n",
       "  \"Who's Who \",\n",
       "  'WhoSampled',\n",
       "  'Wikidata',\n",
       "  'WikiLeaks',\n",
       "  'Wikinews',\n",
       "  'Wikipedia',\n",
       "  'WordPress.com',\n",
       "  'WorldNetDaily',\n",
       "  'Worldometer',\n",
       "  'YouTube',\n",
       "  'Zero Hedge',\n",
       "  'ZoomInfo',\n",
       "  '70 News',\n",
       "  'ABCnews.com.co',\n",
       "  'American News',\n",
       "  'banned.video',\n",
       "  \"Before It's News\",\n",
       "  'bients.com',\n",
       "  'Bipartisan Report',\n",
       "  'bizstandardnews.com',\n",
       "  'Bloomberg.ma',\n",
       "  'The Boston Tribune',\n",
       "  'Breaking-CNN.com',\n",
       "  'BVA News',\n",
       "  'Cairns News',\n",
       "  'Celebtricity',\n",
       "  'CBSnews.com.co',\n",
       "  'cnn-trending.com',\n",
       "  'Conservative 101',\n",
       "  'Conservative Frontline',\n",
       "  'CountyNewsroom.info',\n",
       "  'Daily USA Update',\n",
       "  'Disclose.tv',\n",
       "  'DrudgeReport.com.co',\n",
       "  'Empire Herald',\n",
       "  'Empire News',\n",
       "  'Empire Sports',\n",
       "  'The Exposé',\n",
       "  'Fox-news24.com',\n",
       "  'The Gateway Pundit',\n",
       "  'Global Associated News',\n",
       "  'Globalresearch.ca',\n",
       "  'Gossip Mill Mzansi',\n",
       "  'The Grayzone',\n",
       "  'Guerilla News',\n",
       "  'Gummy Post',\n",
       "  'Houston Chronicle TV',\n",
       "  'Huzlers',\n",
       "  'InfoWars',\n",
       "  'Judicial Watch',\n",
       "  'ΚΒΟΙ2.com',\n",
       "  'KMT 11 News',\n",
       "  'The Last Line of Defense',\n",
       "  'Law Enforcement Today',\n",
       "  'Liberal Society',\n",
       "  'Liberty Writers News',\n",
       "  'LinkBeef',\n",
       "  'MV-media',\n",
       "  'Naha Daily',\n",
       "  'National Insider Politics',\n",
       "  'NationalReport.net',\n",
       "  'Natural News',\n",
       "  'NBCNews.com.co',\n",
       "  'News Breaks Here',\n",
       "  'NewsBuzzDaily',\n",
       "  'News Examiner',\n",
       "  'News Hound',\n",
       "  'The News Nerd',\n",
       "  'NewsPunch',\n",
       "  'NewsWatch33',\n",
       "  'The New York Evening',\n",
       "  'Next News Network',\n",
       "  'Now 8 News',\n",
       "  'Oneworld.press',\n",
       "  'OpIndia',\n",
       "  'Palmer Report',\n",
       "  'Peace Data',\n",
       "  'Postcard News',\n",
       "  'The Predicted',\n",
       "  'Prntly',\n",
       "  'React 365',\n",
       "  'The Reporterz',\n",
       "  'Snoopack',\n",
       "  'Spin Zone',\n",
       "  'St George Gazette',\n",
       "  'Stuppid',\n",
       "  'Super Station 95',\n",
       "  'TruNews',\n",
       "  'TrueTrumpers.com',\n",
       "  'UConservative',\n",
       "  'UndergroundNewsReport.com',\n",
       "  'The Unhived Mind',\n",
       "  'United Media Publishing',\n",
       "  'USA Daily Info',\n",
       "  'usatoday.com.co',\n",
       "  'US Postman',\n",
       "  'washingtonpost.com.co',\n",
       "  'WorldNetDaily',\n",
       "  'World News Daily Report'],\n",
       " 'green': ['ABC News',\n",
       "  'The Age',\n",
       "  'Agence France-Presse',\n",
       "  'Al Jazeera',\n",
       "  'Amnesty International',\n",
       "  'Anti-Defamation League',\n",
       "  'Aon',\n",
       "  'Ars Technica',\n",
       "  'Associated Press',\n",
       "  'The Atlantic',\n",
       "  'The Australian',\n",
       "  'The A.V. Club',\n",
       "  'AVN',\n",
       "  'Axios',\n",
       "  'BBC',\n",
       "  'Behind the Voice Actors',\n",
       "  'Bellingcat',\n",
       "  'Bloomberg',\n",
       "  \"Burke's Peerage\",\n",
       "  'BuzzFeed News',\n",
       "  'The Christian Science Monitor',\n",
       "  'Climate Feedback',\n",
       "  'CNET ',\n",
       "  'CNN',\n",
       "  'Coda Media',\n",
       "  'Common Sense Media',\n",
       "  'The Conversation',\n",
       "  'The Daily Telegraph',\n",
       "  'Deadline Hollywood',\n",
       "  \"Debrett's\",\n",
       "  'Deseret News',\n",
       "  'Deutsche Welle',\n",
       "  'Digital Spy',\n",
       "  'The Diplomat',\n",
       "  'The Economist',\n",
       "  'Encyclopædia Iranica',\n",
       "  'Engadget',\n",
       "  'Entertainment Weekly',\n",
       "  'Financial Times',\n",
       "  'Forbes',\n",
       "  'Fox News',\n",
       "  'Game Developer',\n",
       "  'Game Informer',\n",
       "  'Gazeta Wyborcza',\n",
       "  'gnis-coord',\n",
       "  'Gizmodo',\n",
       "  'The Globe and Mail',\n",
       "  'The Guardian',\n",
       "  'Haaretz',\n",
       "  'The Hill',\n",
       "  'The Hindu',\n",
       "  'The Hollywood Reporter',\n",
       "  'HuffPost',\n",
       "  'Idolator',\n",
       "  'IGN',\n",
       "  'The Independent',\n",
       "  'The Indian Express',\n",
       "  'Insider culture',\n",
       "  'Inter Press Service',\n",
       "  'The Intercept',\n",
       "  'International Fact-Checking Network',\n",
       "  'Jacobin',\n",
       "  'JAMA',\n",
       "  'The Jewish Chronicle',\n",
       "  'Kirkus Reviews',\n",
       "  'Kommersant',\n",
       "  'Los Angeles Times',\n",
       "  'Mail & Guardian',\n",
       "  'The Mary Sue',\n",
       "  'Metacritic',\n",
       "  'Le Monde diplomatique',\n",
       "  'Mother Jones',\n",
       "  'MSNBC',\n",
       "  'The Nation',\n",
       "  'National Geographic',\n",
       "  'NBC News',\n",
       "  'The New Republic',\n",
       "  'New York',\n",
       "  'New York Daily News',\n",
       "  'The New York Times',\n",
       "  'The New Yorker',\n",
       "  'The New Zealand Herald',\n",
       "  'Newslaundry',\n",
       "  'Newsweek ',\n",
       "  'NPR',\n",
       "  'People',\n",
       "  'Pew Research Center',\n",
       "  'People Make Games',\n",
       "  'PinkNews',\n",
       "  'Playboy',\n",
       "  'Politico',\n",
       "  'PolitiFact',\n",
       "  'Polygon',\n",
       "  'ProPublica',\n",
       "  'Quartz',\n",
       "  'Radio Free Asia',\n",
       "  'Rappler',\n",
       "  'Reason',\n",
       "  'The Register',\n",
       "  'Religion News Service',\n",
       "  'Reuters',\n",
       "  'Rolling Stone',\n",
       "  'Rotten Tomatoes',\n",
       "  'Science-Based Medicine',\n",
       "  'Scientific American',\n",
       "  'SCOTUSblog',\n",
       "  'Sky News UK',\n",
       "  'Snopes',\n",
       "  'SCMP',\n",
       "  'Southern Poverty Law Center',\n",
       "  'Space.com',\n",
       "  'Der Spiegel',\n",
       "  'The Sydney Morning Herald',\n",
       "  'TheWrap',\n",
       "  'Time',\n",
       "  'The Times',\n",
       "  'TorrentFreak',\n",
       "  'TV Guide',\n",
       "  'U.S. News & World Report',\n",
       "  'USA Today',\n",
       "  'Vanity Fair',\n",
       "  'Variety',\n",
       "  'VentureBeat',\n",
       "  'The Verge',\n",
       "  'Vogue',\n",
       "  'Voice of America',\n",
       "  'Vox',\n",
       "  'The Wall Street Journal',\n",
       "  'The Washington Post',\n",
       "  'The Weekly Standard',\n",
       "  'Wired',\n",
       "  'Yahoo News',\n",
       "  'ZDNet'],\n",
       " 'yellow': ['Alexa Internet',\n",
       "  'AllSides',\n",
       "  'The American Conservative',\n",
       "  'Anadolu Agency',\n",
       "  'Apple Daily',\n",
       "  'Arab News',\n",
       "  'Asian News International',\n",
       "  'AskMen',\n",
       "  'Asian News International',\n",
       "  'Ballotpedia',\n",
       "  'Biography.com',\n",
       "  'Bloomberg profiles',\n",
       "  'Boing Boing',\n",
       "  'Bustle',\n",
       "  'BuzzFeed',\n",
       "  'Cato Institute',\n",
       "  'Center for Economic and Policy Research',\n",
       "  'China Daily',\n",
       "  'CliffsNotes',\n",
       "  'CNET ',\n",
       "  'Cosmopolitan',\n",
       "  'The Daily Beast',\n",
       "  'The Daily Dot',\n",
       "  'Daily Mirror',\n",
       "  'Daily NK',\n",
       "  'Democracy Now!',\n",
       "  'Dotdash',\n",
       "  'Encyclopædia Britannica',\n",
       "  'Entrepreneur',\n",
       "  'Evening Standard',\n",
       "  'Fairness and Accuracy in Reporting',\n",
       "  'Fox News ',\n",
       "  'Genius',\n",
       "  'gns-coord',\n",
       "  'Google Maps',\n",
       "  'The Green Papers',\n",
       "  'The Guardian blogs',\n",
       "  'Guinness World Records',\n",
       "  'Hansard',\n",
       "  'Heavy.com',\n",
       "  'Hope not Hate',\n",
       "  'HuffPost ',\n",
       "  'Human Events',\n",
       "  'Independent Journal Review',\n",
       "  'Insider',\n",
       "  'IslamQA.info',\n",
       "  'Jezebel',\n",
       "  'Mashable',\n",
       "  'MDPI',\n",
       "  'Media Matters for America',\n",
       "  'Mediaite',\n",
       "  'MetalSucks',\n",
       "  'Middle East Media Research Institute',\n",
       "  'Mondoweiss',\n",
       "  'Morning Star',\n",
       "  'National Review',\n",
       "  'The Needle Drop',\n",
       "  'Newsweek ',\n",
       "  'The Next Web',\n",
       "  'Pride.com',\n",
       "  'Quackwatch',\n",
       "  'RealClearPolitics',\n",
       "  'RhythmOne',\n",
       "  'RIA Novosti',\n",
       "  'Salon',\n",
       "  'ScienceBlogs',\n",
       "  'Screen Rant',\n",
       "  'Sherdog',\n",
       "  'Sixth Tone',\n",
       "  \"The Skeptic's Dictionary\",\n",
       "  'Sky News Australia',\n",
       "  'SparkNotes',\n",
       "  'The Spectator',\n",
       "  'The Straits Times',\n",
       "  'TechCrunch',\n",
       "  'ThinkProgress',\n",
       "  'The Times of India',\n",
       "  'TMZ',\n",
       "  'Townhall',\n",
       "  'TRT World',\n",
       "  'Us Weekly',\n",
       "  'Vice Media',\n",
       "  'Washington Examiner',\n",
       "  'The Washington Times',\n",
       "  'Wikidata transcluded statements',\n",
       "  'World Socialist Web Site',\n",
       "  'XBIZ',\n",
       "  'Xinhua News Agency']}"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#add the fake news list to the dictionary\n",
    "non_scholarly_flags[\"red\"] += red_news\n",
    "\n",
    "non_scholarly_flags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export to a JSON file\n",
    "import json\n",
    "\n",
    "with open(\"non_scholarly_flags.json\", \"w\") as outfile:\n",
    "    json.dump(non_scholarly_flags, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: 1, 2: 2, 3: 3, 4: 4, 5: 5, 6: 6}"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = {1:1,2:2,3:3}\n",
    "b = {4:4,5:5,6:6}\n",
    "a.update(b)\n",
    "a"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}