ORCID_Usage.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100% (1034 of 1034) |####################| Elapsed Time: 0:00:03 Time:  0:00:03\n"
     ]
    }
   ],
   "source": [
    "#Get metadata from CaltechDATA\n",
    "\n",
    "import os\n",
    "from ames.harvesters import get_caltechdata\n",
    "\n",
    "if os.path.isdir('data') == False:\n",
    "    os.mkdir('data')\n",
    "os.chdir('data')\n",
    "\n",
    "production = True\n",
    "collection = 'caltechdata.ds'\n",
    "\n",
    "get_caltechdata(collection,production)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total ORCIDS:  423\n",
      "Unique ORCIDS:  124\n",
      "Records with at least one ORCID:  217\n",
      "Total records:  1034\n",
      "Percent of records with at least one ORCID: 21%\n",
      "Modern records (post 2012 publication date): 305\n",
      "Percent of modern records with at least one ORCID: 71%\n"
     ]
    }
   ],
   "source": [
    "#Determine percentage of records have authors with ORCIDS in CaltechDATA\n",
    "\n",
    "from ames.harvesters import get_records\n",
    "from py_dataset import dataset\n",
    "\n",
    "dot_paths = ['.dates','.resourceType','.subjects','.publicationYear','.creators']\n",
    "keys = dataset.keys(collection)\n",
    "author_metadata = get_records(dot_paths,'dois',collection,keys)\n",
    "dates = []\n",
    "categories = []\n",
    "keys = []\n",
    "orcid_records = 0\n",
    "new_records = 0\n",
    "total_orcids = 0\n",
    "records = 0\n",
    "orcids = set()\n",
    "for record in author_metadata:\n",
    "    records += 1\n",
    "    orcid_count = 0\n",
    "    for c in record['creators']:\n",
    "        if 'nameIdentifiers' in c:\n",
    "            for n in c['nameIdentifiers']:\n",
    "                if n['nameIdentifierScheme'] == 'ORCID':\n",
    "                    orcid_count += 1\n",
    "                    orcids.add(n['nameIdentifier'])\n",
    "    total_orcids += orcid_count\n",
    "    if orcid_count > 0:\n",
    "        orcid_records += 1\n",
    "    if int(record['publicationYear']) > 2012:\n",
    "        new_records += 1\n",
    "\n",
    "print(\"Total ORCIDS: \",total_orcids)\n",
    "print(\"Unique ORCIDS: \",len(orcids))\n",
    "print(\"Records with at least one ORCID: \",orcid_records)\n",
    "print(\"Total records: \",records)\n",
    "print(f\"Percent of records with at least one ORCID: {100*(orcid_records/records):.0f}%\")\n",
    "print(f\"Modern records (post 2012 publication date): {new_records}\")\n",
    "print(f\"Percent of modern records with at least one ORCID: {100*(orcid_records/new_records):.0f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100% (40 of 40) |########################| Elapsed Time: 0:00:46 Time:  0:00:46\n",
      " 92% (95 of 103) |#####################  | Elapsed Time: 0:02:13 ETA:   0:00:10"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10.7907/z93r0qt5 already exists in collection datacite.ds\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100% (103 of 103) |######################| Elapsed Time: 0:02:24 Time:  0:02:24\n"
     ]
    }
   ],
   "source": [
    "# Alternativly, harvest all metadata from DataCite\n",
    "\n",
    "import requests, shutil\n",
    "from progressbar import progressbar\n",
    "\n",
    "def validate_response(response):\n",
    "    if (response.status_code != 200):\n",
    "        print(str(response.status_code) + \" \" + response.text)\n",
    "        exit()\n",
    "    else:\n",
    "        return response.json()\n",
    "\n",
    "clients = ['tind.caltech','caltech.library']\n",
    "\n",
    "collection = 'datacite.ds'\n",
    "#Always make a new collection\n",
    "shutil.rmtree(collection)\n",
    "dataset.init(collection)\n",
    "\n",
    "for client in clients:\n",
    "    endpoint = 'https://api.datacite.org/dois'\n",
    "    query = endpoint + '?client-id=' + client\n",
    "    response = requests.get(query)\n",
    "    response = validate_response(response)\n",
    "    pages = response['meta']['totalPages']\n",
    "    records = response['data']\n",
    "\n",
    "    orcid_records = 0\n",
    "    new_records = 0\n",
    "    record_num = 0\n",
    "    page = 1\n",
    "    for p in progressbar(range(pages)):\n",
    "        for r in records:\n",
    "            err = dataset.create(collection,r['id'],r['attributes'])\n",
    "            if err != '':\n",
    "                print(err)\n",
    "            record_num += 1\n",
    "        page = page + 1\n",
    "        response = requests.get(query + '&page[number]='+str(page))\n",
    "        response = validate_response(response)\n",
    "        records = response['data']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total ORCIDS:  1125\n",
      "Unique ORCIDS:  780\n",
      "Records with at least one ORCID:  915\n",
      "Total DOIs:  3549\n",
      "Percent of DOIs with at least one ORCID: 26%\n",
      "Modern DOIs (post 2012 publication date): 1855\n",
      "Percent of modern DOIs with at least one ORCID: 49%\n"
     ]
    }
   ],
   "source": [
    "#Now process metadata to get stats from DataCite\n",
    "\n",
    "orcid_records = 0\n",
    "total_orcids = 0\n",
    "new_records = 0\n",
    "record_num = 0\n",
    "orcids = set()\n",
    "page = 1\n",
    "keys = dataset.keys(collection)\n",
    "for k in keys:\n",
    "    record,err = dataset.read(collection,k)\n",
    "    record_num += 1\n",
    "    orcid_count = 0\n",
    "    for c in record['creators']:\n",
    "        if 'nameIdentifiers' in c:\n",
    "            for n in c['nameIdentifiers']:\n",
    "                if n['nameIdentifierScheme'] == 'ORCID':\n",
    "                    if 'nameIdentifier' in n:\n",
    "                        orcid_count += 1\n",
    "                        orcids.add(n['nameIdentifier'])\n",
    "    total_orcids += orcid_count\n",
    "    if orcid_count > 0:\n",
    "        orcid_records += 1\n",
    "    if record['publicationYear'] != None:\n",
    "        if int(record['publicationYear']) > 2012:\n",
    "            new_records += 1\n",
    "\n",
    "print(\"Total ORCIDS: \",total_orcids)\n",
    "print(\"Unique ORCIDS: \",len(orcids))\n",
    "print(\"Records with at least one ORCID: \",orcid_records)\n",
    "print(\"Total DOIs: \",record_num)\n",
    "print(f\"Percent of DOIs with at least one ORCID: {100*(orcid_records/record_num):.0f}%\")\n",
    "print(f\"Modern DOIs (post 2012 publication date): {new_records}\")\n",
    "print(f\"Percent of modern DOIs with at least one ORCID: {100*(orcid_records/new_records):.0f}%\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}