Skip to content

Commit 5c56915

Browse files
fccoelhofccoelho
committed
Corrigi muitos bugs e atualizei código
1 parent c9c093a commit 5c56915

File tree

12 files changed

+1810
-20445
lines changed

12 files changed

+1810
-20445
lines changed

‎Book/Capitulo_1.ipynb‎

Lines changed: 557 additions & 18068 deletions
Large diffs are not rendered by default.

‎Book/Capitulo_2.ipynb‎

Lines changed: 297 additions & 389 deletions
Large diffs are not rendered by default.

‎Book/Capitulo_3.ipynb‎

Lines changed: 187 additions & 284 deletions
Large diffs are not rendered by default.

‎Book/Capitulo_4.ipynb‎

Lines changed: 613 additions & 741 deletions
Large diffs are not rendered by default.

‎Book/Capitulo_5.ipynb‎

Lines changed: 126 additions & 926 deletions
Large diffs are not rendered by default.

‎Book/Capitulo_6.ipynb‎

Lines changed: 9 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -13,39 +13,25 @@
1313
},
1414
{
1515
"cell_type": "code",
16-
"execution_count": 33,
16+
"execution_count": null,
1717
"id": "88ed1d45-d9f3-4008-ae6f-d0b681d7b28f",
1818
"metadata": {
1919
"tags": [
2020
"hide-output"
21-
]
21+
],
22+
"is_executing": true,
23+
"ExecuteTime": {
24+
"start_time": "2023-10-11T14:40:36.447117094Z"
25+
}
2226
},
2327
"outputs": [
2428
{
2529
"name": "stdout",
2630
"output_type": "stream",
2731
"text": [
28-
"Fetching results for covid19\n",
29-
"Found 500 items\n",
30-
" 26%|██████████▍ | 128/500 [01:32<03:51, 1.61it/s]Downloading of 36571519 failed. HTTP Error 400: Bad Request\n",
31-
" 99%|████████████████████████████████████████▌| 494/500 [05:53<00:04, 1.46it/s]Downloading of 36567860 failed. HTTP Error 400: Bad Request\n",
32-
"100%|█████████████████████████████████████████| 500/500 [05:58<00:00, 1.40it/s]\n",
33-
"Fetching results for covid-19\n",
34-
"Found 500 items\n",
35-
"100%|█████████████████████████████████████████| 500/500 [00:31<00:00, 15.84it/s]\n",
36-
"Fetching results for sars-cov-2\n",
37-
"Found 500 items\n",
38-
" 45%|██████████████████▍ | 225/500 [00:55<03:13, 1.42it/s]Downloading of 36564060 failed. HTTP Error 400: Bad Request\n",
39-
" 65%|██████████████████████████▋ | 326/500 [02:08<02:25, 1.20it/s]Downloading of 36560681 failed. HTTP Error 400: Bad Request\n",
40-
"100%|█████████████████████████████████████████| 500/500 [04:26<00:00, 1.88it/s]\n",
41-
"Fetching results for 2019-ncov\n",
42-
"Found 500 items\n",
43-
"100%|████████████████████████████████████████| 500/500 [00:01<00:00, 372.41it/s]\n",
44-
"Fetching results for novel coronavirus\n",
45-
"Found 500 items\n",
46-
"100%|█████████████████████████████████████████| 500/500 [00:10<00:00, 49.21it/s]\n",
47-
"Updating citations...\n",
48-
"100%|███████████████████████████████████████| 2054/2054 [18:36<00:00, 1.84it/s]\n"
32+
"Fetching results for covid19\r\n",
33+
"Found 500 items\r\n",
34+
" 27%|███████████▏ | 136/500 [01:48<04:48, 1.26it/s]"
4935
]
5036
}
5137
],

dhbb.w2v renamed to Book/dhbb.w2v

File renamed without changes.

‎Book/fetch2sql.py‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from concurrent.futures import ThreadPoolExecutor, as_completed
1515
import pandas as pd
1616
from datetime import date
17-
from ratelimiter import RateLimiter
17+
# from ratelimiter import RateLimiter
1818
import warnings
1919
warnings.filterwarnings('ignore')
2020

@@ -74,16 +74,16 @@ def __init__(self, email, search_term, count=9999, collection='articles'):
7474
self.citation_table = 'citations' if collection == 'articles' else f"citations_{collection}"
7575
with engine.connect() as connection:
7676
connection.execute(
77-
f"create table IF NOT EXISTS {self.articles_table}(pmid integer unique, title varchar(1024), abstract text, journal varchar(512), pubdate date);")
77+
text(f"create table IF NOT EXISTS {self.articles_table}(pmid integer unique, title varchar(1024), abstract text, journal varchar(512), pubdate date);"))
7878
connection.execute(
79-
f"create table IF NOT EXISTS {self.citation_table}(pmid integer unique, cited_by text);")
79+
text(f"create table IF NOT EXISTS {self.citation_table}(pmid integer unique, cited_by text);"))
8080
connection.execute(
81-
f"create unique index IF NOT EXISTS pmid_idx on {self.articles_table} (pmid)")
81+
text(f"create unique index IF NOT EXISTS pmid_idx on {self.articles_table} (pmid)"))
8282
connection.execute(
83-
f"create unique index IF NOT EXISTS pmid_idx on {self.citation_table} (pmid)")
83+
text(f"create unique index IF NOT EXISTS pmid_idx on {self.citation_table} (pmid)"))
8484

8585

86-
@RateLimiter(max_calls=4, period=1)
86+
# @RateLimiter(max_calls=4, period=1)
8787
def _fetch(self, pmid):
8888
try:
8989
handle = Entrez.efetch(db="pubmed", id=pmid, retmode='xml')
@@ -99,7 +99,7 @@ def _fetch(self, pmid):
9999
def _get_old_ids(self):
100100
with engine.connect() as connection:
101101
res = connection.execute(
102-
f"select pmid from {self.articles_table};")
102+
text(f"select pmid from {self.articles_table};"))
103103
oldids = res.fetchall()
104104

105105
return [i[0] for i in oldids]
@@ -121,9 +121,9 @@ def _get_citations(self, pmid):
121121

122122
def create_FT_index(self):
123123
with engine.connect() as connection:
124-
query = f"""CREATE VIRTUAL TABLE IF NOT EXISTS article_fts USING fts5(title, abstract, content={self.articles_table},tokenize='porter ascii', content_rowid='pmid');
125-
"""
126-
query2 = f"INSERT INTO article_fts (rowid, title, abstract) SELECT pmid, title, abstract FROM {self.articles_table};"
124+
query = text(f"""CREATE VIRTUAL TABLE IF NOT EXISTS article_fts USING fts5(title, abstract, content={self.articles_table},tokenize='porter ascii', content_rowid='pmid');
125+
""")
126+
query2 = text(f"INSERT INTO article_fts (rowid, title, abstract) SELECT pmid, title, abstract FROM {self.articles_table};")
127127
connection.execute(query)
128128
connection.execute(query2)
129129

@@ -143,8 +143,8 @@ def update_citations_concurrently(self):
143143
else:
144144
with engine.connect() as connection:
145145
cits = '|'.join(cits)
146-
connection.execute(f"insert into {self.citation_table} VALUES(%s, %s) \
147-
on conflict (pmid) do update set cited_by=%s;", (pmid, cits, cits))
146+
connection.execute(text(f"insert into {self.citation_table} VALUES(%s, %s) \
147+
on conflict (pmid) do update set cited_by=%s;", (pmid, cits, cits)))
148148
except Exception as e:
149149
print(f'{pmid} generated an exception: {e}')
150150

‎Book/intro.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ A seleção de assuntos para inclusão neste livro é baseada na demanda dos lei
77
Este livro está baseado em uma combinação de cursos sobre o assunto ministrados por [mim](https://github.com/fccoelho) ao longo dos anos. Procuro manter o conteúdo acessível a pessoas com menos experiência em programação em Python. Caso você, leitor, se depare com trechos técnicamente muito avançados, fique à vontade para abrir uma `issue` explicando como acha que podemos tornar o conteúdo mais acessível.
88

99
Caso queira apoiar este projeto, faz um PIX de qualquer valor, usando o código QR abaixo:
10+
<img src="./PIX_QR_only.jpg" width=300px>
1011

11-
![doação](./PIX_QR_only.jpg)
1212

1313

‎Book/minha_tabela.csv.gz‎

26 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)