Skip to content

Commit e588d7c

Browse files
iliusilius
committed
add WordFrequency.info reader
1 parent 1b316a1 commit e588d7c

6 files changed

Lines changed: 378 additions & 0 deletions

File tree

‎doc/p/__index__.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
| Tabfile (.txt, .dic) | Tabfile | [tabfile.md](./tabfile.md) |
4646
| TMX (.tmx) | TMX | [tmx.md](./tmx.md) |
4747
| Wiktextract (.jsonl) | Wiktextract | [wiktextract.md](./wiktextract.md) |
48+
| WordFrequency.info COCA lemma list (.wordfrequency) | WordFrequency | [wordfrequency.md](./wordfrequency.md) |
4849
| WordNet | Wordnet | [wordnet.md](./wordnet.md) |
4950
| Wordset.org JSON directory | Wordset | [wordset.md](./wordset.md) |
5051
| XDXF (.xdxf) | Xdxf | [xdxf.md](./xdxf.md) |

‎doc/p/wordfrequency.md‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
## WordFrequency.info COCA lemma list (.wordfrequency)
2+
3+
<!--
4+
This document is generated from source code. Do NOT edit.
5+
To update, modify plugins/wordfrequency/__init__.py file, then run ./scripts/gen
6+
-->
7+
8+
### General Information
9+
10+
| Attribute | Value |
11+
| --------------- | --------------------------------------------------------------- |
12+
| Name | WordFrequency |
13+
| snake_case_name | wordfrequency |
14+
| Description | WordFrequency.info COCA lemma list (.wordfrequency) |
15+
| Extensions | `.wordfrequency` |
16+
| Read support | Yes |
17+
| Write support | No |
18+
| Single-file | Yes |
19+
| Kind | 📝 text |
20+
| Wiki | ― |
21+
| Website | [Word frequency data (COCA)](<https://www.wordfrequency.info/>) |
22+
23+
### Read options
24+
25+
| Name | Default | Type | Comment |
26+
| ---------- | ------- | ---- | ---------------- |
27+
| encoding | `utf-8` | str | Encoding/charset |
28+
| gram_color | `green` | str | Grammar color |
29+
30+
### Input format
31+
32+
Tab-separated lemma frequency lists from [WordFrequency.info](https://www.wordfrequency.info/)
33+
(COCA corpus), e.g. `lemmas_60k.txt`.
34+
35+
Use the `.wordfrequency` extension, or pass `formatName=WordFrequencyInfo` for `.txt` files.

‎plugins-meta/index.json‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2041,6 +2041,41 @@
20412041
"lzma"
20422042
]
20432043
},
2044+
{
2045+
"module": "wordfrequency_info",
2046+
"lname": "wordfrequency",
2047+
"name": "WordFrequency",
2048+
"description": "WordFrequency.info COCA lemma list (.wordfrequency)",
2049+
"extensions": [
2050+
".wordfrequency"
2051+
],
2052+
"singleFile": true,
2053+
"optionsProp": {
2054+
"encoding": {
2055+
"class": "EncodingOption",
2056+
"type": "str",
2057+
"customValue": true,
2058+
"comment": "Encoding/charset"
2059+
},
2060+
"gram_color": {
2061+
"class": "StrOption",
2062+
"type": "str",
2063+
"customValue": true,
2064+
"comment": "Grammar color"
2065+
}
2066+
},
2067+
"canRead": true,
2068+
"canWrite": false,
2069+
"readOptions": {
2070+
"encoding": "utf-8",
2071+
"gram_color": "green"
2072+
},
2073+
"readCompressions": [
2074+
"gz",
2075+
"bz2",
2076+
"lzma"
2077+
]
2078+
},
20442079
{
20452080
"module": "wordnet",
20462081
"lname": "wordnet",

‎pyglossary/plugins/wordfrequency_info/__init__.py‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
# -*- coding: utf-8 -*-
2+
from __future__ import annotations
3+
4+
from typing import TYPE_CHECKING
5+
6+
from pyglossary.option import EncodingOption, StrOption
7+
8+
from .reader import Reader
9+
10+
if TYPE_CHECKING:
11+
from pyglossary.option import Option
12+
13+
__all__ = [
14+
"Reader",
15+
"description",
16+
"enable",
17+
"extensionCreate",
18+
"extensions",
19+
"kind",
20+
"lname",
21+
"name",
22+
"optionsProp",
23+
"singleFile",
24+
"website",
25+
"wiki",
26+
]
27+
28+
enable = True
29+
lname = "wordfrequency"
30+
name = "WordFrequency"
31+
description = "WordFrequency.info COCA lemma list (.wordfrequency)"
32+
extensions = (".wordfrequency",)
33+
extensionCreate = ".wordfrequency"
34+
singleFile = True
35+
kind = "text"
36+
wiki = ""
37+
website = (
38+
"https://www.wordfrequency.info/",
39+
"Word frequency data (COCA)",
40+
)
41+
42+
optionsProp: dict[str, Option] = {
43+
"encoding": EncodingOption(),
44+
"gram_color": StrOption(
45+
comment="Grammar color",
46+
),
47+
}
48+
49+
docTail = """\
50+
### Input format
51+
52+
Tab-separated lemma frequency lists from [WordFrequency.info](https://www.wordfrequency.info/)
53+
(COCA corpus), e.g. `lemmas_60k.txt`.
54+
55+
Use the `.wordfrequency` extension,
56+
or pass `formatName=WordFrequencyInfo` for `.txt` files.
57+
"""
58+
59+
60+
# Some words have multiple PoS rows: PoS rows
61+
# blue: noun (n), verb (v)
62+
# board: verb (v), preposition (i)
63+
# design: noun (n), verb (v)
64+
# hammer: noun (n), verb (v)
65+
# light: verb (v), preposition (i)

‎pyglossary/plugins/wordfrequency_info/reader.py‎

Lines changed: 238 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,238 @@
1+
# -*- coding: utf-8 -*-
2+
from __future__ import annotations
3+
4+
import html
5+
from typing import TYPE_CHECKING
6+
7+
from pyglossary.text_reader import TextGlossaryReader
8+
9+
if TYPE_CHECKING:
10+
from pyglossary.glossary_types import ReaderGlossaryType
11+
from pyglossary.text_reader import nextBlockResultType
12+
13+
__all__ = ["Reader"]
14+
15+
# CLAWS7 tag first letter; see https://ucrel.lancs.ac.uk/claws7tags.html
16+
POS_NAMES: dict[str, str] = {
17+
"a": "determiner",
18+
"c": "conjunction",
19+
"d": "WH-determiner",
20+
"e": "adverb",
21+
"g": "genitive",
22+
"i": "preposition",
23+
"j": "adjective",
24+
"m": "cardinal numeral",
25+
"n": "noun",
26+
"p": "pronoun",
27+
"r": "adverb",
28+
"s": "subordinating conjunction",
29+
"t": "infinitive marker",
30+
"u": "interjection",
31+
"v": "verb",
32+
"x": "negative",
33+
"y": "pronoun",
34+
"z": "spelling",
35+
}
36+
37+
GENRES: tuple[str, ...] = (
38+
"blog",
39+
"web",
40+
"TVM",
41+
"spok",
42+
"fic",
43+
"mag",
44+
"news",
45+
"acad",
46+
)
47+
48+
GENRE_LABELS: dict[str, str] = {
49+
"blog": "Blog",
50+
"web": "Web",
51+
"TVM": "TV/Movies",
52+
"spok": "Spoken",
53+
"fic": "Fiction",
54+
"mag": "Magazine",
55+
"news": "News",
56+
"acad": "Academic",
57+
}
58+
59+
DEFI_TEMPLATE = """\
60+
<div class="wordfrequency">
61+
<style>.wordfrequency th,.wordfrequency td{{text-align:left}}.wordfrequency th:not(:last-child),.wordfrequency td:not(:last-child){{padding-right:1.5em}}</style>
62+
<h3><font class="pos grammar" color="{gram_color}">{pos_name}</font></h3>
63+
<table>
64+
<tbody>
65+
<tr><td>Rank</td><td>{rank}</td></tr>
66+
<tr><td>Frequency</td><td>{freq}</td></tr>
67+
<tr><td>Per million</td><td>{per_mil}</td></tr>
68+
<tr><td>% capitalized</td><td>{pct_caps}</td></tr>
69+
<tr><td>% all caps</td><td>{pct_allc}</td></tr>
70+
<tr><td>Range</td><td>{range_}</td></tr>
71+
<tr><td>Dispersion</td><td>{disp}</td></tr>
72+
</tbody>
73+
</table>
74+
<h4>Frequency by genre</h4>
75+
<table>
76+
<thead><tr><th>Genre</th><th>Count</th><th>Per million</th></tr></thead>
77+
<tbody>
78+
{genre_rows}
79+
</tbody>
80+
</table>
81+
</div>
82+
"""
83+
84+
GENRE_ROW_TEMPLATE = "<tr><td>{label}</td><td>{count}</td><td>{per_mil}</td></tr>\n"
85+
86+
87+
def _parse_int(value: str) -> int:
88+
try:
89+
return int(value.replace(",", ""))
90+
except ValueError:
91+
return 0
92+
93+
94+
def _format_int(value: str) -> str:
95+
try:
96+
return f"{int(value):,}"
97+
except ValueError:
98+
return value
99+
100+
101+
def _pos_name(pos: str) -> str:
102+
if not pos:
103+
return ""
104+
key = pos[0].lower()
105+
name = POS_NAMES.get(key)
106+
if name:
107+
return name.title()
108+
return pos
109+
110+
111+
class Reader(TextGlossaryReader):
112+
useByteProgress = True
113+
_gram_color: str = "green"
114+
115+
def __init__(self, glos: ReaderGlossaryType, hasInfo: bool = False) -> None:
116+
TextGlossaryReader.__init__(self, glos, hasInfo=hasInfo)
117+
self._columns: list[str] = []
118+
self._colIndex: dict[str, int] = {}
119+
120+
def open(self, filename: str) -> None:
121+
self._glos.setDefaultDefiFormat("h")
122+
self._columns = []
123+
self._colIndex = {}
124+
TextGlossaryReader.open(self, filename)
125+
self._readPreamble()
126+
127+
def newEntry(self, word, defi: str): # noqa: ANN001, ANN201
128+
entry = TextGlossaryReader.newEntry(self, word, defi)
129+
entry.defiFormat = "h"
130+
return entry
131+
132+
def _setColumns(self, columns: list[str]) -> None:
133+
self._columns = columns
134+
self._colIndex = {name: i for i, name in enumerate(columns)}
135+
136+
def _readPreamble(self) -> None:
137+
description_lines: list[str] = []
138+
while True:
139+
line = self.readline()
140+
if not line:
141+
return
142+
line = line.rstrip("\n")
143+
if not line:
144+
continue
145+
if line.startswith("*"):
146+
description_lines.append(line.lstrip("* ").strip())
147+
continue
148+
if line.startswith("-----"):
149+
continue
150+
if line.startswith("rank\t"):
151+
self._setColumns(line.split("\t"))
152+
if description_lines:
153+
self.setInfo("description", "\n".join(description_lines))
154+
self.setInfo("name", "Word Frequency (COCA)")
155+
self.setInfo("website", "https://www.wordfrequency.info/")
156+
return
157+
158+
def _col(self, parts: list[str], name: str) -> str:
159+
index = self._colIndex.get(name)
160+
if index is None or index >= len(parts):
161+
return ""
162+
return parts[index]
163+
164+
def _renderDefinition(self, parts: list[str]) -> str:
165+
pos = self._col(parts, "PoS")
166+
genre_items: list[tuple[str, str, str]] = []
167+
for genre in GENRES:
168+
count = self._col(parts, genre)
169+
per_mil = self._col(parts, f"{genre}PM")
170+
if _parse_int(count) == 0:
171+
continue
172+
genre_items.append((genre, count, per_mil))
173+
genre_items.sort(key=lambda item: _parse_int(item[1]), reverse=True)
174+
genre_rows: list[str] = []
175+
for genre, count, per_mil in genre_items:
176+
genre_rows.append(
177+
GENRE_ROW_TEMPLATE.format(
178+
label=html.escape(GENRE_LABELS.get(genre, genre)),
179+
count=html.escape(_format_int(count)),
180+
per_mil=html.escape(per_mil),
181+
),
182+
)
183+
return DEFI_TEMPLATE.format(
184+
pos_name=html.escape(_pos_name(pos)),
185+
gram_color=html.escape(self._gram_color),
186+
rank=html.escape(self._col(parts, "rank")),
187+
freq=html.escape(_format_int(self._col(parts, "freq"))),
188+
per_mil=html.escape(self._col(parts, "perMil")),
189+
pct_caps=html.escape(self._col(parts, "%caps")),
190+
pct_allc=html.escape(self._col(parts, "%allC")),
191+
range_=html.escape(_format_int(self._col(parts, "range"))),
192+
disp=html.escape(self._col(parts, "disp")),
193+
genre_rows="".join(genre_rows),
194+
)
195+
196+
def _makeBlock(self, parts: list[str]) -> nextBlockResultType:
197+
lemma = self._col(parts, "lemma")
198+
if not lemma:
199+
return None
200+
return lemma, self._renderDefinition(parts), None
201+
202+
def nextBlock(self) -> nextBlockResultType:
203+
if not self._file:
204+
raise StopIteration
205+
while True:
206+
line = self.readline()
207+
if not line:
208+
raise StopIteration
209+
line = line.rstrip("\n")
210+
if not line:
211+
continue
212+
if line.startswith(("*", "-----")):
213+
continue
214+
if line.startswith("rank\t"):
215+
self._setColumns(line.split("\t"))
216+
continue
217+
parts = line.split("\t")
218+
if len(parts) < 3:
219+
continue
220+
if not self._colIndex:
221+
self._setColumns(
222+
[
223+
"rank",
224+
"lemma",
225+
"PoS",
226+
"freq",
227+
"perMil",
228+
"%caps",
229+
"%allC",
230+
"range",
231+
"disp",
232+
*GENRES,
233+
*(f"{g}PM" for g in GENRES),
234+
],
235+
)
236+
block = self._makeBlock(parts)
237+
if block:
238+
return block

0 commit comments

Comments
 (0)