Skip to content

Commit 7c08343

Browse files
authored
of Menu 0.7.0: safer cleanup, disk usage, --json mode and translation dictionary (#8673)
* of Menu fixes and api * i18n: keep catalogue files LF; tests tolerate CRLF checkouts (Git Bash) * i18n: one strings.tsv with a column per language (replaces en.tsv and per-language files) - header key<TAB>en<TAB>de<TAB>..., one row per string; empty cells fall back to English - i18n_extract.py rewrites key + en from the code and keeps every language column - of i18n import adds or merges a language column (validated, other languages untouched); check / list / export read the same file - tests cover several languages in one file, merge, CRLF checkouts
1 parent 9adefb0 commit 7c08343

10 files changed

Lines changed: 3134 additions & 479 deletions

File tree

‎.gitattributes‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
# Files with LF line endings for Unix-like systems
1818
*.sh text eol=lf
19+
*.tsv text eol=lf
1920

2021
# Binary files (no EOL normalization)
2122
*.png binary
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
name: of cleanup safety
2+
3+
on:
4+
pull_request:
5+
paths:
6+
- "scripts/of.sh"
7+
- "scripts/ui.sh"
8+
- "scripts/dev/tests/test-cleanup-safety.sh"
9+
- "scripts/dev/tests/test-of-json.sh"
10+
- "scripts/dev/tests/test-i18n.sh"
11+
- "scripts/dev/i18n_extract.py"
12+
- "scripts/i18n/**"
13+
- ".github/workflows/cleanup-safety.yml"
14+
push:
15+
branches: [master]
16+
paths:
17+
- "scripts/of.sh"
18+
- "scripts/ui.sh"
19+
- "scripts/dev/tests/test-cleanup-safety.sh"
20+
- "scripts/dev/tests/test-of-json.sh"
21+
- "scripts/dev/tests/test-i18n.sh"
22+
- "scripts/dev/i18n_extract.py"
23+
- "scripts/i18n/**"
24+
workflow_dispatch:
25+
26+
concurrency:
27+
group: cleanup-safety-${{ github.event.pull_request.number || github.sha }}
28+
cancel-in-progress: true
29+
30+
jobs:
31+
test:
32+
name: ${{ matrix.name }}
33+
runs-on: ${{ matrix.os }}
34+
strategy:
35+
fail-fast: false
36+
matrix:
37+
include:
38+
- { name: Linux, os: ubuntu-24.04 }
39+
- { name: macOS, os: macos-latest }
40+
- { name: Windows (Git Bash), os: windows-latest }
41+
steps:
42+
- uses: actions/checkout@v7.0.1
43+
- name: Cleanup safety tests
44+
shell: bash
45+
run: bash scripts/dev/tests/test-cleanup-safety.sh
46+
- name: of --json contract tests
47+
shell: bash
48+
run: bash scripts/dev/tests/test-of-json.sh
49+
- name: Translation dictionary tests
50+
shell: bash
51+
run: bash scripts/dev/tests/test-i18n.sh
52+
- name: Cleanup safety tests (macOS system bash 3.2)
53+
if: runner.os == 'macOS'
54+
shell: bash
55+
run: |
56+
/bin/bash scripts/dev/tests/test-cleanup-safety.sh
57+
/bin/bash scripts/dev/tests/test-of-json.sh
58+
/bin/bash scripts/dev/tests/test-i18n.sh

‎scripts/dev/i18n_extract.py‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
#!/usr/bin/env python3
2+
"""Keep the English column of scripts/i18n/strings.tsv in step with the code.
3+
4+
python3 scripts/dev/i18n_extract.py # update strings.tsv
5+
python3 scripts/dev/i18n_extract.py --check # exit 1 if it is out of date
6+
7+
Every user-facing string in scripts/of.sh and scripts/ui.sh is written
8+
$(t some.key "English text with %s placeholders" arg ...)
9+
strings.tsv is the single dictionary file: a header `key<TAB>en<TAB>de<TAB>...`
10+
and one row per string, one column per language. This tool rewrites the key and
11+
`en` columns from the code and keeps every other language column as it is
12+
(new strings get empty cells, removed strings lose their row). Empty cells fall
13+
back to English at runtime.
14+
"""
15+
import os, re, sys
16+
17+
ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..'))
18+
SOURCES = ['scripts/of.sh', 'scripts/ui.sh']
19+
OUT = os.path.join(ROOT, 'scripts', 'i18n', 'strings.tsv')
20+
CALL = re.compile(r'\$\(t ([A-Za-z0-9_.]+) "((?:[^"\\]|\\.)*)"')
21+
22+
def unescape(text):
23+
# what the shell turns the double-quoted text into
24+
return re.sub(r'\\([\"$`\\])', r'\1', text)
25+
26+
def extract():
27+
found = {}
28+
for rel in SOURCES:
29+
with open(os.path.join(ROOT, rel), encoding='utf-8') as f:
30+
src = f.read()
31+
for m in CALL.finditer(src):
32+
key, text = m.group(1), unescape(m.group(2))
33+
if key in found and found[key] != text:
34+
sys.exit(f'{rel}: key {key!r} used with two different texts:\n {found[key]!r}\n {text!r}')
35+
found[key] = text
36+
for key, text in found.items():
37+
if '\t' in text or '\n' in text:
38+
sys.exit(f'key {key!r}: tabs and newlines are not allowed in catalogue text')
39+
return found
40+
41+
def read_existing():
42+
"""-> (languages other than en, {key: {lang: text}})"""
43+
langs, rows = [], {}
44+
try:
45+
with open(OUT, encoding='utf-8', newline='') as f:
46+
lines = f.read().replace('\r\n', '\n').split('\n')
47+
except FileNotFoundError:
48+
return langs, rows
49+
header = lines[0].split('\t') if lines and lines[0] else ['key', 'en']
50+
langs = [l for l in header[1:] if l != 'en']
51+
for line in lines[1:]:
52+
if not line:
53+
continue
54+
cells = line.split('\t')
55+
row = dict(zip(header[1:], cells[1:]))
56+
rows[cells[0]] = row
57+
return langs, rows
58+
59+
def render(found, langs, rows):
60+
out = ['\t'.join(['key', 'en'] + langs)]
61+
for key in sorted(found):
62+
old = rows.get(key, {})
63+
out.append('\t'.join([key, found[key]] + [old.get(l, '') for l in langs]))
64+
return '\n'.join(out) + '\n'
65+
66+
def main():
67+
found = extract()
68+
langs, rows = read_existing()
69+
body = render(found, langs, rows)
70+
if '--check' in sys.argv:
71+
try:
72+
current = open(OUT, encoding='utf-8', newline='').read().replace('\r\n', '\n')
73+
except FileNotFoundError:
74+
current = ''
75+
if current != body:
76+
print('scripts/i18n/strings.tsv is out of date: run python3 scripts/dev/i18n_extract.py', file=sys.stderr)
77+
sys.exit(1)
78+
print(f'strings.tsv is up to date ({len(found)} strings, languages: en{"".join(", " + l for l in langs)})')
79+
return
80+
os.makedirs(os.path.dirname(OUT), exist_ok=True)
81+
with open(OUT, 'w', encoding='utf-8', newline='\n') as f:
82+
f.write(body)
83+
print(f'wrote {OUT} ({len(found)} strings, languages: en{"".join(", " + l for l in langs)})')
84+
85+
main()

0 commit comments

Comments
 (0)