|
| 1 | +#!/usr/bin/env bash |
| 2 | + |
| 3 | +set -ex |
| 4 | + |
| 5 | +log() { |
| 6 | + # This function is from espnet |
| 7 | + local fname=${BASH_SOURCE[1]##*/} |
| 8 | + echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*" |
| 9 | +} |
| 10 | + |
| 11 | +echo "EXE is $EXE" |
| 12 | +echo "PATH: $PATH" |
| 13 | + |
| 14 | +which $EXE |
| 15 | + |
| 16 | +log "------------------------------------------------------------" |
| 17 | +log "Download the CATT (Encoder-Only) diacritization model " |
| 18 | +log "------------------------------------------------------------" |
| 19 | + |
| 20 | +tmp_dir=catt_eo_model_onnx |
| 21 | +mkdir -p $tmp_dir |
| 22 | +cd $tmp_dir |
| 23 | +curl -SL -O https://github.com/abjadai/catt/releases/download/v2/eo_model_onnx.zip |
| 24 | +unzip -o eo_model_onnx.zip |
| 25 | +rm eo_model_onnx.zip |
| 26 | +cd .. |
| 27 | + |
| 28 | +assert_diacritized() { |
| 29 | + local input="$1" |
| 30 | + local output="$2" |
| 31 | + python - "$input" "$output" <<'PY' |
| 32 | +import sys |
| 33 | +inp, out = sys.argv[1], sys.argv[2] |
| 34 | +assert out, "output is empty" |
| 35 | +assert out != inp, "output equals input (no diacritization applied)" |
| 36 | +# 0x064B fathatan, 0x0652 sukun, 0x0670 superscript alef |
| 37 | +tashkeel = set(range(0x064B, 0x0653)) | {0x0670} |
| 38 | +assert any(ord(c) in tashkeel for c in out), \ |
| 39 | + "output contains no tashkeel codepoint" |
| 40 | +print("OK") |
| 41 | +PY |
| 42 | +} |
| 43 | + |
| 44 | +first_input="وقالت مجلة نيوزويك الأمريكية التحديث الجديد ل إنستجرام يمكن أن يساهم في إيقاف وكشف الحسابات المزورة بسهولة شديدة" |
| 45 | +first_output=$($EXE \ |
| 46 | + --debug=1 \ |
| 47 | + --catt-encoder=$tmp_dir/encoder.onnx \ |
| 48 | + --catt-decoder=$tmp_dir/decoder.onnx \ |
| 49 | + "$first_input") |
| 50 | +assert_diacritized "$first_input" "$first_output" |
| 51 | + |
| 52 | +second_input="اللغة العربية من أقدم اللغات السامية" |
| 53 | +second_output=$($EXE \ |
| 54 | + --debug=1 \ |
| 55 | + --catt-encoder=$tmp_dir/encoder.onnx \ |
| 56 | + --catt-decoder=$tmp_dir/decoder.onnx \ |
| 57 | + "$second_input") |
| 58 | +assert_diacritized "$second_input" "$second_output" |
| 59 | + |
| 60 | +rm -rf $tmp_dir |
0 commit comments