Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion ext/js/language/language-descriptors.js
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ import {addSerboCroatianDiacritics, removeSerboCroatianAccentMarks} from './sh/s
import {albanianTransforms} from './sq/albanian-transforms.js';
import {capitalizeFirstLetter, decapitalize, removeAlphabeticDiacritics} from './text-processors.js';
import {tagalogTransforms} from './tl/tagalog-transforms.js';
import {removeUkrainianDiacritics, ukrainianApostropheVariants} from './uk/ukrainian-text-preprocessors.js';
import {ukrainianTransforms} from './uk/ukrainian-transforms.js';
import {normalizeDiacritics} from './vi/viet-text-preprocessors.js';
import {convertFinalLetters, convertYiddishLigatures} from './yi/yiddish-text-postprocessors.js';
import {combineYiddishLigatures, removeYiddishDiacritics} from './yi/yiddish-text-preprocessors.js';
Expand Down Expand Up @@ -551,7 +553,12 @@ const languageDescriptors = [
iso639_3: 'ukr',
name: 'Ukrainian',
exampleText: 'читати',
textPreprocessors: capitalizationPreprocessors,
textPreprocessors: {
...capitalizationPreprocessors,
removeUkrainianDiacritics,
ukrainianApostropheVariants,
},
languageTransforms: ukrainianTransforms,
},
{
iso: 'vi',
Expand Down
47 changes: 47 additions & 0 deletions ext/js/language/uk/ukrainian-text-preprocessors.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
/*
* Copyright (C) 2026 Yomitan Authors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/

/**
* Ukrainian uses several visually similar characters for the apostrophe which separates a labial
* consonant from a following iotated vowel, as in "п'ять". Dictionaries and the text they are
* scanned against rarely agree on which one to use, so all of them are treated as equivalent.
*/
const apostropheVariantsRegExp = /['‘’ʼ`´]/g;

/**
* Dictionary terms are indexed exactly as the dictionary spells them, so a variant has to be
* produced for every character an entry might realistically be stored with, not just for the one
* the scanned text happens to use. These three all occur as headword spellings in practice.
*/
const apostropheNormalizations = ['\'', '’', 'ʼ'];

/** @type {import('language').TextProcessor} */
export const removeUkrainianDiacritics = {
name: 'Remove diacritics',
description: 'Á → A, á → a',
process: (str) => [str, str.replace(/́/g, '')],
};

/** @type {import('language').TextProcessor} */
export const ukrainianApostropheVariants = {
name: 'Search for apostrophe variants',
description: '’ → \', ʼ → \' and vice versa',
process: (str) => [
str,
...apostropheNormalizations.map((apostrophe) => str.replace(apostropheVariantsRegExp, apostrophe)),
],
Comment thread
JayXT marked this conversation as resolved.
};
Loading