Skip to content

Add configurable custom MCM regex limits #3

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vanmatten wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Add configurable custom MCM regex limits #3

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion Misc/customTxtDefinition.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
#CustomTypeIsFallback_1=false define if the pattern is the fallback if no pattern in the list can match.
#CustomTypeSaveLangSuffixe_1=true if true, rebuild the output name with the target language suffixe (ex: mcmcfile_en.txt)
#CustomTypeSearchByLine_1=true if true, analyse the text with a single match per line, if false, process a match-again routine over the whole text in a raw. The later process might be very slow on big files.
#CustomTypeLineLimit_1=50 define the maximum number of lines to look for a regex match, starting from the beginning.
#CustomTypeSizeLimit_1=2048 define the maximum number of characters to look for a regex match, starting from the first one.
#---------
SupportedTxtExt=*.txt;*.ini;*.csv;*.pas;*.xml;*.json;*.pot;
#---------
Expand All @@ -19,6 +21,8 @@ CustomTypeRestrictExt_1=.txt
CustomTypeSaveLangSuffixe_1=true
CustomTypeIsFallback_1=false
CustomTypeSearchByLine_1=true
CustomTypeLineLimit_1=50
CustomTypeSizeLimit_1=2048
#---------
CustomTypeRegex_2=^(?![;#])([^=]+)====(?!.*={2,})(.*)$
CustomTypeLabel_2=res
Expand All @@ -28,6 +32,8 @@ CustomTypeRestrictExt_2=.ini
CustomTypeSaveLangSuffixe_2=false
CustomTypeIsFallback_2=false
CustomTypeSearchByLine_2=true
CustomTypeLineLimit_2=50
CustomTypeSizeLimit_2=2048
#---------
CustomTypeRegex_3=^(?!#)(.+~[0-9a-fA-F]+)\|.*?name\("(.+?)"\)(?:\|.*?name\("(.+?)"\))?
CustomTypeLabel_3=rftp
Expand All @@ -37,6 +43,8 @@ CustomTypeRestrictExt_3=.txt
CustomTypeSaveLangSuffixe_3=false
CustomTypeIsFallback_3=false
CustomTypeSearchByLine_3=true
CustomTypeLineLimit_3=50
CustomTypeSizeLimit_3=2048
#---------
CustomTypeRegex_4=^(0x.+~[^\|]+)\|([^\|]+)
CustomTypeLabel_4=descfw
Expand All @@ -46,6 +54,8 @@ CustomTypeRestrictExt_4=.txt
CustomTypeSaveLangSuffixe_4=false
CustomTypeIsFallback_4=false
CustomTypeSearchByLine_4=true
CustomTypeLineLimit_4=50
CustomTypeSizeLimit_4=2048
#-----fallback: catch each line directly--------
CustomTypeRegex_5=^(.*)$
CustomTypeLabel_5=Generic
Expand All @@ -55,6 +65,8 @@ CustomTypeRestrictExt_5=*
CustomTypeSaveLangSuffixe_5=false
CustomTypeIsFallback_5=true
CustomTypeSearchByLine_5=true
CustomTypeLineLimit_5=50
CustomTypeSizeLimit_5=2048
#---------
CustomTypeRegex_6=<(description|moduleName)>(.*?)<\/\1>
CustomTypeLabel_6=fomod
Expand All @@ -64,6 +76,8 @@ CustomTypeRestrictExt_6=.txt
CustomTypeSaveLangSuffixe_6=false
CustomTypeIsFallback_6=false
CustomTypeSearchByLine_6=false
CustomTypeLineLimit_6=50
CustomTypeSizeLimit_6=2048
#---------testPOT
#CustomTypeRegex_7=^(?!#)msgctxt "(.*?)"$^(?!#)msgid "(.*?)"$
#CustomTypeRegex_7=msgctxt "(.*?)".*?msgid "(.*?)"
Expand All @@ -74,6 +88,8 @@ CustomTypeSearchByLine_6=false
#CustomTypeSaveLangSuffixe_7=false
#CustomTypeIsFallback_7=false
#CustomTypeSearchByLine_7=false
#CustomTypeLineLimit_7=50
#CustomTypeSizeLimit_7=2048
#---------test
#CustomTypeRegex_9=^(?!#)msgctxt "(.*?)"$^(?!#)msgid "(.*?)"$
#CustomTypeRegex_9=msgid "(.*?)"$
Expand All @@ -84,6 +100,8 @@ CustomTypeSearchByLine_6=false
#CustomTypeSaveLangSuffixe_9=false
#CustomTypeIsFallback_9=false
#CustomTypeSearchByLine_9=true
#CustomTypeLineLimit_9=50
#CustomTypeSizeLimit_9=2048
#---------test
#CustomTypeRegex_8="form_id": "(.*?)",.*?"original": "(.*?)",
#CustomTypeRegex_8="original": "(.*?)",
Expand All @@ -94,4 +112,5 @@ CustomTypeSearchByLine_6=false
#CustomTypeSaveLangSuffixe_8=false
#CustomTypeIsFallback_8=false
#CustomTypeSearchByLine_8=true

#CustomTypeLineLimit_8=50
#CustomTypeSizeLimit_8=2048
35 changes: 27 additions & 8 deletions TESVT_Const.pas
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,8 @@ rcustomTxtDefinition = record
iIdBackref: integer;
bUseSaveLangSuffixe: boolean;
stats: integer;
iLineLimit: integer;
iSizeLimit: integer;
end;

pcustomTxtDefinition = ^rcustomTxtDefinition;
Expand All @@ -123,6 +125,8 @@ tCustomTxt = class
function getrefId(i: integer): integer;
function getLabel(i: integer): string;
function getEncoding(i: integer): tencoding;
function getLineLimit(i: integer): integer;
function getSizeLimit(i: integer): integer;
function mcmDecider(f: tstringList): integer;
function isValidTxtExt(ext: string): boolean;
function isSearchByLine(i: integer): boolean;
Expand Down Expand Up @@ -1091,6 +1095,8 @@ constructor tCustomTxt.Create(l: tstringList);
p.bIsFallback := strtobooldef(l.Values[format('CustomTypeIsFallback_%d', [i])], false);
p.bUseSaveLangSuffixe := strtobooldef(l.Values[format('CustomTypeSaveLangSuffixe_%d', [i])], false);
p.bSearchByLine := strtobooldef(l.Values[format('CustomTypeSearchByLine_%d', [i])], true);
p.iLineLimit := strtointdef(l.Values[format('CustomTypeLineLimit_%d', [i])], 50);
p.iSizeLimit := strtointdef(l.Values[format('CustomTypeSizeLimit_%d', [i])], 2048);

end;
// define fallback
Expand Down Expand Up @@ -1195,13 +1201,26 @@ procedure tCustomTxt.resetstat;
pcustomTxtDefinition(lCustomTxtParamList[i]).stats := 0;
end;

function tCustomTxt.getLineLimit(i: integer): integer;
var
p: pcustomTxtDefinition;
begin
p := lCustomTxtParamList[i];
Result := p.iLineLimit;
end;

function tCustomTxt.getSizeLimit(i: integer): integer;
var
p: pcustomTxtDefinition;
begin
p := lCustomTxtParamList[i];
Result := p.iSizeLimit;
end;

function tCustomTxt.mcmDecider(f: tstringList): integer;
var
i, j, m: integer;
RegEx: tperlRegex;
const
iLineLimit = 50;
iSizeLimit = 2048;
begin
Result := -1;
if lCustomTxtParamList.Count = 0 then
Expand All @@ -1226,13 +1245,13 @@ function tCustomTxt.mcmDecider(f: tstringList): integer;
RegEx.subject := f[i];
if RegEx.Match then
incStat(j);
if i > iLineLimit then
if i > getLineLimit(j) then
break;
end;
end
else
begin
RegEx.subject := copy(f.text, 1, iSizeLimit);
RegEx.subject := copy(f.text, 1, getSizeLimit(j));
if RegEx.Match then
repeat
incStat(j);
Expand Down Expand Up @@ -1569,7 +1588,7 @@ procedure enableToolBar(tb: TToolBar; b: boolean);
function isChar_WordDelimiter(c: char): boolean;
begin
case c of
#0 .. #32, '.', ',', '?', '!', ':', ';', '(', ')', '[', ']', '{', '}', '�', '"', '/', '\', '|', '+', '*', '=', '%', '&', '>', '<', '�', '�', '$', '@', '�', '#', '~', '^', '_',
#0 .. #32, '.', ',', '?', '!', ':', ';', '(', ')', '[', ']', '{', '}', '°', '"', '/', '\', '|', '+', '*', '=', '%', '&', '>', '<', '¡', '¿', '$', '@', '§', '#', '~', '^', '_',
#160, #8239,
// Non-breaking spaces ,
#8230, // Suspension points
Expand All @@ -1587,15 +1606,15 @@ function isChar_WordDelimiterExtended(c: char): boolean;
Result := isChar_WordDelimiter(c);
if not Result then
case c of
'-', '''', #8209, '�', #8720:
'-', '''', #8209, '’', #8720:
Result := true;
end;
end;

function isChar_WordDelimitersEndLine(c: char): boolean;
begin
case c of
'.', '?', '!', '�', '�', #8230:
'.', '?', '!', '¡', '¿', #8230:
Result := true;
else
Result := false;
Expand Down