Skip to content

Commit 092f91a

Browse files
HoppingTappyHoppingTappy
committed
Merge remote-tracking branch 'upstream/master'
2 parents 9b19f1f + 0345a2f commit 092f91a

16 files changed

+189
-13
lines changed

‎.github/workflows/sonarscan.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -111,10 +111,10 @@ jobs:
111111
$env:CMD_TESTS1 `
112112
--gtest_output=xml:${{ github.workspace }}\tests1-googletest.xml
113113
114-
- name: Set up JDK 11
114+
- name: Set up JDK 17
115115
uses: actions/setup-java@v1
116116
with:
117-
java-version: 11
117+
java-version: 17
118118

119119
- name: Fetch SonarScanner
120120
working-directory: ${{ github.workspace }}\.sonar

‎sakura_core/charset/CCodeBase.cpp

+11-2
Original file line numberDiff line numberDiff line change
@@ -54,8 +54,17 @@ std::wstring CCodeBase::CodeToHex(const CNativeW& cSrc, const CommonSetting_Stat
5454
// 表示用16進表示 UNICODE → Hex 変換 2008/6/9 Uchi
5555
EConvertResult CCodeBase::UnicodeToHex(const wchar_t* cSrc, const int iSLen, WCHAR* pDst, const CommonSetting_Statusbar* psStatusbar)
5656
{
57-
if (IsUTF16High(cSrc[0]) && iSLen >= 2 && IsUTF16Low(cSrc[1])) {
58-
// サロゲートペア
57+
// IVS
58+
if (iSLen >= 3 && IsVariationSelector(cSrc + 1)) {
59+
if (psStatusbar->m_bDispSPCodepoint) {
60+
auto_sprintf(pDst, L"%04X, U+%05X", cSrc[0], ConvertToUtf32(cSrc + 1));
61+
}
62+
else {
63+
auto_sprintf(pDst, L"%04X, %04X%04X", cSrc[0], cSrc[1], cSrc[2]);
64+
}
65+
}
66+
// サロゲートペア
67+
else if (iSLen >= 2 && IsSurrogatePair(cSrc)) {
5968
if (psStatusbar->m_bDispSPCodepoint) {
6069
auto_sprintf( pDst, L"U+%05X", 0x10000 + ((cSrc[0] & 0x3FF)<<10) + (cSrc[1] & 0x3FF));
6170
}

‎sakura_core/charset/CUtf8.cpp

+3
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,9 @@ EConvertResult CUtf8::_UnicodeToHex(const wchar_t* cSrc, const int iSLen, WCHAR*
227227
if (IsUTF16High(cSrc[0]) && iSLen >= 2 && IsUTF16Low(cSrc[1])) {
228228
cBuff._GetMemory()->SetRawDataHoldBuffer(cSrc, 4);
229229
}
230+
else if (iSLen >= 3 && IsVariationSelector(cSrc + 1)) {
231+
cBuff._GetMemory()->SetRawDataHoldBuffer(cSrc, sizeof(wchar_t) * 3);
232+
}
230233
else {
231234
cBuff._GetMemory()->SetRawDataHoldBuffer(cSrc, 2);
232235
if( IsBinaryOnSurrogate(cSrc[0]) ){

‎sakura_core/charset/codechecker.h

+37
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,43 @@ inline bool IsUTF16Low( wchar_t c ){
359359
return IsUtf16SurrogLow(c);
360360
}
361361

362+
/*!
363+
* 文字列がサロゲートペアで始まっているか判定する
364+
*/
365+
inline bool IsSurrogatePair(std::wstring_view text) {
366+
return 2 <= text.length() && IsUTF16High(text[0]) && IsUTF16Low(text[1]);
367+
}
368+
369+
/*!
370+
* UTF16文字列の先頭1文字をUTF32コードポイントに変換する
371+
*
372+
* @return 文字列の先頭1文字のコードポイント
373+
*/
374+
inline
375+
_Success_(return != 0)
376+
char32_t ConvertToUtf32(std::wstring_view text) {
377+
if (text.empty()) {
378+
return 0;
379+
}
380+
if (IsSurrogatePair(text)) {
381+
return 0x10000 + ((text[0] & 0x3ff) << 10) + (text[1] & 0x3ff);
382+
}
383+
if (const auto ch = text[0];
384+
!(ch & 0xfc00))
385+
{
386+
return ch;
387+
}
388+
return 0;
389+
}
390+
391+
/*!
392+
* 文字列がIVSの異体字セレクタで始まっているか判定する
393+
*/
394+
inline bool IsVariationSelector(std::wstring_view text) {
395+
const auto cp = ConvertToUtf32(text);
396+
return 0xe0100 <= cp && cp <= 0xe01ef;
397+
}
398+
362399
//! 上位バイトと下位バイトを交換 (主に UTF-16 LE/BE 向け)
363400
inline unsigned short _SwapHLByte( const unsigned short wc ){
364401
unsigned short wc1 = static_cast<unsigned short>( (static_cast<unsigned int>(wc) << 8) & 0x0000ffff );

‎sakura_core/mem/CNativeW.cpp

+6
Original file line numberDiff line numberDiff line change
@@ -396,6 +396,12 @@ CLogicInt CNativeW::GetSizeOfChar( const wchar_t* pData, int nDataLen, int nIdx
396396
}
397397
}
398398

399+
// IVSの異体字セレクタチェック
400+
if (IsVariationSelector(pData + nIdx + 1)) {
401+
// 正字 + 異体字セレクタで3個分
402+
return CLogicInt(3);
403+
}
404+
399405
return CLogicInt(1);
400406
}
401407

‎sakura_core/parse/CWordParse.cpp

+11-7
Original file line numberDiff line numberDiff line change
@@ -140,11 +140,10 @@ ECharKind CWordParse::WhatKindOfChar(
140140
{
141141
using namespace WCODE;
142142

143-
int nCharChars = CNativeW::GetSizeOfChar( pData, pDataLen, nIdx );
144-
if( nCharChars == 0 ){
145-
return CK_NULL; // NULL
146-
}
147-
else if( nCharChars == 1 ){
143+
ECharKind ret = CK_NULL;
144+
if(const auto nCharChars = CNativeW::GetSizeOfChar(pData, pDataLen, nIdx);
145+
nCharChars == 1)
146+
{
148147
wchar_t c=pData[nIdx];
149148

150149
//今までの半角
@@ -186,9 +185,14 @@ ECharKind CWordParse::WhatKindOfChar(
186185
}
187186
return CK_ETC; // 半角のその他
188187
}
189-
else{
190-
return CK_NULL; // NULL
188+
// IVS（正字 + 異体字セレクタ）
189+
else if (nCharChars == 3 &&
190+
IsVariationSelector(pData + nIdx + 1))
191+
{
192+
ret = CK_ZEN_ETC; // 全角のその他(漢字など)
191193
}
194+
195+
return ret;
192196
}
193197

194198
//! 二つの文字を結合したものの種類を調べる

‎sakura_core/view/CEditView_Command_New.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -613,7 +613,7 @@ void CEditView::DeleteData(
613613
nNxtPos = GetCaret().GetCaretLayoutPos().GetX() + CLayoutInt(pcLayout->GetLayoutEol().GetLen()>0?1+m_pcEditDoc->m_cLayoutMgr.GetCharSpacing():0);
614614
}
615615
else{
616-
nNxtIdx = CLogicInt(CNativeW::GetCharNext( pLine, nLineLen, &pLine[nCurIdx] ) - pLine);
616+
nNxtIdx = nCurIdx + CNativeW::GetSizeOfChar( pLine, nLineLen, nCurIdx);
617617
// 指定された行のデータ内の位置に対応する桁の位置を調べる
618618
nNxtPos = LineIndexToColumn( pcLayout, nNxtIdx );
619619
}

‎sakura_core/view/CTextDrawer.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ void CTextDrawer::DispText( HDC hdc, DispPos* pDispPos, int marginy, const wchar
132132
nWorkWidth += pDrawDxArray[nDrawLength++];
133133
}
134134
// サロゲートペア対策 2008/7/5 Uchi Update 7/8 Uchi
135-
if (nDrawLength < nDrawDataMaxLength && pDrawDxArray[nDrawLength] == 0) {
135+
while (nDrawLength < nDrawDataMaxLength && pDrawDxArray[nDrawLength] == 0) {
136136
nDrawLength++;
137137
}
138138

‎sakura_core/view/CTextMetrics.cpp

+6
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,12 @@ const int* CTextMetrics::GenerateDxArray(
108108
}
109109
vResultArray.push_back(cache.CalcPxWidthByFont(pText[i]) + spacing);
110110
nIndent += vResultArray.back();
111+
112+
if (IsVariationSelector(pText + i + 1)) {
113+
vResultArray.push_back(0);
114+
vResultArray.push_back(0);
115+
i += 2;
116+
}
111117
}
112118
return vResultArray.data();
113119
}

‎tests/unittests/test-ccodebase.cpp

+23
Original file line numberDiff line numberDiff line change
@@ -813,6 +813,9 @@ TEST(CCodeBase, Utf8ToHex)
813813

814814
// カラー絵文字「男性のシンボル」（サロゲートペア）
815815
EXPECT_STREQ(L"F09F9AB9", pCodeBase->CodeToHex(L"\U0001F6B9", sStatusbar).c_str());
816+
817+
// IVS(Ideographic Variation Sequence) 「葛󠄀」（葛󠄀城市の葛󠄀、下がヒ）
818+
EXPECT_STREQ(L"E8919BF3A08480", pCodeBase->CodeToHex(L"葛󠄀", sStatusbar).c_str());
816819
}
817820

818821
/*!
@@ -837,3 +840,23 @@ TEST(CCodeBase, Latin1ToHex)
837840
// カラー絵文字「男性のシンボル」（サロゲートペア）
838841
EXPECT_STREQ(L"D83DDEB9", pCodeBase->CodeToHex(L"\U0001F6B9", sStatusbar).c_str());
839842
}
843+
844+
TEST(CCodeBase, UnicodeToHex)
845+
{
846+
const auto eCodeType = CODE_UNICODE;
847+
auto pCodeBase = CCodeFactory::CreateCodeBase(eCodeType);
848+
849+
// 特定コードのマルチバイトを表示する設定
850+
CommonSetting_Statusbar sStatusbar;
851+
sStatusbar.m_bDispUniInSjis = false;
852+
sStatusbar.m_bDispUniInJis = false;
853+
sStatusbar.m_bDispUniInEuc = false;
854+
sStatusbar.m_bDispUtf8Codepoint = false;
855+
sStatusbar.m_bDispSPCodepoint = false;
856+
857+
sStatusbar.m_bDispSPCodepoint = true;
858+
EXPECT_STREQ(L"845B, U+E0100", pCodeBase->CodeToHex(L"葛󠄀", sStatusbar).c_str());
859+
860+
sStatusbar.m_bDispSPCodepoint = false;
861+
EXPECT_STREQ(L"845B, DB40DD00", pCodeBase->CodeToHex(L"葛󠄀", sStatusbar).c_str());
862+
}

‎tests/unittests/test-cnative.cpp

+20
Original file line numberDiff line numberDiff line change
@@ -792,6 +792,26 @@ TEST(CNativeW, GetSizeOfChar)
792792
EXPECT_EQ(CNativeW::GetSizeOfChar(CStringRef(L"\xd83c\xdf38", 2), 1), 1);
793793
}
794794

795+
TEST(CNativeW, GetSizeOfChar_Empty)
796+
{
797+
const auto& s = L"";
798+
EXPECT_EQ(0, CNativeW::GetSizeOfChar(s, _countof(s) - 1, 0));
799+
}
800+
801+
TEST(CNativeW, GetSizeOfChar_SurrogatePair)
802+
{
803+
// 絵文字　男性のシンボル
804+
const auto& s = L"\U0001f6b9";
805+
EXPECT_EQ(2, CNativeW::GetSizeOfChar(s, _countof(s) - 1, 0));
806+
}
807+
808+
TEST(CNativeW, GetSizeOfChar_IVS)
809+
{
810+
// 葛󠄀城市(先頭の文字が異体字)
811+
const auto& s = L"葛󠄀城市";
812+
EXPECT_EQ(3, CNativeW::GetSizeOfChar(s, _countof(s) - 1, 0));
813+
}
814+
795815
/*!
796816
* @brief GetKetaOfCharの仕様
797817
* @remark 指定した文字の桁数を返す。

‎tests/unittests/test-codechecker.cpp

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
Copyright (C) 2023, Sakura Editor Organization
3+
4+
This software is provided 'as-is', without any express or implied
5+
warranty. In no event will the authors be held liable for any damages
6+
arising from the use of this software.
7+
8+
Permission is granted to anyone to use this software for any purpose,
9+
including commercial applications, and to alter it and redistribute it
10+
freely, subject to the following restrictions:
11+
12+
1. The origin of this software must not be misrepresented;
13+
you must not claim that you wrote the original software.
14+
If you use this software in a product, an acknowledgment
15+
in the product documentation would be appreciated but is
16+
not required.
17+
18+
2. Altered source versions must be plainly marked as such,
19+
and must not be misrepresented as being the original software.
20+
21+
3. This notice may not be removed or altered from any source
22+
distribution.
23+
*/
24+
25+
#include <gtest/gtest.h>
26+
#include "charset/codechecker.h"
27+
28+
TEST(ConvertToUtf32, SurrogatePair)
29+
{
30+
// 絵文字　男性のシンボル
31+
const auto& s = L"\U0001f6b9";
32+
EXPECT_EQ(0x1f6b9, ConvertToUtf32(s));
33+
}
34+
35+
TEST(ConvertToUtf32, VariationSelector)
36+
{
37+
// 異体字セレクタ　VS17
38+
const auto& s = L"\U000e0100";
39+
EXPECT_EQ(0xe0100, ConvertToUtf32(s));
40+
}
41+
42+
TEST(ConvertToUtf32, BinaryOnSurrogate)
43+
{
44+
// 独自仕様　変換できない文字は1byteずつ下位サロゲートに詰める
45+
const auto& s = L"\xdcff";
46+
EXPECT_EQ(0, ConvertToUtf32(s));
47+
}

‎tests/unittests/test-ctextmetrics.cpp

+11
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,17 @@ TEST(CTextMetrics, GenerateDxArray7)
220220
EXPECT_EQ(v[3], 99);
221221
}
222222

223+
TEST(CTextMetrics, GenerateDxArray8)
224+
{
225+
// IVSのVariantSelectorが続く文字列は先頭1文字 + 幅0×2で生成する
226+
std::vector<int> v;
227+
FakeCache1 cache;
228+
CTextMetrics::GenerateDxArray(&v, L"葛󠄀", 2, 0, 0, 0, 10, cache);
229+
EXPECT_TRUE(v[0]);
230+
EXPECT_FALSE(v[1]);
231+
EXPECT_FALSE(v[2]);
232+
}
233+
223234
TEST(CTextMetrics, CalcTextWidth)
224235
{
225236
int dx[] = {1, 2, 3};

‎tests/unittests/test-cwordparse.cpp

+6
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,12 @@ TEST(WhatKindOfChar, SurrogatePairs)
140140
EXPECT_EQ(CK_ZEN_ETC, CWordParse::WhatKindOfChar(L"\xd842\xdfb7", 2, 0));
141141
}
142142

143+
TEST(WhatKindOfChar, IVS)
144+
{
145+
// EXPECT_EQ(CK_ETC, CWordParse::WhatKindOfChar(L"葛󠄀", 3, 0));
146+
EXPECT_EQ(CK_ZEN_ETC, CWordParse::WhatKindOfChar(L"葛󠄀", 3, 0));
147+
}
148+
143149
TEST(WhatKindOfTwoChars, ReturnsSameKindIfTwoKindsAreIdentical)
144150
{
145151
EXPECT_EQ(CK_HIRA, CWordParse::WhatKindOfTwoChars(CK_HIRA, CK_HIRA));

‎tests/unittests/tests1.vcxproj

+1
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@
110110
<ClCompile Include="code-main.cpp" />
111111
<ClCompile Include="test-cdocline.cpp" />
112112
<ClCompile Include="test-cdoclinemgr.cpp" />
113+
<ClCompile Include="test-codechecker.cpp" />
113114
<ClCompile Include="test-cppa.cpp" />
114115
<ClCompile Include="test-csearchagent.cpp" />
115116
<ClCompile Include="test-crunningtimer.cpp" />

‎tests/unittests/tests1.vcxproj.filters

+3
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,9 @@
172172
<ClCompile Include="test-extmodules.cpp">
173173
<Filter>Test Files</Filter>
174174
</ClCompile>
175+
<ClCompile Include="test-codechecker.cpp">
176+
<Filter>Test Files</Filter>
177+
</ClCompile>
175178
</ItemGroup>
176179
<ItemGroup>
177180
<ClInclude Include="StartEditorProcessForTest.h">

0 commit comments

Comments
 (0)