Skip to content

Commit 77b1ef5

Browse files
fix: leave unicode normalizations in paths untouched (#12039)
* fix: bypass Qt's QFile::encodeName() in csync * Move decode name methods to the `OCC::FileSystem` namespace * Use custom mkpath to ensure no normalization is done on names * Stop the socket API from normalizing file names * fix: implement FakeFolder::fromDisk using csync functions to bypass QFile::encodeName() * Use `std::filesystem::rename` on non-Windows systems To prevent `QFile::rename` doing normalization changes to the file name. * Add file/dir name normalization test Check that a file/directory name with NFC encoding on the server ends up with the same encoding on the client, and that a subsequent discovery+sync will not upload differently encoded files. Same for an NFD encoded file/directory name. * Fix mkpath --------- Co-authored-by: Erik Verbruggen <[email protected]>
1 parent 37ff08f commit 77b1ef5

10 files changed

+223
-48
lines changed

src/common/filesystembase.cpp

+33-18
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@
2626
#include <QSettings>
2727
#include <QStorageInfo>
2828

29+
#include <filesystem>
30+
2931
#include <sys/stat.h>
3032
#include <sys/types.h>
3133

@@ -49,6 +51,16 @@ namespace OCC {
4951

5052
Q_LOGGING_CATEGORY(lcFileSystem, "sync.filesystem", QtInfoMsg)
5153

54+
QByteArray FileSystem::encodeFileName(const QString &fileName)
55+
{
56+
return fileName.toLocal8Bit();
57+
}
58+
59+
QString FileSystem::decodeFileName(const char *localFileName)
60+
{
61+
return QString::fromLocal8Bit(localFileName);
62+
}
63+
5264
QString FileSystem::longWinPath(const QString &inpath)
5365
{
5466
#ifndef Q_OS_WIN
@@ -197,28 +209,15 @@ bool FileSystem::uncheckedRenameReplace(const QString &originFileName,
197209
QString *errorString)
198210
{
199211
Q_ASSERT(errorString);
212+
200213
#ifndef Q_OS_WIN
201-
bool success;
202-
QFile orig(originFileName);
203-
// We want a rename that also overwrites. QFile::rename does not overwrite.
204-
// Qt 5.1 has QSaveFile::renameOverwrite we could use.
205-
// ### FIXME
206-
success = true;
207-
bool destExists = fileExists(destinationFileName);
208-
if (destExists && !QFile::remove(destinationFileName)) {
209-
*errorString = orig.errorString();
210-
qCWarning(lcFileSystem) << "Target file could not be removed.";
211-
success = false;
212-
}
213-
if (success) {
214-
success = orig.rename(destinationFileName);
215-
}
216-
if (!success) {
217-
*errorString = orig.errorString();
214+
std::error_code err;
215+
std::filesystem::rename(originFileName.toStdString(), destinationFileName.toStdString(), err);
216+
if (err) {
217+
*errorString = QString::fromStdString(err.message());
218218
qCWarning(lcFileSystem) << "Renaming temp file to final failed: " << *errorString;
219219
return false;
220220
}
221-
222221
#else //Q_OS_WIN
223222
// You can not overwrite a read-only file on windows.
224223

@@ -351,6 +350,22 @@ bool FileSystem::fileExists(const QString &filename, const QFileInfo &fileInfo)
351350
return re;
352351
}
353352

353+
bool FileSystem::mkpath(const QString &parent, const QString &newDir)
354+
{
355+
#ifdef Q_OS_WIN
356+
return QDir(parent).mkpath(newDir);
357+
#else // POSIX
358+
std::error_code err;
359+
QString fullPath = parent;
360+
if (!fullPath.endsWith(u'/')) {
361+
fullPath += u'/';
362+
}
363+
fullPath += newDir;
364+
std::filesystem::create_directories(fullPath.toStdString(), err);
365+
return err.value() == 0;
366+
#endif
367+
}
368+
354369
QString FileSystem::fileSystemForPath(const QString &path)
355370
{
356371
QString p = path;

src/common/filesystembase.h

+5
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,9 @@ OCSYNC_EXPORT Q_DECLARE_LOGGING_CATEGORY(lcFileSystem)
4747
namespace FileSystem {
4848
OCSYNC_EXPORT Q_NAMESPACE;
4949

50+
QByteArray OCSYNC_EXPORT encodeFileName(const QString &fileName);
51+
QString OCSYNC_EXPORT decodeFileName(const char *localFileName);
52+
5053
/**
5154
* List of characters not allowd in filenames on Windows
5255
*/
@@ -105,6 +108,8 @@ namespace FileSystem {
105108
*/
106109
bool OCSYNC_EXPORT fileExists(const QString &filename, const QFileInfo & = QFileInfo());
107110

111+
bool OCSYNC_EXPORT mkpath(const QString &parent, const QString &newDir);
112+
108113
/**
109114
* @brief Rename the file \a originFileName to \a destinationFileName.
110115
*

src/csync/std/c_time.cpp

+1-3
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,9 @@
2323

2424
#include "common/filesystembase.h"
2525

26-
#include <QFile>
27-
2826
#ifdef HAVE_UTIMES
2927
int c_utimes(const QString &uri, const struct timeval *times) {
30-
int ret = utimes(QFile::encodeName(uri).constData(), times);
28+
int ret = utimes(uri.toLocal8Bit().constData(), times);
3129
return ret;
3230
}
3331
#else // HAVE_UTIMES

src/csync/vio/csync_vio_local_unix.cpp

+6-6
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,11 @@
2828

2929
#include "csync.h"
3030

31-
#include "vio/csync_vio_local.h"
31+
#include "common/filesystembase.h"
3232
#include "common/vfs.h"
33+
#include "vio/csync_vio_local.h"
3334

3435
#include <QtCore/QLoggingCategory>
35-
#include <QtCore/QFile>
3636

3737
Q_LOGGING_CATEGORY(lcCSyncVIOLocal, "sync.csync.vio_local", QtInfoMsg)
3838

@@ -48,7 +48,7 @@ struct csync_vio_handle_t {
4848
csync_vio_handle_t *csync_vio_local_opendir(const QString &name) {
4949
std::unique_ptr<csync_vio_handle_t> handle(new csync_vio_handle_t{});
5050

51-
auto dirname = QFile::encodeName(name);
51+
auto dirname = OCC::FileSystem::encodeFileName(name);
5252

5353
handle->dh = opendir(dirname.constData());
5454
if (!handle->dh) {
@@ -77,7 +77,7 @@ std::unique_ptr<csync_file_stat_t> csync_vio_local_readdir(csync_vio_handle_t *h
7777
} while (qstrcmp(dirent->d_name, ".") == 0 || qstrcmp(dirent->d_name, "..") == 0);
7878

7979
file_stat.reset(new csync_file_stat_t);
80-
file_stat->path = QFile::decodeName(dirent->d_name);
80+
file_stat->path = OCC::FileSystem::decodeFileName(dirent->d_name);
8181

8282
/* Check for availability of d_type, see manpage. */
8383
#if defined(_DIRENT_HAVE_D_TYPE) || defined(__APPLE__)
@@ -120,8 +120,8 @@ int csync_vio_local_stat(const QString &uri, csync_file_stat_t *buf)
120120
{
121121
struct stat sb;
122122

123-
if (lstat(QFile::encodeName(uri).constData(), &sb) < 0) {
124-
return -1;
123+
if (lstat(OCC::FileSystem::encodeFileName(uri).constData(), &sb) < 0) {
124+
return -1;
125125
}
126126

127127
switch (sb.st_mode & S_IFMT) {

src/gui/socketapi/socketapi.cpp

+1-3
Original file line numberDiff line numberDiff line change
@@ -235,9 +235,7 @@ void SocketApi::slotReadSocket()
235235
static auto invalidListener = QSharedPointer<SocketListener>::create(nullptr);
236236
const auto listener = _listeners.value(socket, invalidListener);
237237
while (socket->canReadLine()) {
238-
// Make sure to normalize the input from the socket to
239-
// make sure that the path will match, especially on OS X.
240-
QString line = QString::fromUtf8(socket->readLine()).normalized(QString::NormalizationForm_C);
238+
QString line = QString::fromUtf8(socket->readLine());
241239
// Note: do NOT use QString::trimmed() here! That will also remove any trailing spaces (which _are_ part of the filename)!
242240
line.chop(1); // remove the '\n'
243241

src/libsync/propagatorjobs.cpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -178,8 +178,8 @@ void PropagateLocalMkdir::start()
178178
done(SyncFileItem::NormalError, tr("Can not create local folder %1 because of a local file name clash with %2").arg(newDirStr, QDir::toNativeSeparators(clash.get())));
179179
return;
180180
}
181-
QDir localDir(propagator()->localPath());
182-
if (!localDir.mkpath(_item->_file)) {
181+
182+
if (!FileSystem::mkpath(propagator()->localPath(), _item->_file)) {
183183
done(SyncFileItem::NormalError, tr("could not create folder %1").arg(newDirStr));
184184
return;
185185
}

test/testfilesystem.cpp

+68
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,74 @@ private Q_SLOTS:
137137

138138
QVERIFY(tmp.remove());
139139
}
140+
141+
void testMkdir_data()
142+
{
143+
QTest::addColumn<QString>("name");
144+
145+
const unsigned char a_umlaut_composed_bytes[] = {0xc3, 0xa4, 0x00};
146+
const QString a_umlaut_composed = QString::fromUtf8(reinterpret_cast<const char *>(a_umlaut_composed_bytes));
147+
const QString a_umlaut_decomposed = a_umlaut_composed.normalized(QString::NormalizationForm_D);
148+
149+
QTest::newRow("a-umlaut composed") << a_umlaut_composed;
150+
QTest::newRow("a-umlaut decomposed") << a_umlaut_decomposed;
151+
}
152+
153+
// This is not a full test, it is meant to verify that no nomalization changes are done.
154+
// The implementation of `OCC::FileSystem::mkpath` relies on either Qt (Windows) or
155+
// `std::filesystem` (POSIX), which should be covered by tests of their own.
156+
void testMkdir()
157+
{
158+
QFETCH(QString, name);
159+
160+
auto tmp = OCC::TestUtils::createTempDir();
161+
QVERIFY(OCC::FileSystem::mkpath(tmp.path(), name));
162+
csync_file_stat_t buf;
163+
auto dh = csync_vio_local_opendir(tmp.path());
164+
QVERIFY(dh);
165+
while (auto fs = csync_vio_local_readdir(dh, nullptr)) {
166+
QCOMPARE(fs->path, name);
167+
QCOMPARE(fs->type, ItemTypeDirectory);
168+
}
169+
csync_vio_local_closedir(dh);
170+
}
171+
172+
void testRename_data()
173+
{
174+
QTest::addColumn<QString>("name");
175+
176+
const unsigned char a_umlaut_composed_bytes[] = {0xc3, 0xa4, 0x00};
177+
const QString a_umlaut_composed = QString::fromUtf8(reinterpret_cast<const char *>(a_umlaut_composed_bytes));
178+
const QString a_umlaut_decomposed = a_umlaut_composed.normalized(QString::NormalizationForm_D);
179+
180+
QTest::newRow("a-umlaut composed") << a_umlaut_composed;
181+
QTest::newRow("a-umlaut decomposed") << a_umlaut_decomposed;
182+
}
183+
184+
// This is not a full test, it is meant to verify that no nomalization changes are done.
185+
void testRename()
186+
{
187+
QFETCH(QString, name);
188+
189+
auto tmp = OCC::TestUtils::createTempDir();
190+
QFile f(tmp.path() + u"/abc");
191+
QVERIFY(f.open(QFile::WriteOnly));
192+
QByteArray data("abc");
193+
QCOMPARE(f.write(data), data.size());
194+
f.close();
195+
196+
QString err;
197+
QVERIFY(OCC::FileSystem::uncheckedRenameReplace(f.fileName(), tmp.path() + u'/' + name, &err));
198+
199+
csync_file_stat_t buf;
200+
auto dh = csync_vio_local_opendir(tmp.path());
201+
QVERIFY(dh);
202+
while (auto fs = csync_vio_local_readdir(dh, nullptr)) {
203+
QCOMPARE(fs->path, name);
204+
QCOMPARE(fs->type, ItemTypeFile);
205+
}
206+
csync_vio_local_closedir(dh);
207+
}
140208
};
141209

142210
QTEST_GUILESS_MAIN(TestFileSystem)

test/testlocaldiscovery.cpp

+77
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,83 @@ private Q_SLOTS:
238238
QVERIFY(!fakeFolder.currentRemoteState().find(QStringLiteral("C/.foo")));
239239
QVERIFY(!fakeFolder.currentRemoteState().find(QStringLiteral("C/bar")));
240240
}
241+
242+
void testNameNormalization_data()
243+
{
244+
QTest::addColumn<QString>("correct");
245+
QTest::addColumn<QString>("incorrect");
246+
247+
const unsigned char a_umlaut_composed_bytes[] = {0xc3, 0xa4, 0x00};
248+
const QString a_umlaut_composed = QString::fromUtf8(reinterpret_cast<const char *>(a_umlaut_composed_bytes));
249+
const QString a_umlaut_decomposed = a_umlaut_composed.normalized(QString::NormalizationForm_D);
250+
251+
QTest::newRow("a_umlaut decomposed") << a_umlaut_decomposed << a_umlaut_composed;
252+
QTest::newRow("a_umlaut composed") << a_umlaut_composed << a_umlaut_decomposed;
253+
}
254+
255+
// Test that when a file/directory name on the remote is encoded in NFC, the local name is encoded
256+
// in the same way, and that a subsequent sync does not change anything. And the same for NFD.
257+
void testNameNormalization()
258+
{
259+
QFETCH_GLOBAL(Vfs::Mode, vfsMode);
260+
QFETCH_GLOBAL(bool, filesAreDehydrated);
261+
262+
QFETCH(QString, correct);
263+
QFETCH(QString, incorrect);
264+
265+
// Create an empty remote folder
266+
FakeFolder fakeFolder({FileInfo{}}, vfsMode, filesAreDehydrated);
267+
OperationCounter counter(fakeFolder);
268+
269+
// Create a file with an a-umlout in the "correct" normalization:
270+
fakeFolder.remoteModifier().mkdir(QStringLiteral("P"));
271+
fakeFolder.remoteModifier().mkdir(QStringLiteral("P/A"));
272+
fakeFolder.remoteModifier().insert(QStringLiteral("P/A/") + correct);
273+
274+
// Same for a directory, holding a "normal" file:
275+
fakeFolder.remoteModifier().mkdir(QStringLiteral("P/B") + correct);
276+
fakeFolder.remoteModifier().insert(QStringLiteral("P/B") + correct + QStringLiteral("/b"));
277+
278+
LocalDiscoveryTracker tracker;
279+
connect(&fakeFolder.syncEngine(), &SyncEngine::itemCompleted, &tracker, &LocalDiscoveryTracker::slotItemCompleted);
280+
connect(&fakeFolder.syncEngine(), &SyncEngine::finished, &tracker, &LocalDiscoveryTracker::slotSyncFinished);
281+
282+
// First sync: discover that there are files/directories on the server that are not yet synced to the local end
283+
QVERIFY(fakeFolder.applyLocalModificationsAndSync());
284+
285+
// Check that locally we have the file and the directory with the correct names:
286+
{
287+
auto localState = fakeFolder.currentLocalState();
288+
QVERIFY(localState.find(QStringLiteral("P/A/") + correct) != nullptr); // check if the file exists
289+
QVERIFY(localState.find(QStringLiteral("P/B") + correct + QStringLiteral("/b")) != nullptr); // check if the file exists
290+
}
291+
292+
counter.reset();
293+
294+
qDebug() << "*** MARK"; // Log marker to check if a PUT/DELETE shows up in the second sync
295+
296+
// Force a full local discovery on the next sync, which forces a walk of the (local) file system, reading back names (and file sizes/mtimes/etc.)...
297+
fakeFolder.syncEngine().setLocalDiscoveryOptions(LocalDiscoveryStyle::DatabaseAndFilesystem, {QStringLiteral("P")});
298+
tracker.startSyncFullDiscovery();
299+
300+
// ... and start the second sync:
301+
QVERIFY(fakeFolder.applyLocalModificationsAndSync());
302+
303+
// If the normalization of the file/directory name did not change, no rename/move/etc. should have been detected, so check that the client didn't issue
304+
// any of these operations:
305+
QCOMPARE(counter.nDELETE, 0);
306+
QCOMPARE(counter.nMOVE, 0);
307+
QCOMPARE(counter.nPUT, 0);
308+
309+
// Check that the remote names are unchanged, and that no "incorrect" names have been introduced:
310+
FileInfo &remoteState = fakeFolder.currentRemoteState();
311+
QVERIFY(remoteState.find(QStringLiteral("P/A/") + correct) != nullptr); // check if the file still exists in the original normalization
312+
QVERIFY(remoteState.find(QStringLiteral("P/A/") + incorrect) == nullptr); // there should NOT be a file with another normalization
313+
QVERIFY(remoteState.find(QStringLiteral("P/B") + correct + QStringLiteral("/b"))
314+
!= nullptr); // check if the directory still exists in the original normalization
315+
QVERIFY(remoteState.find(QStringLiteral("P/B") + incorrect + QStringLiteral("/b"))
316+
== nullptr); // there should NOT be a directory with another normalization
317+
}
241318
};
242319

243320
QTEST_GUILESS_MAIN(TestLocalDiscovery)

0 commit comments

Comments
 (0)