Add tag support in sync process

src/common/preparedsqlquerymanager.h

@@ -108,6 +108,8 @@ class OCSYNC_EXPORT PreparedSqlQueryManager
         GetE2EeLockedFoldersQuery,
         DeleteE2EeLockedFolderQuery,
         ListAllTopLevelE2eeFoldersStatusLessThanQuery,
+        UpdateTagListQuery,
+        GetTagListQuery,
 
         PreparedQueryCount
     };

src/common/syncjournaldb.cpp

@@ -49,7 +49,7 @@ Q_LOGGING_CATEGORY(lcDb, "nextcloud.sync.database", QtInfoMsg)
 #define GET_FILE_RECORD_QUERY \
         "SELECT path, inode, modtime, type, md5, fileid, remotePerm, filesize," \
         "  ignoredChildrenRemote, contentchecksumtype.name || ':' || contentChecksum, e2eMangledName, isE2eEncrypted, " \
-        "  lock, lockOwnerDisplayName, lockOwnerId, lockType, lockOwnerEditor, lockTime, lockTimeout, isShared, lastShareStateFetchedTimestmap, sharedByMe" \
+        "  lock, lockOwnerDisplayName, lockOwnerId, lockType, lockOwnerEditor, lockTime, lockTimeout, isShared, lastShareStateFetchedTimestmap, sharedByMe, tagList" \
         " FROM metadata" \
         "  LEFT JOIN checksumtype as contentchecksumtype ON metadata.contentChecksumTypeId == contentchecksumtype.id"
 
@@ -77,6 +77,7 @@ static void fillFileRecordFromGetQuery(SyncJournalFileRecord &rec, SqlQuery &que
     rec._isShared = query.intValue(19) > 0;
     rec._lastShareStateFetchedTimestamp = query.int64Value(20);
     rec._sharedByMe = query.intValue(21) > 0;
+    rec._tagList = query.baValue(22);
 }
 
 static QByteArray defaultJournalMode(const QString &dbPath)
@@ -832,6 +833,8 @@ bool SyncJournalDb::updateMetadataTableStructure()
     }
     commitInternal(QStringLiteral("update database structure: add basePath index"));
 
+    addColumn(QStringLiteral("tagList"), QStringLiteral("TEXT"));
+
     return re;
 }
 
@@ -958,7 +961,8 @@ Result<void, QString> SyncJournalDb::setFileRecord(const SyncJournalFileRecord &
                  << "lock editor:" << record._lockstate._lockEditorApp
                  << "sharedByMe:" << record._sharedByMe
                  << "isShared:" << record._isShared
-                 << "lastShareStateFetchedTimestamp:" << record._lastShareStateFetchedTimestamp;
+                 << "lastShareStateFetchedTimestamp:" << record._lastShareStateFetchedTimestamp
+                 << "tagList:"<<record._tagList;
 
     const qint64 phash = getPHash(record._path);
     if (!checkConnect()) {
@@ -984,8 +988,8 @@ Result<void, QString> SyncJournalDb::setFileRecord(const SyncJournalFileRecord &
     const auto query = _queryManager.get(PreparedSqlQueryManager::SetFileRecordQuery, QByteArrayLiteral("INSERT OR REPLACE INTO metadata "
                                                                                                         "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, "
                                                                                                         "contentChecksum, contentChecksumTypeId, e2eMangledName, isE2eEncrypted, lock, lockType, lockOwnerDisplayName, lockOwnerId, "
-                                                                                                        "lockOwnerEditor, lockTime, lockTimeout, isShared, lastShareStateFetchedTimestmap, sharedByMe) "
-                                                                                                        "VALUES (?1 , ?2, ?3 , ?4 , ?5 , ?6 , ?7,  ?8 , ?9 , ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21, ?22, ?23, ?24, ?25, ?26, ?27, ?28);"),
+                                                                                                        "lockOwnerEditor, lockTime, lockTimeout, isShared, lastShareStateFetchedTimestmap, sharedByMe, tagList) "
+                                                                                                        "VALUES (?1 , ?2, ?3 , ?4 , ?5 , ?6 , ?7,  ?8 , ?9 , ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21, ?22, ?23, ?24, ?25, ?26, ?27, ?28, ?29);"),
         _db);
     if (!query) {
         return query->error();
@@ -1019,6 +1023,7 @@ Result<void, QString> SyncJournalDb::setFileRecord(const SyncJournalFileRecord &
     query->bindValue(26, record._isShared);
     query->bindValue(27, record._lastShareStateFetchedTimestamp);
     query->bindValue(28, record._sharedByMe);
+    query->bindValue(29, record._tagList);
 
     if (!query->exec()) {
         return query->error();
@@ -1497,6 +1502,58 @@ int SyncJournalDb::getFileRecordCount()
     return -1;
 }
 
+bool SyncJournalDb::updateMetadataTagList(const QString &filename,const QByteArray* tagList)
+{
+    QMutexLocker locker(&_mutex);
+
+    qCInfo(lcDb) << "Updating file tag list " << filename;
+
+    if (!checkConnect()) {
+        qCWarning(lcDb) << "Failed to connect database.";
+        return false;
+    }
+
+    const qint64 phash = getPHash(filename.toUtf8());
+
+    const auto query = _queryManager.get(PreparedSqlQueryManager::UpdateTagListQuery,
+                                         QByteArrayLiteral("UPDATE metadata"
+                                                           " SET tagList = ?2"
+                                                           " WHERE phash = ?1;"),
+                                          _db);
+    if (!query) {
+        return false;
+    }
+    query->bindValue(1, phash);
+    query->bindValue(2, *tagList);
+    return query->exec();
+}
+
+QByteArray SyncJournalDb::tagList(const QString &file)
+{
+    QMutexLocker locker(&_mutex);
+    if (!checkConnect())
+        return QByteArray();
+
+    const qint64 phash = getPHash(file.toUtf8());
+
+    const auto query = _queryManager.get(PreparedSqlQueryManager::GetTagListQuery,
+                                         QByteArrayLiteral("SELECT tagList FROM metadata"
+                                                           " WHERE phash = ?1;"),
+                                         _db);
+    if (!query) {
+        return QByteArray();
+    }
+
+    query->bindValue(1, phash);
+    if (!query->exec())return QByteArray();
+
+    auto next = query->next();
+    if (!next.ok || !next.hasData)return QByteArray();
+
+    QByteArray result = query->baValue(0);
+    return result;
+}
+
 bool SyncJournalDb::updateFileRecordChecksum(const QString &filename,
     const QByteArray &contentChecksum,
     const QByteArray &contentChecksumType)

src/common/syncjournaldb.h

@@ -246,6 +246,16 @@ class OCSYNC_EXPORT SyncJournalDb : public QObject
     void setDataFingerprint(const QByteArray &dataFingerprint);
     QByteArray dataFingerprint();
 
+    /*!
+    * \brief Update the taglist in the metadata table.
+     */
+    bool updateMetadataTagList(const QString &file,
+                               const QByteArray* tagList);
+
+    /*!
+     * \brief Returns the taglist of the queried file.
+     */
+    QByteArray tagList(const QString &file);
 
     // Conflict record functions
 

src/common/syncjournalfilerecord.cpp

@@ -51,6 +51,7 @@ bool operator==(const SyncJournalFileRecord &lhs,
         && lhs._fileSize == rhs._fileSize
         && lhs._remotePerm == rhs._remotePerm
         && lhs._serverHasIgnoredFiles == rhs._serverHasIgnoredFiles
-        && lhs._checksumHeader == rhs._checksumHeader;
+        && lhs._checksumHeader == rhs._checksumHeader
+        && lhs._tagList == rhs._tagList;
 }
 }

src/common/syncjournalfilerecord.h

@@ -87,6 +87,7 @@ class OCSYNC_EXPORT SyncJournalFileRecord
     bool _isShared = false;
     qint64 _lastShareStateFetchedTimestamp = 0;
     bool _sharedByMe = false;
+    QByteArray _tagList;
 };
 
 QDebug& operator<<(QDebug &stream, const SyncJournalFileRecord::EncryptionStatus status);

src/csync/csync.h

@@ -221,6 +221,8 @@ struct OCSYNC_EXPORT csync_file_stat_s {
   QByteArray directDownloadCookies;
   QByteArray original_path; // only set if locale conversion fails
 
+  QByteArray tagList; // \n-separated List of tags
+
   // In the local tree, this can hold a checksum and its type if it is
   //   computed during discovery for some reason.
   // In the remote tree, this will have the server checksum, if available.

src/csync/vio/csync_vio_local_unix.cpp

@@ -28,6 +28,15 @@
 
 #include <memory>
 
+#ifdef __APPLE__
+#include <CoreFoundation/CFBundle.h>
+#include <CoreServices/CoreServices.h>
+#endif
+
+#ifdef __linux__
+#include <sys/xattr.h>
+#endif
+
 #include "c_private.h"
 #include "c_lib.h"
 #include "csync.h"
@@ -120,6 +129,65 @@ std::unique_ptr<csync_file_stat_t> csync_vio_local_readdir(csync_vio_handle_t *h
       file_stat->type = ItemTypeSkip;
   }
 
+
+#ifdef __APPLE__
+
+  // Create necessary system related objects
+  CFStringRef cfstr = CFStringCreateWithCString(kCFAllocatorDefault,
+                                                fullPath.constData(),
+                                                kCFStringEncodingUTF8);
+  CFURLRef urlref = CFURLCreateWithFileSystemPath(kCFAllocatorDefault,
+                                                  cfstr,
+                                                  kCFURLPOSIXPathStyle,
+                                                  dirent->d_type == DT_DIR);
+
+  // Query tags
+  CFArrayRef labels=NULL;
+  Boolean result = CFURLCopyResourcePropertyForKey(urlref,
+                                                   kCFURLTagNamesKey,
+                                                   &labels,
+                                                   NULL);
+
+  if(result==true && labels != NULL){
+    // Extract the labels to our array
+    int count = (int) CFArrayGetCount(labels);
+
+    if(count>0){
+      QStringList tagarray;
+
+      for(int index=0;index<count;index++){
+        CFStringRef str = (CFStringRef)CFArrayGetValueAtIndex(labels,index);
+        if(CFStringGetLength(str)>0)tagarray << QString::fromCFString(str);
+      }
+      tagarray.sort(Qt::CaseInsensitive);
+      QString tagList = tagarray.join(QChar(0x000A));
+      file_stat->tagList=tagList.toUtf8();
+    }
+  }
+
+  // Clean up
+  CFRelease(cfstr);
+  CFRelease(urlref);
+  if(labels!=NULL)CFRelease(labels);
+
+#endif
+
+#ifdef __linux__
+	const int sz=4096;
+	char buffer[sz];
+	int result = getxattr(fullPath.constData(),"user.xdg.tags",buffer,sz);
+
+	if(result>0)
+	{
+		// Data is store usually as comma(,) separated list.
+		// So we just need to replace ',' by '\n'.
+		QByteArray tagList = QByteArray(buffer,result);
+		tagList.replace(',','\n');
+		printf("%s\n",buffer);
+		file_stat->tagList= tagList;
+	}
+#endif
+
   // Override type for virtual files if desired
   if (vfs) {
       // Directly modifies file_stat->type.

src/gui/editlocallyjob.cpp

@@ -213,7 +213,9 @@ void EditLocallyJob::fetchRemoteFileParentInfo()
                                   QByteArrayLiteral("http://owncloud.org/ns:size"),
                                   QByteArrayLiteral("http://owncloud.org/ns:id"),
                                   QByteArrayLiteral("http://owncloud.org/ns:permissions"),
-                                  QByteArrayLiteral("http://owncloud.org/ns:checksums")};
+                                  QByteArrayLiteral("http://owncloud.org/ns:checksums"),
+                                  QByteArrayLiteral("http://owncloud.org/ns:tags"),
+                                  QByteArrayLiteral("http://nextcloud.org/ns:system-tags")};
 
     job->setProperties(props);
     connect(job, &LsColJob::directoryListingIterated, this, &EditLocallyJob::slotDirectoryListingIterated);

src/gui/folder.cpp

@@ -558,7 +558,6 @@ void Folder::slotWatchedPathChanged(const QString &path, ChangeReason reason)
         qCDebug(lcFolder) << "Changed path is not contained in folder, ignoring:" << path;
         return;
     }
-
     auto relativePath = path.midRef(this->path().size());
 
     if (pathIsIgnored(path)) {
@@ -610,7 +609,7 @@ void Folder::slotWatchedPathChanged(const QString &path, ChangeReason reason)
         // an attribute change (pin state) that caused the notification
         bool spurious = false;
         if (record.isValid()
-            && !FileSystem::fileChanged(path, record._fileSize, record._modtime)) {
+            && !FileSystem::fileChanged(path, record._fileSize, record._modtime,record._tagList)) {
             spurious = true;
 
             if (auto pinState = _vfs->pinState(relativePath.toString())) {
@@ -628,7 +627,6 @@ void Folder::slotWatchedPathChanged(const QString &path, ChangeReason reason)
         }
     }
     warnOnNewExcludedItem(record, relativePath);
-
     emit watchedFileChangedExternally(path);
 
     // Also schedule this folder for a sync, but only after some delay:

src/gui/folderwatcher_mac.cpp

@@ -71,8 +71,9 @@ static void callback(
         | kFSEventStreamEventFlagItemInodeMetaMod // for mtime change
         | kFSEventStreamEventFlagItemRenamed // also coming for moves to trash in finder
         | kFSEventStreamEventFlagItemModified // for content change
-        | kFSEventStreamEventFlagItemCloned; // for cloned items (since 10.13)
-    //We ignore other flags, e.g. for owner change, xattr change, Finder label change etc
+        | kFSEventStreamEventFlagItemCloned // for cloned items (since 10.13)
+        | kFSEventStreamEventFlagItemXattrMod; // for tags change (which are stored as xattr)
+        //We ignore other flags, e.g. for owner change etc.
 
     QStringList paths;
     CFArrayRef eventPaths = (CFArrayRef)eventPathsVoid;
@@ -111,7 +112,7 @@ void FolderWatcherPrivate::startWatching()
         pathsToWatch,
         kFSEventStreamEventIdSinceNow,
         0, // latency
-        kFSEventStreamCreateFlagUseCFTypes | kFSEventStreamCreateFlagFileEvents | kFSEventStreamCreateFlagIgnoreSelf);
+        kFSEventStreamCreateFlagUseCFTypes | kFSEventStreamCreateFlagFileEvents | kFSEventStreamCreateFlagIgnoreSelf | kFSEventStreamEventFlagItemXattrMod);
 
     CFRelease(pathsToWatch);
     CFRelease(folderCF);

src/libsync/CMakeLists.txt

@@ -46,6 +46,8 @@ set(libsync_SRCS
     encryptedfoldermetadatahandler.cpp
     filesystem.h
     filesystem.cpp
+    filetags.h
+    filetags.cpp
     helpers.cpp
     httplogger.h
     httplogger.cpp

src/libsync/discovery.cpp

@@ -17,6 +17,7 @@
 #include "common/filesystembase.h"
 #include "common/syncjournaldb.h"
 #include "filesystem.h"
+#include "filetags.h"
 #include "syncfileitem.h"
 #include "progressdispatcher.h"
 #include <QDebug>
@@ -482,7 +483,8 @@ void ProcessDirectoryJob::processFile(PathTuple path,
                               << " | e2ee: " << dbEntry.isE2eEncrypted() << "/" << serverEntry.isE2eEncrypted()
                               << " | e2eeMangledName: " << dbEntry.e2eMangledName() << "/" << serverEntry.e2eMangledName
                               << " | file lock: " << localFileIsLocked << "//" << serverFileIsLocked
-                              << " | metadata missing: /" << localEntry.isMetadataMissing << '/';
+                              << " | metadata missing: /" << localEntry.isMetadataMissing << '/'
+                              << " | tagList: " << dbEntry._tagList << "//" << localEntry.tagList << "//" << serverEntry.tagList;
 
     if (localEntry.isValid()
         && !serverEntry.isValid()
@@ -491,6 +493,19 @@ void ProcessDirectoryJob::processFile(PathTuple path,
         qCWarning(lcDisco) << "File" << path._original << "was modified before the last sync run and is not in the sync journal and server";
     }
 
+    if(localEntry.isValid()
+        && dbEntry.isValid()) {
+
+        // Tag synchronization
+        FileTagManager::syncTags(_discoveryData->_account,
+                                 _discoveryData->_statedb,
+                                 _discoveryData->_localDir,
+                                 path._original,
+                                 localEntry,
+                                 serverEntry,
+                                 dbEntry);
+    }
+
     if (_discoveryData->isRenamed(path._original)) {
         qCDebug(lcDisco) << "Ignoring renamed";
         return; // Ignore this.
@@ -656,6 +671,7 @@ void ProcessDirectoryJob::processFileAnalyzeRemoteInfo(const SyncFileItemPtr &it
     item->_lockEditorApp = serverEntry.lockEditorApp;
     item->_lockTime = serverEntry.lockTime;
     item->_lockTimeout = serverEntry.lockTimeout;
+    item->_tagList = serverEntry.tagList;
 
     qCDebug(lcDisco()) << "item lock for:" << item->_file
                        << item->_locked