feat: SMB crawler

.gitignore

@@ -1,8 +1,13 @@
+.DS_Store
 target
 /.settings
 /.classpath
 /.project
+.settings
+.classpath
+.project
 .idea
 *.iml
 /.run
 /logs/
+/venv/

README.md

@@ -7,7 +7,7 @@ This crawler helps to index binary documents such as PDF, Open Office, MS Office
 **Main features**:
 
 * Local file system (or a mounted drive) crawling and index new files, update existing ones and removes old ones.
-* Remote file system over SSH crawling.
+* Remote file system over SSH/FTP/SMB crawling.
 * REST interface to let you "upload" your binary documents to elasticsearch.
 
 You need to install a version matching your Elasticsearch version:

cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java

@@ -34,6 +34,7 @@
 import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings;
 import fr.pilato.elasticsearch.crawler.fs.settings.FsSettingsFileHandler;
 import fr.pilato.elasticsearch.crawler.fs.settings.FsSettingsParser;
+import fr.pilato.elasticsearch.crawler.fs.settings.Server.PROTOCOL;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
@@ -208,6 +209,20 @@ public static void main(String[] args) throws Exception {
             if (fsSettings.getFs() == null) {
                 fsSettings.setFs(Fs.DEFAULT);
             }
+
+            if (fsSettings.getServer() != null) {
+                if (fsSettings.getServer().getProtocol().equals(PROTOCOL.FTP) && fsSettings.getServer().getPort() == PROTOCOL.SSH_PORT) {
+                    fsSettings.getServer().setPort(PROTOCOL.FTP_PORT);
+                }
+                if (fsSettings.getServer().getProtocol().equals(PROTOCOL.FTP) && StringUtils.isEmpty(fsSettings.getServer().getUsername())) {
+                    fsSettings.getServer().setUsername("anonymous");
+                }
+
+                if (fsSettings.getServer().getProtocol().equals(PROTOCOL.SMB) && StringUtils.isEmpty(fsSettings.getServer().getUsername())) {
+                    fsSettings.getServer().setUsername("Guest");
+                }
+            }
+
             if (fsSettings.getElasticsearch() == null) {
                 fsSettings.setElasticsearch(Elasticsearch.DEFAULT());
             }

core/pom.xml

@@ -69,10 +69,18 @@
             <groupId>fr.pilato.elasticsearch.crawler</groupId>
             <artifactId>fscrawler-crawler-fs</artifactId>
         </dependency>
+        <dependency>
+            <groupId>fr.pilato.elasticsearch.crawler</groupId>
+            <artifactId>fscrawler-crawler-ftp</artifactId>
+        </dependency>
         <dependency>
             <groupId>fr.pilato.elasticsearch.crawler</groupId>
             <artifactId>fscrawler-crawler-ssh</artifactId>
         </dependency>
+        <dependency>
+            <groupId>fr.pilato.elasticsearch.crawler</groupId>
+            <artifactId>fscrawler-crawler-smb</artifactId>
+        </dependency>
 
         <!-- Our Elasticsearch Client -->
         <dependency>

core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java

@@ -123,9 +123,15 @@ public void start() throws Exception {
             if (settings.getServer() == null || Server.PROTOCOL.LOCAL.equals(settings.getServer().getProtocol())) {
                 // Local FS
                 fsParser = new FsParserLocal(settings, config, managementService, documentService, loop);
+            } else if (Server.PROTOCOL.SMB.equals(settings.getServer().getProtocol())) {
+                // Remote SMB FS
+                fsParser = new FsParserSmb(settings, config, managementService, documentService, loop);
             } else if (Server.PROTOCOL.SSH.equals(settings.getServer().getProtocol())) {
                 // Remote SSH FS
                 fsParser = new FsParserSsh(settings, config, managementService, documentService, loop);
+            } else if (Server.PROTOCOL.FTP.equals(settings.getServer().getProtocol())) {
+                // Remote FTP FS
+                fsParser = new FsParserFTP(settings, config, managementService, documentService, loop);
             } else {
                 // Non supported protocol
                 throw new RuntimeException(settings.getServer().getProtocol() + " is not supported yet. Please use " +
@@ -146,7 +152,7 @@ public void close() throws InterruptedException, IOException {
         if (fsParser != null) {
             fsParser.close();
 
-            synchronized(fsParser.getSemaphore()) {
+            synchronized (fsParser.getSemaphore()) {
                 fsParser.getSemaphore().notifyAll();
             }
         }

core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java

@@ -28,21 +28,23 @@
 import fr.pilato.elasticsearch.crawler.fs.beans.ScanStatistic;
 import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractModel;
 import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor;
+import fr.pilato.elasticsearch.crawler.fs.crawler.fs.FileAbstractorFile;
 import fr.pilato.elasticsearch.crawler.fs.framework.ByteSizeValue;
 import fr.pilato.elasticsearch.crawler.fs.framework.FSCrawlerLogger;
+import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil;
 import fr.pilato.elasticsearch.crawler.fs.framework.OsValidator;
 import fr.pilato.elasticsearch.crawler.fs.framework.SignTool;
 import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService;
 import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementService;
 import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerService;
 import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings;
+import fr.pilato.elasticsearch.crawler.fs.settings.Server.PROTOCOL;
 import fr.pilato.elasticsearch.crawler.fs.tika.TikaDocParser;
 import fr.pilato.elasticsearch.crawler.fs.tika.XmlDocParser;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
 import java.io.BufferedReader;
-import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
@@ -97,12 +99,10 @@ public abstract class FsParserAbstract extends FsParser {
             messageDigest = null;
         }
 
-        // On Windows, when using SSH server, we need to force the "Linux" separator
-        if (OsValidator.WINDOWS && fsSettings.getServer() != null) {
-            logger.debug("We are running on Windows with SSH Server settings so we need to force the Linux separator.");
-            pathSeparator = "/";
-        } else {
-            pathSeparator = File.separator;
+        pathSeparator = FsCrawlerUtil.getPathSeparator(fsSettings.getFs().getUrl());
+        if (OsValidator.WINDOWS && fsSettings.getServer() == null) {
+            logger.debug("We are running on Windows without Server settings so we use the separator in accordance with fs.url");
+            FileAbstractorFile.separator = pathSeparator;
         }
     }
 
@@ -257,7 +257,7 @@ private void addFilesRecursively(FileAbstractor<?> path, String filepath, LocalD
                     logger.trace("FileAbstractModel = {}", child);
                     String filename = child.getName();
 
-                    String virtualFileName = computeVirtualPathName(stats.getRootPath(), new File(filepath, filename).toString());
+                    String virtualFileName = computeVirtualPathName(stats.getRootPath(), computeRealPathName(filepath, filename));
 
                     // https://github.com/dadoonet/fscrawler/issues/1 : Filter documents
                     boolean isIndexable = isIndexable(child.isDirectory(), virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes());
@@ -274,9 +274,9 @@ private void addFilesRecursively(FileAbstractor<?> path, String filepath, LocalD
                                         indexFile(child, stats, filepath,
                                                 fsSettings.getFs().isIndexContent() || fsSettings.getFs().isStoreSource() ? path.getInputStream(child) : null, child.getSize());
                                         stats.addFile();
-                                    } catch (java.io.FileNotFoundException e) {
+                                    } catch (Exception e) {
                                         if (fsSettings.getFs().isContinueOnError()) {
-                                            logger.warn("Unable to open Input Stream for {}, skipping...: {}", filename, e.getMessage());
+                                            logger.warn("Unable to index {}, skipping...: {}", filename, e.getMessage());
                                         } else {
                                             throw e;
                                         }
@@ -319,7 +319,7 @@ private void addFilesRecursively(FileAbstractor<?> path, String filepath, LocalD
             for (String esfile : esFiles) {
                 logger.trace("Checking file [{}]", esfile);
 
-                String virtualFileName = computeVirtualPathName(stats.getRootPath(), new File(filepath, esfile).toString());
+                String virtualFileName = computeVirtualPathName(stats.getRootPath(), computeRealPathName(filepath, esfile));
                 if (isIndexable(false, virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes())
                         && !fsFiles.contains(esfile)) {
                     logger.trace("Removing file [{}] in elasticsearch/workplace", esfile);
@@ -334,7 +334,7 @@ private void addFilesRecursively(FileAbstractor<?> path, String filepath, LocalD
 
                 // for the delete folder
                 for (String esfolder : esFolders) {
-                    String virtualFileName = computeVirtualPathName(stats.getRootPath(), new File(filepath, esfolder).toString());
+                    String virtualFileName = computeVirtualPathName(stats.getRootPath(), computeRealPathName(filepath, esfolder));
                     if (isIndexable(true, virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes())) {
                         logger.trace("Checking directory [{}]", esfolder);
                         if (!fsFolders.contains(esfolder)) {
@@ -377,7 +377,7 @@ private void indexFile(FileAbstractModel fileAbstractModel, ScanStatistic stats,
         final long size = fileAbstractModel.getSize();
 
         logger.debug("fetching content from [{}],[{}]", dirname, filename);
-        String fullFilename = new File(dirname, filename).toString();
+        String fullFilename = computeRealPathName(dirname, filename);
 
         try {
             // Create the Doc object (only needed when we have add_as_inner_object: true (default) or when we don't index json or xml)
@@ -393,7 +393,11 @@ private void indexFile(FileAbstractModel fileAbstractModel, ScanStatistic stats,
                 doc.getFile().setLastModified(localDateTimeToDate(lastModified));
                 doc.getFile().setLastAccessed(localDateTimeToDate(lastAccessed));
                 doc.getFile().setIndexingDate(localDateTimeToDate(LocalDateTime.now()));
-                doc.getFile().setUrl("file://" + fullFilename);
+                if (fsSettings.getServer() == null) {
+                    doc.getFile().setUrl("file://" + fullFilename);
+                } else if (fsSettings.getServer().getProtocol().equals(PROTOCOL.FTP)) {
+                    doc.getFile().setUrl(String.format("ftp://%s:%d%s", fsSettings.getServer().getHostname(), fsSettings.getServer().getPort(), fullFilename));
+                }
                 doc.getFile().setExtension(extension);
                 if (fsSettings.getFs().isAddFilesize()) {
                     doc.getFile().setFilesize(size);
@@ -493,7 +497,10 @@ private void indexFile(FileAbstractModel fileAbstractModel, ScanStatistic stats,
     }
 
     private String generateIdFromFilename(String filename, String filepath) throws NoSuchAlgorithmException {
-        return fsSettings.getFs().isFilenameAsId() ? filename : SignTool.sign((new File(filepath, filename)).toString());
+        String filepathForId = filepath.replace("\\", "/");
+        String filenameForId = filename.replace("\\", "").replace("/", "");
+        String idSource = filepathForId.endsWith("/") ? filepathForId.concat(filenameForId) : filepathForId.concat("/").concat(filenameForId);
+        return fsSettings.getFs().isFilenameAsId() ? filename : SignTool.sign(idSource);
     }
 
     private String read(InputStream input) throws IOException {
@@ -518,7 +525,7 @@ private void indexDirectory(String id, Folder folder) throws IOException {
 
     /**
      * Index a directory
-     * @param path complete path like /path/to/subdir
+     * @param path complete path like "/", "/path/to/subdir", "C:\\dir", "C:/dir", "/C:/dir", "//SOMEONE/dir"
      */
     private void indexDirectory(String path) throws Exception {
         String name = path.substring(path.lastIndexOf(pathSeparator) + 1);

core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserFTP.java

@@ -0,0 +1,39 @@
+/*
+ * Licensed to David Pilato (the "Author") under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Author licenses this
+ * file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package fr.pilato.elasticsearch.crawler.fs;
+
+import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor;
+import fr.pilato.elasticsearch.crawler.fs.crawler.ftp.FileAbstractorFTP;
+import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService;
+import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementService;
+import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings;
+import java.nio.file.Path;
+
+public class FsParserFTP extends FsParserAbstract {
+
+    public FsParserFTP(FsSettings fsSettings, Path config, FsCrawlerManagementService managementService,
+                         FsCrawlerDocumentService documentService, Integer loop) {
+        super(fsSettings, config, managementService, documentService, loop);
+    }
+
+    protected FileAbstractor<?> buildFileAbstractor() {
+        return new FileAbstractorFTP(fsSettings);
+    }
+}

core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserSmb.java

@@ -0,0 +1,22 @@
+package fr.pilato.elasticsearch.crawler.fs;
+
+import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor;
+import fr.pilato.elasticsearch.crawler.fs.crawler.smb.FileAbstractorSMB;
+import fr.pilato.elasticsearch.crawler.fs.crawler.ssh.FileAbstractorSSH;
+import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService;
+import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementService;
+import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings;
+import java.nio.file.Path;
+
+public class FsParserSmb extends FsParserAbstract{
+
+    public FsParserSmb(FsSettings fsSettings, Path config, FsCrawlerManagementService managementService,
+                       FsCrawlerDocumentService documentService, Integer loop){
+        super(fsSettings, config, managementService, documentService, loop);
+    }
+
+    @Override
+    protected FileAbstractor<?> buildFileAbstractor() {
+        return new FileAbstractorSMB(fsSettings);
+    }
+}

crawler/crawler-fs/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/fs/FileAbstractorFile.java

@@ -48,6 +48,15 @@ public FileAbstractorFile(FsSettings fsSettings) {
         super(fsSettings);
     }
 
+    public static String separator = File.separator;
+
+    private String resolveSeparator(String path) {
+        if (separator.equals("/")) {
+            return path.replace("\\", "/");
+        }
+        return path.replace("/", "\\");
+    }
+
     @Override
     public FileAbstractModel toFileAbstractModel(String path, File file) {
         return new FileAbstractModel(
@@ -57,8 +66,8 @@ public FileAbstractModel toFileAbstractModel(String path, File file) {
                 getCreationTime(file),
                 getLastAccessTime(file),
                 getFileExtension(file),
-                path,
-                file.getAbsolutePath(),
+                resolveSeparator(path),
+                resolveSeparator(file.getAbsolutePath()),
                 file.length(),
                 getOwnerName(file),
                 getGroupName(file),
@@ -67,7 +76,7 @@ public FileAbstractModel toFileAbstractModel(String path, File file) {
 
     @Override
     public InputStream getInputStream(FileAbstractModel file) throws Exception {
-        return new FileInputStream(new File(file.getFullpath()));
+        return new FileInputStream(file.getFullpath());
     }
 
     @Override