diff --git a/.gitignore b/.gitignore index c9b58b2ef..6ad5f8103 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,13 @@ +.DS_Store target /.settings /.classpath /.project +.settings +.classpath +.project .idea *.iml /.run /logs/ +/venv/ diff --git a/README.md b/README.md index acccdb8ce..4fa1b999b 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This crawler helps to index binary documents such as PDF, Open Office, MS Office **Main features**: * Local file system (or a mounted drive) crawling and index new files, update existing ones and removes old ones. -* Remote file system over SSH crawling. +* Remote file system over SSH/FTP/SMB crawling. * REST interface to let you "upload" your binary documents to elasticsearch. You need to install a version matching your Elasticsearch version: diff --git a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java index 5f13de4ea..b0c4a208a 100644 --- a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java +++ b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java @@ -34,6 +34,7 @@ import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettingsFileHandler; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettingsParser; +import fr.pilato.elasticsearch.crawler.fs.settings.Server.PROTOCOL; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; @@ -208,6 +209,20 @@ public static void main(String[] args) throws Exception { if (fsSettings.getFs() == null) { fsSettings.setFs(Fs.DEFAULT); } + + if (fsSettings.getServer() != null) { + if (fsSettings.getServer().getProtocol().equals(PROTOCOL.FTP) && fsSettings.getServer().getPort() == PROTOCOL.SSH_PORT) { + fsSettings.getServer().setPort(PROTOCOL.FTP_PORT); + } + if (fsSettings.getServer().getProtocol().equals(PROTOCOL.FTP) && StringUtils.isEmpty(fsSettings.getServer().getUsername())) { + fsSettings.getServer().setUsername("anonymous"); + } + + if (fsSettings.getServer().getProtocol().equals(PROTOCOL.SMB) && StringUtils.isEmpty(fsSettings.getServer().getUsername())) { + fsSettings.getServer().setUsername("Guest"); + } + } + if (fsSettings.getElasticsearch() == null) { fsSettings.setElasticsearch(Elasticsearch.DEFAULT()); } diff --git a/core/pom.xml b/core/pom.xml index 5f1bdfe4e..e4253b57f 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -69,10 +69,18 @@ fr.pilato.elasticsearch.crawler fscrawler-crawler-fs + + fr.pilato.elasticsearch.crawler + fscrawler-crawler-ftp + fr.pilato.elasticsearch.crawler fscrawler-crawler-ssh + + fr.pilato.elasticsearch.crawler + fscrawler-crawler-smb + diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java index f8bdf4f12..97b9b0e2c 100644 --- a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java @@ -123,9 +123,15 @@ public void start() throws Exception { if (settings.getServer() == null || Server.PROTOCOL.LOCAL.equals(settings.getServer().getProtocol())) { // Local FS fsParser = new FsParserLocal(settings, config, managementService, documentService, loop); + } else if (Server.PROTOCOL.SMB.equals(settings.getServer().getProtocol())) { + // Remote SMB FS + fsParser = new FsParserSmb(settings, config, managementService, documentService, loop); } else if (Server.PROTOCOL.SSH.equals(settings.getServer().getProtocol())) { // Remote SSH FS fsParser = new FsParserSsh(settings, config, managementService, documentService, loop); + } else if (Server.PROTOCOL.FTP.equals(settings.getServer().getProtocol())) { + // Remote FTP FS + fsParser = new FsParserFTP(settings, config, managementService, documentService, loop); } else { // Non supported protocol throw new RuntimeException(settings.getServer().getProtocol() + " is not supported yet. Please use " + @@ -146,7 +152,7 @@ public void close() throws InterruptedException, IOException { if (fsParser != null) { fsParser.close(); - synchronized(fsParser.getSemaphore()) { + synchronized (fsParser.getSemaphore()) { fsParser.getSemaphore().notifyAll(); } } diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java index afdc2ede9..6908ec8dc 100644 --- a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java @@ -28,21 +28,23 @@ import fr.pilato.elasticsearch.crawler.fs.beans.ScanStatistic; import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractModel; import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor; +import fr.pilato.elasticsearch.crawler.fs.crawler.fs.FileAbstractorFile; import fr.pilato.elasticsearch.crawler.fs.framework.ByteSizeValue; import fr.pilato.elasticsearch.crawler.fs.framework.FSCrawlerLogger; +import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil; import fr.pilato.elasticsearch.crawler.fs.framework.OsValidator; import fr.pilato.elasticsearch.crawler.fs.framework.SignTool; import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementService; import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerService; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import fr.pilato.elasticsearch.crawler.fs.settings.Server.PROTOCOL; import fr.pilato.elasticsearch.crawler.fs.tika.TikaDocParser; import fr.pilato.elasticsearch.crawler.fs.tika.XmlDocParser; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.io.BufferedReader; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -97,12 +99,10 @@ public abstract class FsParserAbstract extends FsParser { messageDigest = null; } - // On Windows, when using SSH server, we need to force the "Linux" separator - if (OsValidator.WINDOWS && fsSettings.getServer() != null) { - logger.debug("We are running on Windows with SSH Server settings so we need to force the Linux separator."); - pathSeparator = "/"; - } else { - pathSeparator = File.separator; + pathSeparator = FsCrawlerUtil.getPathSeparator(fsSettings.getFs().getUrl()); + if (OsValidator.WINDOWS && fsSettings.getServer() == null) { + logger.debug("We are running on Windows without Server settings so we use the separator in accordance with fs.url"); + FileAbstractorFile.separator = pathSeparator; } } @@ -257,7 +257,7 @@ private void addFilesRecursively(FileAbstractor path, String filepath, LocalD logger.trace("FileAbstractModel = {}", child); String filename = child.getName(); - String virtualFileName = computeVirtualPathName(stats.getRootPath(), new File(filepath, filename).toString()); + String virtualFileName = computeVirtualPathName(stats.getRootPath(), computeRealPathName(filepath, filename)); // https://github.com/dadoonet/fscrawler/issues/1 : Filter documents boolean isIndexable = isIndexable(child.isDirectory(), virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes()); @@ -274,9 +274,9 @@ private void addFilesRecursively(FileAbstractor path, String filepath, LocalD indexFile(child, stats, filepath, fsSettings.getFs().isIndexContent() || fsSettings.getFs().isStoreSource() ? path.getInputStream(child) : null, child.getSize()); stats.addFile(); - } catch (java.io.FileNotFoundException e) { + } catch (Exception e) { if (fsSettings.getFs().isContinueOnError()) { - logger.warn("Unable to open Input Stream for {}, skipping...: {}", filename, e.getMessage()); + logger.warn("Unable to index {}, skipping...: {}", filename, e.getMessage()); } else { throw e; } @@ -319,7 +319,7 @@ private void addFilesRecursively(FileAbstractor path, String filepath, LocalD for (String esfile : esFiles) { logger.trace("Checking file [{}]", esfile); - String virtualFileName = computeVirtualPathName(stats.getRootPath(), new File(filepath, esfile).toString()); + String virtualFileName = computeVirtualPathName(stats.getRootPath(), computeRealPathName(filepath, esfile)); if (isIndexable(false, virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes()) && !fsFiles.contains(esfile)) { logger.trace("Removing file [{}] in elasticsearch/workplace", esfile); @@ -334,7 +334,7 @@ private void addFilesRecursively(FileAbstractor path, String filepath, LocalD // for the delete folder for (String esfolder : esFolders) { - String virtualFileName = computeVirtualPathName(stats.getRootPath(), new File(filepath, esfolder).toString()); + String virtualFileName = computeVirtualPathName(stats.getRootPath(), computeRealPathName(filepath, esfolder)); if (isIndexable(true, virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes())) { logger.trace("Checking directory [{}]", esfolder); if (!fsFolders.contains(esfolder)) { @@ -377,7 +377,7 @@ private void indexFile(FileAbstractModel fileAbstractModel, ScanStatistic stats, final long size = fileAbstractModel.getSize(); logger.debug("fetching content from [{}],[{}]", dirname, filename); - String fullFilename = new File(dirname, filename).toString(); + String fullFilename = computeRealPathName(dirname, filename); try { // Create the Doc object (only needed when we have add_as_inner_object: true (default) or when we don't index json or xml) @@ -393,7 +393,11 @@ private void indexFile(FileAbstractModel fileAbstractModel, ScanStatistic stats, doc.getFile().setLastModified(localDateTimeToDate(lastModified)); doc.getFile().setLastAccessed(localDateTimeToDate(lastAccessed)); doc.getFile().setIndexingDate(localDateTimeToDate(LocalDateTime.now())); - doc.getFile().setUrl("file://" + fullFilename); + if (fsSettings.getServer() == null) { + doc.getFile().setUrl("file://" + fullFilename); + } else if (fsSettings.getServer().getProtocol().equals(PROTOCOL.FTP)) { + doc.getFile().setUrl(String.format("ftp://%s:%d%s", fsSettings.getServer().getHostname(), fsSettings.getServer().getPort(), fullFilename)); + } doc.getFile().setExtension(extension); if (fsSettings.getFs().isAddFilesize()) { doc.getFile().setFilesize(size); @@ -493,7 +497,10 @@ private void indexFile(FileAbstractModel fileAbstractModel, ScanStatistic stats, } private String generateIdFromFilename(String filename, String filepath) throws NoSuchAlgorithmException { - return fsSettings.getFs().isFilenameAsId() ? filename : SignTool.sign((new File(filepath, filename)).toString()); + String filepathForId = filepath.replace("\\", "/"); + String filenameForId = filename.replace("\\", "").replace("/", ""); + String idSource = filepathForId.endsWith("/") ? filepathForId.concat(filenameForId) : filepathForId.concat("/").concat(filenameForId); + return fsSettings.getFs().isFilenameAsId() ? filename : SignTool.sign(idSource); } private String read(InputStream input) throws IOException { @@ -518,7 +525,7 @@ private void indexDirectory(String id, Folder folder) throws IOException { /** * Index a directory - * @param path complete path like /path/to/subdir + * @param path complete path like "/", "/path/to/subdir", "C:\\dir", "C:/dir", "/C:/dir", "//SOMEONE/dir" */ private void indexDirectory(String path) throws Exception { String name = path.substring(path.lastIndexOf(pathSeparator) + 1); diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserFTP.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserFTP.java new file mode 100644 index 000000000..d04934503 --- /dev/null +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserFTP.java @@ -0,0 +1,39 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs; + +import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor; +import fr.pilato.elasticsearch.crawler.fs.crawler.ftp.FileAbstractorFTP; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementService; +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import java.nio.file.Path; + +public class FsParserFTP extends FsParserAbstract { + + public FsParserFTP(FsSettings fsSettings, Path config, FsCrawlerManagementService managementService, + FsCrawlerDocumentService documentService, Integer loop) { + super(fsSettings, config, managementService, documentService, loop); + } + + protected FileAbstractor buildFileAbstractor() { + return new FileAbstractorFTP(fsSettings); + } +} diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserSmb.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserSmb.java new file mode 100644 index 000000000..b319b4a5f --- /dev/null +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserSmb.java @@ -0,0 +1,22 @@ +package fr.pilato.elasticsearch.crawler.fs; + +import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor; +import fr.pilato.elasticsearch.crawler.fs.crawler.smb.FileAbstractorSMB; +import fr.pilato.elasticsearch.crawler.fs.crawler.ssh.FileAbstractorSSH; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementService; +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import java.nio.file.Path; + +public class FsParserSmb extends FsParserAbstract{ + + public FsParserSmb(FsSettings fsSettings, Path config, FsCrawlerManagementService managementService, + FsCrawlerDocumentService documentService, Integer loop){ + super(fsSettings, config, managementService, documentService, loop); + } + + @Override + protected FileAbstractor buildFileAbstractor() { + return new FileAbstractorSMB(fsSettings); + } +} diff --git a/core/src/test/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerUtilTest.java b/core/src/test/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerUtilTest.java deleted file mode 100644 index bcd812888..000000000 --- a/core/src/test/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerUtilTest.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to David Pilato (the "Author") under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Author licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package fr.pilato.elasticsearch.crawler.fs; - -import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerMetadataTestCase; -import org.junit.Test; - -import java.io.File; -import java.time.LocalDateTime; -import java.util.TimeZone; - -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.computeVirtualPathName; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getFileExtension; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.localDateTimeToDate; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.is; - -/** - * We want to test some utilities - */ -public class FsCrawlerUtilTest extends AbstractFSCrawlerMetadataTestCase { - - @Test - public void testComputePathLinux() { - testHelper("/tmp", "/tmp", "/"); - testHelper("/tmp", "/tmp/dir", "/dir"); - testHelper("/tmp", "/tmp/dir/subdir", "/dir/subdir"); - testHelper("/tmp", "/tmp/file.txt", "/file.txt"); - testHelper("/tmp", "/tmp/dir/file.txt", "/dir/file.txt"); - testHelper("/tmp", "/tmp/dir/subdir/file.txt", "/dir/subdir/file.txt"); - } - - @Test - public void testComputePathWindows() { - testHelper("C:\\tmp", "C:\\tmp", "/"); - testHelper("C:\\tmp", "C:\\tmp\\dir", "/dir"); - testHelper("C:\\tmp", "C:\\tmp\\dir\\subdir", "/dir/subdir"); - testHelper("C:\\tmp", "C:\\tmp\\file.txt", "/file.txt"); - testHelper("C:\\tmp", "C:\\tmp\\dir\\file.txt", "/dir/file.txt"); - testHelper("C:\\tmp", "C:\\tmp\\dir\\subdir\\file.txt", "/dir/subdir/file.txt"); - } - - private void testHelper(String rootPath, String realPath, String expectedPath) { - assertThat(computeVirtualPathName(rootPath, realPath), is(expectedPath)); - } - - @Test - public void testGetFileExtension() { - assertThat(getFileExtension(new File("foo.bar")), is("bar")); - assertThat(getFileExtension(new File("foo")), is("")); - assertThat(getFileExtension(new File("foo.bar.baz")), is("baz")); - } - - @Test - public void testLocalDateToDate() { - LocalDateTime now = LocalDateTime.now(); - logger.info("Current Time [{}] in [{}] is actually [{}]", now, TimeZone.getDefault().getDisplayName(), localDateTimeToDate(now)); - } -} diff --git a/crawler/crawler-fs/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/fs/FileAbstractorFile.java b/crawler/crawler-fs/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/fs/FileAbstractorFile.java index 4453f6a84..55b864ad8 100644 --- a/crawler/crawler-fs/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/fs/FileAbstractorFile.java +++ b/crawler/crawler-fs/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/fs/FileAbstractorFile.java @@ -48,6 +48,15 @@ public FileAbstractorFile(FsSettings fsSettings) { super(fsSettings); } + public static String separator = File.separator; + + private String resolveSeparator(String path) { + if (separator.equals("/")) { + return path.replace("\\", "/"); + } + return path.replace("/", "\\"); + } + @Override public FileAbstractModel toFileAbstractModel(String path, File file) { return new FileAbstractModel( @@ -57,8 +66,8 @@ public FileAbstractModel toFileAbstractModel(String path, File file) { getCreationTime(file), getLastAccessTime(file), getFileExtension(file), - path, - file.getAbsolutePath(), + resolveSeparator(path), + resolveSeparator(file.getAbsolutePath()), file.length(), getOwnerName(file), getGroupName(file), @@ -67,7 +76,7 @@ public FileAbstractModel toFileAbstractModel(String path, File file) { @Override public InputStream getInputStream(FileAbstractModel file) throws Exception { - return new FileInputStream(new File(file.getFullpath())); + return new FileInputStream(file.getFullpath()); } @Override diff --git a/crawler/crawler-ftp/pom.xml b/crawler/crawler-ftp/pom.xml new file mode 100644 index 000000000..f02dd6a18 --- /dev/null +++ b/crawler/crawler-ftp/pom.xml @@ -0,0 +1,37 @@ + + + + fscrawler-crawler + fr.pilato.elasticsearch.crawler + 2.7-SNAPSHOT + + 4.0.0 + + fscrawler-crawler-ftp + FSCrawler Crawlers: FTP + + + + fr.pilato.elasticsearch.crawler + fscrawler-crawler-abstract + + + + + commons-net + commons-net + + + org.mockftpserver + MockFtpServer + test + + + org.apache.logging.log4j + log4j-iostreams + + + + diff --git a/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FTPUtils.java b/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FTPUtils.java new file mode 100644 index 000000000..96cda2ba8 --- /dev/null +++ b/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FTPUtils.java @@ -0,0 +1,37 @@ +package fr.pilato.elasticsearch.crawler.fs.crawler.ftp; + +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.toOctalPermission; + +import org.apache.commons.net.ftp.FTPFile; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class FTPUtils { + private static final Logger logger = LogManager.getLogger(FTPUtils.class); + + /** + * Determines FTPFile permissions. + */ + public static int getFilePermissions(final FTPFile file) { + try { + int user = toOctalPermission( + file.hasPermission(FTPFile.USER_ACCESS, FTPFile.READ_PERMISSION), + file.hasPermission(FTPFile.USER_ACCESS, FTPFile.WRITE_PERMISSION), + file.hasPermission(FTPFile.USER_ACCESS, FTPFile.EXECUTE_PERMISSION)); + int group = toOctalPermission( + file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.READ_PERMISSION), + file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.WRITE_PERMISSION), + file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.EXECUTE_PERMISSION)); + int others = toOctalPermission( + file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.READ_PERMISSION), + file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.WRITE_PERMISSION), + file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.EXECUTE_PERMISSION)); + + return user * 100 + group * 10 + others; + } catch (Exception e) { + logger.warn("Failed to determine 'permissions' of {}: {}", file, e.getMessage()); + return -1; + } + } + +} diff --git a/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java b/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java new file mode 100644 index 000000000..cfbe7e4aa --- /dev/null +++ b/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java @@ -0,0 +1,208 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.crawler.ftp; + +import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractModel; +import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor; +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import fr.pilato.elasticsearch.crawler.fs.settings.Server; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.net.PrintCommandListener; +import org.apache.commons.net.ftp.FTP; +import org.apache.commons.net.ftp.FTPClient; +import org.apache.commons.net.ftp.FTPFile; +import org.apache.commons.net.ftp.FTPReply; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.io.IoBuilder; + +import java.io.InputStream; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Collection; +import java.util.stream.Collectors; + +public class FileAbstractorFTP extends FileAbstractor { + private final Logger logger = LogManager.getLogger(FileAbstractorFTP.class); + + private FTPClient ftp; + + private final OutputStream loggerOutputStream = IoBuilder.forLogger(logger).buildOutputStream(); + + private final PrintCommandListener ftpListener = new PrintCommandListener(new PrintWriter(loggerOutputStream)); + + private boolean isUtf8 = false; + + private static final String ALTERNATIVE_ENCODING = "GBK"; + + public FileAbstractorFTP(FsSettings fsSettings) { + super(fsSettings); + } + + @Override + public FileAbstractModel toFileAbstractModel(String path, FTPFile file) { + String filename = file.getName(); + String extension = FilenameUtils.getExtension(filename); + + String toEncoding = ALTERNATIVE_ENCODING; + if (isUtf8) { + toEncoding = StandardCharsets.UTF_8.displayName(); + } + try { + filename = new String(filename.getBytes(FTP.DEFAULT_CONTROL_ENCODING), toEncoding); + path = new String(path.getBytes(FTP.DEFAULT_CONTROL_ENCODING), toEncoding); + } catch (UnsupportedEncodingException e) { + logger.error("Error during encoding: {}", e.getMessage()); + } + + return new FileAbstractModel( + filename, + file.isFile(), + // We are using here the local TimeZone as a reference. If the remote system is under another TZ, this might cause issues + LocalDateTime.ofInstant(Instant.ofEpochMilli(file.getTimestamp().getTimeInMillis()), ZoneId.systemDefault()), + // We don't have the creation date + null, + // We don't have the access date + null, + extension, + path, + path.equals("/") ? path.concat(filename) : path.concat("/").concat(filename), + file.getSize(), + file.getUser(), + file.getGroup(), + FTPUtils.getFilePermissions(file)); + } + + @Override + public InputStream getInputStream(FileAbstractModel file) throws IOException { + String fullPath = file.getFullpath(); + if (isUtf8) { + fullPath = new String(fullPath.getBytes(StandardCharsets.UTF_8), FTP.DEFAULT_CONTROL_ENCODING); + } else { + fullPath = new String(fullPath.getBytes(ALTERNATIVE_ENCODING), FTP.DEFAULT_CONTROL_ENCODING); + } + + InputStream inputStream = ftp.retrieveFileStream(fullPath); + if (inputStream != null) { + ftp.completePendingCommand(); + return inputStream; + } else { + throw new IOException(String.format("FTP client can not retrieve stream for [%s]", file.getFullpath())); + } + } + + @Override + public Collection getFiles(String dir) throws IOException { + logger.debug("Listing files from {}", dir); + if (isUtf8) { + dir = new String(dir.getBytes(StandardCharsets.UTF_8), FTP.DEFAULT_CONTROL_ENCODING); + } else { + dir = new String(dir.getBytes(ALTERNATIVE_ENCODING), FTP.DEFAULT_CONTROL_ENCODING); + } + ftp.enterLocalPassiveMode(); + FTPFile[] ftpFiles = ftp.listFiles(dir); + if (ftpFiles == null) return null; + List files = Arrays.stream(ftpFiles).filter(file -> { + if (fsSettings.getFs().isFollowSymlinks()) return true; + return !file.isSymbolicLink(); + }).collect(Collectors.toList()); + + Collection result = new ArrayList<>(files.size()); + // Iterate other files + // We ignore here all files like . and .. + String finalDir = dir; + result.addAll(files.stream().filter(file -> !".".equals(file.getName()) && + !"..".equals(file.getName())) + .map(file -> toFileAbstractModel(finalDir, file)) + .collect(Collectors.toList())); + + logger.debug("{} local files found", result.size()); + return result; + } + + @Override + public boolean exists(String dir) { + try { + logger.debug("Checking dir existence: " + dir); + if (isUtf8) { + dir = new String(dir.getBytes(StandardCharsets.UTF_8), FTP.DEFAULT_CONTROL_ENCODING); + } else { + dir = new String(dir.getBytes(ALTERNATIVE_ENCODING), FTP.DEFAULT_CONTROL_ENCODING); + } + return ftp.changeWorkingDirectory(dir); + } catch (IOException e) { + return false; + } + } + + @Override + public void open() throws IOException { + Server server = fsSettings.getServer(); + logger.debug("Opening FTP connection to {}@{}", server.getUsername(), server.getHostname()); + + ftp = new FTPClient(); + ftp.addProtocolCommandListener(ftpListener); + // send a safe command (i.e. NOOP) over the control connection to reset the router's idle timer + ftp.setControlKeepAliveTimeout(300); + openFTPConnection(); + } + + @Override + public void close() throws IOException { + ftp.logout(); + ftp.disconnect(); + } + + private void openFTPConnection() throws IOException { + Server server = fsSettings.getServer(); + ftp.connect(server.getHostname(), server.getPort()); + + // checking FTP client connection. + int reply = ftp.getReplyCode(); + if (!FTPReply.isPositiveCompletion(reply)) { + ftp.disconnect(); + logger.warn("Cannot connect with FTP to {}@{}", server.getUsername(), + server.getHostname()); + throw new RuntimeException("Can not connect to " + server.getUsername() + "@" + server.getHostname()); + } + + if (!ftp.login(server.getUsername(), server.getPassword())) { + ftp.disconnect(); + throw new RuntimeException("Please check ftp user or password"); + } + + int utf8Reply = ftp.sendCommand("OPTS UTF8", "ON"); + if (FTPReply.isPositiveCompletion(utf8Reply)) { + isUtf8 = true; + } + ftp.setFileType(FTPClient.BINARY_FILE_TYPE); + + logger.debug("FTP connection successful"); + } +} diff --git a/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java b/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java new file mode 100644 index 000000000..1b893f379 --- /dev/null +++ b/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java @@ -0,0 +1,225 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.crawler.ftp; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractModel; +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import fr.pilato.elasticsearch.crawler.fs.settings.Server; +import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collection; + +import java.util.List; +import java.util.stream.Collectors; +import org.apache.commons.io.IOUtils; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.mockftpserver.fake.FakeFtpServer; +import org.mockftpserver.fake.UserAccount; +import org.mockftpserver.fake.filesystem.DirectoryEntry; +import org.mockftpserver.fake.filesystem.FileEntry; +import org.mockftpserver.fake.filesystem.FileSystem; +import org.mockftpserver.fake.filesystem.Permissions; +import org.mockftpserver.fake.filesystem.UnixFakeFileSystem; + +public class FileAbstractorFTPTest extends AbstractFSCrawlerTestCase { + private FakeFtpServer fakeFtpServer; + private final String nestedDir = "/nested"; + private final String permissionDir = "/permission"; + private final String user = "user"; + private final String pass = "pass"; + + @Before + public void setup() { + fakeFtpServer = new FakeFtpServer(); + fakeFtpServer.setServerControlPort(5968); + fakeFtpServer.addUserAccount(new UserAccount(user, pass, "/")); + FileSystem fileSystem = new UnixFakeFileSystem(); + + fileSystem.add(new DirectoryEntry(nestedDir)); + fileSystem.add(new FileEntry(nestedDir + "/foo.txt", "文件名不支持中文")); + fileSystem.add(new FileEntry(nestedDir + "/bar.txt", "filename doesn't support utf-8")); + + fileSystem.add(new DirectoryEntry(nestedDir + "/buzz")); + fileSystem.add(new FileEntry(nestedDir + "/buzz/hello.txt", "hello")); + fileSystem.add(new FileEntry(nestedDir + "/buzz/world.txt", "world")); + + fileSystem.add(new DirectoryEntry(permissionDir)); + FileEntry fileAllPermissions = new FileEntry(permissionDir + "/all.txt", "123"); + fileAllPermissions.setPermissions(Permissions.ALL); + fileSystem.add(fileAllPermissions); + FileEntry fileNonePermissions = new FileEntry(permissionDir + "/none.txt", "456"); + fileNonePermissions.setPermissions(Permissions.NONE); + fileSystem.add(fileNonePermissions); + + fakeFtpServer.setFileSystem(fileSystem); + fakeFtpServer.start(); + } + + @After + public void shutDown() { + fakeFtpServer.stop(); + } + + @Test + public void testConnectToFakeFTPServer() throws Exception { + int port = fakeFtpServer.getServerControlPort(); + FsSettings fsSettings = FsSettings.builder("fake") + .setServer( + Server.builder() + .setHostname("localhost") + .setUsername(user) + .setPassword(pass) + .setPort(port) + .build() + ) + .build(); + + FileAbstractorFTP ftp = new FileAbstractorFTP(fsSettings); + ftp.open(); + boolean exists = ftp.exists(nestedDir); + assertThat(exists, is(true)); + Collection files = ftp.getFiles(nestedDir); + assertThat(files.size(), is(3)); + + for (FileAbstractModel file : files) { + if (file.isDirectory()) { + assertThat(file.getName(), is("buzz")); + Collection subDirFiles = ftp.getFiles(file.getFullpath()); + assertThat(subDirFiles.size(), is(2)); + logger.debug("Found {} files in sub dir", subDirFiles.size()); + for (FileAbstractModel subDirFile : subDirFiles) { + try (InputStream inputStream = ftp.getInputStream(subDirFile)) { + String content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + logger.debug("[{}] - {}: {}", file.getName(), subDirFile.getName(), content); + } + } + } else { + try (InputStream inputStream = ftp.getInputStream(file)) { + String content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + logger.debug(" - {}: {}", file.getName(), content); + } + } + } + + ftp.close(); + } + + /** + * FakeFtpServer doesn't support utf-8 + * You have to adapt this test to your own system + * So this test is disabled by default + */ + @Test @Ignore + public void testConnectToFTPServer() throws Exception { + String path = "/中文目录"; + FsSettings fsSettings = FsSettings.builder("local_utf8_test") + .setServer( + Server.builder() + .setHostname("192.168.18.207") + .setUsername("helsonxiao") + .setPassword("123456") + .setPort(21) + .build() + ) + .build(); + + FileAbstractorFTP ftp = new FileAbstractorFTP(fsSettings); + ftp.open(); + boolean exists = ftp.exists(path); + assertThat(exists, is(true)); + Collection files = ftp.getFiles(path); + logger.debug("Found {} files", files.size()); + + for (FileAbstractModel file : files) { + if (file.isDirectory()) { + Collection subDirFiles = ftp.getFiles(file.getFullpath()); + logger.debug("Found {} files in sub dir", subDirFiles.size()); + for (FileAbstractModel subDirFile : subDirFiles) { + if (subDirFile.isFile()) { + try (InputStream inputStream = ftp.getInputStream(subDirFile)) { + String content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + logger.debug("[{}] - {}: {}", file.getName(), subDirFile.getName(), content); + } + } + } + } else { + try (InputStream inputStream = ftp.getInputStream(file)) { + String content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + logger.debug(" - {}: {}", file.getName(), content); + } + } + } + + ftp.close(); + } + + @Test + public void testFTPFilePermissions() throws IOException { + int port = fakeFtpServer.getServerControlPort(); + FsSettings fsSettings = FsSettings.builder("fake") + .setServer( + Server.builder() + .setHostname("localhost") + .setUsername(user) + .setPassword(pass) + .setPort(port) + .build() + ) + .build(); + + FileAbstractorFTP ftp = new FileAbstractorFTP(fsSettings); + ftp.open(); + + Collection files = ftp.getFiles(permissionDir); + assertThat(files.size(), is(2)); + List filenames = files.stream().map(FileAbstractModel::getName).collect(Collectors.toList()); + assertThat(filenames.contains("all.txt"), is(true)); + assertThat(filenames.contains("none.txt"), is(true)); + for (FileAbstractModel file : files) { + if (file.getName().equals("all.txt")) { + assertThat(file.getPermissions(), is(777)); + try (InputStream inputStream = ftp.getInputStream(file)) { + String content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + logger.debug(" - {}: {}", file.getName(), content); + } + } else if (file.getName().equals("none.txt")) { + assertThat(file.getPermissions(), is(0)); + boolean errorOccurred = false; + try (InputStream ignored = ftp.getInputStream(file)) { + logger.error(ignored); + } catch (IOException e) { + errorOccurred = true; + logger.error(e.getMessage()); + } + assertThat(errorOccurred, is(true)); + } + } + + ftp.close(); + } +} diff --git a/crawler/crawler-ftp/src/test/resources/log4j2.xml b/crawler/crawler-ftp/src/test/resources/log4j2.xml new file mode 100644 index 000000000..c551aa429 --- /dev/null +++ b/crawler/crawler-ftp/src/test/resources/log4j2.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/crawler/crawler-smb/pom.xml b/crawler/crawler-smb/pom.xml new file mode 100644 index 000000000..a8043eb15 --- /dev/null +++ b/crawler/crawler-smb/pom.xml @@ -0,0 +1,38 @@ + + + + fscrawler-crawler + fr.pilato.elasticsearch.crawler + 2.7-SNAPSHOT + + 4.0.0 + + + fscrawler-crawler-smb + FSCrawler Crawlers: SMB + + + + + fr.pilato.elasticsearch.crawler + fscrawler-crawler-abstract + + + + + com.hierynomus + smbj + 0.11.1 + + + org.slf4j + slf4j-api + + + + + + + \ No newline at end of file diff --git a/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java b/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java new file mode 100644 index 000000000..0385487dc --- /dev/null +++ b/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java @@ -0,0 +1,194 @@ +package fr.pilato.elasticsearch.crawler.fs.crawler.smb; + +import com.hierynomus.msdtyp.AccessMask; +import com.hierynomus.msdtyp.SecurityInformation; +import com.hierynomus.msfscc.fileinformation.FileIdBothDirectoryInformation; +import com.hierynomus.mssmb2.SMB2CreateDisposition; +import com.hierynomus.mssmb2.SMB2ShareAccess; +import com.hierynomus.mssmb2.SMBApiException; +import com.hierynomus.protocol.commons.EnumWithValue; +import com.hierynomus.security.bc.BCSecurityProvider; +import com.hierynomus.smbj.SMBClient; +import com.hierynomus.smbj.SmbConfig; +import com.hierynomus.smbj.auth.AuthenticationContext; +import com.hierynomus.smbj.connection.Connection; +import com.hierynomus.smbj.session.Session; +import com.hierynomus.smbj.share.Directory; +import com.hierynomus.smbj.share.DiskEntry; +import com.hierynomus.smbj.share.DiskShare; +import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractModel; +import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor; +import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil; +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import fr.pilato.elasticsearch.crawler.fs.settings.Server; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class FileAbstractorSMB extends FileAbstractor { + + + private final Logger logger = LogManager.getLogger(FileAbstractorSMB.class); + + public FileAbstractorSMB(FsSettings fsSettings) { + super(fsSettings); + } + + private DiskShare share; + + private SMBClient client; + + @Override + public FileAbstractModel toFileAbstractModel(String path, DiskEntry file) { + + int permissions = 777; + + //TODO 修正权限 file.getFileInformation().getAccessInformation().getAccessFlags() + + EnumSet list = EnumWithValue.EnumUtils.toEnumSet(file.getFileInformation().getAccessInformation().getAccessFlags(), AccessMask.class); + + + //此处这样取文件/文件夹名的原因为：file.getFileInformation().getNameInformation() 取到的值永远为null + String fileName = FsCrawlerUtil.getFileName(file.getUncPath()); + String extension = FilenameUtils.getExtension(fileName); + + return new FileAbstractModel( + fileName, + !file.getFileInformation().getStandardInformation().isDirectory(), + // We are using here the local TimeZone as a reference. If the remote system is under another TZ, this might cause issues + LocalDateTime.ofInstant(Instant.ofEpochMilli(file.getFileInformation().getBasicInformation().getLastWriteTime().toEpochMillis()), ZoneId.systemDefault()), + // We don't have the creation date + null, + // We are using here the local TimeZone as a reference. If the remote system is under another TZ, this might cause issues + LocalDateTime.ofInstant(Instant.ofEpochMilli(file.getFileInformation().getBasicInformation().getCreationTime().toEpochMillis()), ZoneId.systemDefault()), + extension, + path, + path.concat("/").concat(fileName), + file.getFileInformation().getStandardInformation().getAllocationSize(), + file.getSecurityInformation(Collections.singleton(SecurityInformation.OWNER_SECURITY_INFORMATION)).getOwnerSid().toString(), + file.getSecurityInformation(Collections.singleton(SecurityInformation.GROUP_SECURITY_INFORMATION)).getGroupSid().toString(), + permissions); + } + + @Override + public InputStream getInputStream(FileAbstractModel file) throws IOException { + if (file.isFile()) { + String fullPath = file.getFullpath(); + fullPath = FsCrawlerUtil.getRelativePath(fullPath); + + try { + return share.openFile(fullPath, EnumSet.of(AccessMask.GENERIC_READ), + null, + SMB2ShareAccess.ALL, + SMB2CreateDisposition.FILE_OPEN, + null).getInputStream(); + } catch (SMBApiException e) { + logger.error("SMB client can not retrieve stream for {} , e: {}", fullPath , e); + throw new IOException(String.format("SMB client can not retrieve stream for [%s] ",fullPath)); + } + } else { + return new ByteArrayInputStream(file.getName().getBytes()); + } + } + + @Override + public Collection getFiles(String dir) { + + String relativeDir = FsCrawlerUtil.getRelativePath(dir); + logger.debug("Listing smb files from {}", relativeDir); + List ls; + + Directory directory = share.openDirectory(relativeDir, EnumSet.of(AccessMask.GENERIC_READ), + null, + SMB2ShareAccess.ALL, + SMB2CreateDisposition.FILE_OPEN, + null); + + ls = directory.list(); + if (ls == null) { + return null; + } + + Collection result = new ArrayList<>(ls.size()); + // Iterate other files + // We ignore here all files like . and .. + result.addAll(ls.stream().filter(file -> !".".equals(file.getFileName()) && + !"..".equals(file.getFileName())) + .map(file -> toFileAbstractModel(dir, share.open(relativeDir + "/" + file.getFileName(), EnumSet.of(AccessMask.GENERIC_READ), + null, + SMB2ShareAccess.ALL, + SMB2CreateDisposition.FILE_OPEN, + null))) + .collect(Collectors.toList())); + + logger.debug("{} local files found", result.size()); + return result; + } + + @Override + public boolean exists(String dir) { + dir = FsCrawlerUtil.getRelativePath(dir); + return share.folderExists(dir); + } + + @Override + public void open() throws Exception { + share = openSMBConnection(fsSettings.getServer()); + } + + @Override + public void close() throws Exception { + share.close(); + client.close(); + } + + + private DiskShare openSMBConnection(Server server) throws IOException { + logger.debug("Opening SMB connection to {}@{}", server.getUsername(), server.getHostname()); + + + Session session; + AuthenticationContext ac = new AuthenticationContext(server.getUsername(), server.getPassword().toCharArray(), server.getHostname()); + try { + logger.debug("Start trying to connect through SMB2"); + SmbConfig smbConfig = SmbConfig.builder() + .withTimeout(12, TimeUnit.SECONDS) // Timeout sets Read, Write, and Transact timeouts (default is 60 seconds) + .withSoTimeout(18, TimeUnit.SECONDS) // Socket Timeout (default is 0 seconds, blocks forever) + .build(); + + client = new SMBClient(smbConfig); + Connection connection = client.connect(server.getHostname()); + session = connection.authenticate(ac); + } catch (UnsupportedOperationException | SMBApiException e) { + logger.debug("Start trying to connect through SMB3"); + //close client + client.close(); + SmbConfig smbConfig = SmbConfig.builder() + //SMB3.0 use BCSecurityProvider + .withSecurityProvider(new BCSecurityProvider()) + .withTimeout(12, TimeUnit.SECONDS) // Timeout sets Read, Write, and Transact timeouts (default is 60 seconds) + .withSoTimeout(18, TimeUnit.SECONDS) // Socket Timeout (default is 0 seconds, blocks forever) + .build(); + client = new SMBClient(smbConfig); + Connection connection = client.connect(server.getHostname()); + session = connection.authenticate(ac); + } + String url = fsSettings.getFs().getUrl(); + String serverName = FsCrawlerUtil.getServerName(url); + return (DiskShare) session.connectShare(serverName); + + } +} diff --git a/crawler/crawler-smb/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMBTest.java b/crawler/crawler-smb/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMBTest.java new file mode 100644 index 000000000..062480253 --- /dev/null +++ b/crawler/crawler-smb/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMBTest.java @@ -0,0 +1,49 @@ +package fr.pilato.elasticsearch.crawler.fs.crawler.smb; + +import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractModel; +import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import fr.pilato.elasticsearch.crawler.fs.settings.Server; +import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase; +import java.util.Collection; +import junit.framework.TestCase; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import org.junit.Ignore; +import org.junit.Test; + +public class FileAbstractorSMBTest extends AbstractFSCrawlerTestCase { + + @Test + @Ignore + public void testConnectToWindows() throws Exception { + String[] paths = {"","folder","文件夹","folder/文件夹","文件夹/folder"}; + String host = "192.168.31.45"; + String user = "lzwcyd"; + String pass = "123456"; + String url = "//Desktop/win10_share_test"; + FsSettings fsSettings = FsSettings.builder("foo") + .setServer( + Server.builder() + .setHostname(host) + .setUsername(user) + .setPassword(pass) + .build() + ).setFs(Fs.builder() + .setUrl(url) + .build()) + .build(); + FileAbstractorSMB smb = new FileAbstractorSMB(fsSettings); + smb.open(); + for (String path : paths) { + boolean exists = smb.exists(path); + assertThat(exists, is(true)); + Collection files = smb.getFiles(path); + logger.debug("Found {} files", files.size()); + for (FileAbstractModel file : files) { + logger.debug(" - {}", file); + } + } + smb.close(); + } +} \ No newline at end of file diff --git a/crawler/crawler-ssh/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ssh/FileAbstractorSSH.java b/crawler/crawler-ssh/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ssh/FileAbstractorSSH.java index 2c95c94f8..a872def6e 100644 --- a/crawler/crawler-ssh/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ssh/FileAbstractorSSH.java +++ b/crawler/crawler-ssh/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ssh/FileAbstractorSSH.java @@ -62,7 +62,7 @@ public FileAbstractModel toFileAbstractModel(String path, ChannelSftp.LsEntry fi LocalDateTime.ofInstant(Instant.ofEpochMilli(file.getAttrs().getATime()*1000L), ZoneId.systemDefault()), FilenameUtils.getExtension(file.getFilename()), path, - path.concat("/").concat(file.getFilename()), + path.equals("/") ? path.concat(file.getFilename()) : path.concat("/").concat(file.getFilename()), file.getAttrs().getSize(), Integer.toString(file.getAttrs().getUId()), Integer.toString(file.getAttrs().getGId()), diff --git a/crawler/pom.xml b/crawler/pom.xml index 64d02cb9f..ee7874a63 100644 --- a/crawler/pom.xml +++ b/crawler/pom.xml @@ -16,7 +16,9 @@ crawler-abstract crawler-fs + crawler-ftp crawler-ssh + crawler-smb diff --git a/distribution/pom.xml b/distribution/pom.xml index c021f95d7..898a411b2 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -21,6 +21,7 @@ + ${env.DOCKER_SKIP} build dadoonet diff --git a/docs/source/admin/fs/ftp.rst b/docs/source/admin/fs/ftp.rst new file mode 100644 index 000000000..c8b570b4d --- /dev/null +++ b/docs/source/admin/fs/ftp.rst @@ -0,0 +1,48 @@ +.. _ftp-settings: + +FTP settings +------------ + +You can index files remotely using FTP. + +Here is a list of FTP settings (under ``server.`` prefix): + ++-----------------------+-----------------------+-----------------------+ +| Name | Default value | Documentation | ++=======================+=======================+=======================+ +| ``server.hostname`` | ``null`` | Hostname | ++-----------------------+-----------------------+-----------------------+ +| ``server.port`` | ``21`` | Port | ++-----------------------+-----------------------+-----------------------+ +| ``server.username`` | ``anonymous`` | :ref:`ftp_login` | ++-----------------------+-----------------------+-----------------------+ +| ``server.password`` | ``null`` | :ref:`ftp_login` | ++-----------------------+-----------------------+-----------------------+ +| ``server.protocol`` | ``"local"`` | Set it to ``ftp`` | ++-----------------------+-----------------------+-----------------------+ + +.. _ftp_login: + +Username / Password +~~~~~~~~~~~~~~~~~~~ + +Let’s say you want to index from a remote server using FTP: + +- FS URL: ``/path/to/data/dir/on/server`` +- Server: ``mynode.mydomain.com`` +- Username: ``username`` (default to ``anonymous``) +- Password: ``password`` +- Protocol: ``ftp`` (default to ``local``) +- Port: ``21`` (default to ``21``) + +.. code:: yaml + + name: "test" + fs: + url: "/path/to/data/dir/on/server" + server: + hostname: "mynode.mydomain.com" + port: 21 + username: "username" + password: "password" + protocol: "ftp" diff --git a/docs/source/admin/fs/smb.rst b/docs/source/admin/fs/smb.rst new file mode 100644 index 000000000..facc31445 --- /dev/null +++ b/docs/source/admin/fs/smb.rst @@ -0,0 +1,44 @@ +.. _smb-settings: + +SMB settings +------------ + +You can index files remotely using SMB. + +Here is a list of SMB settings (under ``server.`` prefix): + ++-----------------------+-----------------------+-----------------------+ +| Name | Default value | Documentation | ++=======================+=======================+=======================+ +| ``server.hostname`` | ``null`` | Hostname | ++-----------------------+-----------------------+-----------------------+ +| ``server.username`` | ``Guest`` | :ref:`smb_login` | ++-----------------------+-----------------------+-----------------------+ +| ``server.password`` | ``null`` | :ref:`smb_login` | ++-----------------------+-----------------------+-----------------------+ +| ``server.protocol`` | ``"local"`` | Set it to ``smb`` | ++-----------------------+-----------------------+-----------------------+ + +.. _smb_login: + +Username / Password +~~~~~~~~~~~~~~~~~~~ + +Let’s say you want to index from a remote server using SMB: + +- FS URL: ``/path/to/data/dir/on/server`` +- Server: ``mynode.mydomain.com`` +- Username: ``username`` (default to ``anonymous``) +- Password: ``password`` +- Protocol: ``smb`` (default to ``local``) + +.. code:: yaml + + name: "test" + fs: + url: "/path/to/data/dir/on/server" + server: + hostname: "mynode.mydomain.com" + username: "username" + password: "password" + protocol: "smb" diff --git a/docs/source/admin/fs/ssh.rst b/docs/source/admin/fs/ssh.rst index 2653deb3a..48f7683cd 100644 --- a/docs/source/admin/fs/ssh.rst +++ b/docs/source/admin/fs/ssh.rst @@ -5,7 +5,7 @@ SSH settings You can index files remotely using SSH. -Here is a list of SSH settings (under ``server.`` prefix)`: +Here is a list of SSH settings (under ``server.`` prefix): +-----------------------+-----------------------+-----------------------+ | Name | Default value | Documentation | @@ -93,3 +93,20 @@ To specify the drive, you need to use the following format: username: "username" password: "password" protocol: "ssh" + +Windows shared folder +~~~~~~~~~~~~~~~~~~~~~ + +When using Windows shared folder, you need to use the following format: + +.. code:: yaml + + name: "test" + fs: + url: "//DESKTOP-NAME/path/to/data/dir/on/server" + server: + hostname: "mynode.mydomain.com" + port: 22 + username: "username" + password: "password" + protocol: "ssh" diff --git a/docs/source/conf.py b/docs/source/conf.py index a3c5c0aad..93d39326e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -16,7 +16,7 @@ # import sys # sys.path.insert(0, os.path.abspath('.')) import os -import ConfigParser +import configparser from datetime import date from os.path import join, dirname @@ -38,7 +38,7 @@ # built documents. # -config = ConfigParser.RawConfigParser() +config = configparser.RawConfigParser() config.read(join(dirname(__file__), "fscrawler.ini")) # development versions always have the suffix '-SNAPSHOT' diff --git a/docs/source/dev/build.rst b/docs/source/dev/build.rst index 8a3964e28..cc23c7017 100644 --- a/docs/source/dev/build.rst +++ b/docs/source/dev/build.rst @@ -51,6 +51,12 @@ But you need first to specify the Maven profile to use and rebuild the project. * ``es-7x`` for Elasticsearch 7.x * ``es-6x`` for Elasticsearch 6.x +Run a specific test from your Terminal +"""""""""""""""""""""""""""""""""""""" + +To run a specific integration test, just run:: + + mvn verify -am -Dtests.class=fr.pilato.elasticsearch.crawler.fs.test.integration.CLASS_NAME -Dtests.method="METHOD_NAME" Run tests with an external cluster """""""""""""""""""""""""""""""""" diff --git a/docs/source/dev/doc.rst b/docs/source/dev/doc.rst index cf002380b..5226e75ab 100644 --- a/docs/source/dev/doc.rst +++ b/docs/source/dev/doc.rst @@ -4,12 +4,11 @@ Writing documentation This project uses `ReadTheDocs `_ to build and serve the documentation. If you want to run the generation of documentation (recommended!), you need -to have Python installed. Then install ``sphinx`` -$ pip install sphinx sphinx-autobuild +to have Python3 installed. -Assuming you have `Python `_ already, install `Sphinx `_:: +Assuming you have `Python3 `_ already, install `Sphinx `_:: - $ pip install sphinx sphinx-autobuild + $ pip install sphinx sphinx-autobuild sphinx_rtd_theme recommonmark Go to the ``docs`` directory and build the html documentation:: diff --git a/docs/source/index.rst b/docs/source/index.rst index 7ae8598fc..738029d12 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -15,7 +15,7 @@ This crawler helps to index binary documents such as PDF, Open Office, MS Office **Main features**: * Local file system (or a mounted drive) crawling and index new files, update existing ones and removes old ones. -* Remote file system over SSH crawling. +* Remote file system over SSH/FTP/SMB crawling. * REST interface to let you "upload" your binary documents to elasticsearch. .. note:: @@ -56,6 +56,7 @@ This crawler helps to index binary documents such as PDF, Open Office, MS Office admin/fs/simple admin/fs/local-fs admin/fs/ssh + admin/fs/ftp admin/fs/elasticsearch admin/fs/wpsearch admin/fs/rest diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 955dd3287..1ede539ae 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -184,7 +184,7 @@ Then, you can run Elasticsearch. .. code:: sh - docker-compose up -d elasticsearch elasticsearch2 + docker-compose up -d elasticsearch docker-compose logs -f elasticsearch Wait for elasticsearch to be started: diff --git a/docs/source/user/options.rst b/docs/source/user/options.rst index aebc6d9f8..2984a0f01 100644 --- a/docs/source/user/options.rst +++ b/docs/source/user/options.rst @@ -40,5 +40,7 @@ You will find more information about settings in the following sections: - :ref:`cli-options` - :ref:`local-fs-settings` - :ref:`ssh-settings` +- :ref:`ftp-settings` +- :ref:`smb-settings` - :ref:`elasticsearch-settings` diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java index 1f74a541d..bc32b3dfd 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java @@ -29,7 +29,6 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; -import java.nio.charset.StandardCharsets; import java.nio.file.CopyOption; import java.nio.file.FileAlreadyExistsException; import java.nio.file.FileSystem; @@ -282,11 +281,34 @@ public static boolean isIndexable(String content, List filters) { return true; } + public static String getPathSeparator(String path) { + if (path.contains("/") && !path.contains("\\")) { + return "/"; + } + + if (!path.contains("/") && (path.contains("\\") || path.contains(":"))) { + return "\\"; + } + + return File.separator; + } + + public static String computeRealPathName(String _dirname, String filename) { + // new File(dirname, filename).toString() is not suitable for server + String separator = getPathSeparator(_dirname); + String dirname = _dirname.endsWith(separator) ? _dirname : _dirname.concat(separator); + return dirname + filename; + } + public static String computeVirtualPathName(String rootPath, String realPath) { - String result = "/"; - if (realPath != null && realPath.length() > rootPath.length()) { - result = realPath.substring(rootPath.length()) - .replace("\\", "/"); + String result = getPathSeparator(rootPath); + if (realPath.startsWith(rootPath) && realPath.length() > rootPath.length()) { + if (rootPath.equals("/")) { + // "/" is very common for FTP + result = realPath; + } else { + result = realPath.substring(rootPath.length()); + } } logger.debug("computeVirtualPathName({}, {}) = {}", rootPath, realPath, result); @@ -415,12 +437,22 @@ public static int getFilePermissions(final File file) { return user * 100 + group * 10 + others; } catch(Exception e) { - logger.warn("Failed to determine 'owner' of {}: {}", file, e.getMessage()); + logger.warn("Failed to determine 'permissions' of {}: {}", file, e.getMessage()); return -1; } } - private static int toOctalPermission(boolean read, boolean write, boolean execute) { + /** + * This method is used to get the file/folder name from the path, only for SMB Crawler + * because file.getFileInformation().getNameInformation() always equal null + * @param uncPath file uncPath + * @return fileName + */ + public static String getFileName(String uncPath) { + return uncPath.substring(uncPath.lastIndexOf("\\") + 1); + } + + public static int toOctalPermission(boolean read, boolean write, boolean execute) { return (read ? 4 : 0) + (write ? 2 : 0) + (execute ? 1 : 0); } @@ -576,7 +608,9 @@ public static void createDirIfMissing(Path root) { if (Files.notExists(root)) { Files.createDirectory(root); } - } catch (IOException ignored) { } + } catch (IOException ignored) { + logger.error("Failed to create config dir"); + } } /** @@ -629,4 +663,29 @@ public static String extractMajorVersion(String version) { public static String extractMinorVersion(String version) { return version.split("\\.")[1]; } + + /** + * obtain server name through server url（SMB） + * @param url serverUrl + * @return serverName + */ + public static String getServerName(String url) { + return url.split("/")[url.startsWith("//") ? 3 : url.startsWith("/") ? 1 : 0]; + } + + + /** + * get relative path (SMB) //desktopName/shareName //desktopName/shareName/test /shareName /shareName/test + * @param dir dir + * @return relative path + */ + public static String getRelativePath(String dir) { + String[] path = dir.split("/"); + if (dir.startsWith("//")) { + dir = dir.substring(3 + path[2].length() + path[3].length()); + } else if (dir.startsWith("/") && path.length >= 2) { + dir = dir.substring(1 + path[1].length()); + } + return dir; + } } diff --git a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java index d420837b5..4f4e9d675 100644 --- a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java +++ b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java @@ -20,6 +20,8 @@ package fr.pilato.elasticsearch.crawler.fs.framework; import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase; +import java.time.LocalDateTime; +import java.util.TimeZone; import org.junit.BeforeClass; import org.junit.Test; @@ -33,12 +35,19 @@ import java.util.Set; import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.computeRealPathName; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.computeVirtualPathName; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.extractMajorVersion; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.extractMinorVersion; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getFileExtension; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getFileName; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getFilePermissions; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getGroupName; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getOwnerName; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getServerName; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getRelativePath; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.isFileSizeUnderLimit; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.localDateTimeToDate; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.isEmptyOrNullString; @@ -78,6 +87,17 @@ public void testPermissions() { assertThat(permissions, is(700)); } + @Test + public void testGetFileName(){ + assertThat(getFileName("\\test\\test.txt"),is("test.txt")); + assertThat(getFileName("\\test.txt"),is("test.txt")); + assertThat(getFileName("test.txt"),is("test.txt")); + assertThat(getFileName("\\test\\test\\test.txt"),is("test.txt")); + assertThat(getFileName("\\test\\test"),is("test")); + assertThat(getFileName("\\test"),is("test")); + assertThat(getFileName("test"),is("test")); + } + @Test public void testIsFileSizeUnderLimit() { assertThat(isFileSizeUnderLimit(ByteSizeValue.parseBytesSizeValue("1mb"), 1), is(true)); @@ -97,4 +117,141 @@ public void testExtractMinorVersion() { assertThat(extractMinorVersion("7.2.0"), is("2")); assertThat(extractMinorVersion("10.1.0"), is("1")); } + + @Test + public void testGetRealPathNameWindows() { + testRealPath("/C:", "test-windows.txt", "/C:/test-windows.txt"); + testRealPath("/C:/", "test-windows.txt", "/C:/test-windows.txt"); + testRealPath("/C:/dir", "test-windows.txt", "/C:/dir/test-windows.txt"); + testRealPath("/C:/dir/", "test-windows.txt", "/C:/dir/test-windows.txt"); + + testRealPath("C:/", "test-windows.txt", "C:/test-windows.txt"); + testRealPath("C:/dir", "test-windows.txt", "C:/dir/test-windows.txt"); + testRealPath("C:/dir/", "test-windows.txt", "C:/dir/test-windows.txt"); + + testRealPath("C:", "test-windows.txt", "C:\\test-windows.txt"); + testRealPath("C:\\", "test-windows.txt", "C:\\test-windows.txt"); + testRealPath("C:\\dir", "test-windows.txt", "C:\\dir\\test-windows.txt"); + testRealPath("C:\\dir\\", "test-windows.txt", "C:\\dir\\test-windows.txt"); + + testRealPath("\\\\SOMEONE", "test-smb.txt", "\\\\SOMEONE\\test-smb.txt"); + testRealPath("\\\\SOMEONE\\", "test-smb.txt", "\\\\SOMEONE\\test-smb.txt"); + testRealPath("\\\\SOMEONE\\share", "test-smb.txt", "\\\\SOMEONE\\share\\test-smb.txt"); + testRealPath("\\\\SOMEONE\\share\\", "test-smb.txt", "\\\\SOMEONE\\share\\test-smb.txt"); + } + + @Test + public void testGetRealPathNameLinux() { + // Local Linux / FTP + testRealPath("/", "test-linux.txt", "/test-linux.txt"); + testRealPath("/dir", "test-linux.txt", "/dir/test-linux.txt"); + testRealPath("/dir/", "test-linux.txt", "/dir/test-linux.txt"); + + // SMB + testRealPath("//SOMEONE", "test-smb.txt", "//SOMEONE/test-smb.txt"); + testRealPath("//SOMEONE/", "test-smb.txt", "//SOMEONE/test-smb.txt"); + testRealPath("//SOMEONE/share", "test-smb.txt", "//SOMEONE/share/test-smb.txt"); + testRealPath("//SOMEONE/share/", "test-smb.txt", "//SOMEONE/share/test-smb.txt"); + } + + private void testRealPath(String dirname, String filename, String expectedPath) { + assertThat(computeRealPathName(dirname, filename), is(expectedPath)); + } + + @Test + public void testComputePathLinux() { + // Local Linux / FTP + testVirtualPath("/", "/", "/"); + testVirtualPath("/", "/dir", "/dir"); + testVirtualPath("/", "/dir/subdir", "/dir/subdir"); + testVirtualPath("/", "/file.txt", "/file.txt"); + testVirtualPath("/", "/dir/file.txt", "/dir/file.txt"); + testVirtualPath("/", "/dir/subdir/file.txt", "/dir/subdir/file.txt"); + + testVirtualPath("/tmp", "/tmp", "/"); + testVirtualPath("/tmp", "/tmp/dir", "/dir"); + testVirtualPath("/tmp", "/tmp/dir/subdir", "/dir/subdir"); + testVirtualPath("/tmp", "/tmp/file.txt", "/file.txt"); + testVirtualPath("/tmp", "/tmp/dir/file.txt", "/dir/file.txt"); + testVirtualPath("/tmp", "/tmp/dir/subdir/file.txt", "/dir/subdir/file.txt"); + + // SMB + testVirtualPath("//SOMEONE/share", "//SOMEONE/share", "/"); + testVirtualPath("//SOMEONE/share", "//SOMEONE/share/dir", "/dir"); + testVirtualPath("//SOMEONE/share", "//SOMEONE/share/dir/subdir", "/dir/subdir"); + testVirtualPath("//SOMEONE/share", "//SOMEONE/share/file.txt", "/file.txt"); + testVirtualPath("//SOMEONE/share", "//SOMEONE/share/dir/file.txt", "/dir/file.txt"); + testVirtualPath("//SOMEONE/share", "//SOMEONE/share/dir/subdir/file.txt", "/dir/subdir/file.txt"); + } + + @Test + public void testComputePathWindows() { + testVirtualPath("C:", "C:", "\\"); + testVirtualPath("C:", "C:\\dir", "\\dir"); + testVirtualPath("C:", "C:\\dir\\subdir", "\\dir\\subdir"); + testVirtualPath("C:", "C:\\file.txt", "\\file.txt"); + testVirtualPath("C:", "C:\\dir\\file.txt", "\\dir\\file.txt"); + testVirtualPath("C:", "C:\\dir\\subdir\\file.txt", "\\dir\\subdir\\file.txt"); + + testVirtualPath("C:\\tmp", "C:\\tmp", "\\"); + testVirtualPath("C:\\tmp", "C:\\tmp\\dir", "\\dir"); + testVirtualPath("C:\\tmp", "C:\\tmp\\dir\\subdir", "\\dir\\subdir"); + testVirtualPath("C:\\tmp", "C:\\tmp\\file.txt", "\\file.txt"); + testVirtualPath("C:\\tmp", "C:\\tmp\\dir\\file.txt", "\\dir\\file.txt"); + testVirtualPath("C:\\tmp", "C:\\tmp\\dir\\subdir\\file.txt", "\\dir\\subdir\\file.txt"); + + testVirtualPath("C:/tmp", "C:/tmp", "/"); + testVirtualPath("C:/tmp", "C:/tmp/dir", "/dir"); + testVirtualPath("C:/tmp", "C:/tmp/dir/subdir", "/dir/subdir"); + testVirtualPath("C:/tmp", "C:/tmp/file.txt", "/file.txt"); + testVirtualPath("C:/tmp", "C:/tmp/dir/file.txt", "/dir/file.txt"); + testVirtualPath("C:/tmp", "C:/tmp/dir/subdir/file.txt", "/dir/subdir/file.txt"); + + testVirtualPath("/C:/tmp", "/C:/tmp", "/"); + testVirtualPath("/C:/tmp", "/C:/tmp/dir", "/dir"); + testVirtualPath("/C:/tmp", "/C:/tmp/dir/subdir", "/dir/subdir"); + testVirtualPath("/C:/tmp", "/C:/tmp/file.txt", "/file.txt"); + testVirtualPath("/C:/tmp", "/C:/tmp/dir/file.txt", "/dir/file.txt"); + testVirtualPath("/C:/tmp", "/C:/tmp/dir/subdir/file.txt", "/dir/subdir/file.txt"); + + testVirtualPath("\\\\SOMEONE\\share", "\\\\SOMEONE\\share", "\\"); + testVirtualPath("\\\\SOMEONE\\share", "\\\\SOMEONE\\share\\dir", "\\dir"); + testVirtualPath("\\\\SOMEONE\\share", "\\\\SOMEONE\\share\\dir\\subdir", "\\dir\\subdir"); + testVirtualPath("\\\\SOMEONE\\share", "\\\\SOMEONE\\share\\file.txt", "\\file.txt"); + testVirtualPath("\\\\SOMEONE\\share", "\\\\SOMEONE\\share\\dir\\file.txt", "\\dir\\file.txt"); + testVirtualPath("\\\\SOMEONE\\share", "\\\\SOMEONE\\share\\dir\\subdir\\file.txt", "\\dir\\subdir\\file.txt"); + } + + private void testVirtualPath(String rootPath, String realPath, String expectedPath) { + assertThat(computeVirtualPathName(rootPath, realPath), is(expectedPath)); + } + + @Test + public void testGetFileExtension() { + assertThat(getFileExtension(new File("foo.bar")), is("bar")); + assertThat(getFileExtension(new File("foo")), is("")); + assertThat(getFileExtension(new File("foo.bar.baz")), is("baz")); + } + + @Test + public void testLocalDateToDate() { + LocalDateTime now = LocalDateTime.now(); + logger.info("Current Time [{}] in [{}] is actually [{}]", now, TimeZone.getDefault().getDisplayName(), localDateTimeToDate(now)); + } + + @Test + public void testGetServerName(){ + assertThat(getServerName("//desttop-123/test"), is("test")); + } + + @Test + public void testGetRelativePath(){ + assertThat(getRelativePath("//desktopName/shareName"), is("")); + assertThat(getRelativePath("//desktopName/shareName/test"), is("/test")); + assertThat(getRelativePath("//desktopName/shareName/test.txt"), is("/test.txt")); + assertThat(getRelativePath("//desktopName/shareName/folder/test.txt"), is("/folder/test.txt")); + assertThat(getRelativePath("/shareName"), is("")); + assertThat(getRelativePath("/shareName/test"), is("/test")); + } + } diff --git a/integration-tests/it-common/pom.xml b/integration-tests/it-common/pom.xml index ceeb5174a..952cdddde 100644 --- a/integration-tests/it-common/pom.xml +++ b/integration-tests/it-common/pom.xml @@ -31,6 +31,11 @@ fr.pilato.elasticsearch.crawler fscrawler-test-documents + + org.mockftpserver + MockFtpServer + compile + + ${DOCKER_USERNAME} + ${DOCKER_PASSWORD} ${env.DOCKER_USERNAME} ${env.DOCKER_PASSWORD} @@ -551,11 +553,21 @@ fscrawler-crawler-fs 2.7-SNAPSHOT + + fr.pilato.elasticsearch.crawler + fscrawler-crawler-ftp + 2.7-SNAPSHOT + fr.pilato.elasticsearch.crawler fscrawler-crawler-ssh 2.7-SNAPSHOT + + fr.pilato.elasticsearch.crawler + fscrawler-crawler-smb + 2.7-SNAPSHOT + fr.pilato.elasticsearch.crawler fscrawler-tika @@ -794,13 +806,33 @@ - + + com.jcraft jsch 0.1.55 + + + commons-net + commons-net + 3.8.0 + + + org.mockftpserver + MockFtpServer + 2.8.0 + test + + + org.slf4j + slf4j-api + + + + com.beust @@ -888,6 +920,12 @@ ${log4j.version} true + + org.apache.logging.log4j + log4j-iostreams + ${log4j.version} + true + org.fusesource.jansi jansi diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java index 081e441e9..4b6f086b4 100644 --- a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java @@ -62,19 +62,26 @@ public static boolean validateSettings(Logger logger, FsSettings settings, boole // Checking protocol if (settings.getServer() != null) { if (!Server.PROTOCOL.LOCAL.equals(settings.getServer().getProtocol()) && - !Server.PROTOCOL.SSH.equals(settings.getServer().getProtocol())) { + !Server.PROTOCOL.SSH.equals(settings.getServer().getProtocol()) && !Server.PROTOCOL.FTP.equals(settings.getServer().getProtocol()) && !Server.PROTOCOL.SMB.equals(settings.getServer().getProtocol())) { // Non supported protocol logger.error(settings.getServer().getProtocol() + " is not supported yet. Please use " + - Server.PROTOCOL.LOCAL + " or " + Server.PROTOCOL.SSH + ". Disabling crawler"); + Server.PROTOCOL.LOCAL + " or " + Server.PROTOCOL.SSH + " or " + Server.PROTOCOL.FTP + " or " + Server.PROTOCOL.SMB + ". Disabling crawler"); return true; } // Checking username/password if (Server.PROTOCOL.SSH.equals(settings.getServer().getProtocol()) && FsCrawlerUtil.isNullOrEmpty(settings.getServer().getUsername())) { - // Non supported protocol logger.error("When using SSH, you need to set a username and probably a password or a pem file. Disabling crawler"); return true; + } else if (Server.PROTOCOL.FTP.equals(settings.getServer().getProtocol()) && + FsCrawlerUtil.isNullOrEmpty(settings.getServer().getUsername())) { + logger.error("When using FTP, you need to set a username and probably a password. Disabling crawler"); + return true; + } else if (Server.PROTOCOL.SMB.equals(settings.getServer().getProtocol()) && + FsCrawlerUtil.isNullOrEmpty(settings.getServer().getUsername())) { + logger.error("When using SMB, you need to set a username and probably a password. Disabling crawler"); + return true; } } diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java index 270392e0e..5bf059f84 100644 --- a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java @@ -28,8 +28,11 @@ public class Server { public static final class PROTOCOL { public static final String LOCAL = "local"; + public static final String SMB = "smb"; public static final String SSH = "ssh"; + public static final String FTP = "ftp"; public static final int SSH_PORT = 22; + public static final int FTP_PORT = 21; } public Server() { diff --git a/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidatorTest.java b/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidatorTest.java index b59164255..70214a59a 100644 --- a/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidatorTest.java +++ b/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidatorTest.java @@ -66,6 +66,10 @@ public void testSettingsValidation() { settings = buildSettings(null, Server.builder().setProtocol(Server.PROTOCOL.SSH).build()); assertThat(FsCrawlerValidator.validateSettings(logger, settings, false), is(true)); + // Checking username / password when FTP + settings = buildSettings(null, Server.builder().setProtocol(Server.PROTOCOL.FTP).build()); + assertThat(FsCrawlerValidator.validateSettings(logger, settings, false), is(true)); + // Checking That we don't try to do both xml and json settings = buildSettings(Fs.builder().setJsonSupport(true).setXmlSupport(true).build(), null); assertThat(FsCrawlerValidator.validateSettings(logger, settings, false), is(true));