From 1705d4509a7db2c11a8d71f2756d04e034222549 Mon Sep 17 00:00:00 2001 From: helson Date: Sun, 18 Jul 2021 15:24:47 +0800 Subject: [PATCH 01/26] feat: FTP crawler * fix: gitignore * doc: writing documentation * doc: how to test specific module * feat: ftp crawler * doc: ftp settings --- .gitignore | 5 + .../crawler/fs/cli/FsCrawlerCli.java | 9 + core/pom.xml | 4 + .../crawler/fs/FsCrawlerImpl.java | 3 + .../crawler/fs/FsParserAbstract.java | 16 +- .../elasticsearch/crawler/fs/FsParserFTP.java | 39 ++++ .../crawler/fs/FsCrawlerUtilTest.java | 76 ------- crawler/crawler-ftp/pom.xml | 37 ++++ .../fs/crawler/ftp/FileAbstractorFTP.java | 207 ++++++++++++++++++ .../fs/crawler/ftp/FileAbstractorFTPTest.java | 162 ++++++++++++++ .../crawler-ftp/src/test/resources/log4j2.xml | 22 ++ .../fs/crawler/ssh/FileAbstractorSSH.java | 2 +- crawler/pom.xml | 1 + docs/source/admin/fs/ftp.rst | 48 ++++ docs/source/admin/fs/ssh.rst | 17 +- docs/source/conf.py | 4 +- docs/source/dev/build.rst | 6 + docs/source/dev/doc.rst | 7 +- docs/source/index.rst | 3 +- docs/source/installation.rst | 2 +- docs/source/user/options.rst | 1 + framework/pom.xml | 11 + .../crawler/fs/framework/FsCrawlerUtil.java | 59 ++++- .../fs/framework/FsCrawlerUtilTest.java | 151 +++++++++++++ .../elasticsearch/FsCrawlerTestFTPIT.java | 81 +++++++ pom.xml | 32 +++ .../fs/settings/FsCrawlerValidator.java | 9 +- .../crawler/fs/settings/Server.java | 2 + .../fs/settings/FsCrawlerValidatorTest.java | 4 + 29 files changed, 919 insertions(+), 101 deletions(-) create mode 100644 core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserFTP.java delete mode 100644 core/src/test/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerUtilTest.java create mode 100644 crawler/crawler-ftp/pom.xml create mode 100644 crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java create mode 100644 crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java create mode 100644 crawler/crawler-ftp/src/test/resources/log4j2.xml create mode 100644 docs/source/admin/fs/ftp.rst create mode 100644 integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestFTPIT.java diff --git a/.gitignore b/.gitignore index c9b58b2ef..6ad5f8103 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,13 @@ +.DS_Store target /.settings /.classpath /.project +.settings +.classpath +.project .idea *.iml /.run /logs/ +/venv/ diff --git a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java index 5f13de4ea..899507a65 100644 --- a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java +++ b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java @@ -34,6 +34,7 @@ import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettingsFileHandler; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettingsParser; +import fr.pilato.elasticsearch.crawler.fs.settings.Server.PROTOCOL; import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; @@ -208,6 +209,14 @@ public static void main(String[] args) throws Exception { if (fsSettings.getFs() == null) { fsSettings.setFs(Fs.DEFAULT); } + + if (fsSettings.getServer().getProtocol().equals(PROTOCOL.FTP) && fsSettings.getServer().getPort() == PROTOCOL.SSH_PORT) { + fsSettings.getServer().setPort(PROTOCOL.FTP_PORT); + } + if (fsSettings.getServer().getProtocol().equals(PROTOCOL.FTP) && StringUtils.isEmpty(fsSettings.getServer().getUsername())) { + fsSettings.getServer().setUsername("anonymous"); + } + if (fsSettings.getElasticsearch() == null) { fsSettings.setElasticsearch(Elasticsearch.DEFAULT()); } diff --git a/core/pom.xml b/core/pom.xml index 5f1bdfe4e..8e18d5381 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -69,6 +69,10 @@ fr.pilato.elasticsearch.crawler fscrawler-crawler-fs + + fr.pilato.elasticsearch.crawler + fscrawler-crawler-ftp + fr.pilato.elasticsearch.crawler fscrawler-crawler-ssh diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java index f8bdf4f12..32da3dc5c 100644 --- a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java @@ -126,6 +126,9 @@ public void start() throws Exception { } else if (Server.PROTOCOL.SSH.equals(settings.getServer().getProtocol())) { // Remote SSH FS fsParser = new FsParserSsh(settings, config, managementService, documentService, loop); + } else if (Server.PROTOCOL.FTP.equals(settings.getServer().getProtocol())) { + // Remote FTP FS + fsParser = new FsParserFTP(settings, config, managementService, documentService, loop); } else { // Non supported protocol throw new RuntimeException(settings.getServer().getProtocol() + " is not supported yet. Please use " + diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java index b891a90bb..dbf297d2e 100644 --- a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java @@ -29,6 +29,7 @@ import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor; import fr.pilato.elasticsearch.crawler.fs.framework.ByteSizeValue; import fr.pilato.elasticsearch.crawler.fs.framework.FSCrawlerLogger; +import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil; import fr.pilato.elasticsearch.crawler.fs.framework.OsValidator; import fr.pilato.elasticsearch.crawler.fs.framework.SignTool; import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; @@ -96,9 +97,9 @@ public abstract class FsParserAbstract extends FsParser { messageDigest = null; } - // On Windows, when using SSH server, we need to force the "Linux" separator + // On Windows, when using server, we need to force the "Linux" separator if (OsValidator.WINDOWS && fsSettings.getServer() != null) { - logger.debug("We are running on Windows with SSH Server settings so we need to force the Linux separator."); + logger.debug("We are running on Windows with Server settings so we need to force the Linux separator."); pathSeparator = "/"; } else { pathSeparator = File.separator; @@ -256,7 +257,7 @@ private void addFilesRecursively(FileAbstractor path, String filepath, LocalD logger.trace("FileAbstractModel = {}", child); String filename = child.getName(); - String virtualFileName = computeVirtualPathName(stats.getRootPath(), new File(filepath, filename).toString()); + String virtualFileName = computeVirtualPathName(stats.getRootPath(), FsCrawlerUtil.computeRealPathName(filepath, filename)); // https://github.com/dadoonet/fscrawler/issues/1 : Filter documents boolean isIndexable = isIndexable(child.isDirectory(), virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes()); @@ -318,7 +319,7 @@ private void addFilesRecursively(FileAbstractor path, String filepath, LocalD for (String esfile : esFiles) { logger.trace("Checking file [{}]", esfile); - String virtualFileName = computeVirtualPathName(stats.getRootPath(), new File(filepath, esfile).toString()); + String virtualFileName = computeVirtualPathName(stats.getRootPath(), FsCrawlerUtil.computeRealPathName(filepath, esfile)); if (isIndexable(false, virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes()) && !fsFiles.contains(esfile)) { logger.trace("Removing file [{}] in elasticsearch/workplace", esfile); @@ -333,7 +334,7 @@ private void addFilesRecursively(FileAbstractor path, String filepath, LocalD // for the delete folder for (String esfolder : esFolders) { - String virtualFileName = computeVirtualPathName(stats.getRootPath(), new File(filepath, esfolder).toString()); + String virtualFileName = computeVirtualPathName(stats.getRootPath(), FsCrawlerUtil.computeRealPathName(filepath, esfolder)); if (isIndexable(true, virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes())) { logger.trace("Checking directory [{}]", esfolder); if (!fsFolders.contains(esfolder)) { @@ -376,7 +377,7 @@ private void indexFile(FileAbstractModel fileAbstractModel, ScanStatistic stats, final long size = fileAbstractModel.getSize(); logger.debug("fetching content from [{}],[{}]", dirname, filename); - String fullFilename = new File(dirname, filename).toString(); + String fullFilename = FsCrawlerUtil.computeRealPathName(dirname, filename); try { // Create the Doc object (only needed when we have add_as_inner_object: true (default) or when we don't index json or xml) @@ -392,6 +393,7 @@ private void indexFile(FileAbstractModel fileAbstractModel, ScanStatistic stats, doc.getFile().setLastModified(localDateTimeToDate(lastModified)); doc.getFile().setLastAccessed(localDateTimeToDate(lastAccessed)); doc.getFile().setIndexingDate(localDateTimeToDate(LocalDateTime.now())); + // TODO: how about just set for local fs? doc.getFile().setUrl("file://" + fullFilename); doc.getFile().setExtension(extension); if (fsSettings.getFs().isAddFilesize()) { @@ -517,7 +519,7 @@ private void indexDirectory(String id, fr.pilato.elasticsearch.crawler.fs.beans. /** * Index a directory - * @param path complete path like /path/to/subdir + * @param path complete path like "/", "/path/to/subdir", "/C:/dir", "//SOMEONE/dir" */ private void indexDirectory(String path) throws Exception { fr.pilato.elasticsearch.crawler.fs.beans.Path pathObject = new fr.pilato.elasticsearch.crawler.fs.beans.Path(); diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserFTP.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserFTP.java new file mode 100644 index 000000000..d04934503 --- /dev/null +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserFTP.java @@ -0,0 +1,39 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs; + +import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor; +import fr.pilato.elasticsearch.crawler.fs.crawler.ftp.FileAbstractorFTP; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementService; +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import java.nio.file.Path; + +public class FsParserFTP extends FsParserAbstract { + + public FsParserFTP(FsSettings fsSettings, Path config, FsCrawlerManagementService managementService, + FsCrawlerDocumentService documentService, Integer loop) { + super(fsSettings, config, managementService, documentService, loop); + } + + protected FileAbstractor buildFileAbstractor() { + return new FileAbstractorFTP(fsSettings); + } +} diff --git a/core/src/test/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerUtilTest.java b/core/src/test/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerUtilTest.java deleted file mode 100644 index bcd812888..000000000 --- a/core/src/test/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerUtilTest.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to David Pilato (the "Author") under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. Author licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package fr.pilato.elasticsearch.crawler.fs; - -import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerMetadataTestCase; -import org.junit.Test; - -import java.io.File; -import java.time.LocalDateTime; -import java.util.TimeZone; - -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.computeVirtualPathName; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getFileExtension; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.localDateTimeToDate; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.is; - -/** - * We want to test some utilities - */ -public class FsCrawlerUtilTest extends AbstractFSCrawlerMetadataTestCase { - - @Test - public void testComputePathLinux() { - testHelper("/tmp", "/tmp", "/"); - testHelper("/tmp", "/tmp/dir", "/dir"); - testHelper("/tmp", "/tmp/dir/subdir", "/dir/subdir"); - testHelper("/tmp", "/tmp/file.txt", "/file.txt"); - testHelper("/tmp", "/tmp/dir/file.txt", "/dir/file.txt"); - testHelper("/tmp", "/tmp/dir/subdir/file.txt", "/dir/subdir/file.txt"); - } - - @Test - public void testComputePathWindows() { - testHelper("C:\\tmp", "C:\\tmp", "/"); - testHelper("C:\\tmp", "C:\\tmp\\dir", "/dir"); - testHelper("C:\\tmp", "C:\\tmp\\dir\\subdir", "/dir/subdir"); - testHelper("C:\\tmp", "C:\\tmp\\file.txt", "/file.txt"); - testHelper("C:\\tmp", "C:\\tmp\\dir\\file.txt", "/dir/file.txt"); - testHelper("C:\\tmp", "C:\\tmp\\dir\\subdir\\file.txt", "/dir/subdir/file.txt"); - } - - private void testHelper(String rootPath, String realPath, String expectedPath) { - assertThat(computeVirtualPathName(rootPath, realPath), is(expectedPath)); - } - - @Test - public void testGetFileExtension() { - assertThat(getFileExtension(new File("foo.bar")), is("bar")); - assertThat(getFileExtension(new File("foo")), is("")); - assertThat(getFileExtension(new File("foo.bar.baz")), is("baz")); - } - - @Test - public void testLocalDateToDate() { - LocalDateTime now = LocalDateTime.now(); - logger.info("Current Time [{}] in [{}] is actually [{}]", now, TimeZone.getDefault().getDisplayName(), localDateTimeToDate(now)); - } -} diff --git a/crawler/crawler-ftp/pom.xml b/crawler/crawler-ftp/pom.xml new file mode 100644 index 000000000..f02dd6a18 --- /dev/null +++ b/crawler/crawler-ftp/pom.xml @@ -0,0 +1,37 @@ + + + + fscrawler-crawler + fr.pilato.elasticsearch.crawler + 2.7-SNAPSHOT + + 4.0.0 + + fscrawler-crawler-ftp + FSCrawler Crawlers: FTP + + + + fr.pilato.elasticsearch.crawler + fscrawler-crawler-abstract + + + + + commons-net + commons-net + + + org.mockftpserver + MockFtpServer + test + + + org.apache.logging.log4j + log4j-iostreams + + + + diff --git a/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java b/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java new file mode 100644 index 000000000..fe4f69243 --- /dev/null +++ b/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java @@ -0,0 +1,207 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.crawler.ftp; + +import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractModel; +import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor; +import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil; +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import fr.pilato.elasticsearch.crawler.fs.settings.Server; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.net.PrintCommandListener; +import org.apache.commons.net.ftp.FTP; +import org.apache.commons.net.ftp.FTPClient; +import org.apache.commons.net.ftp.FTPFile; +import org.apache.commons.net.ftp.FTPReply; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.io.IoBuilder; + +import java.io.InputStream; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Collection; +import java.util.stream.Collectors; + +public class FileAbstractorFTP extends FileAbstractor { + private final Logger logger = LogManager.getLogger(FileAbstractorFTP.class); + + private FTPClient ftp; + + private final OutputStream loggerOutputStream = IoBuilder.forLogger(logger).buildOutputStream(); + + private final PrintCommandListener ftpListener = new PrintCommandListener(new PrintWriter(loggerOutputStream)); + + private String controlEncoding = FTP.DEFAULT_CONTROL_ENCODING; + + public FileAbstractorFTP(FsSettings fsSettings) { + super(fsSettings); + } + + @Override + public FileAbstractModel toFileAbstractModel(String _path, FTPFile file) { + String filename = file.getName(); + String extension = FilenameUtils.getExtension(filename); + String path = _path; + + // if server is not using utf-8 + if (controlEncoding.equals(FTP.DEFAULT_CONTROL_ENCODING)) { + if (file.isFile()) { + try { + filename = new String(filename.getBytes(controlEncoding), StandardCharsets.UTF_8); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + try { + path = new String(_path.getBytes(controlEncoding), StandardCharsets.UTF_8); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + } + } + + return new FileAbstractModel( + filename, + file.isFile(), + // We are using here the local TimeZone as a reference. If the remote system is under another TZ, this might cause issues + LocalDateTime.ofInstant(Instant.ofEpochMilli(file.getTimestamp().getTimeInMillis()), ZoneId.systemDefault()), + // We don't have the creation date + null, + // We don't have the access date + null, + extension, + path, + path.equals("/") ? path.concat(filename) : path.concat("/").concat(filename), + file.getSize(), + file.getUser(), + file.getGroup(), + FsCrawlerUtil.getFilePermissions(file)); + } + + @Override + public InputStream getInputStream(FileAbstractModel file) throws Exception { + // FTP data connection could be closed after transfer process. + openFTPConnection(); + + ftp.enterLocalPassiveMode(); + String fullPath = file.getFullpath(); + if (controlEncoding.equals(FTP.DEFAULT_CONTROL_ENCODING)) { + fullPath = new String(fullPath.getBytes(StandardCharsets.UTF_8), FTP.DEFAULT_CONTROL_ENCODING); + } + return ftp.retrieveFileStream(fullPath); + } + + @Override + public Collection getFiles(String dir) throws IOException { + // FTP data connection could be closed after transfer process. + openFTPConnection(); + + if (controlEncoding.equals(FTP.DEFAULT_CONTROL_ENCODING)) { + dir = new String(dir.getBytes(StandardCharsets.UTF_8), FTP.DEFAULT_CONTROL_ENCODING); + } + logger.debug("Listing local files from {}", dir); + + ftp.enterLocalPassiveMode(); + FTPFile[] ftpFiles = ftp.listFiles(dir); + if (ftpFiles == null) return null; + List files = Arrays.stream(ftpFiles).filter(file -> { + if (fsSettings.getFs().isFollowSymlinks()) return true; + return !file.isSymbolicLink(); + }).collect(Collectors.toList()); + + Collection result = new ArrayList<>(files.size()); + // Iterate other files + // We ignore here all files like . and .. + String finalDir = dir; + result.addAll(files.stream().filter(file -> !".".equals(file.getName()) && + !"..".equals(file.getName())) + .map(file -> toFileAbstractModel(finalDir, file)) + .collect(Collectors.toList())); + + logger.debug("{} local files found", result.size()); + return result; + } + + @Override + public boolean exists(String dir) { + try { + if (controlEncoding.equals(FTP.DEFAULT_CONTROL_ENCODING)) { + dir = new String(dir.getBytes(StandardCharsets.UTF_8), FTP.DEFAULT_CONTROL_ENCODING); + } + return ftp.changeWorkingDirectory(dir); + } catch (IOException e) { + return false; + } + } + + @Override + public void open() throws IOException { + Server server = fsSettings.getServer(); + logger.debug("Opening FTP connection to {}@{}", server.getUsername(), server.getHostname()); + + ftp = new FTPClient(); + ftp.addProtocolCommandListener(ftpListener); + // send a safe command (i.e. NOOP) over the control connection to reset the router's idle timer + ftp.setControlKeepAliveTimeout(300); + openFTPConnection(); + } + + @Override + public void close() throws IOException { + ftp.logout(); + ftp.disconnect(); + } + + private void openFTPConnection() throws IOException { + Server server = fsSettings.getServer(); + ftp.connect(server.getHostname(), server.getPort()); + + // checking FTP client connection. + int reply = ftp.getReplyCode(); + if (!FTPReply.isPositiveCompletion(reply)) { + ftp.disconnect(); + logger.warn("Cannot connect with FTP to {}@{}", server.getUsername(), + server.getHostname()); + throw new RuntimeException("Can not connect to " + server.getUsername() + "@" + server.getHostname()); + } + + if (!ftp.login(server.getUsername(), server.getPassword())) { + ftp.disconnect(); + throw new RuntimeException("Please check ftp user or password"); + } + + int utf8Reply = ftp.sendCommand("OPTS UTF8", "ON"); + if (FTPReply.isPositiveCompletion(utf8Reply)) { + controlEncoding = StandardCharsets.UTF_8.displayName(); + ftp.setControlEncoding(controlEncoding); + } + + logger.debug("FTP connection successful"); + } +} diff --git a/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java b/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java new file mode 100644 index 000000000..474f9ee3a --- /dev/null +++ b/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java @@ -0,0 +1,162 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.crawler.ftp; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractModel; +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import fr.pilato.elasticsearch.crawler.fs.settings.Server; +import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collection; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.net.ftp.FTPClient; +import org.junit.After; +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; +import org.mockftpserver.fake.FakeFtpServer; +import org.mockftpserver.fake.UserAccount; +import org.mockftpserver.fake.filesystem.DirectoryEntry; +import org.mockftpserver.fake.filesystem.FileEntry; +import org.mockftpserver.fake.filesystem.FileSystem; +import org.mockftpserver.fake.filesystem.UnixFakeFileSystem; + +public class FileAbstractorFTPTest extends AbstractFSCrawlerTestCase { + private FakeFtpServer fakeFtpServer; + private final String path = "/data"; + private final String user = "user"; + private final String pass = "password"; + + @Before + public void setup() { + // it doesn't seem to support utf-8 + fakeFtpServer = new FakeFtpServer(); + fakeFtpServer.setServerControlPort(5968); + fakeFtpServer.addUserAccount(new UserAccount(user, pass, path)); + FileSystem fileSystem = new UnixFakeFileSystem(); + + fileSystem.add(new DirectoryEntry("/data")); + fileSystem.add(new FileEntry("/data/foo.txt", "foo")); + fileSystem.add(new FileEntry("/data/bar.txt", "bar")); + + fileSystem.add(new DirectoryEntry("/data/buzz")); + fileSystem.add(new FileEntry("/data/buzz/hello.txt", "hello")); + fileSystem.add(new FileEntry("/data/buzz/world.txt", "world")); + + fakeFtpServer.setFileSystem(fileSystem); + fakeFtpServer.start(); + } + + @After + public void teardown() { + fakeFtpServer.stop(); + } + + @Test + public void testConnectToFakeFTPServer() throws Exception { + int port = fakeFtpServer.getServerControlPort(); + FsSettings fsSettings = FsSettings.builder("fake") + .setServer( + Server.builder() + .setHostname("localhost") + .setUsername(user) + .setPassword(pass) + .setPort(port) + .build() + ) + .build(); + + FileAbstractorFTP ftp = new FileAbstractorFTP(fsSettings); + ftp.open(); + boolean exists = ftp.exists(path); + assertThat(exists, is(true)); + Collection files = ftp.getFiles(path); + assertThat(files.size(), is(3)); + + for (FileAbstractModel file : files) { + if (file.isDirectory()) { + assertThat(file.getName(), is("buzz")); + Collection subDirFiles = ftp.getFiles(file.getFullpath()); + assertThat(subDirFiles.size(), is(2)); + logger.debug("Found {} files in sub dir", subDirFiles.size()); + for (FileAbstractModel subDirFile : subDirFiles) { + try (InputStream inputStream = ftp.getInputStream(subDirFile)) { + String content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + logger.debug("[sub dir] - {}: {}", subDirFile.getName(), content); + } + } + } else { + try (InputStream inputStream = ftp.getInputStream(file)) { + String content = IOUtils.toString(inputStream, FTPClient.DEFAULT_CONTROL_ENCODING); + logger.debug(" - {}: {}", file.getName(), content); + } + } + } + + ftp.close(); + } + + @Test @Ignore + public void testConnectToFTPServer() throws Exception { + String path = "/"; + FsSettings fsSettings = FsSettings.builder("local_test") + .setServer( + Server.builder() + .setHostname("192.168.18.207") + .setUsername("username") + .setPassword("password") + .setPort(21) + .build() + ) + .build(); + + FileAbstractorFTP ftp = new FileAbstractorFTP(fsSettings); + ftp.open(); + boolean exists = ftp.exists(path); + assertThat(exists, is(true)); + Collection files = ftp.getFiles(path); + logger.debug("Found {} files", files.size()); + + for (FileAbstractModel file : files) { + if (file.isDirectory()) { + Collection subDirFiles = ftp.getFiles(file.getFullpath()); + logger.debug("Found {} files in sub dir", subDirFiles.size()); + for (FileAbstractModel subDirFile : subDirFiles) { + try (InputStream inputStream = ftp.getInputStream(subDirFile)) { + String content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + logger.debug("[sub dir] - {}: {}", subDirFile.getName(), content); + } + } + } else { + try (InputStream inputStream = ftp.getInputStream(file)) { + String content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + logger.debug(" - {}: {}", file.getName(), content); + } + } + } + + ftp.close(); + } +} diff --git a/crawler/crawler-ftp/src/test/resources/log4j2.xml b/crawler/crawler-ftp/src/test/resources/log4j2.xml new file mode 100644 index 000000000..c551aa429 --- /dev/null +++ b/crawler/crawler-ftp/src/test/resources/log4j2.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/crawler/crawler-ssh/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ssh/FileAbstractorSSH.java b/crawler/crawler-ssh/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ssh/FileAbstractorSSH.java index 2c95c94f8..a872def6e 100644 --- a/crawler/crawler-ssh/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ssh/FileAbstractorSSH.java +++ b/crawler/crawler-ssh/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ssh/FileAbstractorSSH.java @@ -62,7 +62,7 @@ public FileAbstractModel toFileAbstractModel(String path, ChannelSftp.LsEntry fi LocalDateTime.ofInstant(Instant.ofEpochMilli(file.getAttrs().getATime()*1000L), ZoneId.systemDefault()), FilenameUtils.getExtension(file.getFilename()), path, - path.concat("/").concat(file.getFilename()), + path.equals("/") ? path.concat(file.getFilename()) : path.concat("/").concat(file.getFilename()), file.getAttrs().getSize(), Integer.toString(file.getAttrs().getUId()), Integer.toString(file.getAttrs().getGId()), diff --git a/crawler/pom.xml b/crawler/pom.xml index 64d02cb9f..6d4f73f80 100644 --- a/crawler/pom.xml +++ b/crawler/pom.xml @@ -16,6 +16,7 @@ crawler-abstract crawler-fs + crawler-ftp crawler-ssh diff --git a/docs/source/admin/fs/ftp.rst b/docs/source/admin/fs/ftp.rst new file mode 100644 index 000000000..c8b570b4d --- /dev/null +++ b/docs/source/admin/fs/ftp.rst @@ -0,0 +1,48 @@ +.. _ftp-settings: + +FTP settings +------------ + +You can index files remotely using FTP. + +Here is a list of FTP settings (under ``server.`` prefix): + ++-----------------------+-----------------------+-----------------------+ +| Name | Default value | Documentation | ++=======================+=======================+=======================+ +| ``server.hostname`` | ``null`` | Hostname | ++-----------------------+-----------------------+-----------------------+ +| ``server.port`` | ``21`` | Port | ++-----------------------+-----------------------+-----------------------+ +| ``server.username`` | ``anonymous`` | :ref:`ftp_login` | ++-----------------------+-----------------------+-----------------------+ +| ``server.password`` | ``null`` | :ref:`ftp_login` | ++-----------------------+-----------------------+-----------------------+ +| ``server.protocol`` | ``"local"`` | Set it to ``ftp`` | ++-----------------------+-----------------------+-----------------------+ + +.. _ftp_login: + +Username / Password +~~~~~~~~~~~~~~~~~~~ + +Let’s say you want to index from a remote server using FTP: + +- FS URL: ``/path/to/data/dir/on/server`` +- Server: ``mynode.mydomain.com`` +- Username: ``username`` (default to ``anonymous``) +- Password: ``password`` +- Protocol: ``ftp`` (default to ``local``) +- Port: ``21`` (default to ``21``) + +.. code:: yaml + + name: "test" + fs: + url: "/path/to/data/dir/on/server" + server: + hostname: "mynode.mydomain.com" + port: 21 + username: "username" + password: "password" + protocol: "ftp" diff --git a/docs/source/admin/fs/ssh.rst b/docs/source/admin/fs/ssh.rst index 2653deb3a..a534bab1b 100644 --- a/docs/source/admin/fs/ssh.rst +++ b/docs/source/admin/fs/ssh.rst @@ -5,7 +5,7 @@ SSH settings You can index files remotely using SSH. -Here is a list of SSH settings (under ``server.`` prefix)`: +Here is a list of SSH settings (under ``server.`` prefix): +-----------------------+-----------------------+-----------------------+ | Name | Default value | Documentation | @@ -93,3 +93,18 @@ To specify the drive, you need to use the following format: username: "username" password: "password" protocol: "ssh" + +Windows shared folders +~~~~~~~~~~~~~~ + +.. code:: yaml + + name: "test" + fs: + url: "//DESKTOP-NAME/path/to/data/dir/on/server" + server: + hostname: "mynode.mydomain.com" + port: 22 + username: "username" + password: "password" + protocol: "ssh" diff --git a/docs/source/conf.py b/docs/source/conf.py index a3c5c0aad..93d39326e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -16,7 +16,7 @@ # import sys # sys.path.insert(0, os.path.abspath('.')) import os -import ConfigParser +import configparser from datetime import date from os.path import join, dirname @@ -38,7 +38,7 @@ # built documents. # -config = ConfigParser.RawConfigParser() +config = configparser.RawConfigParser() config.read(join(dirname(__file__), "fscrawler.ini")) # development versions always have the suffix '-SNAPSHOT' diff --git a/docs/source/dev/build.rst b/docs/source/dev/build.rst index f1d0a6cc7..99ccdd379 100644 --- a/docs/source/dev/build.rst +++ b/docs/source/dev/build.rst @@ -51,6 +51,12 @@ But you need first to specify the Maven profile to use and rebuild the project. * ``es-7x`` for Elasticsearch 7.x * ``es-6x`` for Elasticsearch 6.x +Run specific module tests from your Terminal +"""""""""""""""""""""""""""""" + +To run integration tests for a specific module, just run:: + + mvn test -am -DfailIfNoTests=false -pl [module_name_or_folder_path] Run tests with an external cluster """""""""""""""""""""""""""""""""" diff --git a/docs/source/dev/doc.rst b/docs/source/dev/doc.rst index cf002380b..5226e75ab 100644 --- a/docs/source/dev/doc.rst +++ b/docs/source/dev/doc.rst @@ -4,12 +4,11 @@ Writing documentation This project uses `ReadTheDocs `_ to build and serve the documentation. If you want to run the generation of documentation (recommended!), you need -to have Python installed. Then install ``sphinx`` -$ pip install sphinx sphinx-autobuild +to have Python3 installed. -Assuming you have `Python `_ already, install `Sphinx `_:: +Assuming you have `Python3 `_ already, install `Sphinx `_:: - $ pip install sphinx sphinx-autobuild + $ pip install sphinx sphinx-autobuild sphinx_rtd_theme recommonmark Go to the ``docs`` directory and build the html documentation:: diff --git a/docs/source/index.rst b/docs/source/index.rst index 7ae8598fc..c32d9c9d7 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -15,7 +15,7 @@ This crawler helps to index binary documents such as PDF, Open Office, MS Office **Main features**: * Local file system (or a mounted drive) crawling and index new files, update existing ones and removes old ones. -* Remote file system over SSH crawling. +* Remote file system over SSH/FTP/SMB(WIP) crawling. * REST interface to let you "upload" your binary documents to elasticsearch. .. note:: @@ -56,6 +56,7 @@ This crawler helps to index binary documents such as PDF, Open Office, MS Office admin/fs/simple admin/fs/local-fs admin/fs/ssh + admin/fs/ftp admin/fs/elasticsearch admin/fs/wpsearch admin/fs/rest diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 4383da446..dc8e7e9e4 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -171,7 +171,7 @@ Then, you can run Elasticsearch. .. code:: sh - docker-compose up -d elasticsearch elasticsearch2 + docker-compose up -d elasticsearch docker-compose logs -f elasticsearch Wait for elasticsearch to be started: diff --git a/docs/source/user/options.rst b/docs/source/user/options.rst index aebc6d9f8..c63ea50ee 100644 --- a/docs/source/user/options.rst +++ b/docs/source/user/options.rst @@ -40,5 +40,6 @@ You will find more information about settings in the following sections: - :ref:`cli-options` - :ref:`local-fs-settings` - :ref:`ssh-settings` +- :ref:`ftp-settings` - :ref:`elasticsearch-settings` diff --git a/framework/pom.xml b/framework/pom.xml index 176472c21..bddd6a6f1 100644 --- a/framework/pom.xml +++ b/framework/pom.xml @@ -39,6 +39,17 @@ commons-io + + + commons-net + commons-net + + + org.mockftpserver + MockFtpServer + test + + com.fasterxml.jackson.core diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java index 1f74a541d..b63b03511 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; +import org.apache.commons.net.ftp.FTPFile; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -282,15 +283,38 @@ public static boolean isIndexable(String content, List filters) { return true; } + public static String computeRealPathName(String dirname, String filename) { + if (dirname != null) { + dirname = dirname.replace("\\", "/"); + } + + String fullFilename = new File(dirname, filename).toString(); + // fix windows share folder path: "/server/dir" -> "//server/dir" + if (dirname != null && dirname.startsWith("//") && !fullFilename.startsWith("//")) { + fullFilename = "/".concat(fullFilename); + } + return fullFilename.startsWith("/") ? fullFilename : "/".concat(fullFilename); + } + public static String computeVirtualPathName(String rootPath, String realPath) { + if (rootPath != null) { + rootPath = rootPath.replace("\\", "/"); + } + if (realPath != null) { + realPath = realPath.replace("\\", "/"); + } + String result = "/"; - if (realPath != null && realPath.length() > rootPath.length()) { - result = realPath.substring(rootPath.length()) - .replace("\\", "/"); + if (realPath != null && rootPath != null && realPath.length() > rootPath.length()) { + if (rootPath.equals("/")) { + result = realPath; + } else { + result = realPath.substring(rootPath.length()); + } } logger.debug("computeVirtualPathName({}, {}) = {}", rootPath, realPath, result); - return result; + return result.startsWith("/") ? result : "/".concat(result); } public static LocalDateTime getCreationTime(File file) { @@ -415,7 +439,32 @@ public static int getFilePermissions(final File file) { return user * 100 + group * 10 + others; } catch(Exception e) { - logger.warn("Failed to determine 'owner' of {}: {}", file, e.getMessage()); + logger.warn("Failed to determine 'permissions' of {}: {}", file, e.getMessage()); + return -1; + } + } + + /** + * Determines FTPFile permissions. + */ + public static int getFilePermissions(final FTPFile file) { + try { + int user = toOctalPermission( + file.hasPermission(FTPFile.USER_ACCESS, FTPFile.READ_PERMISSION), + file.hasPermission(FTPFile.USER_ACCESS, FTPFile.WRITE_PERMISSION), + file.hasPermission(FTPFile.USER_ACCESS, FTPFile.EXECUTE_PERMISSION)); + int group = toOctalPermission( + file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.READ_PERMISSION), + file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.WRITE_PERMISSION), + file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.EXECUTE_PERMISSION)); + int others = toOctalPermission( + file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.READ_PERMISSION), + file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.WRITE_PERMISSION), + file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.EXECUTE_PERMISSION)); + + return user * 100 + group * 10 + others; + } catch (Exception e) { + logger.warn("Failed to determine 'permissions' of {}: {}", file, e.getMessage()); return -1; } } diff --git a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java index d420837b5..ec6740d37 100644 --- a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java +++ b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java @@ -20,7 +20,16 @@ package fr.pilato.elasticsearch.crawler.fs.framework; import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase; +import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.List; +import java.util.TimeZone; +import java.util.stream.Collectors; +import org.apache.commons.net.ftp.FTPClient; +import org.apache.commons.net.ftp.FTPFile; +import org.apache.commons.net.ftp.FTPReply; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import java.io.File; @@ -31,14 +40,25 @@ import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; import java.util.Set; +import org.mockftpserver.fake.FakeFtpServer; +import org.mockftpserver.fake.UserAccount; +import org.mockftpserver.fake.filesystem.DirectoryEntry; +import org.mockftpserver.fake.filesystem.FileEntry; +import org.mockftpserver.fake.filesystem.FileSystem; +import org.mockftpserver.fake.filesystem.Permissions; +import org.mockftpserver.fake.filesystem.UnixFakeFileSystem; import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.computeRealPathName; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.computeVirtualPathName; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.extractMajorVersion; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.extractMinorVersion; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getFileExtension; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getFilePermissions; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getGroupName; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getOwnerName; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.isFileSizeUnderLimit; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.localDateTimeToDate; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.isEmptyOrNullString; @@ -78,6 +98,49 @@ public void testPermissions() { assertThat(permissions, is(700)); } + @Test + public void testFTPFilePermissions() throws IOException { + String user = "user"; + String password = "password"; + FakeFtpServer fakeFtpServer = new FakeFtpServer(); + fakeFtpServer.setServerControlPort(5968); + fakeFtpServer.addUserAccount(new UserAccount(user, password, "/data")); + FileSystem fileSystem = new UnixFakeFileSystem(); + fileSystem.add(new DirectoryEntry("/data")); + FileEntry fileAllPermissions = new FileEntry("/data/all.txt", "123"); + fileAllPermissions.setPermissions(Permissions.ALL); + fileSystem.add(fileAllPermissions); + FileEntry fileNonePermissions = new FileEntry("/data/none.txt", "456"); + fileNonePermissions.setPermissions(Permissions.NONE); + fileSystem.add(fileNonePermissions); + fakeFtpServer.setFileSystem(fileSystem); + fakeFtpServer.start(); + + FTPClient ftp = new FTPClient(); + ftp.connect("localhost", fakeFtpServer.getServerControlPort()); + int reply = ftp.getReplyCode(); + if (!FTPReply.isPositiveCompletion(reply)) { + ftp.disconnect(); + throw new IOException("Exception in connecting to FTP Server"); + } + ftp.login(user, password); + + FTPFile[] files = ftp.listFiles("/data"); + List filenames = Arrays.stream(files).map(FTPFile::getName).collect(Collectors.toList()); + assertThat(filenames.contains("all.txt"), is(true)); + assertThat(filenames.contains("none.txt"), is(true)); + for (FTPFile file : files) { + if (file.getName().equals("all.txt")) { + assertThat(getFilePermissions(file), is(777)); + } else if (file.getName().equals("none.txt")) { + assertThat(getFilePermissions(file), is(0)); + } + } + + ftp.disconnect(); + fakeFtpServer.stop(); + } + @Test public void testIsFileSizeUnderLimit() { assertThat(isFileSizeUnderLimit(ByteSizeValue.parseBytesSizeValue("1mb"), 1), is(true)); @@ -97,4 +160,92 @@ public void testExtractMinorVersion() { assertThat(extractMinorVersion("7.2.0"), is("2")); assertThat(extractMinorVersion("10.1.0"), is("1")); } + + @Test + public void testGetRealPathName() { + testRealPath("/", "test-linux.txt", "/test-linux.txt"); + testRealPath("/dir", "test-linux.txt", "/dir/test-linux.txt"); + testRealPath("/dir/", "test-linux.txt", "/dir/test-linux.txt"); + + testRealPath("C:", "test-windows.txt", "/C:/test-windows.txt"); + testRealPath("C:\\", "test-windows.txt", "/C:/test-windows.txt"); + + testRealPath("/C:", "test-windows.txt", "/C:/test-windows.txt"); + testRealPath("/C:/", "test-windows.txt", "/C:/test-windows.txt"); + + testRealPath("//SOMEONE/share", "test-smb.txt", "//SOMEONE/share/test-smb.txt"); + testRealPath("//SOMEONE/share/", "test-smb.txt", "//SOMEONE/share/test-smb.txt"); + } + + private void testRealPath(String dirname, String filename, String expectedPath) { + assertThat(computeRealPathName(dirname, filename), is(expectedPath)); + } + + @Test + public void testComputePathLinux() { + testVirtualPath("/", "/", "/"); + testVirtualPath("/", "/dir", "/dir"); + testVirtualPath("/", "/dir/subdir", "/dir/subdir"); + testVirtualPath("/", "/file.txt", "/file.txt"); + testVirtualPath("/", "/dir/file.txt", "/dir/file.txt"); + testVirtualPath("/", "/dir/subdir/file.txt", "/dir/subdir/file.txt"); + + testVirtualPath("/tmp", "/tmp", "/"); + testVirtualPath("/tmp", "/tmp/dir", "/dir"); + testVirtualPath("/tmp", "/tmp/dir/subdir", "/dir/subdir"); + testVirtualPath("/tmp", "/tmp/file.txt", "/file.txt"); + testVirtualPath("/tmp", "/tmp/dir/file.txt", "/dir/file.txt"); + testVirtualPath("/tmp", "/tmp/dir/subdir/file.txt", "/dir/subdir/file.txt"); + } + + @Test + public void testComputePathWindows() { + testVirtualPath("C:", "C:", "/"); + testVirtualPath("C:", "C:\\dir", "/dir"); + testVirtualPath("C:", "C:\\dir\\subdir", "/dir/subdir"); + testVirtualPath("C:", "C:\\file.txt", "/file.txt"); + testVirtualPath("C:", "C:\\dir\\file.txt", "/dir/file.txt"); + testVirtualPath("C:", "C:\\dir\\subdir\\file.txt", "/dir/subdir/file.txt"); + + testVirtualPath("C:\\tmp", "C:\\tmp", "/"); + testVirtualPath("C:\\tmp", "C:\\tmp\\dir", "/dir"); + testVirtualPath("C:\\tmp", "C:\\tmp\\dir\\subdir", "/dir/subdir"); + testVirtualPath("C:\\tmp", "C:\\tmp\\file.txt", "/file.txt"); + testVirtualPath("C:\\tmp", "C:\\tmp\\dir\\file.txt", "/dir/file.txt"); + testVirtualPath("C:\\tmp", "C:\\tmp\\dir\\subdir\\file.txt", "/dir/subdir/file.txt"); + + testVirtualPath("/C:/tmp", "/C:/tmp", "/"); + testVirtualPath("/C:/tmp", "/C:/tmp/dir", "/dir"); + testVirtualPath("/C:/tmp", "/C:/tmp/dir/subdir", "/dir/subdir"); + testVirtualPath("/C:/tmp", "/C:/tmp/file.txt", "/file.txt"); + testVirtualPath("/C:/tmp", "/C:/tmp/dir/file.txt", "/dir/file.txt"); + testVirtualPath("/C:/tmp", "/C:/tmp/dir/subdir/file.txt", "/dir/subdir/file.txt"); + } + + @Test + public void testComputePathSmb() { + testVirtualPath("//SOMEONE/share", "//SOMEONE/share", "/"); + testVirtualPath("//SOMEONE/share", "//SOMEONE/share/dir", "/dir"); + testVirtualPath("//SOMEONE/share", "//SOMEONE/share/dir/subdir", "/dir/subdir"); + testVirtualPath("//SOMEONE/share", "//SOMEONE/share/file.txt", "/file.txt"); + testVirtualPath("//SOMEONE/share", "//SOMEONE/share/dir/file.txt", "/dir/file.txt"); + testVirtualPath("//SOMEONE/share", "//SOMEONE/share/dir/subdir/file.txt", "/dir/subdir/file.txt"); + } + + private void testVirtualPath(String rootPath, String realPath, String expectedPath) { + assertThat(computeVirtualPathName(rootPath, realPath), is(expectedPath)); + } + + @Test + public void testGetFileExtension() { + assertThat(getFileExtension(new File("foo.bar")), is("bar")); + assertThat(getFileExtension(new File("foo")), is("")); + assertThat(getFileExtension(new File("foo.bar.baz")), is("baz")); + } + + @Test + public void testLocalDateToDate() { + LocalDateTime now = LocalDateTime.now(); + logger.info("Current Time [{}] in [{}] is actually [{}]", now, TimeZone.getDefault().getDisplayName(), localDateTimeToDate(now)); + } } diff --git a/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestFTPIT.java b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestFTPIT.java new file mode 100644 index 000000000..2f8eba6b8 --- /dev/null +++ b/integration-tests/it-common/src/main/java/fr/pilato/elasticsearch/crawler/fs/test/integration/elasticsearch/FsCrawlerTestFTPIT.java @@ -0,0 +1,81 @@ +/* + * Licensed to David Pilato (the "Author") under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. Author licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package fr.pilato.elasticsearch.crawler.fs.test.integration.elasticsearch; + +import fr.pilato.elasticsearch.crawler.fs.client.ESSearchRequest; +import fr.pilato.elasticsearch.crawler.fs.settings.Fs; +import fr.pilato.elasticsearch.crawler.fs.settings.Server; +import fr.pilato.elasticsearch.crawler.fs.settings.Server.PROTOCOL; +import fr.pilato.elasticsearch.crawler.fs.test.integration.AbstractFsCrawlerITCase; +import org.junit.Ignore; +import org.junit.Test; + +/** + * Test crawler with FTP + * TODO: test framework is broken? + */ +public class FsCrawlerTestFTPIT extends AbstractFsCrawlerITCase { + + /** + * You have to adapt this test to your own system + * So this test is disabled by default + */ + @Test @Ignore + public void test_ftp() throws Exception { + String hostname = "192.168.18.207"; + String username = "anonymous"; + String password = ""; + + Fs fs = startCrawlerDefinition().build(); + Server server = Server.builder() + .setHostname(hostname) + .setUsername(username) + .setPassword(password) + .setProtocol(Server.PROTOCOL.FTP) + .setPort(PROTOCOL.FTP_PORT) + .build(); + startCrawler(getCrawlerName(), fs, endCrawlerDefinition(getCrawlerName()), server); + + countTestHelper(new ESSearchRequest().withIndex(getCrawlerName()), 2L, null); + } + + /** + * You have to adapt this test to your own system + * So this test is disabled by default + */ + @Test @Ignore + public void test_ftp_with_user() throws Exception { + String hostname = "192.168.18.207"; + String username = "helsonxiao"; + String password = "123456"; + + Fs fs = startCrawlerDefinition().build(); + Server server = Server.builder() + .setHostname(hostname) + .setUsername(username) + .setPassword(password) + .setProtocol(Server.PROTOCOL.FTP) + .setPort(PROTOCOL.FTP_PORT) + .build(); + startCrawler(getCrawlerName(), fs, endCrawlerDefinition(getCrawlerName()), server); + + countTestHelper(new ESSearchRequest().withIndex(getCrawlerName()), 1L, null); + } +} diff --git a/pom.xml b/pom.xml index 8ff597c84..967435d9a 100644 --- a/pom.xml +++ b/pom.xml @@ -104,7 +104,9 @@ ${env.SONATYPE_PASS} + ${DOCKER_USERNAME} + ${DOCKER_PASSWORD} ${env.DOCKER_USERNAME} ${env.DOCKER_PASSWORD} @@ -551,6 +553,11 @@ fscrawler-crawler-fs 2.7-SNAPSHOT + + fr.pilato.elasticsearch.crawler + fscrawler-crawler-ftp + 2.7-SNAPSHOT + fr.pilato.elasticsearch.crawler fscrawler-crawler-ssh @@ -888,6 +895,12 @@ ${log4j.version} true + + org.apache.logging.log4j + log4j-iostreams + ${log4j.version} + true + org.fusesource.jansi jansi @@ -917,6 +930,25 @@ 2.11.0 + + + commons-net + commons-net + 3.8.0 + + + org.mockftpserver + MockFtpServer + 2.8.0 + test + + + org.slf4j + slf4j-api + + + + org.apache.httpcomponents diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java index 081e441e9..eb661b30a 100644 --- a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java @@ -62,19 +62,22 @@ public static boolean validateSettings(Logger logger, FsSettings settings, boole // Checking protocol if (settings.getServer() != null) { if (!Server.PROTOCOL.LOCAL.equals(settings.getServer().getProtocol()) && - !Server.PROTOCOL.SSH.equals(settings.getServer().getProtocol())) { + !Server.PROTOCOL.SSH.equals(settings.getServer().getProtocol()) && !Server.PROTOCOL.FTP.equals(settings.getServer().getProtocol())) { // Non supported protocol logger.error(settings.getServer().getProtocol() + " is not supported yet. Please use " + - Server.PROTOCOL.LOCAL + " or " + Server.PROTOCOL.SSH + ". Disabling crawler"); + Server.PROTOCOL.LOCAL + " or " + Server.PROTOCOL.SSH + " or " + Server.PROTOCOL.FTP + ". Disabling crawler"); return true; } // Checking username/password if (Server.PROTOCOL.SSH.equals(settings.getServer().getProtocol()) && FsCrawlerUtil.isNullOrEmpty(settings.getServer().getUsername())) { - // Non supported protocol logger.error("When using SSH, you need to set a username and probably a password or a pem file. Disabling crawler"); return true; + } else if (Server.PROTOCOL.FTP.equals(settings.getServer().getProtocol()) && + FsCrawlerUtil.isNullOrEmpty(settings.getServer().getUsername())) { + logger.error("When using FTP, you need to set a username and probably a password. Disabling crawler"); + return true; } } diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java index 270392e0e..e7580e7d9 100644 --- a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java @@ -29,7 +29,9 @@ public class Server { public static final class PROTOCOL { public static final String LOCAL = "local"; public static final String SSH = "ssh"; + public static final String FTP = "ftp"; public static final int SSH_PORT = 22; + public static final int FTP_PORT = 21; } public Server() { diff --git a/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidatorTest.java b/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidatorTest.java index b59164255..70214a59a 100644 --- a/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidatorTest.java +++ b/settings/src/test/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidatorTest.java @@ -66,6 +66,10 @@ public void testSettingsValidation() { settings = buildSettings(null, Server.builder().setProtocol(Server.PROTOCOL.SSH).build()); assertThat(FsCrawlerValidator.validateSettings(logger, settings, false), is(true)); + // Checking username / password when FTP + settings = buildSettings(null, Server.builder().setProtocol(Server.PROTOCOL.FTP).build()); + assertThat(FsCrawlerValidator.validateSettings(logger, settings, false), is(true)); + // Checking That we don't try to do both xml and json settings = buildSettings(Fs.builder().setJsonSupport(true).setXmlSupport(true).build(), null); assertThat(FsCrawlerValidator.validateSettings(logger, settings, false), is(true)); From 20217725c9b0850daf61571fc6658629e2c1d101 Mon Sep 17 00:00:00 2001 From: helsonxiao Date: Sun, 18 Jul 2021 23:25:57 +0800 Subject: [PATCH 02/26] fix: ftp file encoding --- .../fs/crawler/ftp/FileAbstractorFTP.java | 20 +++++++++---------- .../fs/crawler/ftp/FileAbstractorFTPTest.java | 12 ++++++----- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java b/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java index fe4f69243..155fd90f7 100644 --- a/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java +++ b/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java @@ -72,17 +72,15 @@ public FileAbstractModel toFileAbstractModel(String _path, FTPFile file) { // if server is not using utf-8 if (controlEncoding.equals(FTP.DEFAULT_CONTROL_ENCODING)) { - if (file.isFile()) { - try { - filename = new String(filename.getBytes(controlEncoding), StandardCharsets.UTF_8); - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - } - try { - path = new String(_path.getBytes(controlEncoding), StandardCharsets.UTF_8); - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - } + try { + filename = new String(filename.getBytes(controlEncoding), StandardCharsets.UTF_8); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + try { + path = new String(_path.getBytes(controlEncoding), StandardCharsets.UTF_8); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); } } diff --git a/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java b/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java index 474f9ee3a..0d4a32913 100644 --- a/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java +++ b/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java @@ -125,8 +125,8 @@ public void testConnectToFTPServer() throws Exception { .setServer( Server.builder() .setHostname("192.168.18.207") - .setUsername("username") - .setPassword("password") + .setUsername("helsonxiao") + .setPassword("123456") .setPort(21) .build() ) @@ -144,9 +144,11 @@ public void testConnectToFTPServer() throws Exception { Collection subDirFiles = ftp.getFiles(file.getFullpath()); logger.debug("Found {} files in sub dir", subDirFiles.size()); for (FileAbstractModel subDirFile : subDirFiles) { - try (InputStream inputStream = ftp.getInputStream(subDirFile)) { - String content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); - logger.debug("[sub dir] - {}: {}", subDirFile.getName(), content); + if (subDirFile.isFile()) { + try (InputStream inputStream = ftp.getInputStream(subDirFile)) { + String content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + logger.debug("[sub dir] - {}: {}", subDirFile.getName(), content); + } } } } else { From 042cfe431aa1a9d56fe3f8d09ce040d68eacb087 Mon Sep 17 00:00:00 2001 From: helsonxiao Date: Tue, 20 Jul 2021 23:49:25 +0800 Subject: [PATCH 03/26] doc: ftp support --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bbfd4b1e7..769be2e74 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This crawler helps to index binary documents such as PDF, Open Office, MS Office **Main features**: * Local file system (or a mounted drive) crawling and index new files, update existing ones and removes old ones. -* Remote file system over SSH crawling. +* Remote file system over SSH/FTP/SMB(WIP) crawling. * REST interface to let you "upload" your binary documents to elasticsearch. You need to install a version matching your Elasticsearch version: From 5696b6d5a5aefe799161a6f0e5faf376af93ff8d Mon Sep 17 00:00:00 2001 From: helsonxiao Date: Wed, 21 Jul 2021 23:05:37 +0800 Subject: [PATCH 04/26] fix: cli default server checking --- .../elasticsearch/crawler/fs/cli/FsCrawlerCli.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java index 899507a65..6cfcc7810 100644 --- a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java +++ b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java @@ -210,11 +210,13 @@ public static void main(String[] args) throws Exception { fsSettings.setFs(Fs.DEFAULT); } - if (fsSettings.getServer().getProtocol().equals(PROTOCOL.FTP) && fsSettings.getServer().getPort() == PROTOCOL.SSH_PORT) { - fsSettings.getServer().setPort(PROTOCOL.FTP_PORT); - } - if (fsSettings.getServer().getProtocol().equals(PROTOCOL.FTP) && StringUtils.isEmpty(fsSettings.getServer().getUsername())) { - fsSettings.getServer().setUsername("anonymous"); + if (fsSettings.getServer() != null) { + if (fsSettings.getServer().getProtocol().equals(PROTOCOL.FTP) && fsSettings.getServer().getPort() == PROTOCOL.SSH_PORT) { + fsSettings.getServer().setPort(PROTOCOL.FTP_PORT); + } + if (fsSettings.getServer().getProtocol().equals(PROTOCOL.FTP) && StringUtils.isEmpty(fsSettings.getServer().getUsername())) { + fsSettings.getServer().setUsername("anonymous"); + } } if (fsSettings.getElasticsearch() == null) { From 39bdb7f43ba6ae4137d12a9c556a74b59d399417 Mon Sep 17 00:00:00 2001 From: helson Date: Fri, 23 Jul 2021 16:11:02 +0800 Subject: [PATCH 05/26] fix: separator --- .../crawler/fs/FsParserAbstract.java | 25 ++--- .../fs/crawler/fs/FileAbstractorFile.java | 2 +- .../crawler/fs/framework/FsCrawlerUtil.java | 36 ++++---- .../fs/framework/FsCrawlerUtilTest.java | 92 +++++++++++++------ 4 files changed, 94 insertions(+), 61 deletions(-) diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java index dbf297d2e..d73f4f428 100644 --- a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java @@ -30,12 +30,13 @@ import fr.pilato.elasticsearch.crawler.fs.framework.ByteSizeValue; import fr.pilato.elasticsearch.crawler.fs.framework.FSCrawlerLogger; import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil; -import fr.pilato.elasticsearch.crawler.fs.framework.OsValidator; import fr.pilato.elasticsearch.crawler.fs.framework.SignTool; import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementService; import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerService; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import fr.pilato.elasticsearch.crawler.fs.settings.Server; +import fr.pilato.elasticsearch.crawler.fs.settings.Server.PROTOCOL; import fr.pilato.elasticsearch.crawler.fs.tika.TikaDocParser; import fr.pilato.elasticsearch.crawler.fs.tika.XmlDocParser; import org.apache.logging.log4j.LogManager; @@ -97,13 +98,7 @@ public abstract class FsParserAbstract extends FsParser { messageDigest = null; } - // On Windows, when using server, we need to force the "Linux" separator - if (OsValidator.WINDOWS && fsSettings.getServer() != null) { - logger.debug("We are running on Windows with Server settings so we need to force the Linux separator."); - pathSeparator = "/"; - } else { - pathSeparator = File.separator; - } + pathSeparator = FsCrawlerUtil.getPathSeparator(fsSettings.getFs().getUrl()); } protected abstract FileAbstractor buildFileAbstractor(); @@ -393,8 +388,11 @@ private void indexFile(FileAbstractModel fileAbstractModel, ScanStatistic stats, doc.getFile().setLastModified(localDateTimeToDate(lastModified)); doc.getFile().setLastAccessed(localDateTimeToDate(lastAccessed)); doc.getFile().setIndexingDate(localDateTimeToDate(LocalDateTime.now())); - // TODO: how about just set for local fs? - doc.getFile().setUrl("file://" + fullFilename); + if (fsSettings.getServer() == null) { + doc.getFile().setUrl("file://" + fullFilename); + } else if (fsSettings.getServer().getProtocol().equals(PROTOCOL.FTP)) { + doc.getFile().setUrl(String.format("ftp://%s:%d%s", fsSettings.getServer().getHostname(), fsSettings.getServer().getPort(), fullFilename)); + } doc.getFile().setExtension(extension); if (fsSettings.getFs().isAddFilesize()) { doc.getFile().setFilesize(size); @@ -493,8 +491,11 @@ private void indexFile(FileAbstractModel fileAbstractModel, ScanStatistic stats, } } - private String generateIdFromFilename(String filename, String filepath) throws NoSuchAlgorithmException { - return fsSettings.getFs().isFilenameAsId() ? filename : SignTool.sign((new File(filepath, filename)).toString()); + private String generateIdFromFilename(String _filename, String _filepath) throws NoSuchAlgorithmException { + String filepathForId = _filepath.replace("\\", "/"); + String filename = _filename.replace("\\", "").replace("/", ""); + String fullFilename = filepathForId.endsWith("/") ? filepathForId.concat(filename) : filepathForId.concat("/").concat(filename); + return fsSettings.getFs().isFilenameAsId() ? filename : SignTool.sign(fullFilename); } private String read(InputStream input) throws IOException { diff --git a/crawler/crawler-fs/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/fs/FileAbstractorFile.java b/crawler/crawler-fs/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/fs/FileAbstractorFile.java index 4453f6a84..3e4dd6f43 100644 --- a/crawler/crawler-fs/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/fs/FileAbstractorFile.java +++ b/crawler/crawler-fs/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/fs/FileAbstractorFile.java @@ -67,7 +67,7 @@ public FileAbstractModel toFileAbstractModel(String path, File file) { @Override public InputStream getInputStream(FileAbstractModel file) throws Exception { - return new FileInputStream(new File(file.getFullpath())); + return new FileInputStream(file.getFullpath()); } @Override diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java index b63b03511..65b9b15cc 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java @@ -283,30 +283,30 @@ public static boolean isIndexable(String content, List filters) { return true; } - public static String computeRealPathName(String dirname, String filename) { - if (dirname != null) { - dirname = dirname.replace("\\", "/"); + public static String getPathSeparator(String path) { + if (path.contains("/") && !path.contains("\\")) { + return "/"; } - String fullFilename = new File(dirname, filename).toString(); - // fix windows share folder path: "/server/dir" -> "//server/dir" - if (dirname != null && dirname.startsWith("//") && !fullFilename.startsWith("//")) { - fullFilename = "/".concat(fullFilename); + if (!path.contains("/") && (path.contains("\\") || path.contains(":"))) { + return "\\"; } - return fullFilename.startsWith("/") ? fullFilename : "/".concat(fullFilename); + + return File.separator; } - public static String computeVirtualPathName(String rootPath, String realPath) { - if (rootPath != null) { - rootPath = rootPath.replace("\\", "/"); - } - if (realPath != null) { - realPath = realPath.replace("\\", "/"); - } + public static String computeRealPathName(String _dirname, String filename) { + // new File(dirname, filename).toString() is not suitable for server + String separator = getPathSeparator(_dirname); + String dirname = _dirname.endsWith(separator) ? _dirname : _dirname.concat(separator); + return dirname + filename; + } - String result = "/"; - if (realPath != null && rootPath != null && realPath.length() > rootPath.length()) { + public static String computeVirtualPathName(String rootPath, String realPath) { + String result = getPathSeparator(rootPath); + if (realPath.startsWith(rootPath) && realPath.length() > rootPath.length()) { if (rootPath.equals("/")) { + // "/" is very common for FTP result = realPath; } else { result = realPath.substring(rootPath.length()); @@ -314,7 +314,7 @@ public static String computeVirtualPathName(String rootPath, String realPath) { } logger.debug("computeVirtualPathName({}, {}) = {}", rootPath, realPath, result); - return result.startsWith("/") ? result : "/".concat(result); + return result; } public static LocalDateTime getCreationTime(File file) { diff --git a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java index ec6740d37..eaf1295f4 100644 --- a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java +++ b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java @@ -29,7 +29,6 @@ import org.apache.commons.net.ftp.FTPFile; import org.apache.commons.net.ftp.FTPReply; import org.junit.BeforeClass; -import org.junit.Ignore; import org.junit.Test; import java.io.File; @@ -162,17 +161,37 @@ public void testExtractMinorVersion() { } @Test - public void testGetRealPathName() { + public void testGetRealPathNameWindows() { + testRealPath("/C:", "test-windows.txt", "/C:/test-windows.txt"); + testRealPath("/C:/", "test-windows.txt", "/C:/test-windows.txt"); + testRealPath("/C:/dir", "test-windows.txt", "/C:/dir/test-windows.txt"); + testRealPath("/C:/dir/", "test-windows.txt", "/C:/dir/test-windows.txt"); + + testRealPath("C:/", "test-windows.txt", "C:/test-windows.txt"); + testRealPath("C:/dir", "test-windows.txt", "C:/dir/test-windows.txt"); + testRealPath("C:/dir/", "test-windows.txt", "C:/dir/test-windows.txt"); + + testRealPath("C:", "test-windows.txt", "C:\\test-windows.txt"); + testRealPath("C:\\", "test-windows.txt", "C:\\test-windows.txt"); + testRealPath("C:\\dir", "test-windows.txt", "C:\\dir\\test-windows.txt"); + testRealPath("C:\\dir\\", "test-windows.txt", "C:\\dir\\test-windows.txt"); + + testRealPath("\\\\SOMEONE", "test-smb.txt", "\\\\SOMEONE\\test-smb.txt"); + testRealPath("\\\\SOMEONE\\", "test-smb.txt", "\\\\SOMEONE\\test-smb.txt"); + testRealPath("\\\\SOMEONE\\share", "test-smb.txt", "\\\\SOMEONE\\share\\test-smb.txt"); + testRealPath("\\\\SOMEONE\\share\\", "test-smb.txt", "\\\\SOMEONE\\share\\test-smb.txt"); + } + + @Test + public void testGetRealPathNameLinux() { + // Local Linux / FTP testRealPath("/", "test-linux.txt", "/test-linux.txt"); testRealPath("/dir", "test-linux.txt", "/dir/test-linux.txt"); testRealPath("/dir/", "test-linux.txt", "/dir/test-linux.txt"); - testRealPath("C:", "test-windows.txt", "/C:/test-windows.txt"); - testRealPath("C:\\", "test-windows.txt", "/C:/test-windows.txt"); - - testRealPath("/C:", "test-windows.txt", "/C:/test-windows.txt"); - testRealPath("/C:/", "test-windows.txt", "/C:/test-windows.txt"); - + // SMB + testRealPath("//SOMEONE", "test-smb.txt", "//SOMEONE/test-smb.txt"); + testRealPath("//SOMEONE/", "test-smb.txt", "//SOMEONE/test-smb.txt"); testRealPath("//SOMEONE/share", "test-smb.txt", "//SOMEONE/share/test-smb.txt"); testRealPath("//SOMEONE/share/", "test-smb.txt", "//SOMEONE/share/test-smb.txt"); } @@ -183,6 +202,7 @@ private void testRealPath(String dirname, String filename, String expectedPath) @Test public void testComputePathLinux() { + // Local Linux / FTP testVirtualPath("/", "/", "/"); testVirtualPath("/", "/dir", "/dir"); testVirtualPath("/", "/dir/subdir", "/dir/subdir"); @@ -196,23 +216,38 @@ public void testComputePathLinux() { testVirtualPath("/tmp", "/tmp/file.txt", "/file.txt"); testVirtualPath("/tmp", "/tmp/dir/file.txt", "/dir/file.txt"); testVirtualPath("/tmp", "/tmp/dir/subdir/file.txt", "/dir/subdir/file.txt"); + + // SMB + testVirtualPath("//SOMEONE/share", "//SOMEONE/share", "/"); + testVirtualPath("//SOMEONE/share", "//SOMEONE/share/dir", "/dir"); + testVirtualPath("//SOMEONE/share", "//SOMEONE/share/dir/subdir", "/dir/subdir"); + testVirtualPath("//SOMEONE/share", "//SOMEONE/share/file.txt", "/file.txt"); + testVirtualPath("//SOMEONE/share", "//SOMEONE/share/dir/file.txt", "/dir/file.txt"); + testVirtualPath("//SOMEONE/share", "//SOMEONE/share/dir/subdir/file.txt", "/dir/subdir/file.txt"); } @Test public void testComputePathWindows() { - testVirtualPath("C:", "C:", "/"); - testVirtualPath("C:", "C:\\dir", "/dir"); - testVirtualPath("C:", "C:\\dir\\subdir", "/dir/subdir"); - testVirtualPath("C:", "C:\\file.txt", "/file.txt"); - testVirtualPath("C:", "C:\\dir\\file.txt", "/dir/file.txt"); - testVirtualPath("C:", "C:\\dir\\subdir\\file.txt", "/dir/subdir/file.txt"); - - testVirtualPath("C:\\tmp", "C:\\tmp", "/"); - testVirtualPath("C:\\tmp", "C:\\tmp\\dir", "/dir"); - testVirtualPath("C:\\tmp", "C:\\tmp\\dir\\subdir", "/dir/subdir"); - testVirtualPath("C:\\tmp", "C:\\tmp\\file.txt", "/file.txt"); - testVirtualPath("C:\\tmp", "C:\\tmp\\dir\\file.txt", "/dir/file.txt"); - testVirtualPath("C:\\tmp", "C:\\tmp\\dir\\subdir\\file.txt", "/dir/subdir/file.txt"); + testVirtualPath("C:", "C:", "\\"); + testVirtualPath("C:", "C:\\dir", "\\dir"); + testVirtualPath("C:", "C:\\dir\\subdir", "\\dir\\subdir"); + testVirtualPath("C:", "C:\\file.txt", "\\file.txt"); + testVirtualPath("C:", "C:\\dir\\file.txt", "\\dir\\file.txt"); + testVirtualPath("C:", "C:\\dir\\subdir\\file.txt", "\\dir\\subdir\\file.txt"); + + testVirtualPath("C:\\tmp", "C:\\tmp", "\\"); + testVirtualPath("C:\\tmp", "C:\\tmp\\dir", "\\dir"); + testVirtualPath("C:\\tmp", "C:\\tmp\\dir\\subdir", "\\dir\\subdir"); + testVirtualPath("C:\\tmp", "C:\\tmp\\file.txt", "\\file.txt"); + testVirtualPath("C:\\tmp", "C:\\tmp\\dir\\file.txt", "\\dir\\file.txt"); + testVirtualPath("C:\\tmp", "C:\\tmp\\dir\\subdir\\file.txt", "\\dir\\subdir\\file.txt"); + + testVirtualPath("C:/tmp", "C:/tmp", "/"); + testVirtualPath("C:/tmp", "C:/tmp/dir", "/dir"); + testVirtualPath("C:/tmp", "C:/tmp/dir/subdir", "/dir/subdir"); + testVirtualPath("C:/tmp", "C:/tmp/file.txt", "/file.txt"); + testVirtualPath("C:/tmp", "C:/tmp/dir/file.txt", "/dir/file.txt"); + testVirtualPath("C:/tmp", "C:/tmp/dir/subdir/file.txt", "/dir/subdir/file.txt"); testVirtualPath("/C:/tmp", "/C:/tmp", "/"); testVirtualPath("/C:/tmp", "/C:/tmp/dir", "/dir"); @@ -220,16 +255,13 @@ public void testComputePathWindows() { testVirtualPath("/C:/tmp", "/C:/tmp/file.txt", "/file.txt"); testVirtualPath("/C:/tmp", "/C:/tmp/dir/file.txt", "/dir/file.txt"); testVirtualPath("/C:/tmp", "/C:/tmp/dir/subdir/file.txt", "/dir/subdir/file.txt"); - } - @Test - public void testComputePathSmb() { - testVirtualPath("//SOMEONE/share", "//SOMEONE/share", "/"); - testVirtualPath("//SOMEONE/share", "//SOMEONE/share/dir", "/dir"); - testVirtualPath("//SOMEONE/share", "//SOMEONE/share/dir/subdir", "/dir/subdir"); - testVirtualPath("//SOMEONE/share", "//SOMEONE/share/file.txt", "/file.txt"); - testVirtualPath("//SOMEONE/share", "//SOMEONE/share/dir/file.txt", "/dir/file.txt"); - testVirtualPath("//SOMEONE/share", "//SOMEONE/share/dir/subdir/file.txt", "/dir/subdir/file.txt"); + testVirtualPath("\\\\SOMEONE\\share", "\\\\SOMEONE\\share", "\\"); + testVirtualPath("\\\\SOMEONE\\share", "\\\\SOMEONE\\share\\dir", "\\dir"); + testVirtualPath("\\\\SOMEONE\\share", "\\\\SOMEONE\\share\\dir\\subdir", "\\dir\\subdir"); + testVirtualPath("\\\\SOMEONE\\share", "\\\\SOMEONE\\share\\file.txt", "\\file.txt"); + testVirtualPath("\\\\SOMEONE\\share", "\\\\SOMEONE\\share\\dir\\file.txt", "\\dir\\file.txt"); + testVirtualPath("\\\\SOMEONE\\share", "\\\\SOMEONE\\share\\dir\\subdir\\file.txt", "\\dir\\subdir\\file.txt"); } private void testVirtualPath(String rootPath, String realPath, String expectedPath) { From d621f77fba5aba3742f63936991762782e464efe Mon Sep 17 00:00:00 2001 From: helson Date: Fri, 23 Jul 2021 18:49:50 +0800 Subject: [PATCH 06/26] fix: local fs separator on windows --- .../elasticsearch/crawler/fs/FsParserAbstract.java | 8 ++++++-- .../crawler/fs/crawler/fs/FileAbstractorFile.java | 13 +++++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java index d73f4f428..78ecc18a3 100644 --- a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java @@ -27,15 +27,16 @@ import fr.pilato.elasticsearch.crawler.fs.beans.ScanStatistic; import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractModel; import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor; +import fr.pilato.elasticsearch.crawler.fs.crawler.fs.FileAbstractorFile; import fr.pilato.elasticsearch.crawler.fs.framework.ByteSizeValue; import fr.pilato.elasticsearch.crawler.fs.framework.FSCrawlerLogger; import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil; +import fr.pilato.elasticsearch.crawler.fs.framework.OsValidator; import fr.pilato.elasticsearch.crawler.fs.framework.SignTool; import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementService; import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerService; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; -import fr.pilato.elasticsearch.crawler.fs.settings.Server; import fr.pilato.elasticsearch.crawler.fs.settings.Server.PROTOCOL; import fr.pilato.elasticsearch.crawler.fs.tika.TikaDocParser; import fr.pilato.elasticsearch.crawler.fs.tika.XmlDocParser; @@ -43,7 +44,6 @@ import org.apache.logging.log4j.Logger; import java.io.BufferedReader; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -99,6 +99,10 @@ public abstract class FsParserAbstract extends FsParser { } pathSeparator = FsCrawlerUtil.getPathSeparator(fsSettings.getFs().getUrl()); + if (OsValidator.WINDOWS && fsSettings.getServer() == null) { + logger.debug("We are running on Windows without Server settings so we use the separator in accordance with fs.url"); + FileAbstractorFile.separator = pathSeparator; + } } protected abstract FileAbstractor buildFileAbstractor(); diff --git a/crawler/crawler-fs/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/fs/FileAbstractorFile.java b/crawler/crawler-fs/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/fs/FileAbstractorFile.java index 3e4dd6f43..55b864ad8 100644 --- a/crawler/crawler-fs/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/fs/FileAbstractorFile.java +++ b/crawler/crawler-fs/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/fs/FileAbstractorFile.java @@ -48,6 +48,15 @@ public FileAbstractorFile(FsSettings fsSettings) { super(fsSettings); } + public static String separator = File.separator; + + private String resolveSeparator(String path) { + if (separator.equals("/")) { + return path.replace("\\", "/"); + } + return path.replace("/", "\\"); + } + @Override public FileAbstractModel toFileAbstractModel(String path, File file) { return new FileAbstractModel( @@ -57,8 +66,8 @@ public FileAbstractModel toFileAbstractModel(String path, File file) { getCreationTime(file), getLastAccessTime(file), getFileExtension(file), - path, - file.getAbsolutePath(), + resolveSeparator(path), + resolveSeparator(file.getAbsolutePath()), file.length(), getOwnerName(file), getGroupName(file), From 4030fe12823ec080c56fa66a5eeb4b7106f55996 Mon Sep 17 00:00:00 2001 From: helson Date: Fri, 23 Jul 2021 22:13:04 +0800 Subject: [PATCH 07/26] fix: ftp exists --- .../crawler/fs/crawler/ftp/FileAbstractorFTP.java | 10 +++++----- .../crawler/fs/crawler/ftp/FileAbstractorFTPTest.java | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java b/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java index 155fd90f7..e76e7cca6 100644 --- a/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java +++ b/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java @@ -75,12 +75,12 @@ public FileAbstractModel toFileAbstractModel(String _path, FTPFile file) { try { filename = new String(filename.getBytes(controlEncoding), StandardCharsets.UTF_8); } catch (UnsupportedEncodingException e) { - e.printStackTrace(); + logger.error("Error during filename encoding: {}", e.getMessage()); } try { path = new String(_path.getBytes(controlEncoding), StandardCharsets.UTF_8); } catch (UnsupportedEncodingException e) { - e.printStackTrace(); + logger.error("Error during path encoding: {}", e.getMessage()); } } @@ -149,9 +149,9 @@ public Collection getFiles(String dir) throws IOException { @Override public boolean exists(String dir) { try { - if (controlEncoding.equals(FTP.DEFAULT_CONTROL_ENCODING)) { - dir = new String(dir.getBytes(StandardCharsets.UTF_8), FTP.DEFAULT_CONTROL_ENCODING); - } + logger.debug("Checking dir existence: " + dir); + // changeWorkingDirectory don't know utf-8 + dir = new String(dir.getBytes(StandardCharsets.UTF_8), FTP.DEFAULT_CONTROL_ENCODING); return ftp.changeWorkingDirectory(dir); } catch (IOException e) { return false; diff --git a/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java b/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java index 0d4a32913..9ac82bbe3 100644 --- a/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java +++ b/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java @@ -120,8 +120,8 @@ public void testConnectToFakeFTPServer() throws Exception { @Test @Ignore public void testConnectToFTPServer() throws Exception { - String path = "/"; - FsSettings fsSettings = FsSettings.builder("local_test") + String path = "/中文目录"; + FsSettings fsSettings = FsSettings.builder("local_utf8_test") .setServer( Server.builder() .setHostname("192.168.18.207") From 0455ec4c9df61b0dc3047817384f0c263578e43e Mon Sep 17 00:00:00 2001 From: helson Date: Sun, 25 Jul 2021 13:12:11 +0800 Subject: [PATCH 08/26] fix: encoding --- .../fs/crawler/ftp/FileAbstractorFTP.java | 47 ++++++++++--------- .../fs/crawler/ftp/FileAbstractorFTPTest.java | 25 +++++----- 2 files changed, 38 insertions(+), 34 deletions(-) diff --git a/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java b/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java index e76e7cca6..43dfe10db 100644 --- a/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java +++ b/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java @@ -58,7 +58,9 @@ public class FileAbstractorFTP extends FileAbstractor { private final PrintCommandListener ftpListener = new PrintCommandListener(new PrintWriter(loggerOutputStream)); - private String controlEncoding = FTP.DEFAULT_CONTROL_ENCODING; + private boolean isUtf8 = false; + + private static final String ALTERNATIVE_ENCODING = "GBK"; public FileAbstractorFTP(FsSettings fsSettings) { super(fsSettings); @@ -70,18 +72,15 @@ public FileAbstractModel toFileAbstractModel(String _path, FTPFile file) { String extension = FilenameUtils.getExtension(filename); String path = _path; - // if server is not using utf-8 - if (controlEncoding.equals(FTP.DEFAULT_CONTROL_ENCODING)) { - try { - filename = new String(filename.getBytes(controlEncoding), StandardCharsets.UTF_8); - } catch (UnsupportedEncodingException e) { - logger.error("Error during filename encoding: {}", e.getMessage()); - } - try { - path = new String(_path.getBytes(controlEncoding), StandardCharsets.UTF_8); - } catch (UnsupportedEncodingException e) { - logger.error("Error during path encoding: {}", e.getMessage()); - } + String toEncoding = ALTERNATIVE_ENCODING; + if (isUtf8) { + toEncoding = StandardCharsets.UTF_8.displayName(); + } + try { + filename = new String(filename.getBytes(FTP.DEFAULT_CONTROL_ENCODING), toEncoding); + path = new String(_path.getBytes(FTP.DEFAULT_CONTROL_ENCODING), toEncoding); + } catch (UnsupportedEncodingException e) { + logger.error("Error during encoding: {}", e.getMessage()); } return new FileAbstractModel( @@ -109,8 +108,10 @@ public InputStream getInputStream(FileAbstractModel file) throws Exception { ftp.enterLocalPassiveMode(); String fullPath = file.getFullpath(); - if (controlEncoding.equals(FTP.DEFAULT_CONTROL_ENCODING)) { + if (isUtf8) { fullPath = new String(fullPath.getBytes(StandardCharsets.UTF_8), FTP.DEFAULT_CONTROL_ENCODING); + } else { + fullPath = new String(fullPath.getBytes(ALTERNATIVE_ENCODING), FTP.DEFAULT_CONTROL_ENCODING); } return ftp.retrieveFileStream(fullPath); } @@ -120,11 +121,12 @@ public Collection getFiles(String dir) throws IOException { // FTP data connection could be closed after transfer process. openFTPConnection(); - if (controlEncoding.equals(FTP.DEFAULT_CONTROL_ENCODING)) { + logger.debug("Listing local files from {}", dir); + if (isUtf8) { dir = new String(dir.getBytes(StandardCharsets.UTF_8), FTP.DEFAULT_CONTROL_ENCODING); + } else { + dir = new String(dir.getBytes(ALTERNATIVE_ENCODING), FTP.DEFAULT_CONTROL_ENCODING); } - logger.debug("Listing local files from {}", dir); - ftp.enterLocalPassiveMode(); FTPFile[] ftpFiles = ftp.listFiles(dir); if (ftpFiles == null) return null; @@ -150,8 +152,11 @@ public Collection getFiles(String dir) throws IOException { public boolean exists(String dir) { try { logger.debug("Checking dir existence: " + dir); - // changeWorkingDirectory don't know utf-8 - dir = new String(dir.getBytes(StandardCharsets.UTF_8), FTP.DEFAULT_CONTROL_ENCODING); + if (isUtf8) { + dir = new String(dir.getBytes(StandardCharsets.UTF_8), FTP.DEFAULT_CONTROL_ENCODING); + } else { + dir = new String(dir.getBytes(ALTERNATIVE_ENCODING), FTP.DEFAULT_CONTROL_ENCODING); + } return ftp.changeWorkingDirectory(dir); } catch (IOException e) { return false; @@ -196,9 +201,9 @@ private void openFTPConnection() throws IOException { int utf8Reply = ftp.sendCommand("OPTS UTF8", "ON"); if (FTPReply.isPositiveCompletion(utf8Reply)) { - controlEncoding = StandardCharsets.UTF_8.displayName(); - ftp.setControlEncoding(controlEncoding); + isUtf8 = true; } + ftp.setFileType(FTPClient.BINARY_FILE_TYPE); logger.debug("FTP connection successful"); } diff --git a/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java b/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java index 9ac82bbe3..e5c9cbaf7 100644 --- a/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java +++ b/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java @@ -31,7 +31,6 @@ import java.util.Collection; import org.apache.commons.io.IOUtils; -import org.apache.commons.net.ftp.FTPClient; import org.junit.After; import org.junit.Before; import org.junit.Ignore; @@ -45,25 +44,25 @@ public class FileAbstractorFTPTest extends AbstractFSCrawlerTestCase { private FakeFtpServer fakeFtpServer; - private final String path = "/data"; + private final String home = "/home"; private final String user = "user"; private final String pass = "password"; @Before public void setup() { - // it doesn't seem to support utf-8 + // it doesn't support utf-8 fakeFtpServer = new FakeFtpServer(); fakeFtpServer.setServerControlPort(5968); - fakeFtpServer.addUserAccount(new UserAccount(user, pass, path)); + fakeFtpServer.addUserAccount(new UserAccount(user, pass, home)); FileSystem fileSystem = new UnixFakeFileSystem(); - fileSystem.add(new DirectoryEntry("/data")); - fileSystem.add(new FileEntry("/data/foo.txt", "foo")); - fileSystem.add(new FileEntry("/data/bar.txt", "bar")); + fileSystem.add(new DirectoryEntry(home)); + fileSystem.add(new FileEntry(home + "/foo.txt", "文件名不支持中文")); + fileSystem.add(new FileEntry(home + "/bar.txt", "bar")); - fileSystem.add(new DirectoryEntry("/data/buzz")); - fileSystem.add(new FileEntry("/data/buzz/hello.txt", "hello")); - fileSystem.add(new FileEntry("/data/buzz/world.txt", "world")); + fileSystem.add(new DirectoryEntry(home + "/buzz")); + fileSystem.add(new FileEntry(home + "/buzz/hello.txt", "hello")); + fileSystem.add(new FileEntry(home + "/buzz/world.txt", "world")); fakeFtpServer.setFileSystem(fileSystem); fakeFtpServer.start(); @@ -90,9 +89,9 @@ public void testConnectToFakeFTPServer() throws Exception { FileAbstractorFTP ftp = new FileAbstractorFTP(fsSettings); ftp.open(); - boolean exists = ftp.exists(path); + boolean exists = ftp.exists(home); assertThat(exists, is(true)); - Collection files = ftp.getFiles(path); + Collection files = ftp.getFiles(home); assertThat(files.size(), is(3)); for (FileAbstractModel file : files) { @@ -109,7 +108,7 @@ public void testConnectToFakeFTPServer() throws Exception { } } else { try (InputStream inputStream = ftp.getInputStream(file)) { - String content = IOUtils.toString(inputStream, FTPClient.DEFAULT_CONTROL_ENCODING); + String content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); logger.debug(" - {}: {}", file.getName(), content); } } From 6e5e92af4f7bf6960a74d004faa39b2d1bb9b809 Mon Sep 17 00:00:00 2001 From: helson Date: Sun, 25 Jul 2021 20:59:09 +0800 Subject: [PATCH 09/26] fix: unnecessary connection --- .../fs/crawler/ftp/FileAbstractorFTP.java | 16 +++++----------- .../fs/crawler/ftp/FileAbstractorFTPTest.java | 4 ++-- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java b/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java index 43dfe10db..c211f30bf 100644 --- a/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java +++ b/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java @@ -67,10 +67,9 @@ public FileAbstractorFTP(FsSettings fsSettings) { } @Override - public FileAbstractModel toFileAbstractModel(String _path, FTPFile file) { + public FileAbstractModel toFileAbstractModel(String path, FTPFile file) { String filename = file.getName(); String extension = FilenameUtils.getExtension(filename); - String path = _path; String toEncoding = ALTERNATIVE_ENCODING; if (isUtf8) { @@ -78,7 +77,7 @@ public FileAbstractModel toFileAbstractModel(String _path, FTPFile file) { } try { filename = new String(filename.getBytes(FTP.DEFAULT_CONTROL_ENCODING), toEncoding); - path = new String(_path.getBytes(FTP.DEFAULT_CONTROL_ENCODING), toEncoding); + path = new String(path.getBytes(FTP.DEFAULT_CONTROL_ENCODING), toEncoding); } catch (UnsupportedEncodingException e) { logger.error("Error during encoding: {}", e.getMessage()); } @@ -103,24 +102,19 @@ public FileAbstractModel toFileAbstractModel(String _path, FTPFile file) { @Override public InputStream getInputStream(FileAbstractModel file) throws Exception { - // FTP data connection could be closed after transfer process. - openFTPConnection(); - - ftp.enterLocalPassiveMode(); String fullPath = file.getFullpath(); if (isUtf8) { fullPath = new String(fullPath.getBytes(StandardCharsets.UTF_8), FTP.DEFAULT_CONTROL_ENCODING); } else { fullPath = new String(fullPath.getBytes(ALTERNATIVE_ENCODING), FTP.DEFAULT_CONTROL_ENCODING); } - return ftp.retrieveFileStream(fullPath); + InputStream inputStream = ftp.retrieveFileStream(fullPath); + ftp.completePendingCommand(); + return inputStream; } @Override public Collection getFiles(String dir) throws IOException { - // FTP data connection could be closed after transfer process. - openFTPConnection(); - logger.debug("Listing local files from {}", dir); if (isUtf8) { dir = new String(dir.getBytes(StandardCharsets.UTF_8), FTP.DEFAULT_CONTROL_ENCODING); diff --git a/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java b/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java index e5c9cbaf7..21dfff30c 100644 --- a/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java +++ b/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java @@ -103,7 +103,7 @@ public void testConnectToFakeFTPServer() throws Exception { for (FileAbstractModel subDirFile : subDirFiles) { try (InputStream inputStream = ftp.getInputStream(subDirFile)) { String content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); - logger.debug("[sub dir] - {}: {}", subDirFile.getName(), content); + logger.debug("[{}] - {}: {}", file.getName(), subDirFile.getName(), content); } } } else { @@ -146,7 +146,7 @@ public void testConnectToFTPServer() throws Exception { if (subDirFile.isFile()) { try (InputStream inputStream = ftp.getInputStream(subDirFile)) { String content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); - logger.debug("[sub dir] - {}: {}", subDirFile.getName(), content); + logger.debug("[{}] - {}: {}", file.getName(), subDirFile.getName(), content); } } } From 4b563ea7cedf3865219b8ba45083358a2eedf32f Mon Sep 17 00:00:00 2001 From: farmer <1153595464@qq.com> Date: Sat, 17 Jul 2021 09:04:55 +0800 Subject: [PATCH 10/26] feat(SMB): FileAbstractorSMB,FsParserSmb --- .../crawler/fs/FsCrawlerImpl.java | 16 +- .../elasticsearch/crawler/fs/FsParserSmb.java | 21 +++ crawler/crawler-smb/pom.xml | 32 ++++ .../fs/crawler/smb/FileAbstractorSMB.java | 139 ++++++++++++++++++ .../fs/crawler/smb/FileAbstractorSMBTest.java | 45 ++++++ pom.xml | 1 + .../crawler/fs/settings/Server.java | 25 +++- 7 files changed, 267 insertions(+), 12 deletions(-) create mode 100644 core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserSmb.java create mode 100644 crawler/crawler-smb/pom.xml create mode 100644 crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java create mode 100644 crawler/crawler-smb/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMBTest.java diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java index 32da3dc5c..8151120ab 100644 --- a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java @@ -21,20 +21,15 @@ import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil; import fr.pilato.elasticsearch.crawler.fs.framework.TimeValue; -import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; -import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentServiceElasticsearchImpl; -import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentServiceWorkplaceSearchImpl; -import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementService; -import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementServiceElasticsearchImpl; +import fr.pilato.elasticsearch.crawler.fs.service.*; import fr.pilato.elasticsearch.crawler.fs.settings.FsCrawlerValidator; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; import fr.pilato.elasticsearch.crawler.fs.settings.Server; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; /** * @author dadoonet (David Pilato) @@ -123,6 +118,9 @@ public void start() throws Exception { if (settings.getServer() == null || Server.PROTOCOL.LOCAL.equals(settings.getServer().getProtocol())) { // Local FS fsParser = new FsParserLocal(settings, config, managementService, documentService, loop); + } else if (Server.PROTOCOL.SMB.equals(settings.getServer().getProtocol())) { + // Remote SMB FS + fsParser = new FsParserSmb(settings, config, managementService, documentService, loop); } else if (Server.PROTOCOL.SSH.equals(settings.getServer().getProtocol())) { // Remote SSH FS fsParser = new FsParserSsh(settings, config, managementService, documentService, loop); @@ -149,7 +147,7 @@ public void close() throws InterruptedException, IOException { if (fsParser != null) { fsParser.close(); - synchronized(fsParser.getSemaphore()) { + synchronized (fsParser.getSemaphore()) { fsParser.getSemaphore().notifyAll(); } } diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserSmb.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserSmb.java new file mode 100644 index 000000000..fe9bc3cd5 --- /dev/null +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserSmb.java @@ -0,0 +1,21 @@ +package fr.pilato.elasticsearch.crawler.fs; + +import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor; +import fr.pilato.elasticsearch.crawler.fs.crawler.ssh.FileAbstractorSSH; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementService; +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import java.nio.file.Path; + +public class FsParserSmb extends FsParserAbstract{ + + public FsParserSmb(FsSettings fsSettings, Path config, FsCrawlerManagementService managementService, + FsCrawlerDocumentService documentService, Integer loop){ + super(fsSettings, config, managementService, documentService, loop); + } + + @Override + protected FileAbstractor buildFileAbstractor() { + return new FileAbstractorSSH(fsSettings); + } +} diff --git a/crawler/crawler-smb/pom.xml b/crawler/crawler-smb/pom.xml new file mode 100644 index 000000000..28ce9210c --- /dev/null +++ b/crawler/crawler-smb/pom.xml @@ -0,0 +1,32 @@ + + + + fscrawler-crawler + fr.pilato.elasticsearch.crawler + 2.7-SNAPSHOT + + 4.0.0 + + + fscrawler-crawler-smb + FSCrawler Crawlers: SMB + + + + + fr.pilato.elasticsearch.crawler + fscrawler-crawler-abstract + + + + + com.hierynomus + smbj + 0.11.1 + + + + + \ No newline at end of file diff --git a/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java b/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java new file mode 100644 index 000000000..ac838d1f0 --- /dev/null +++ b/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java @@ -0,0 +1,139 @@ +package fr.pilato.elasticsearch.crawler.fs.crawler.smb; + +import com.hierynomus.msdtyp.AccessMask; +import com.hierynomus.msdtyp.SecurityInformation; +import com.hierynomus.msfscc.fileinformation.FileIdBothDirectoryInformation; +import com.hierynomus.mssmb2.SMB2CreateDisposition; +import com.hierynomus.mssmb2.SMB2ShareAccess; +import com.hierynomus.security.bc.BCSecurityProvider; +import com.hierynomus.smbj.SMBClient; +import com.hierynomus.smbj.SmbConfig; +import com.hierynomus.smbj.auth.AuthenticationContext; +import com.hierynomus.smbj.connection.Connection; +import com.hierynomus.smbj.session.Session; +import com.hierynomus.smbj.share.Directory; +import com.hierynomus.smbj.share.DiskEntry; +import com.hierynomus.smbj.share.DiskShare; +import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractModel; +import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor; +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import fr.pilato.elasticsearch.crawler.fs.settings.Server; +import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.*; +import java.util.stream.Collectors; +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class FileAbstractorSMB extends FileAbstractor { + + + private final Logger logger = LogManager.getLogger(FileAbstractorSMB.class); + + public FileAbstractorSMB(FsSettings fsSettings) { + super(fsSettings); + } + + private DiskShare share; + + private SMBClient client; + + @Override + public FileAbstractModel toFileAbstractModel(String path, DiskEntry file) { + return new FileAbstractModel( + file.getFileInformation().getNameInformation(), + !file.getFileInformation().getStandardInformation().isDirectory(), + // We are using here the local TimeZone as a reference. If the remote system is under another TZ, this might cause issues + LocalDateTime.ofInstant(Instant.ofEpochMilli(file.getFileInformation().getBasicInformation().getLastWriteTime().toEpochMillis()), ZoneId.systemDefault()), + // We don't have the creation date + null, + // We are using here the local TimeZone as a reference. If the remote system is under another TZ, this might cause issues + LocalDateTime.ofInstant(Instant.ofEpochMilli(file.getFileInformation().getBasicInformation().getCreationTime().toEpochMillis()), ZoneId.systemDefault()), + FilenameUtils.getExtension(file.getFileInformation().getNameInformation()), + path, + path.concat("/").concat(file.getFileInformation().getNameInformation()), + file.getFileInformation().getStandardInformation().getAllocationSize(), + file.getSecurityInformation(Collections.singleton(SecurityInformation.OWNER_SECURITY_INFORMATION)).getOwnerSid().toString(), + file.getSecurityInformation(Collections.singleton(SecurityInformation.GROUP_SECURITY_INFORMATION)).getGroupSid().toString(), + file.getFileInformation().getAccessInformation().getAccessFlags()); + } + + @Override + public InputStream getInputStream(FileAbstractModel file) throws Exception { + return share.openFile(file.getFullpath(), EnumSet.of(AccessMask.GENERIC_READ), + null, + SMB2ShareAccess.ALL, + SMB2CreateDisposition.FILE_OPEN, + null).getInputStream(); + } + + @Override + public Collection getFiles(String dir) throws Exception { + + logger.debug("Listing local files from {}", dir); + List ls; + + Directory directory = share.openDirectory(dir, EnumSet.of(AccessMask.GENERIC_READ), + null, + SMB2ShareAccess.ALL, + SMB2CreateDisposition.FILE_OPEN, + null); + + ls = directory.list(); + if (ls == null) { + return null; + } + + Collection result = new ArrayList<>(ls.size()); + // Iterate other files + // We ignore here all files like . and .. + result.addAll(ls.stream().filter(file -> !".".equals(file.getFileName()) && + !"..".equals(file.getFileName())) + .map(file -> toFileAbstractModel(dir, share.open(dir + "/" + file.getFileName(), EnumSet.of(AccessMask.GENERIC_READ), + null, + SMB2ShareAccess.ALL, + SMB2CreateDisposition.FILE_OPEN, + null))) + .collect(Collectors.toList())); + + logger.debug("{} local files found", result.size()); + return result; + } + + @Override + public boolean exists(String dir) { + return share.fileExists(dir) || share.folderExists(dir); + } + + @Override + public void open() throws Exception { + share = openSMBConnection(fsSettings.getServer()); + } + + @Override + public void close() throws Exception { + share.close(); + client.close(); + } + + + private DiskShare openSMBConnection(Server server) throws IOException { + logger.debug("Opening SMB connection to {}@{}", server.getUsername(), server.getHostname()); + + SmbConfig smbConfig = SmbConfig.builder() + //SMB3.0 use BCSecurityProvider + .withSecurityProvider(new BCSecurityProvider()) + .build(); + + client = new SMBClient(smbConfig); + AuthenticationContext ac = new AuthenticationContext(server.getUsername(), server.getPassword().toCharArray(), server.getHostname()); + Connection connection = client.connect(server.getHostname()); + Session session = connection.authenticate(ac); + return (DiskShare) session.connectShare(fsSettings.getServer().getServerName()); + + } +} diff --git a/crawler/crawler-smb/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMBTest.java b/crawler/crawler-smb/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMBTest.java new file mode 100644 index 000000000..75815ea6b --- /dev/null +++ b/crawler/crawler-smb/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMBTest.java @@ -0,0 +1,45 @@ +package fr.pilato.elasticsearch.crawler.fs.crawler.smb; + +import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractModel; +import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; +import fr.pilato.elasticsearch.crawler.fs.settings.Server; +import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase; +import java.util.Collection; +import junit.framework.TestCase; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import org.junit.Ignore; +import org.junit.Test; + +public class FileAbstractorSMBTest extends AbstractFSCrawlerTestCase { + + @Test + @Ignore + public void testConnectToWindows() throws Exception { + String path = "test"; + String host = "10.211.55.7"; + String user = "lzwcyd"; + String pass = "123456"; + String serverName = "model"; + FsSettings fsSettings = FsSettings.builder("foo") + .setServer( + Server.builder() + .setHostname(host) + .setUsername(user) + .setPassword(pass) + .setServerName(serverName) + .build() + ) + .build(); + FileAbstractorSMB smb = new FileAbstractorSMB(fsSettings); + smb.open(); + boolean exists = smb.exists(path); + assertThat(exists, is(true)); + Collection files = smb.getFiles(path); + logger.debug("Found {} files", files.size()); + for (FileAbstractModel file : files) { + logger.debug(" - {}", file); + } + smb.close(); + } +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index fcc46e966..6dd7ae83c 100644 --- a/pom.xml +++ b/pom.xml @@ -22,6 +22,7 @@ crawler rest docs + crawler-smb FSCrawler https://github.com/dadoonet/fscrawler/ diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java index e7580e7d9..761fbc53a 100644 --- a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java @@ -21,13 +21,13 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Objects; public class Server { public static final class PROTOCOL { public static final String LOCAL = "local"; + public static final String SMB = "smb"; public static final String SSH = "ssh"; public static final String FTP = "ftp"; public static final int SSH_PORT = 22; @@ -38,12 +38,13 @@ public Server() { } - private Server(String hostname, int port, String username, String password, String protocol, String pemPath) { + private Server(String hostname, int port, String username, String password, String protocol, String serverName, String pemPath) { this.hostname = hostname; this.port = port; this.username = username; this.password = password; this.protocol = protocol; + this.serverName = serverName; this.pemPath = pemPath; } @@ -53,6 +54,7 @@ private Server(String hostname, int port, String username, String password, Stri @JsonIgnore private String password; private String protocol = PROTOCOL.LOCAL; + private String serverName; private String pemPath; public String getHostname() { @@ -97,6 +99,14 @@ public void setProtocol(String protocol) { this.protocol = protocol; } + public String getServerName() { + return serverName; + } + + public void setServerName(String serverName) { + this.serverName = serverName; + } + public String getPemPath() { return pemPath; } @@ -115,6 +125,7 @@ public static class Builder { private String username = null; private String password = null; private String protocol = PROTOCOL.LOCAL; + private String serverName = null; private String pemPath = null; public Builder setHostname(String hostname) { @@ -142,13 +153,18 @@ public Builder setProtocol(String protocol) { return this; } + public Builder setServerName(String serverName) { + this.serverName = serverName; + return this; + } + public Builder setPemPath(String pemPath) { this.pemPath = pemPath; return this; } public Server build() { - return new Server(hostname, port, username, password, protocol, pemPath); + return new Server(hostname, port, username, password, protocol, serverName, pemPath); } } @@ -164,6 +180,7 @@ public boolean equals(Object o) { if (!Objects.equals(username, server.username)) return false; // We can't really test the password as it may be obfuscated if (!Objects.equals(protocol, server.protocol)) return false; + if (!Objects.equals(serverName, server.serverName)) return false; return Objects.equals(pemPath, server.pemPath); } @@ -174,6 +191,7 @@ public int hashCode() { result = 31 * result + port; result = 31 * result + (username != null ? username.hashCode() : 0); result = 31 * result + (protocol != null ? protocol.hashCode() : 0); + result = 31 * result + (serverName != null ? serverName.hashCode() : 0); result = 31 * result + (pemPath != null ? pemPath.hashCode() : 0); return result; } @@ -184,6 +202,7 @@ public String toString() { ", port=" + port + ", username='" + username + '\'' + ", protocol='" + protocol + '\'' + + ", serverName='" + serverName + '\'' + ", pemPath='" + pemPath + '\'' + '}'; } From b8cf55dc12b77f1b35b9f5b7aa1d239285e63231 Mon Sep 17 00:00:00 2001 From: farmer <1153595464@qq.com> Date: Sat, 17 Jul 2021 23:26:07 +0800 Subject: [PATCH 11/26] feat(SMB): SMB3 Crawler --- core/pom.xml | 4 ++ .../elasticsearch/crawler/fs/FsParserSmb.java | 3 +- crawler/crawler-smb/pom.xml | 6 +++ .../fs/crawler/smb/FileAbstractorSMB.java | 46 +++++++++++++++---- .../fs/crawler/smb/FileAbstractorSMBTest.java | 8 ++-- crawler/pom.xml | 1 + .../crawler/fs/framework/FsCrawlerUtil.java | 28 ++++++----- pom.xml | 6 ++- .../fs/settings/FsCrawlerValidator.java | 17 ++++--- .../crawler/fs/settings/Server.java | 22 +-------- 10 files changed, 87 insertions(+), 54 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 8e18d5381..e4253b57f 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -77,6 +77,10 @@ fr.pilato.elasticsearch.crawler fscrawler-crawler-ssh + + fr.pilato.elasticsearch.crawler + fscrawler-crawler-smb + diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserSmb.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserSmb.java index fe9bc3cd5..b319b4a5f 100644 --- a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserSmb.java +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserSmb.java @@ -1,6 +1,7 @@ package fr.pilato.elasticsearch.crawler.fs; import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor; +import fr.pilato.elasticsearch.crawler.fs.crawler.smb.FileAbstractorSMB; import fr.pilato.elasticsearch.crawler.fs.crawler.ssh.FileAbstractorSSH; import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementService; @@ -16,6 +17,6 @@ public FsParserSmb(FsSettings fsSettings, Path config, FsCrawlerManagementServic @Override protected FileAbstractor buildFileAbstractor() { - return new FileAbstractorSSH(fsSettings); + return new FileAbstractorSMB(fsSettings); } } diff --git a/crawler/crawler-smb/pom.xml b/crawler/crawler-smb/pom.xml index 28ce9210c..a8043eb15 100644 --- a/crawler/crawler-smb/pom.xml +++ b/crawler/crawler-smb/pom.xml @@ -25,6 +25,12 @@ com.hierynomus smbj 0.11.1 + + + org.slf4j + slf4j-api + + diff --git a/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java b/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java index ac838d1f0..8bd103fcf 100644 --- a/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java +++ b/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java @@ -5,6 +5,7 @@ import com.hierynomus.msfscc.fileinformation.FileIdBothDirectoryInformation; import com.hierynomus.mssmb2.SMB2CreateDisposition; import com.hierynomus.mssmb2.SMB2ShareAccess; +import com.hierynomus.protocol.commons.EnumWithValue; import com.hierynomus.security.bc.BCSecurityProvider; import com.hierynomus.smbj.SMBClient; import com.hierynomus.smbj.SmbConfig; @@ -18,12 +19,14 @@ import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; import fr.pilato.elasticsearch.crawler.fs.settings.Server; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.commons.io.FilenameUtils; import org.apache.logging.log4j.LogManager; @@ -44,8 +47,19 @@ public FileAbstractorSMB(FsSettings fsSettings) { @Override public FileAbstractModel toFileAbstractModel(String path, DiskEntry file) { + + int permissions = 777; + + //TODO 修正权限 file.getFileInformation().getAccessInformation().getAccessFlags() + + EnumSet list = EnumWithValue.EnumUtils.toEnumSet(file.getFileInformation().getAccessInformation().getAccessFlags(), AccessMask.class); + + + //此处这样取文件/文件夹名的原因为：file.getFileInformation().getNameInformation() 取到的值永远为null + String fileName = file.getUncPath().substring(file.getUncPath().lastIndexOf("\\") + 1); + return new FileAbstractModel( - file.getFileInformation().getNameInformation(), + fileName, !file.getFileInformation().getStandardInformation().isDirectory(), // We are using here the local TimeZone as a reference. If the remote system is under another TZ, this might cause issues LocalDateTime.ofInstant(Instant.ofEpochMilli(file.getFileInformation().getBasicInformation().getLastWriteTime().toEpochMillis()), ZoneId.systemDefault()), @@ -55,20 +69,24 @@ public FileAbstractModel toFileAbstractModel(String path, DiskEntry file) { LocalDateTime.ofInstant(Instant.ofEpochMilli(file.getFileInformation().getBasicInformation().getCreationTime().toEpochMillis()), ZoneId.systemDefault()), FilenameUtils.getExtension(file.getFileInformation().getNameInformation()), path, - path.concat("/").concat(file.getFileInformation().getNameInformation()), + path.concat("/").concat(fileName), file.getFileInformation().getStandardInformation().getAllocationSize(), file.getSecurityInformation(Collections.singleton(SecurityInformation.OWNER_SECURITY_INFORMATION)).getOwnerSid().toString(), file.getSecurityInformation(Collections.singleton(SecurityInformation.GROUP_SECURITY_INFORMATION)).getGroupSid().toString(), - file.getFileInformation().getAccessInformation().getAccessFlags()); + permissions); } @Override public InputStream getInputStream(FileAbstractModel file) throws Exception { - return share.openFile(file.getFullpath(), EnumSet.of(AccessMask.GENERIC_READ), - null, - SMB2ShareAccess.ALL, - SMB2CreateDisposition.FILE_OPEN, - null).getInputStream(); + if (file.isFile()) { + return share.openFile(file.getFullpath(), EnumSet.of(AccessMask.GENERIC_READ), + null, + SMB2ShareAccess.ALL, + SMB2CreateDisposition.FILE_OPEN, + null).getInputStream(); + } else { + return new ByteArrayInputStream(file.getName().getBytes()); + } } @Override @@ -106,6 +124,11 @@ public Collection getFiles(String dir) throws Exception { @Override public boolean exists(String dir) { + if (dir.startsWith("//")) { + String[] path = dir.split("/"); + dir = dir.substring(3 + path[2].length() + path[3].length()); + logger.info("new dir : {}", dir); + } return share.fileExists(dir) || share.folderExists(dir); } @@ -127,13 +150,18 @@ private DiskShare openSMBConnection(Server server) throws IOException { SmbConfig smbConfig = SmbConfig.builder() //SMB3.0 use BCSecurityProvider .withSecurityProvider(new BCSecurityProvider()) + .withTimeout(12, TimeUnit.SECONDS) // Timeout sets Read, Write, and Transact timeouts (default is 60 seconds) + .withSoTimeout(18, TimeUnit.SECONDS) // Socket Timeout (default is 0 seconds, blocks forever) .build(); client = new SMBClient(smbConfig); AuthenticationContext ac = new AuthenticationContext(server.getUsername(), server.getPassword().toCharArray(), server.getHostname()); Connection connection = client.connect(server.getHostname()); Session session = connection.authenticate(ac); - return (DiskShare) session.connectShare(fsSettings.getServer().getServerName()); + String url = fsSettings.getFs().getUrl(); + // //6E64/model //6E64/model/test + String serverName = url.split("/")[3]; + return (DiskShare) session.connectShare(serverName); } } diff --git a/crawler/crawler-smb/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMBTest.java b/crawler/crawler-smb/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMBTest.java index 75815ea6b..5e302c1c2 100644 --- a/crawler/crawler-smb/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMBTest.java +++ b/crawler/crawler-smb/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMBTest.java @@ -1,6 +1,7 @@ package fr.pilato.elasticsearch.crawler.fs.crawler.smb; import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractModel; +import fr.pilato.elasticsearch.crawler.fs.settings.Fs; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; import fr.pilato.elasticsearch.crawler.fs.settings.Server; import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase; @@ -20,16 +21,17 @@ public void testConnectToWindows() throws Exception { String host = "10.211.55.7"; String user = "lzwcyd"; String pass = "123456"; - String serverName = "model"; + String url = "//Desktop/model"; FsSettings fsSettings = FsSettings.builder("foo") .setServer( Server.builder() .setHostname(host) .setUsername(user) .setPassword(pass) - .setServerName(serverName) .build() - ) + ).setFs(Fs.builder() + .setUrl(url) + .build()) .build(); FileAbstractorSMB smb = new FileAbstractorSMB(fsSettings); smb.open(); diff --git a/crawler/pom.xml b/crawler/pom.xml index 6d4f73f80..ee7874a63 100644 --- a/crawler/pom.xml +++ b/crawler/pom.xml @@ -18,6 +18,7 @@ crawler-fs crawler-ftp crawler-ssh + crawler-smb diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java index 65b9b15cc..62a7960e7 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java @@ -303,19 +303,25 @@ public static String computeRealPathName(String _dirname, String filename) { } public static String computeVirtualPathName(String rootPath, String realPath) { - String result = getPathSeparator(rootPath); - if (realPath.startsWith(rootPath) && realPath.length() > rootPath.length()) { - if (rootPath.equals("/")) { - // "/" is very common for FTP - result = realPath; - } else { - result = realPath.substring(rootPath.length()); - } + if (rootPath != null) { + rootPath = rootPath.replace("\\", "/"); + } + if (realPath != null) { + realPath = realPath.replace("\\", "/"); } - logger.debug("computeVirtualPathName({}, {}) = {}", rootPath, realPath, result); - return result; - } + String result = "/"; + if (realPath != null && rootPath != null && realPath.length() > rootPath.length()) { + if ("/".equals(rootPath)) { + result = realPath; + } else { + result = realPath.substring(rootPath.length()); + } + } + + logger.debug("computeVirtualPathName({}, {}) = {}", rootPath, realPath, result); + return result.startsWith("/") ? result : "/".concat(result); + } public static LocalDateTime getCreationTime(File file) { LocalDateTime time; diff --git a/pom.xml b/pom.xml index 6dd7ae83c..18020c7d4 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,6 @@ crawler rest docs - crawler-smb FSCrawler https://github.com/dadoonet/fscrawler/ @@ -564,6 +563,11 @@ fscrawler-crawler-ssh 2.7-SNAPSHOT + + fr.pilato.elasticsearch.crawler + fscrawler-crawler-smb + 2.7-SNAPSHOT + fr.pilato.elasticsearch.crawler fscrawler-tika diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java index eb661b30a..99b614efa 100644 --- a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java @@ -20,21 +20,20 @@ package fr.pilato.elasticsearch.crawler.fs.settings; import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.INDEX_SUFFIX_FOLDER; import fr.pilato.elasticsearch.crawler.fs.framework.OsValidator; -import org.apache.logging.log4j.Logger; - import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; - -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.INDEX_SUFFIX_FOLDER; +import org.apache.logging.log4j.Logger; public class FsCrawlerValidator { /** * Check if settings are valid. Note that settings can be updated by this method (fallback to defaults if not set) - * @param logger Needed to print warn/errors or info - * @param settings Settings we want to check - * @param rest true If Rest server should be started, so we check Rest settings + * + * @param logger Needed to print warn/errors or info + * @param settings Settings we want to check + * @param rest true If Rest server should be started, so we check Rest settings * @return true if we found fatal errors and should prevent from starting */ public static boolean validateSettings(Logger logger, FsSettings settings, boolean rest) { @@ -62,10 +61,10 @@ public static boolean validateSettings(Logger logger, FsSettings settings, boole // Checking protocol if (settings.getServer() != null) { if (!Server.PROTOCOL.LOCAL.equals(settings.getServer().getProtocol()) && - !Server.PROTOCOL.SSH.equals(settings.getServer().getProtocol()) && !Server.PROTOCOL.FTP.equals(settings.getServer().getProtocol())) { + !Server.PROTOCOL.SSH.equals(settings.getServer().getProtocol()) && !Server.PROTOCOL.SMB.equals(settings.getServer().getProtocol())) { // Non supported protocol logger.error(settings.getServer().getProtocol() + " is not supported yet. Please use " + - Server.PROTOCOL.LOCAL + " or " + Server.PROTOCOL.SSH + " or " + Server.PROTOCOL.FTP + ". Disabling crawler"); + Server.PROTOCOL.LOCAL + " or " + Server.PROTOCOL.SSH + " or " + Server.PROTOCOL.SMB + ". Disabling crawler"); return true; } diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java index 761fbc53a..a5324f7af 100644 --- a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java @@ -38,13 +38,12 @@ public Server() { } - private Server(String hostname, int port, String username, String password, String protocol, String serverName, String pemPath) { + private Server(String hostname, int port, String username, String password, String protocol, String pemPath) { this.hostname = hostname; this.port = port; this.username = username; this.password = password; this.protocol = protocol; - this.serverName = serverName; this.pemPath = pemPath; } @@ -54,7 +53,6 @@ private Server(String hostname, int port, String username, String password, Stri @JsonIgnore private String password; private String protocol = PROTOCOL.LOCAL; - private String serverName; private String pemPath; public String getHostname() { @@ -99,14 +97,6 @@ public void setProtocol(String protocol) { this.protocol = protocol; } - public String getServerName() { - return serverName; - } - - public void setServerName(String serverName) { - this.serverName = serverName; - } - public String getPemPath() { return pemPath; } @@ -125,7 +115,6 @@ public static class Builder { private String username = null; private String password = null; private String protocol = PROTOCOL.LOCAL; - private String serverName = null; private String pemPath = null; public Builder setHostname(String hostname) { @@ -153,10 +142,6 @@ public Builder setProtocol(String protocol) { return this; } - public Builder setServerName(String serverName) { - this.serverName = serverName; - return this; - } public Builder setPemPath(String pemPath) { this.pemPath = pemPath; @@ -164,7 +149,7 @@ public Builder setPemPath(String pemPath) { } public Server build() { - return new Server(hostname, port, username, password, protocol, serverName, pemPath); + return new Server(hostname, port, username, password, protocol, pemPath); } } @@ -180,7 +165,6 @@ public boolean equals(Object o) { if (!Objects.equals(username, server.username)) return false; // We can't really test the password as it may be obfuscated if (!Objects.equals(protocol, server.protocol)) return false; - if (!Objects.equals(serverName, server.serverName)) return false; return Objects.equals(pemPath, server.pemPath); } @@ -191,7 +175,6 @@ public int hashCode() { result = 31 * result + port; result = 31 * result + (username != null ? username.hashCode() : 0); result = 31 * result + (protocol != null ? protocol.hashCode() : 0); - result = 31 * result + (serverName != null ? serverName.hashCode() : 0); result = 31 * result + (pemPath != null ? pemPath.hashCode() : 0); return result; } @@ -202,7 +185,6 @@ public String toString() { ", port=" + port + ", username='" + username + '\'' + ", protocol='" + protocol + '\'' + - ", serverName='" + serverName + '\'' + ", pemPath='" + pemPath + '\'' + '}'; } From 5ee1e9c4e01bccc55594db6314aecb5c83a7b423 Mon Sep 17 00:00:00 2001 From: farmer <1153595464@qq.com> Date: Sun, 18 Apr 2021 17:15:03 +0800 Subject: [PATCH 12/26] =?UTF-8?q?fix(SMB):=20=E4=BF=AE=E6=AD=A3=E8=B7=AF?= =?UTF-8?q?=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fs/crawler/smb/FileAbstractorSMB.java | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java b/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java index 8bd103fcf..476626d28 100644 --- a/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java +++ b/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java @@ -79,7 +79,9 @@ public FileAbstractModel toFileAbstractModel(String path, DiskEntry file) { @Override public InputStream getInputStream(FileAbstractModel file) throws Exception { if (file.isFile()) { - return share.openFile(file.getFullpath(), EnumSet.of(AccessMask.GENERIC_READ), + String fullPath = file.getFullpath(); + fullPath = correctionPath(fullPath); + return share.openFile(fullPath, EnumSet.of(AccessMask.GENERIC_READ), null, SMB2ShareAccess.ALL, SMB2CreateDisposition.FILE_OPEN, @@ -92,7 +94,11 @@ public InputStream getInputStream(FileAbstractModel file) throws Exception { @Override public Collection getFiles(String dir) throws Exception { - logger.debug("Listing local files from {}", dir); + //修正路径 + dir = correctionPath(dir); + + + logger.debug("Listing smb files from {}", dir); List ls; Directory directory = share.openDirectory(dir, EnumSet.of(AccessMask.GENERIC_READ), @@ -109,9 +115,10 @@ public Collection getFiles(String dir) throws Exception { Collection result = new ArrayList<>(ls.size()); // Iterate other files // We ignore here all files like . and .. + String finalDir = dir; result.addAll(ls.stream().filter(file -> !".".equals(file.getFileName()) && !"..".equals(file.getFileName())) - .map(file -> toFileAbstractModel(dir, share.open(dir + "/" + file.getFileName(), EnumSet.of(AccessMask.GENERIC_READ), + .map(file -> toFileAbstractModel(finalDir, share.open(finalDir + "/" + file.getFileName(), EnumSet.of(AccessMask.GENERIC_READ), null, SMB2ShareAccess.ALL, SMB2CreateDisposition.FILE_OPEN, @@ -124,11 +131,7 @@ public Collection getFiles(String dir) throws Exception { @Override public boolean exists(String dir) { - if (dir.startsWith("//")) { - String[] path = dir.split("/"); - dir = dir.substring(3 + path[2].length() + path[3].length()); - logger.info("new dir : {}", dir); - } + dir = correctionPath(dir); return share.fileExists(dir) || share.folderExists(dir); } @@ -144,6 +147,15 @@ public void close() throws Exception { } + private String correctionPath(String dir) { + if (dir.startsWith("//")) { + String[] path = dir.split("/"); + dir = dir.substring(3 + path[2].length() + path[3].length()); + } + return dir; + } + + private DiskShare openSMBConnection(Server server) throws IOException { logger.debug("Opening SMB connection to {}@{}", server.getUsername(), server.getHostname()); From e65d76d77371469b21c1da0effff9cb58fda2480 Mon Sep 17 00:00:00 2001 From: lzwcyd Date: Sun, 18 Jul 2021 22:18:25 +0800 Subject: [PATCH 13/26] fix(SMB): Checking username/password --- .../elasticsearch/crawler/fs/settings/FsCrawlerValidator.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java index 99b614efa..373887474 100644 --- a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java @@ -77,6 +77,10 @@ public static boolean validateSettings(Logger logger, FsSettings settings, boole FsCrawlerUtil.isNullOrEmpty(settings.getServer().getUsername())) { logger.error("When using FTP, you need to set a username and probably a password. Disabling crawler"); return true; + } else if (Server.PROTOCOL.SMB.equals(settings.getServer().getProtocol()) && + FsCrawlerUtil.isNullOrEmpty(settings.getServer().getUsername())) { + logger.error("When using SMB, you need to set a username and probably a password. Disabling crawler"); + return true; } } From 3b0450e8faa60a84f37d4d2e7dadc033628a0624 Mon Sep 17 00:00:00 2001 From: lzwcyd Date: Sun, 18 Jul 2021 23:22:40 +0800 Subject: [PATCH 14/26] doc: smb --- docs/source/admin/fs/smb.rst | 44 ++++++++++++++++++++++++++++++++++++ docs/source/index.rst | 2 +- docs/source/user/options.rst | 1 + 3 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 docs/source/admin/fs/smb.rst diff --git a/docs/source/admin/fs/smb.rst b/docs/source/admin/fs/smb.rst new file mode 100644 index 000000000..71befa90a --- /dev/null +++ b/docs/source/admin/fs/smb.rst @@ -0,0 +1,44 @@ +.. _smb-settings: + +SMB settings +------------ + +You can index files remotely using SMB. + +Here is a list of SMB settings (under ``server.`` prefix): + ++-----------------------+-----------------------+-----------------------+ +| Name | Default value | Documentation | ++=======================+=======================+=======================+ +| ``server.hostname`` | ``null`` | Hostname | ++-----------------------+-----------------------+-----------------------+ +| ``server.username`` | ``anonymous`` | :ref:`smb_login` | ++-----------------------+-----------------------+-----------------------+ +| ``server.password`` | ``null`` | :ref:`smb_login` | ++-----------------------+-----------------------+-----------------------+ +| ``server.protocol`` | ``"local"`` | Set it to ``smb`` | ++-----------------------+-----------------------+-----------------------+ + +.. _smb_login: + +Username / Password +~~~~~~~~~~~~~~~~~~~ + +Let’s say you want to index from a remote server using SMB: + +- FS URL: ``/path/to/data/dir/on/server`` +- Server: ``mynode.mydomain.com`` +- Username: ``username`` (default to ``anonymous``) +- Password: ``password`` +- Protocol: ``smb`` (default to ``local``) + +.. code:: yaml + + name: "test" + fs: + url: "/path/to/data/dir/on/server" + server: + hostname: "mynode.mydomain.com" + username: "username" + password: "password" + protocol: "smb" diff --git a/docs/source/index.rst b/docs/source/index.rst index c32d9c9d7..738029d12 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -15,7 +15,7 @@ This crawler helps to index binary documents such as PDF, Open Office, MS Office **Main features**: * Local file system (or a mounted drive) crawling and index new files, update existing ones and removes old ones. -* Remote file system over SSH/FTP/SMB(WIP) crawling. +* Remote file system over SSH/FTP/SMB crawling. * REST interface to let you "upload" your binary documents to elasticsearch. .. note:: diff --git a/docs/source/user/options.rst b/docs/source/user/options.rst index c63ea50ee..2984a0f01 100644 --- a/docs/source/user/options.rst +++ b/docs/source/user/options.rst @@ -41,5 +41,6 @@ You will find more information about settings in the following sections: - :ref:`local-fs-settings` - :ref:`ssh-settings` - :ref:`ftp-settings` +- :ref:`smb-settings` - :ref:`elasticsearch-settings` From 59a065e1b6b8603f7830f8dec207f09625d689a4 Mon Sep 17 00:00:00 2001 From: lzwcyd Date: Mon, 19 Jul 2021 13:05:04 +0800 Subject: [PATCH 15/26] revert: FsParserAbstract --- .../elasticsearch/crawler/fs/FsParserAbstract.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java index 78ecc18a3..1ec2f9c4a 100644 --- a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java @@ -30,7 +30,6 @@ import fr.pilato.elasticsearch.crawler.fs.crawler.fs.FileAbstractorFile; import fr.pilato.elasticsearch.crawler.fs.framework.ByteSizeValue; import fr.pilato.elasticsearch.crawler.fs.framework.FSCrawlerLogger; -import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil; import fr.pilato.elasticsearch.crawler.fs.framework.OsValidator; import fr.pilato.elasticsearch.crawler.fs.framework.SignTool; import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; @@ -256,7 +255,7 @@ private void addFilesRecursively(FileAbstractor path, String filepath, LocalD logger.trace("FileAbstractModel = {}", child); String filename = child.getName(); - String virtualFileName = computeVirtualPathName(stats.getRootPath(), FsCrawlerUtil.computeRealPathName(filepath, filename)); + String virtualFileName = computeVirtualPathName(stats.getRootPath(), new File(filepath, filename).toString()); // https://github.com/dadoonet/fscrawler/issues/1 : Filter documents boolean isIndexable = isIndexable(child.isDirectory(), virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes()); @@ -318,7 +317,7 @@ private void addFilesRecursively(FileAbstractor path, String filepath, LocalD for (String esfile : esFiles) { logger.trace("Checking file [{}]", esfile); - String virtualFileName = computeVirtualPathName(stats.getRootPath(), FsCrawlerUtil.computeRealPathName(filepath, esfile)); + String virtualFileName = computeVirtualPathName(stats.getRootPath(), new File(filepath, esfile).toString()); if (isIndexable(false, virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes()) && !fsFiles.contains(esfile)) { logger.trace("Removing file [{}] in elasticsearch/workplace", esfile); @@ -333,7 +332,7 @@ private void addFilesRecursively(FileAbstractor path, String filepath, LocalD // for the delete folder for (String esfolder : esFolders) { - String virtualFileName = computeVirtualPathName(stats.getRootPath(), FsCrawlerUtil.computeRealPathName(filepath, esfolder)); + String virtualFileName = computeVirtualPathName(stats.getRootPath(), new File(filepath, esfolder).toString()); if (isIndexable(true, virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes())) { logger.trace("Checking directory [{}]", esfolder); if (!fsFolders.contains(esfolder)) { @@ -376,7 +375,7 @@ private void indexFile(FileAbstractModel fileAbstractModel, ScanStatistic stats, final long size = fileAbstractModel.getSize(); logger.debug("fetching content from [{}],[{}]", dirname, filename); - String fullFilename = FsCrawlerUtil.computeRealPathName(dirname, filename); + String fullFilename = new File(dirname, filename).toString(); try { // Create the Doc object (only needed when we have add_as_inner_object: true (default) or when we don't index json or xml) @@ -524,7 +523,7 @@ private void indexDirectory(String id, fr.pilato.elasticsearch.crawler.fs.beans. /** * Index a directory - * @param path complete path like "/", "/path/to/subdir", "/C:/dir", "//SOMEONE/dir" + * @param path complete path like /path/to/subdir */ private void indexDirectory(String path) throws Exception { fr.pilato.elasticsearch.crawler.fs.beans.Path pathObject = new fr.pilato.elasticsearch.crawler.fs.beans.Path(); @@ -570,4 +569,4 @@ private void esDelete(FsCrawlerService service, String index, String id) { } } -} +} \ No newline at end of file From 5cb779c5bca50532f536f8bdf8c7d4e0581544e3 Mon Sep 17 00:00:00 2001 From: lzwcyd Date: Mon, 19 Jul 2021 13:36:06 +0800 Subject: [PATCH 16/26] fix(SMB): modify method name --- .../crawler/fs/cli/FsCrawlerCli.java | 4 ++++ .../crawler/fs/crawler/smb/FileAbstractorSMB.java | 15 ++++++--------- docs/source/admin/fs/smb.rst | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java index 6cfcc7810..67b1c1954 100644 --- a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java +++ b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java @@ -219,6 +219,10 @@ public static void main(String[] args) throws Exception { } } + if (fsSettings.getServer().getProtocol().equals(PROTOCOL.SMB) && StringUtils.isEmpty(fsSettings.getServer().getUsername())) { + fsSettings.getServer().setUsername("Guest"); + } + if (fsSettings.getElasticsearch() == null) { fsSettings.setElasticsearch(Elasticsearch.DEFAULT()); } diff --git a/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java b/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java index 476626d28..fd4de23ca 100644 --- a/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java +++ b/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java @@ -80,7 +80,7 @@ public FileAbstractModel toFileAbstractModel(String path, DiskEntry file) { public InputStream getInputStream(FileAbstractModel file) throws Exception { if (file.isFile()) { String fullPath = file.getFullpath(); - fullPath = correctionPath(fullPath); + fullPath = getRelativePath(fullPath); return share.openFile(fullPath, EnumSet.of(AccessMask.GENERIC_READ), null, SMB2ShareAccess.ALL, @@ -94,10 +94,7 @@ public InputStream getInputStream(FileAbstractModel file) throws Exception { @Override public Collection getFiles(String dir) throws Exception { - //修正路径 - dir = correctionPath(dir); - - + dir = getRelativePath(dir); logger.debug("Listing smb files from {}", dir); List ls; @@ -131,8 +128,8 @@ public Collection getFiles(String dir) throws Exception { @Override public boolean exists(String dir) { - dir = correctionPath(dir); - return share.fileExists(dir) || share.folderExists(dir); + dir = getRelativePath(dir); + return share.folderExists(dir); } @Override @@ -147,7 +144,7 @@ public void close() throws Exception { } - private String correctionPath(String dir) { + private String getRelativePath(String dir) { if (dir.startsWith("//")) { String[] path = dir.split("/"); dir = dir.substring(3 + path[2].length() + path[3].length()); @@ -172,7 +169,7 @@ private DiskShare openSMBConnection(Server server) throws IOException { Session session = connection.authenticate(ac); String url = fsSettings.getFs().getUrl(); // //6E64/model //6E64/model/test - String serverName = url.split("/")[3]; + String serverName = url.split("/")[url.startsWith("//")?3:0]; return (DiskShare) session.connectShare(serverName); } diff --git a/docs/source/admin/fs/smb.rst b/docs/source/admin/fs/smb.rst index 71befa90a..0139eb9f6 100644 --- a/docs/source/admin/fs/smb.rst +++ b/docs/source/admin/fs/smb.rst @@ -12,7 +12,7 @@ Here is a list of SMB settings (under ``server.`` prefix): +=======================+=======================+=======================+ | ``server.hostname`` | ``null`` | Hostname | +-----------------------+-----------------------+-----------------------+ -| ``server.username`` | ``anonymous`` | :ref:`smb_login` | +| ``server.username`` | ``Guest`` | :ref:`smb_login` | +-----------------------+-----------------------+-----------------------+ | ``server.password`` | ``null`` | :ref:`smb_login` | +-----------------------+-----------------------+-----------------------+ From 102bf6a71dc5aa35a92679cc38c71b7387650cdc Mon Sep 17 00:00:00 2001 From: lzwcyd Date: Mon, 19 Jul 2021 13:39:27 +0800 Subject: [PATCH 17/26] revert: FsParserAbstract --- .../elasticsearch/crawler/fs/FsParserAbstract.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java index 1ec2f9c4a..c8e56b804 100644 --- a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java @@ -30,6 +30,7 @@ import fr.pilato.elasticsearch.crawler.fs.crawler.fs.FileAbstractorFile; import fr.pilato.elasticsearch.crawler.fs.framework.ByteSizeValue; import fr.pilato.elasticsearch.crawler.fs.framework.FSCrawlerLogger; +import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil; import fr.pilato.elasticsearch.crawler.fs.framework.OsValidator; import fr.pilato.elasticsearch.crawler.fs.framework.SignTool; import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; @@ -255,7 +256,7 @@ private void addFilesRecursively(FileAbstractor path, String filepath, LocalD logger.trace("FileAbstractModel = {}", child); String filename = child.getName(); - String virtualFileName = computeVirtualPathName(stats.getRootPath(), new File(filepath, filename).toString()); + String virtualFileName = computeVirtualPathName(stats.getRootPath(), FsCrawlerUtil.computeRealPathName(filepath, filename)); // https://github.com/dadoonet/fscrawler/issues/1 : Filter documents boolean isIndexable = isIndexable(child.isDirectory(), virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes()); @@ -317,7 +318,7 @@ private void addFilesRecursively(FileAbstractor path, String filepath, LocalD for (String esfile : esFiles) { logger.trace("Checking file [{}]", esfile); - String virtualFileName = computeVirtualPathName(stats.getRootPath(), new File(filepath, esfile).toString()); + String virtualFileName = computeVirtualPathName(stats.getRootPath(), FsCrawlerUtil.computeRealPathName(filepath, esfile)); if (isIndexable(false, virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes()) && !fsFiles.contains(esfile)) { logger.trace("Removing file [{}] in elasticsearch/workplace", esfile); @@ -332,7 +333,7 @@ private void addFilesRecursively(FileAbstractor path, String filepath, LocalD // for the delete folder for (String esfolder : esFolders) { - String virtualFileName = computeVirtualPathName(stats.getRootPath(), new File(filepath, esfolder).toString()); + String virtualFileName = computeVirtualPathName(stats.getRootPath(), FsCrawlerUtil.computeRealPathName(filepath, esfolder)); if (isIndexable(true, virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes())) { logger.trace("Checking directory [{}]", esfolder); if (!fsFolders.contains(esfolder)) { @@ -375,7 +376,7 @@ private void indexFile(FileAbstractModel fileAbstractModel, ScanStatistic stats, final long size = fileAbstractModel.getSize(); logger.debug("fetching content from [{}],[{}]", dirname, filename); - String fullFilename = new File(dirname, filename).toString(); + String fullFilename = FsCrawlerUtil.computeRealPathName(dirname, filename); try { // Create the Doc object (only needed when we have add_as_inner_object: true (default) or when we don't index json or xml) @@ -523,7 +524,7 @@ private void indexDirectory(String id, fr.pilato.elasticsearch.crawler.fs.beans. /** * Index a directory - * @param path complete path like /path/to/subdir + * @param path complete path like "/", "/path/to/subdir", "/C:/dir", "//SOMEONE/dir" */ private void indexDirectory(String path) throws Exception { fr.pilato.elasticsearch.crawler.fs.beans.Path pathObject = new fr.pilato.elasticsearch.crawler.fs.beans.Path(); From 2970873ee84704d92ca28b2c0e13b7495d7603d3 Mon Sep 17 00:00:00 2001 From: lzwcyd Date: Mon, 19 Jul 2021 14:26:55 +0800 Subject: [PATCH 18/26] test: FileAbstractorSMBTest --- .../fs/crawler/smb/FileAbstractorSMBTest.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/crawler/crawler-smb/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMBTest.java b/crawler/crawler-smb/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMBTest.java index 5e302c1c2..062480253 100644 --- a/crawler/crawler-smb/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMBTest.java +++ b/crawler/crawler-smb/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMBTest.java @@ -17,11 +17,11 @@ public class FileAbstractorSMBTest extends AbstractFSCrawlerTestCase { @Test @Ignore public void testConnectToWindows() throws Exception { - String path = "test"; - String host = "10.211.55.7"; + String[] paths = {"","folder","文件夹","folder/文件夹","文件夹/folder"}; + String host = "192.168.31.45"; String user = "lzwcyd"; String pass = "123456"; - String url = "//Desktop/model"; + String url = "//Desktop/win10_share_test"; FsSettings fsSettings = FsSettings.builder("foo") .setServer( Server.builder() @@ -35,12 +35,14 @@ public void testConnectToWindows() throws Exception { .build(); FileAbstractorSMB smb = new FileAbstractorSMB(fsSettings); smb.open(); - boolean exists = smb.exists(path); - assertThat(exists, is(true)); - Collection files = smb.getFiles(path); - logger.debug("Found {} files", files.size()); - for (FileAbstractModel file : files) { - logger.debug(" - {}", file); + for (String path : paths) { + boolean exists = smb.exists(path); + assertThat(exists, is(true)); + Collection files = smb.getFiles(path); + logger.debug("Found {} files", files.size()); + for (FileAbstractModel file : files) { + logger.debug(" - {}", file); + } } smb.close(); } From d0f28d7af65583e8367a4cbe6aeee9bfab99f8b1 Mon Sep 17 00:00:00 2001 From: lzwcyd Date: Tue, 20 Jul 2021 16:57:37 +0800 Subject: [PATCH 19/26] feat: support SMB2 --- .../fs/crawler/smb/FileAbstractorSMB.java | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java b/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java index fd4de23ca..e4d04ba8f 100644 --- a/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java +++ b/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java @@ -5,6 +5,7 @@ import com.hierynomus.msfscc.fileinformation.FileIdBothDirectoryInformation; import com.hierynomus.mssmb2.SMB2CreateDisposition; import com.hierynomus.mssmb2.SMB2ShareAccess; +import com.hierynomus.mssmb2.SMBApiException; import com.hierynomus.protocol.commons.EnumWithValue; import com.hierynomus.security.bc.BCSecurityProvider; import com.hierynomus.smbj.SMBClient; @@ -129,7 +130,7 @@ public Collection getFiles(String dir) throws Exception { @Override public boolean exists(String dir) { dir = getRelativePath(dir); - return share.folderExists(dir); + return share.folderExists(dir); } @Override @@ -156,20 +157,36 @@ private String getRelativePath(String dir) { private DiskShare openSMBConnection(Server server) throws IOException { logger.debug("Opening SMB connection to {}@{}", server.getUsername(), server.getHostname()); - SmbConfig smbConfig = SmbConfig.builder() - //SMB3.0 use BCSecurityProvider - .withSecurityProvider(new BCSecurityProvider()) - .withTimeout(12, TimeUnit.SECONDS) // Timeout sets Read, Write, and Transact timeouts (default is 60 seconds) - .withSoTimeout(18, TimeUnit.SECONDS) // Socket Timeout (default is 0 seconds, blocks forever) - .build(); - client = new SMBClient(smbConfig); + Session session; AuthenticationContext ac = new AuthenticationContext(server.getUsername(), server.getPassword().toCharArray(), server.getHostname()); - Connection connection = client.connect(server.getHostname()); - Session session = connection.authenticate(ac); + try { + logger.debug("Start trying to connect through SMB2"); + SmbConfig smbConfig = SmbConfig.builder() + .withTimeout(12, TimeUnit.SECONDS) // Timeout sets Read, Write, and Transact timeouts (default is 60 seconds) + .withSoTimeout(18, TimeUnit.SECONDS) // Socket Timeout (default is 0 seconds, blocks forever) + .build(); + + client = new SMBClient(smbConfig); + Connection connection = client.connect(server.getHostname()); + session = connection.authenticate(ac); + } catch (UnsupportedOperationException |SMBApiException e) { + logger.debug("Start trying to connect through SMB3"); + //close client + client.close(); + SmbConfig smbConfig = SmbConfig.builder() + //SMB3.0 use BCSecurityProvider + .withSecurityProvider(new BCSecurityProvider()) + .withTimeout(12, TimeUnit.SECONDS) // Timeout sets Read, Write, and Transact timeouts (default is 60 seconds) + .withSoTimeout(18, TimeUnit.SECONDS) // Socket Timeout (default is 0 seconds, blocks forever) + .build(); + client = new SMBClient(smbConfig); + Connection connection = client.connect(server.getHostname()); + session = connection.authenticate(ac); + } String url = fsSettings.getFs().getUrl(); // //6E64/model //6E64/model/test - String serverName = url.split("/")[url.startsWith("//")?3:0]; + String serverName = url.split("/")[url.startsWith("//") ? 3 : 0]; return (DiskShare) session.connectShare(serverName); } From d4df1b60d03e8c143074b455be047e1513e6dcb9 Mon Sep 17 00:00:00 2001 From: lzwcyd Date: Thu, 22 Jul 2021 00:32:57 +0800 Subject: [PATCH 20/26] fix: cli default server checking --- .../pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java index 67b1c1954..b0c4a208a 100644 --- a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java +++ b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java @@ -217,10 +217,10 @@ public static void main(String[] args) throws Exception { if (fsSettings.getServer().getProtocol().equals(PROTOCOL.FTP) && StringUtils.isEmpty(fsSettings.getServer().getUsername())) { fsSettings.getServer().setUsername("anonymous"); } - } - if (fsSettings.getServer().getProtocol().equals(PROTOCOL.SMB) && StringUtils.isEmpty(fsSettings.getServer().getUsername())) { - fsSettings.getServer().setUsername("Guest"); + if (fsSettings.getServer().getProtocol().equals(PROTOCOL.SMB) && StringUtils.isEmpty(fsSettings.getServer().getUsername())) { + fsSettings.getServer().setUsername("Guest"); + } } if (fsSettings.getElasticsearch() == null) { From d07a83b059a415f4ec77ed996618a4bdad3db61e Mon Sep 17 00:00:00 2001 From: lzwcyd <56526731+lzwcyd@users.noreply.github.com> Date: Sat, 24 Jul 2021 22:55:11 +0800 Subject: [PATCH 21/26] fix: smb crawler get file extension (#16) --- .../fs/crawler/smb/FileAbstractorSMB.java | 8 +-- .../crawler/fs/framework/FsCrawlerUtil.java | 52 +++++++------------ .../fs/framework/FsCrawlerUtilTest.java | 22 ++++---- 3 files changed, 35 insertions(+), 47 deletions(-) diff --git a/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java b/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java index e4d04ba8f..551e559f7 100644 --- a/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java +++ b/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java @@ -18,6 +18,7 @@ import com.hierynomus.smbj.share.DiskShare; import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractModel; import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor; +import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; import fr.pilato.elasticsearch.crawler.fs.settings.Server; import java.io.ByteArrayInputStream; @@ -57,7 +58,8 @@ public FileAbstractModel toFileAbstractModel(String path, DiskEntry file) { //此处这样取文件/文件夹名的原因为：file.getFileInformation().getNameInformation() 取到的值永远为null - String fileName = file.getUncPath().substring(file.getUncPath().lastIndexOf("\\") + 1); + String fileName = FsCrawlerUtil.getFileName(file.getUncPath()); + String extension = FilenameUtils.getExtension(fileName); return new FileAbstractModel( fileName, @@ -68,7 +70,7 @@ public FileAbstractModel toFileAbstractModel(String path, DiskEntry file) { null, // We are using here the local TimeZone as a reference. If the remote system is under another TZ, this might cause issues LocalDateTime.ofInstant(Instant.ofEpochMilli(file.getFileInformation().getBasicInformation().getCreationTime().toEpochMillis()), ZoneId.systemDefault()), - FilenameUtils.getExtension(file.getFileInformation().getNameInformation()), + extension, path, path.concat("/").concat(fileName), file.getFileInformation().getStandardInformation().getAllocationSize(), @@ -170,7 +172,7 @@ private DiskShare openSMBConnection(Server server) throws IOException { client = new SMBClient(smbConfig); Connection connection = client.connect(server.getHostname()); session = connection.authenticate(ac); - } catch (UnsupportedOperationException |SMBApiException e) { + } catch (UnsupportedOperationException | SMBApiException e) { logger.debug("Start trying to connect through SMB3"); //close client client.close(); diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java index 62a7960e7..1a67b1566 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java @@ -20,48 +20,22 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.net.ftp.FTPFile; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.nio.file.CopyOption; -import java.nio.file.FileAlreadyExistsException; -import java.nio.file.FileSystem; -import java.nio.file.FileSystems; -import java.nio.file.FileVisitOption; -import java.nio.file.FileVisitResult; -import java.nio.file.Files; -import java.nio.file.NoSuchFileException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.StandardCopyOption; -import java.nio.file.attribute.BasicFileAttributeView; -import java.nio.file.attribute.BasicFileAttributes; -import java.nio.file.attribute.FileOwnerAttributeView; -import java.nio.file.attribute.PosixFileAttributeView; -import java.nio.file.attribute.PosixFileAttributes; -import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.*; +import java.nio.file.attribute.*; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import java.util.Date; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.Set; -import java.util.TimeZone; +import java.util.*; import java.util.regex.Pattern; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.net.ftp.FTPFile; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; public class FsCrawlerUtil { public static final String INDEX_SUFFIX_FOLDER = "_folder"; @@ -475,6 +449,16 @@ public static int getFilePermissions(final FTPFile file) { } } + /** + * This method is used to get the file/folder name from the path, only for SMB Crawler + * because file.getFileInformation().getNameInformation() always equal null + * @param uncPath file uncPath + * @return fileName + */ + public static String getFileName(String uncPath) { + return uncPath.substring(uncPath.lastIndexOf("\\") + 1); + } + private static int toOctalPermission(boolean read, boolean write, boolean execute) { return (read ? 4 : 0) + (write ? 2 : 0) + (execute ? 1 : 0); } diff --git a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java index eaf1295f4..80b922542 100644 --- a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java +++ b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java @@ -19,6 +19,7 @@ package fr.pilato.elasticsearch.crawler.fs.framework; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.*; import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase; import java.time.LocalDateTime; import java.util.Arrays; @@ -48,16 +49,6 @@ import org.mockftpserver.fake.filesystem.UnixFakeFileSystem; import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.computeRealPathName; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.computeVirtualPathName; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.extractMajorVersion; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.extractMinorVersion; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getFileExtension; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getFilePermissions; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getGroupName; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getOwnerName; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.isFileSizeUnderLimit; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.localDateTimeToDate; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.isEmptyOrNullString; @@ -140,6 +131,17 @@ public void testFTPFilePermissions() throws IOException { fakeFtpServer.stop(); } + @Test + public void testGetFileName(){ + assertThat(getFileName("\\test\\test.txt"),is("test.txt")); + assertThat(getFileName("\\test.txt"),is("test.txt")); + assertThat(getFileName("test.txt"),is("test.txt")); + assertThat(getFileName("\\test\\test\\test.txt"),is("test.txt")); + assertThat(getFileName("\\test\\test"),is("test")); + assertThat(getFileName("\\test"),is("test")); + assertThat(getFileName("test"),is("test")); + } + @Test public void testIsFileSizeUnderLimit() { assertThat(isFileSizeUnderLimit(ByteSizeValue.parseBytesSizeValue("1mb"), 1), is(true)); From 59b589d09a548be9bbef54022e3b85da9695e4a5 Mon Sep 17 00:00:00 2001 From: helsonxiao Date: Mon, 26 Jul 2021 16:27:09 +0800 Subject: [PATCH 22/26] fix: bad styles --- README.md | 2 +- .../crawler/fs/FsCrawlerImpl.java | 11 ++- .../crawler/fs/FsParserAbstract.java | 2 +- docs/source/admin/fs/smb.rst | 2 +- .../crawler/fs/framework/FsCrawlerUtil.java | 70 ++++++++++++------- .../fs/framework/FsCrawlerUtilTest.java | 12 +++- .../fs/settings/FsCrawlerValidator.java | 17 ++--- .../crawler/fs/settings/Server.java | 2 +- 8 files changed, 77 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 769be2e74..4fb5c4e06 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This crawler helps to index binary documents such as PDF, Open Office, MS Office **Main features**: * Local file system (or a mounted drive) crawling and index new files, update existing ones and removes old ones. -* Remote file system over SSH/FTP/SMB(WIP) crawling. +* Remote file system over SSH/FTP/SMB crawling. * REST interface to let you "upload" your binary documents to elasticsearch. You need to install a version matching your Elasticsearch version: diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java index 8151120ab..97b9b0e2c 100644 --- a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsCrawlerImpl.java @@ -21,15 +21,20 @@ import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil; import fr.pilato.elasticsearch.crawler.fs.framework.TimeValue; -import fr.pilato.elasticsearch.crawler.fs.service.*; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentService; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentServiceElasticsearchImpl; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentServiceWorkplaceSearchImpl; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementService; +import fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerManagementServiceElasticsearchImpl; import fr.pilato.elasticsearch.crawler.fs.settings.FsCrawlerValidator; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; import fr.pilato.elasticsearch.crawler.fs.settings.Server; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; /** * @author dadoonet (David Pilato) diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java index c8e56b804..78ecc18a3 100644 --- a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java @@ -570,4 +570,4 @@ private void esDelete(FsCrawlerService service, String index, String id) { } } -} \ No newline at end of file +} diff --git a/docs/source/admin/fs/smb.rst b/docs/source/admin/fs/smb.rst index 0139eb9f6..facc31445 100644 --- a/docs/source/admin/fs/smb.rst +++ b/docs/source/admin/fs/smb.rst @@ -12,7 +12,7 @@ Here is a list of SMB settings (under ``server.`` prefix): +=======================+=======================+=======================+ | ``server.hostname`` | ``null`` | Hostname | +-----------------------+-----------------------+-----------------------+ -| ``server.username`` | ``Guest`` | :ref:`smb_login` | +| ``server.username`` | ``Guest`` | :ref:`smb_login` | +-----------------------+-----------------------+-----------------------+ | ``server.password`` | ``null`` | :ref:`smb_login` | +-----------------------+-----------------------+-----------------------+ diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java index 1a67b1566..e0eedebf4 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java @@ -20,22 +20,48 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.net.ftp.FTPFile; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.URI; -import java.nio.file.*; -import java.nio.file.attribute.*; +import java.nio.charset.StandardCharsets; +import java.nio.file.CopyOption; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.FileVisitOption; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.BasicFileAttributeView; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileOwnerAttributeView; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.PosixFileAttributes; +import java.nio.file.attribute.PosixFilePermission; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; -import java.util.*; +import java.util.Date; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.TimeZone; import java.util.regex.Pattern; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.net.ftp.FTPFile; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; public class FsCrawlerUtil { public static final String INDEX_SUFFIX_FOLDER = "_folder"; @@ -277,26 +303,20 @@ public static String computeRealPathName(String _dirname, String filename) { } public static String computeVirtualPathName(String rootPath, String realPath) { - if (rootPath != null) { - rootPath = rootPath.replace("\\", "/"); - } - if (realPath != null) { - realPath = realPath.replace("\\", "/"); - } - - String result = "/"; - if (realPath != null && rootPath != null && realPath.length() > rootPath.length()) { - if ("/".equals(rootPath)) { - result = realPath; - } else { - result = realPath.substring(rootPath.length()); - } + String result = getPathSeparator(rootPath); + if (realPath.startsWith(rootPath) && realPath.length() > rootPath.length()) { + if (rootPath.equals("/")) { + // "/" is very common for FTP + result = realPath; + } else { + result = realPath.substring(rootPath.length()); } - - logger.debug("computeVirtualPathName({}, {}) = {}", rootPath, realPath, result); - return result.startsWith("/") ? result : "/".concat(result); } + logger.debug("computeVirtualPathName({}, {}) = {}", rootPath, realPath, result); + return result; + } + public static LocalDateTime getCreationTime(File file) { LocalDateTime time; try { diff --git a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java index 80b922542..412611313 100644 --- a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java +++ b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java @@ -19,7 +19,6 @@ package fr.pilato.elasticsearch.crawler.fs.framework; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.*; import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase; import java.time.LocalDateTime; import java.util.Arrays; @@ -49,6 +48,17 @@ import org.mockftpserver.fake.filesystem.UnixFakeFileSystem; import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.computeRealPathName; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.computeVirtualPathName; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.extractMajorVersion; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.extractMinorVersion; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getFileExtension; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getFileName; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getFilePermissions; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getGroupName; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getOwnerName; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.isFileSizeUnderLimit; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.localDateTimeToDate; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.isEmptyOrNullString; diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java index 373887474..4b6f086b4 100644 --- a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java @@ -20,20 +20,21 @@ package fr.pilato.elasticsearch.crawler.fs.settings; import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.INDEX_SUFFIX_FOLDER; import fr.pilato.elasticsearch.crawler.fs.framework.OsValidator; +import org.apache.logging.log4j.Logger; + import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; -import org.apache.logging.log4j.Logger; + +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.INDEX_SUFFIX_FOLDER; public class FsCrawlerValidator { /** * Check if settings are valid. Note that settings can be updated by this method (fallback to defaults if not set) - * - * @param logger Needed to print warn/errors or info - * @param settings Settings we want to check - * @param rest true If Rest server should be started, so we check Rest settings + * @param logger Needed to print warn/errors or info + * @param settings Settings we want to check + * @param rest true If Rest server should be started, so we check Rest settings * @return true if we found fatal errors and should prevent from starting */ public static boolean validateSettings(Logger logger, FsSettings settings, boolean rest) { @@ -61,10 +62,10 @@ public static boolean validateSettings(Logger logger, FsSettings settings, boole // Checking protocol if (settings.getServer() != null) { if (!Server.PROTOCOL.LOCAL.equals(settings.getServer().getProtocol()) && - !Server.PROTOCOL.SSH.equals(settings.getServer().getProtocol()) && !Server.PROTOCOL.SMB.equals(settings.getServer().getProtocol())) { + !Server.PROTOCOL.SSH.equals(settings.getServer().getProtocol()) && !Server.PROTOCOL.FTP.equals(settings.getServer().getProtocol()) && !Server.PROTOCOL.SMB.equals(settings.getServer().getProtocol())) { // Non supported protocol logger.error(settings.getServer().getProtocol() + " is not supported yet. Please use " + - Server.PROTOCOL.LOCAL + " or " + Server.PROTOCOL.SSH + " or " + Server.PROTOCOL.SMB + ". Disabling crawler"); + Server.PROTOCOL.LOCAL + " or " + Server.PROTOCOL.SSH + " or " + Server.PROTOCOL.FTP + " or " + Server.PROTOCOL.SMB + ". Disabling crawler"); return true; } diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java index a5324f7af..5bf059f84 100644 --- a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/Server.java @@ -21,6 +21,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; + import java.util.Objects; public class Server { @@ -142,7 +143,6 @@ public Builder setProtocol(String protocol) { return this; } - public Builder setPemPath(String pemPath) { this.pemPath = pemPath; return this; From f77c15c7e603428d34b013a0603488828d5d6ae3 Mon Sep 17 00:00:00 2001 From: lzwcyd Date: Sat, 31 Jul 2021 13:49:05 +0800 Subject: [PATCH 23/26] fix: smb crawler url --- .../crawler/fs/crawler/smb/FileAbstractorSMB.java | 5 ++--- .../crawler/fs/framework/FsCrawlerUtil.java | 9 +++++++++ .../crawler/fs/framework/FsCrawlerUtilTest.java | 7 +++++++ .../crawler/fs/settings/FsCrawlerValidator.java | 9 +++++++++ 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java b/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java index 551e559f7..332d0cac8 100644 --- a/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java +++ b/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java @@ -117,7 +117,7 @@ public Collection getFiles(String dir) throws Exception { // We ignore here all files like . and .. String finalDir = dir; result.addAll(ls.stream().filter(file -> !".".equals(file.getFileName()) && - !"..".equals(file.getFileName())) + !"..".equals(file.getFileName())) .map(file -> toFileAbstractModel(finalDir, share.open(finalDir + "/" + file.getFileName(), EnumSet.of(AccessMask.GENERIC_READ), null, SMB2ShareAccess.ALL, @@ -187,8 +187,7 @@ private DiskShare openSMBConnection(Server server) throws IOException { session = connection.authenticate(ac); } String url = fsSettings.getFs().getUrl(); - // //6E64/model //6E64/model/test - String serverName = url.split("/")[url.startsWith("//") ? 3 : 0]; + String serverName = FsCrawlerUtil.getServerName(url); return (DiskShare) session.connectShare(serverName); } diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java index e0eedebf4..4a3901f35 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java @@ -688,4 +688,13 @@ public static String extractMajorVersion(String version) { public static String extractMinorVersion(String version) { return version.split("\\.")[1]; } + + /** + * obtain server name through server url（SMB） + * @param url serverUrl + * @return serverName + */ + public static String getServerName(String url) { + return url.split("/")[url.startsWith("//") ? 3 : url.startsWith("/") ? 1 : 0]; + } } diff --git a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java index 412611313..5d4aac4d6 100644 --- a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java +++ b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java @@ -57,6 +57,7 @@ import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getFilePermissions; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getGroupName; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getOwnerName; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getServerName; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.isFileSizeUnderLimit; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.localDateTimeToDate; import static org.hamcrest.MatcherAssert.assertThat; @@ -292,4 +293,10 @@ public void testLocalDateToDate() { LocalDateTime now = LocalDateTime.now(); logger.info("Current Time [{}] in [{}] is actually [{}]", now, TimeZone.getDefault().getDisplayName(), localDateTimeToDate(now)); } + + @Test + public void testGetServerName(){ + assertThat(getServerName("//desttop-123/test"), is("test")); + } + } diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java index 4b6f086b4..80a00cb72 100644 --- a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java @@ -83,6 +83,15 @@ public static boolean validateSettings(Logger logger, FsSettings settings, boole logger.error("When using SMB, you need to set a username and probably a password. Disabling crawler"); return true; } + + if (Server.PROTOCOL.SMB.equals(settings.getServer().getProtocol())){ + String url = settings.getFs().getUrl(); + String[] path = url.split("/"); + if (!url.startsWith("//") || path.length < 4 || url.endsWith("/")){ + logger.error("When using SMB, The url format should be '//DesktopNameOrIp/shareName' and cannot end with'/' . Disabling crawler"); + return true; + } + } } // Checking Checksum Algorithm From 7da7f8b60e51f82af48847d1240bc557d030461f Mon Sep 17 00:00:00 2001 From: lzwcyd Date: Sat, 31 Jul 2021 15:33:37 +0800 Subject: [PATCH 24/26] fix: handle SMB relative path --- .../fs/crawler/smb/FileAbstractorSMB.java | 26 ++++++------------- .../crawler/fs/framework/FsCrawlerUtil.java | 16 ++++++++++++ .../fs/framework/FsCrawlerUtilTest.java | 11 ++++++++ .../fs/settings/FsCrawlerValidator.java | 9 ------- 4 files changed, 35 insertions(+), 27 deletions(-) diff --git a/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java b/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java index 332d0cac8..ed3b3a3e2 100644 --- a/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java +++ b/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java @@ -80,10 +80,10 @@ public FileAbstractModel toFileAbstractModel(String path, DiskEntry file) { } @Override - public InputStream getInputStream(FileAbstractModel file) throws Exception { + public InputStream getInputStream(FileAbstractModel file) { if (file.isFile()) { String fullPath = file.getFullpath(); - fullPath = getRelativePath(fullPath); + fullPath = FsCrawlerUtil.getRelativePath(fullPath); return share.openFile(fullPath, EnumSet.of(AccessMask.GENERIC_READ), null, SMB2ShareAccess.ALL, @@ -95,13 +95,13 @@ public InputStream getInputStream(FileAbstractModel file) throws Exception { } @Override - public Collection getFiles(String dir) throws Exception { + public Collection getFiles(String dir) { - dir = getRelativePath(dir); - logger.debug("Listing smb files from {}", dir); + String relativeDir = FsCrawlerUtil.getRelativePath(dir); + logger.debug("Listing smb files from {}", relativeDir); List ls; - Directory directory = share.openDirectory(dir, EnumSet.of(AccessMask.GENERIC_READ), + Directory directory = share.openDirectory(relativeDir, EnumSet.of(AccessMask.GENERIC_READ), null, SMB2ShareAccess.ALL, SMB2CreateDisposition.FILE_OPEN, @@ -115,10 +115,9 @@ public Collection getFiles(String dir) throws Exception { Collection result = new ArrayList<>(ls.size()); // Iterate other files // We ignore here all files like . and .. - String finalDir = dir; result.addAll(ls.stream().filter(file -> !".".equals(file.getFileName()) && !"..".equals(file.getFileName())) - .map(file -> toFileAbstractModel(finalDir, share.open(finalDir + "/" + file.getFileName(), EnumSet.of(AccessMask.GENERIC_READ), + .map(file -> toFileAbstractModel(dir, share.open(relativeDir + "/" + file.getFileName(), EnumSet.of(AccessMask.GENERIC_READ), null, SMB2ShareAccess.ALL, SMB2CreateDisposition.FILE_OPEN, @@ -131,7 +130,7 @@ public Collection getFiles(String dir) throws Exception { @Override public boolean exists(String dir) { - dir = getRelativePath(dir); + dir = FsCrawlerUtil.getRelativePath(dir); return share.folderExists(dir); } @@ -147,15 +146,6 @@ public void close() throws Exception { } - private String getRelativePath(String dir) { - if (dir.startsWith("//")) { - String[] path = dir.split("/"); - dir = dir.substring(3 + path[2].length() + path[3].length()); - } - return dir; - } - - private DiskShare openSMBConnection(Server server) throws IOException { logger.debug("Opening SMB connection to {}@{}", server.getUsername(), server.getHostname()); diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java index 4a3901f35..f54fa7555 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java @@ -697,4 +697,20 @@ public static String extractMinorVersion(String version) { public static String getServerName(String url) { return url.split("/")[url.startsWith("//") ? 3 : url.startsWith("/") ? 1 : 0]; } + + + /** + * get relative path (SMB) //desktopName/shareName //desktopName/shareName/test /shareName /shareName/test + * @param dir dir + * @return relative path + */ + public static String getRelativePath(String dir) { + String[] path = dir.split("/"); + if (dir.startsWith("//")) { + dir = dir.substring(3 + path[2].length() + path[3].length()); + } else if (dir.startsWith("/") && path.length >= 2) { + dir = dir.substring(1 + path[1].length()); + } + return dir; + } } diff --git a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java index 5d4aac4d6..98b2daea7 100644 --- a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java +++ b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java @@ -58,6 +58,7 @@ import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getGroupName; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getOwnerName; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getServerName; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.getRelativePath; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.isFileSizeUnderLimit; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.localDateTimeToDate; import static org.hamcrest.MatcherAssert.assertThat; @@ -299,4 +300,14 @@ public void testGetServerName(){ assertThat(getServerName("//desttop-123/test"), is("test")); } + @Test + public void testGetRelativePath(){ + assertThat(getRelativePath("//desktopName/shareName"), is("")); + assertThat(getRelativePath("//desktopName/shareName/test"), is("/test")); + assertThat(getRelativePath("//desktopName/shareName/test.txt"), is("/test.txt")); + assertThat(getRelativePath("//desktopName/shareName/folder/test.txt"), is("/folder/test.txt")); + assertThat(getRelativePath("/shareName"), is("")); + assertThat(getRelativePath("/shareName/test"), is("/test")); + } + } diff --git a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java index 80a00cb72..4b6f086b4 100644 --- a/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java +++ b/settings/src/main/java/fr/pilato/elasticsearch/crawler/fs/settings/FsCrawlerValidator.java @@ -83,15 +83,6 @@ public static boolean validateSettings(Logger logger, FsSettings settings, boole logger.error("When using SMB, you need to set a username and probably a password. Disabling crawler"); return true; } - - if (Server.PROTOCOL.SMB.equals(settings.getServer().getProtocol())){ - String url = settings.getFs().getUrl(); - String[] path = url.split("/"); - if (!url.startsWith("//") || path.length < 4 || url.endsWith("/")){ - logger.error("When using SMB, The url format should be '//DesktopNameOrIp/shareName' and cannot end with'/' . Disabling crawler"); - return true; - } - } } // Checking Checksum Algorithm From bfdfd52335a0e68a36d20ed204f780fbcf51017c Mon Sep 17 00:00:00 2001 From: lzwcyd Date: Sat, 31 Jul 2021 19:11:49 +0800 Subject: [PATCH 25/26] fix: SMB catch SMBApiException --- .../fs/crawler/smb/FileAbstractorSMB.java | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java b/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java index ed3b3a3e2..0385487dc 100644 --- a/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java +++ b/crawler/crawler-smb/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/smb/FileAbstractorSMB.java @@ -27,7 +27,11 @@ import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumSet; +import java.util.List; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import org.apache.commons.io.FilenameUtils; @@ -80,15 +84,21 @@ public FileAbstractModel toFileAbstractModel(String path, DiskEntry file) { } @Override - public InputStream getInputStream(FileAbstractModel file) { + public InputStream getInputStream(FileAbstractModel file) throws IOException { if (file.isFile()) { String fullPath = file.getFullpath(); fullPath = FsCrawlerUtil.getRelativePath(fullPath); - return share.openFile(fullPath, EnumSet.of(AccessMask.GENERIC_READ), - null, - SMB2ShareAccess.ALL, - SMB2CreateDisposition.FILE_OPEN, - null).getInputStream(); + + try { + return share.openFile(fullPath, EnumSet.of(AccessMask.GENERIC_READ), + null, + SMB2ShareAccess.ALL, + SMB2CreateDisposition.FILE_OPEN, + null).getInputStream(); + } catch (SMBApiException e) { + logger.error("SMB client can not retrieve stream for {} , e: {}", fullPath , e); + throw new IOException(String.format("SMB client can not retrieve stream for [%s] ",fullPath)); + } } else { return new ByteArrayInputStream(file.getName().getBytes()); } From d3a39bb9d92915c29edfe985a2409a2fbf537f70 Mon Sep 17 00:00:00 2001 From: helson Date: Sun, 1 Aug 2021 00:18:21 +0800 Subject: [PATCH 26/26] fix: ftp permission & tests (#22) --- README.md | 2 +- .../crawler/fs/FsParserAbstract.java | 24 ++--- .../crawler/fs/crawler/ftp/FTPUtils.java | 37 ++++++++ .../fs/crawler/ftp/FileAbstractorFTP.java | 16 ++-- .../fs/crawler/ftp/FileAbstractorFTPTest.java | 88 ++++++++++++++++--- distribution/pom.xml | 1 + docs/source/admin/fs/ssh.rst | 6 +- docs/source/dev/build.rst | 8 +- docs/source/index.rst | 2 +- framework/pom.xml | 11 --- .../crawler/fs/framework/FsCrawlerUtil.java | 33 +------ .../fs/framework/FsCrawlerUtilTest.java | 56 ------------ integration-tests/it-common/pom.xml | 5 ++ .../elasticsearch/FsCrawlerTestFTPIT.java | 75 +++++++++------- pom.xml | 41 ++++----- 15 files changed, 220 insertions(+), 185 deletions(-) create mode 100644 crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FTPUtils.java diff --git a/README.md b/README.md index fa4abc1eb..00b94402f 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ This crawler helps to index binary documents such as PDF, Open Office, MS Office **Main features**: * Local file system (or a mounted drive) crawling and index new files, update existing ones and removes old ones. -* Remote file system over SSH/FTP/SMB(WIP) crawling. +* Remote file system over SSH/FTP crawling. * REST interface to let you "upload" your binary documents to elasticsearch. You need to install a version matching your Elasticsearch version: diff --git a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java index 3cc39cc3a..6908ec8dc 100644 --- a/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java +++ b/core/src/main/java/fr/pilato/elasticsearch/crawler/fs/FsParserAbstract.java @@ -257,7 +257,7 @@ private void addFilesRecursively(FileAbstractor path, String filepath, LocalD logger.trace("FileAbstractModel = {}", child); String filename = child.getName(); - String virtualFileName = computeVirtualPathName(stats.getRootPath(), FsCrawlerUtil.computeRealPathName(filepath, filename)); + String virtualFileName = computeVirtualPathName(stats.getRootPath(), computeRealPathName(filepath, filename)); // https://github.com/dadoonet/fscrawler/issues/1 : Filter documents boolean isIndexable = isIndexable(child.isDirectory(), virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes()); @@ -274,9 +274,9 @@ private void addFilesRecursively(FileAbstractor path, String filepath, LocalD indexFile(child, stats, filepath, fsSettings.getFs().isIndexContent() || fsSettings.getFs().isStoreSource() ? path.getInputStream(child) : null, child.getSize()); stats.addFile(); - } catch (java.io.FileNotFoundException e) { + } catch (Exception e) { if (fsSettings.getFs().isContinueOnError()) { - logger.warn("Unable to open Input Stream for {}, skipping...: {}", filename, e.getMessage()); + logger.warn("Unable to index {}, skipping...: {}", filename, e.getMessage()); } else { throw e; } @@ -319,7 +319,7 @@ private void addFilesRecursively(FileAbstractor path, String filepath, LocalD for (String esfile : esFiles) { logger.trace("Checking file [{}]", esfile); - String virtualFileName = computeVirtualPathName(stats.getRootPath(), FsCrawlerUtil.computeRealPathName(filepath, esfile)); + String virtualFileName = computeVirtualPathName(stats.getRootPath(), computeRealPathName(filepath, esfile)); if (isIndexable(false, virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes()) && !fsFiles.contains(esfile)) { logger.trace("Removing file [{}] in elasticsearch/workplace", esfile); @@ -334,7 +334,7 @@ private void addFilesRecursively(FileAbstractor path, String filepath, LocalD // for the delete folder for (String esfolder : esFolders) { - String virtualFileName = computeVirtualPathName(stats.getRootPath(), FsCrawlerUtil.computeRealPathName(filepath, esfolder)); + String virtualFileName = computeVirtualPathName(stats.getRootPath(), computeRealPathName(filepath, esfolder)); if (isIndexable(true, virtualFileName, fsSettings.getFs().getIncludes(), fsSettings.getFs().getExcludes())) { logger.trace("Checking directory [{}]", esfolder); if (!fsFolders.contains(esfolder)) { @@ -377,7 +377,7 @@ private void indexFile(FileAbstractModel fileAbstractModel, ScanStatistic stats, final long size = fileAbstractModel.getSize(); logger.debug("fetching content from [{}],[{}]", dirname, filename); - String fullFilename = FsCrawlerUtil.computeRealPathName(dirname, filename); + String fullFilename = computeRealPathName(dirname, filename); try { // Create the Doc object (only needed when we have add_as_inner_object: true (default) or when we don't index json or xml) @@ -496,11 +496,11 @@ private void indexFile(FileAbstractModel fileAbstractModel, ScanStatistic stats, } } - private String generateIdFromFilename(String _filename, String _filepath) throws NoSuchAlgorithmException { - String filepathForId = _filepath.replace("\\", "/"); - String filename = _filename.replace("\\", "").replace("/", ""); - String fullFilename = filepathForId.endsWith("/") ? filepathForId.concat(filename) : filepathForId.concat("/").concat(filename); - return fsSettings.getFs().isFilenameAsId() ? filename : SignTool.sign(fullFilename); + private String generateIdFromFilename(String filename, String filepath) throws NoSuchAlgorithmException { + String filepathForId = filepath.replace("\\", "/"); + String filenameForId = filename.replace("\\", "").replace("/", ""); + String idSource = filepathForId.endsWith("/") ? filepathForId.concat(filenameForId) : filepathForId.concat("/").concat(filenameForId); + return fsSettings.getFs().isFilenameAsId() ? filename : SignTool.sign(idSource); } private String read(InputStream input) throws IOException { @@ -525,7 +525,7 @@ private void indexDirectory(String id, Folder folder) throws IOException { /** * Index a directory - * @param path complete path like "/", "/path/to/subdir", "/C:/dir", "//SOMEONE/dir" + * @param path complete path like "/", "/path/to/subdir", "C:\\dir", "C:/dir", "/C:/dir", "//SOMEONE/dir" */ private void indexDirectory(String path) throws Exception { String name = path.substring(path.lastIndexOf(pathSeparator) + 1); diff --git a/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FTPUtils.java b/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FTPUtils.java new file mode 100644 index 000000000..96cda2ba8 --- /dev/null +++ b/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FTPUtils.java @@ -0,0 +1,37 @@ +package fr.pilato.elasticsearch.crawler.fs.crawler.ftp; + +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.toOctalPermission; + +import org.apache.commons.net.ftp.FTPFile; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class FTPUtils { + private static final Logger logger = LogManager.getLogger(FTPUtils.class); + + /** + * Determines FTPFile permissions. + */ + public static int getFilePermissions(final FTPFile file) { + try { + int user = toOctalPermission( + file.hasPermission(FTPFile.USER_ACCESS, FTPFile.READ_PERMISSION), + file.hasPermission(FTPFile.USER_ACCESS, FTPFile.WRITE_PERMISSION), + file.hasPermission(FTPFile.USER_ACCESS, FTPFile.EXECUTE_PERMISSION)); + int group = toOctalPermission( + file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.READ_PERMISSION), + file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.WRITE_PERMISSION), + file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.EXECUTE_PERMISSION)); + int others = toOctalPermission( + file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.READ_PERMISSION), + file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.WRITE_PERMISSION), + file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.EXECUTE_PERMISSION)); + + return user * 100 + group * 10 + others; + } catch (Exception e) { + logger.warn("Failed to determine 'permissions' of {}: {}", file, e.getMessage()); + return -1; + } + } + +} diff --git a/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java b/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java index c211f30bf..cfbe7e4aa 100644 --- a/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java +++ b/crawler/crawler-ftp/src/main/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTP.java @@ -21,7 +21,6 @@ import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractModel; import fr.pilato.elasticsearch.crawler.fs.crawler.FileAbstractor; -import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; import fr.pilato.elasticsearch.crawler.fs.settings.Server; import java.io.IOException; @@ -97,25 +96,30 @@ public FileAbstractModel toFileAbstractModel(String path, FTPFile file) { file.getSize(), file.getUser(), file.getGroup(), - FsCrawlerUtil.getFilePermissions(file)); + FTPUtils.getFilePermissions(file)); } @Override - public InputStream getInputStream(FileAbstractModel file) throws Exception { + public InputStream getInputStream(FileAbstractModel file) throws IOException { String fullPath = file.getFullpath(); if (isUtf8) { fullPath = new String(fullPath.getBytes(StandardCharsets.UTF_8), FTP.DEFAULT_CONTROL_ENCODING); } else { fullPath = new String(fullPath.getBytes(ALTERNATIVE_ENCODING), FTP.DEFAULT_CONTROL_ENCODING); } + InputStream inputStream = ftp.retrieveFileStream(fullPath); - ftp.completePendingCommand(); - return inputStream; + if (inputStream != null) { + ftp.completePendingCommand(); + return inputStream; + } else { + throw new IOException(String.format("FTP client can not retrieve stream for [%s]", file.getFullpath())); + } } @Override public Collection getFiles(String dir) throws IOException { - logger.debug("Listing local files from {}", dir); + logger.debug("Listing files from {}", dir); if (isUtf8) { dir = new String(dir.getBytes(StandardCharsets.UTF_8), FTP.DEFAULT_CONTROL_ENCODING); } else { diff --git a/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java b/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java index 21dfff30c..1b893f379 100644 --- a/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java +++ b/crawler/crawler-ftp/src/test/java/fr/pilato/elasticsearch/crawler/fs/crawler/ftp/FileAbstractorFTPTest.java @@ -26,10 +26,13 @@ import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; import fr.pilato.elasticsearch.crawler.fs.settings.Server; import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase; +import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; import org.apache.commons.io.IOUtils; import org.junit.After; import org.junit.Before; @@ -40,36 +43,45 @@ import org.mockftpserver.fake.filesystem.DirectoryEntry; import org.mockftpserver.fake.filesystem.FileEntry; import org.mockftpserver.fake.filesystem.FileSystem; +import org.mockftpserver.fake.filesystem.Permissions; import org.mockftpserver.fake.filesystem.UnixFakeFileSystem; public class FileAbstractorFTPTest extends AbstractFSCrawlerTestCase { private FakeFtpServer fakeFtpServer; - private final String home = "/home"; + private final String nestedDir = "/nested"; + private final String permissionDir = "/permission"; private final String user = "user"; - private final String pass = "password"; + private final String pass = "pass"; @Before public void setup() { - // it doesn't support utf-8 fakeFtpServer = new FakeFtpServer(); fakeFtpServer.setServerControlPort(5968); - fakeFtpServer.addUserAccount(new UserAccount(user, pass, home)); + fakeFtpServer.addUserAccount(new UserAccount(user, pass, "/")); FileSystem fileSystem = new UnixFakeFileSystem(); - fileSystem.add(new DirectoryEntry(home)); - fileSystem.add(new FileEntry(home + "/foo.txt", "文件名不支持中文")); - fileSystem.add(new FileEntry(home + "/bar.txt", "bar")); + fileSystem.add(new DirectoryEntry(nestedDir)); + fileSystem.add(new FileEntry(nestedDir + "/foo.txt", "文件名不支持中文")); + fileSystem.add(new FileEntry(nestedDir + "/bar.txt", "filename doesn't support utf-8")); - fileSystem.add(new DirectoryEntry(home + "/buzz")); - fileSystem.add(new FileEntry(home + "/buzz/hello.txt", "hello")); - fileSystem.add(new FileEntry(home + "/buzz/world.txt", "world")); + fileSystem.add(new DirectoryEntry(nestedDir + "/buzz")); + fileSystem.add(new FileEntry(nestedDir + "/buzz/hello.txt", "hello")); + fileSystem.add(new FileEntry(nestedDir + "/buzz/world.txt", "world")); + + fileSystem.add(new DirectoryEntry(permissionDir)); + FileEntry fileAllPermissions = new FileEntry(permissionDir + "/all.txt", "123"); + fileAllPermissions.setPermissions(Permissions.ALL); + fileSystem.add(fileAllPermissions); + FileEntry fileNonePermissions = new FileEntry(permissionDir + "/none.txt", "456"); + fileNonePermissions.setPermissions(Permissions.NONE); + fileSystem.add(fileNonePermissions); fakeFtpServer.setFileSystem(fileSystem); fakeFtpServer.start(); } @After - public void teardown() { + public void shutDown() { fakeFtpServer.stop(); } @@ -89,9 +101,9 @@ public void testConnectToFakeFTPServer() throws Exception { FileAbstractorFTP ftp = new FileAbstractorFTP(fsSettings); ftp.open(); - boolean exists = ftp.exists(home); + boolean exists = ftp.exists(nestedDir); assertThat(exists, is(true)); - Collection files = ftp.getFiles(home); + Collection files = ftp.getFiles(nestedDir); assertThat(files.size(), is(3)); for (FileAbstractModel file : files) { @@ -117,6 +129,11 @@ public void testConnectToFakeFTPServer() throws Exception { ftp.close(); } + /** + * FakeFtpServer doesn't support utf-8 + * You have to adapt this test to your own system + * So this test is disabled by default + */ @Test @Ignore public void testConnectToFTPServer() throws Exception { String path = "/中文目录"; @@ -160,4 +177,49 @@ public void testConnectToFTPServer() throws Exception { ftp.close(); } + + @Test + public void testFTPFilePermissions() throws IOException { + int port = fakeFtpServer.getServerControlPort(); + FsSettings fsSettings = FsSettings.builder("fake") + .setServer( + Server.builder() + .setHostname("localhost") + .setUsername(user) + .setPassword(pass) + .setPort(port) + .build() + ) + .build(); + + FileAbstractorFTP ftp = new FileAbstractorFTP(fsSettings); + ftp.open(); + + Collection files = ftp.getFiles(permissionDir); + assertThat(files.size(), is(2)); + List filenames = files.stream().map(FileAbstractModel::getName).collect(Collectors.toList()); + assertThat(filenames.contains("all.txt"), is(true)); + assertThat(filenames.contains("none.txt"), is(true)); + for (FileAbstractModel file : files) { + if (file.getName().equals("all.txt")) { + assertThat(file.getPermissions(), is(777)); + try (InputStream inputStream = ftp.getInputStream(file)) { + String content = IOUtils.toString(inputStream, StandardCharsets.UTF_8); + logger.debug(" - {}: {}", file.getName(), content); + } + } else if (file.getName().equals("none.txt")) { + assertThat(file.getPermissions(), is(0)); + boolean errorOccurred = false; + try (InputStream ignored = ftp.getInputStream(file)) { + logger.error(ignored); + } catch (IOException e) { + errorOccurred = true; + logger.error(e.getMessage()); + } + assertThat(errorOccurred, is(true)); + } + } + + ftp.close(); + } } diff --git a/distribution/pom.xml b/distribution/pom.xml index c021f95d7..898a411b2 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -21,6 +21,7 @@ + ${env.DOCKER_SKIP} build dadoonet diff --git a/docs/source/admin/fs/ssh.rst b/docs/source/admin/fs/ssh.rst index a534bab1b..48f7683cd 100644 --- a/docs/source/admin/fs/ssh.rst +++ b/docs/source/admin/fs/ssh.rst @@ -94,8 +94,10 @@ To specify the drive, you need to use the following format: password: "password" protocol: "ssh" -Windows shared folders -~~~~~~~~~~~~~~ +Windows shared folder +~~~~~~~~~~~~~~~~~~~~~ + +When using Windows shared folder, you need to use the following format: .. code:: yaml diff --git a/docs/source/dev/build.rst b/docs/source/dev/build.rst index 42c048211..cc23c7017 100644 --- a/docs/source/dev/build.rst +++ b/docs/source/dev/build.rst @@ -51,12 +51,12 @@ But you need first to specify the Maven profile to use and rebuild the project. * ``es-7x`` for Elasticsearch 7.x * ``es-6x`` for Elasticsearch 6.x -Run specific module tests from your Terminal -"""""""""""""""""""""""""""""" +Run a specific test from your Terminal +"""""""""""""""""""""""""""""""""""""" -To run integration tests for a specific module, just run:: +To run a specific integration test, just run:: - mvn test -am -DfailIfNoTests=false -pl [module_name_or_folder_path] + mvn verify -am -Dtests.class=fr.pilato.elasticsearch.crawler.fs.test.integration.CLASS_NAME -Dtests.method="METHOD_NAME" Run tests with an external cluster """""""""""""""""""""""""""""""""" diff --git a/docs/source/index.rst b/docs/source/index.rst index c32d9c9d7..b5cfb3759 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -15,7 +15,7 @@ This crawler helps to index binary documents such as PDF, Open Office, MS Office **Main features**: * Local file system (or a mounted drive) crawling and index new files, update existing ones and removes old ones. -* Remote file system over SSH/FTP/SMB(WIP) crawling. +* Remote file system over SSH/FTP crawling. * REST interface to let you "upload" your binary documents to elasticsearch. .. note:: diff --git a/framework/pom.xml b/framework/pom.xml index bddd6a6f1..176472c21 100644 --- a/framework/pom.xml +++ b/framework/pom.xml @@ -39,17 +39,6 @@ commons-io - - - commons-net - commons-net - - - org.mockftpserver - MockFtpServer - test - - com.fasterxml.jackson.core diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java index 65b9b15cc..32d7c0dd6 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtil.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; -import org.apache.commons.net.ftp.FTPFile; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -30,7 +29,6 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; -import java.nio.charset.StandardCharsets; import java.nio.file.CopyOption; import java.nio.file.FileAlreadyExistsException; import java.nio.file.FileSystem; @@ -444,32 +442,7 @@ public static int getFilePermissions(final File file) { } } - /** - * Determines FTPFile permissions. - */ - public static int getFilePermissions(final FTPFile file) { - try { - int user = toOctalPermission( - file.hasPermission(FTPFile.USER_ACCESS, FTPFile.READ_PERMISSION), - file.hasPermission(FTPFile.USER_ACCESS, FTPFile.WRITE_PERMISSION), - file.hasPermission(FTPFile.USER_ACCESS, FTPFile.EXECUTE_PERMISSION)); - int group = toOctalPermission( - file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.READ_PERMISSION), - file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.WRITE_PERMISSION), - file.hasPermission(FTPFile.GROUP_ACCESS, FTPFile.EXECUTE_PERMISSION)); - int others = toOctalPermission( - file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.READ_PERMISSION), - file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.WRITE_PERMISSION), - file.hasPermission(FTPFile.WORLD_ACCESS, FTPFile.EXECUTE_PERMISSION)); - - return user * 100 + group * 10 + others; - } catch (Exception e) { - logger.warn("Failed to determine 'permissions' of {}: {}", file, e.getMessage()); - return -1; - } - } - - private static int toOctalPermission(boolean read, boolean write, boolean execute) { + public static int toOctalPermission(boolean read, boolean write, boolean execute) { return (read ? 4 : 0) + (write ? 2 : 0) + (execute ? 1 : 0); } @@ -625,7 +598,9 @@ public static void createDirIfMissing(Path root) { if (Files.notExists(root)) { Files.createDirectory(root); } - } catch (IOException ignored) { } + } catch (IOException ignored) { + logger.error("Failed to create config dir"); + } } /** diff --git a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java index eaf1295f4..a7c214051 100644 --- a/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java +++ b/framework/src/test/java/fr/pilato/elasticsearch/crawler/fs/framework/FsCrawlerUtilTest.java @@ -21,13 +21,7 @@ import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase; import java.time.LocalDateTime; -import java.util.Arrays; -import java.util.List; import java.util.TimeZone; -import java.util.stream.Collectors; -import org.apache.commons.net.ftp.FTPClient; -import org.apache.commons.net.ftp.FTPFile; -import org.apache.commons.net.ftp.FTPReply; import org.junit.BeforeClass; import org.junit.Test; @@ -39,13 +33,6 @@ import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; import java.util.Set; -import org.mockftpserver.fake.FakeFtpServer; -import org.mockftpserver.fake.UserAccount; -import org.mockftpserver.fake.filesystem.DirectoryEntry; -import org.mockftpserver.fake.filesystem.FileEntry; -import org.mockftpserver.fake.filesystem.FileSystem; -import org.mockftpserver.fake.filesystem.Permissions; -import org.mockftpserver.fake.filesystem.UnixFakeFileSystem; import static com.carrotsearch.randomizedtesting.RandomizedTest.randomIntBetween; import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.computeRealPathName; @@ -97,49 +84,6 @@ public void testPermissions() { assertThat(permissions, is(700)); } - @Test - public void testFTPFilePermissions() throws IOException { - String user = "user"; - String password = "password"; - FakeFtpServer fakeFtpServer = new FakeFtpServer(); - fakeFtpServer.setServerControlPort(5968); - fakeFtpServer.addUserAccount(new UserAccount(user, password, "/data")); - FileSystem fileSystem = new UnixFakeFileSystem(); - fileSystem.add(new DirectoryEntry("/data")); - FileEntry fileAllPermissions = new FileEntry("/data/all.txt", "123"); - fileAllPermissions.setPermissions(Permissions.ALL); - fileSystem.add(fileAllPermissions); - FileEntry fileNonePermissions = new FileEntry("/data/none.txt", "456"); - fileNonePermissions.setPermissions(Permissions.NONE); - fileSystem.add(fileNonePermissions); - fakeFtpServer.setFileSystem(fileSystem); - fakeFtpServer.start(); - - FTPClient ftp = new FTPClient(); - ftp.connect("localhost", fakeFtpServer.getServerControlPort()); - int reply = ftp.getReplyCode(); - if (!FTPReply.isPositiveCompletion(reply)) { - ftp.disconnect(); - throw new IOException("Exception in connecting to FTP Server"); - } - ftp.login(user, password); - - FTPFile[] files = ftp.listFiles("/data"); - List filenames = Arrays.stream(files).map(FTPFile::getName).collect(Collectors.toList()); - assertThat(filenames.contains("all.txt"), is(true)); - assertThat(filenames.contains("none.txt"), is(true)); - for (FTPFile file : files) { - if (file.getName().equals("all.txt")) { - assertThat(getFilePermissions(file), is(777)); - } else if (file.getName().equals("none.txt")) { - assertThat(getFilePermissions(file), is(0)); - } - } - - ftp.disconnect(); - fakeFtpServer.stop(); - } - @Test public void testIsFileSizeUnderLimit() { assertThat(isFileSizeUnderLimit(ByteSizeValue.parseBytesSizeValue("1mb"), 1), is(true)); diff --git a/integration-tests/it-common/pom.xml b/integration-tests/it-common/pom.xml index ceeb5174a..952cdddde 100644 --- a/integration-tests/it-common/pom.xml +++ b/integration-tests/it-common/pom.xml @@ -31,6 +31,11 @@ fr.pilato.elasticsearch.crawler fscrawler-test-documents + + org.mockftpserver + MockFtpServer + compile + + + com.jcraft jsch 0.1.55 + + + commons-net + commons-net + 3.8.0 + + + org.mockftpserver + MockFtpServer + 2.8.0 + test + + + org.slf4j + slf4j-api + + + + com.beust @@ -930,25 +950,6 @@ 2.11.0 - - - commons-net - commons-net - 3.8.0 - - - org.mockftpserver - MockFtpServer - 2.8.0 - test - - - org.slf4j - slf4j-api - - - - org.apache.httpcomponents